* GUI resource

* MemIf interface for VPP
* Better netmodel integration
* Draft documentation
* New tutorials
* Improved monitoring and error handling
* Refactored IP addresses and prefixes representation
* Improved image mgmt for LXD
* Various bugfixes and code refactoring

Change-Id: I90da6cf7b5716bc7deb6bf4e24d3f9f01b5a9b0f
Signed-off-by: Marcel Enguehard <mengueha+fdio@cisco.com>

3 files changed, 83 insertions, 62 deletions

diff --git a/vicn/resource/ns3/emulated_channel.py b/vicn/resource/ns3/emulated_channel.py
index 5f61960c..3c090a9e 100644
--- a/vicn/resource/ns3/emulated_channel.py
+++ b/vicn/resource/ns3/emulated_channel.py
@@ -16,30 +16,33 @@
 # limitations under the License.
 #
 
+import functools
 import logging
 import random
 
+from netmodel.model.key                 import Key
 from netmodel.model.type                import Integer
 from netmodel.util.socket		import check_port
 from vicn.core.address_mgr              import AddressManager
-from vicn.core.attribute      	        import Attribute, Multiplicity
+from vicn.core.attribute                import Attribute, Multiplicity
 from vicn.core.exception                import ResourceNotFound
 from vicn.core.requirement              import Requirement
 from vicn.core.resource                 import BaseResource
 from vicn.core.resource_mgr             import wait_resources
 from vicn.core.task                     import inline_task, async_task, task
 from vicn.core.task                     import BashTask, run_task
+from vicn.core.task                     import inherit_parent
 from vicn.resource.channel              import Channel
-from vicn.resource.linux.application    import LinuxApplication as Application
+from vicn.resource.linux.application    import LinuxApplication
 from vicn.resource.linux.net_device     import NetDevice
 from vicn.resource.linux.tap_device     import TapDevice
-from vicn.resource.linux.veth_pair      import VethPair
+from vicn.resource.linux.veth_pair_lxc  import VethPairLxc
 from vicn.resource.lxd.lxc_container    import LxcContainer
 from vicn.resource.node                 import Node
 
 log = logging.getLogger(__name__)
 
-class EmulatedChannel(Channel, Application):
+class EmulatedChannel(Channel, LinuxApplication):
     """EmulatedChannel resource
 
     This resources serves as a base class for wireless channels emulated by
@@ -56,7 +59,7 @@ class EmulatedChannel(Channel, Application):
         traffic and prevent loops on the bridge.
       - We also need that all interfaces related to ap and stations are created
         before we run the commandline (currently, dynamically adding/removing
-        AP and stations is not supported by the emulator). This is made 
+        AP and stations is not supported by the emulator). This is made
         possible thanks to the key=True parameter, which makes sure the
         attributes are processed before the __create__ is called.
 
@@ -66,11 +69,15 @@ class EmulatedChannel(Channel, Application):
 
     __resource_type__ = BaseResource
 
-    ap = Attribute(Node, description = 'AP', key = True)
+    ap = Attribute(Node, description = 'AP')
     stations = Attribute(Node, description = 'List of stations',
-            multiplicity = Multiplicity.OneToMany, key = True)
+            multiplicity = Multiplicity.OneToMany)
     control_port = Attribute(Integer,
             description = 'Control port for the simulation')
+    nb_base_stations = Attribute(Integer, description='Number of nodes emulated by the AP',
+            default=1)
+
+    __key__ = Key(ap, stations)
 
     # Overloaded attributes
     node = Attribute(requirements = [
@@ -101,20 +108,29 @@ class EmulatedChannel(Channel, Application):
     # Resource lifecycle
     #--------------------------------------------------------------------------
 
+    @inherit_parent
+    def __initialize__(self):
+        return self.__set_ap() > self.__set_stations()
+
+    @inherit_parent
     @inline_task
     def __get__(self):
         raise ResourceNotFound
 
+
+    @inherit_parent
     def __create__(self):
         # NOTE: http://stackoverflow.com/questions/21141352/python-subprocess-
         #       calling-a-script-which-runs-a-background-process-hanging
         # The output of the background scripts is still going to the same file
         # descriptor as the child script, thus the parent script waits for it
         # to finish.
-        cmd = '(' + self.__app_name__ + ' ' + self._get_cmdline_params() + \
-            '>/dev/null 2>&1) &'
-        return BashTask(self.node, cmd)
+        def get_cmdline(channel):
+            return '(' + channel.__app_name__ + ' ' + \
+                  channel._get_cmdline_params() + '>/dev/null 2>&1) &'
+        return BashTask(self.node, functools.partial(get_cmdline, self))
 
+    @inherit_parent
     def __delete__(self):
         raise NotImplementedError
 
@@ -123,9 +139,8 @@ class EmulatedChannel(Channel, Application):
     #--------------------------------------------------------------------------
 
     @async_task
-    async def _set_ap(self, ap=None):
-        if ap is None:
-            ap = self.ap
+    async def __set_ap(self):
+        ap = self.ap
         if ap is None:
             log.info('Ignored setting ap to None...')
             return
@@ -133,13 +148,13 @@ class EmulatedChannel(Channel, Application):
         # Add a WiFi interface for the AP...
         interfaces = list()
         if isinstance(ap, LxcContainer):
-            # Ideally, We need to create a VethPair for each station
+            # Ideally, We need to create a VethPairLxc for each station
             # This should be monitored for the total channel bw
             host = NetDevice(node = ap.node,
                 device_name='vhh-' + ap.name + '-' + self.name,
                 monitored = False,
                 managed = False)
-            self._ap_if = VethPair(node = self.ap,
+            self._ap_if = VethPairLxc(node = self.ap,
                     name          = 'vh-' + ap.name + '-' + self.name,
                     device_name   = 'vh-' + ap.name + '-' + self.name,
                     host          = host,
@@ -171,37 +186,35 @@ class EmulatedChannel(Channel, Application):
         # Add interfaces to bridge
         vlan = AddressManager().get('vlan', self, tag='ap')
 
-        # AS the container has created the VethPair already without Vlan, we
+        # AS the container has created the VethPairLxc already without Vlan, we
         # need to delete and recreate it
         task = self.node.bridge._remove_interface(self._ap_bridged)
         await run_task(task, self._state.manager)
         task = self.node.bridge._add_interface(self._ap_bridged, vlan = vlan)
         await run_task(task, self._state.manager)
 
+        task = self.node.bridge._remove_interface(self._ap_tap)
+        await run_task(task, self._state.manager)
         task = self.node.bridge._add_interface(self._ap_tap, vlan = vlan)
         await run_task(task, self._state.manager)
 
     @inline_task
-    def _get_ap(self):
+    def __get_ap(self):
         return {'ap': None}
 
     @inline_task
-    def _get_stations(self):
+    def __get_stations(self):
         return {'stations': list()}
 
     @async_task
-    async def _set_stations(self, stations=None):
-        print('adding stations...')
-        if stations is None:
-            stations = self.stations
-
-        for station in stations:
+    async def __set_stations(self):
+        for station in self.stations:
             await self._add_station(station)
 
-    def _add_stations(self, stations):
+    def __add_stations(self, stations):
         raise NotImplementedError
 
     @inline_task
-    def _remove_stations(self, station):
+    def __remove_stations(self, station):
         raise NotImplementedError
 
diff --git a/vicn/resource/ns3/emulated_lte_channel.py b/vicn/resource/ns3/emulated_lte_channel.py
index bf0f7097..0847a403 100644
--- a/vicn/resource/ns3/emulated_lte_channel.py
+++ b/vicn/resource/ns3/emulated_lte_channel.py
@@ -16,11 +16,15 @@
 # limitations under the License.
 #
 
+import math
+
 from vicn.core.address_mgr              import AddressManager
-from vicn.core.resource_mgr             import wait_resources
-from vicn.core.task                     import run_task
+from vicn.core.resource_mgr             import wait_resources, wait_resource_task
+from vicn.core.task                     import run_task, EmptyTask
 from vicn.resource.ns3.emulated_channel import EmulatedChannel
 from vicn.resource.linux.net_device     import NetDevice
+from vicn.core.attribute                import Attribute
+from netmodel.model.type                import Integer
 
 DEFAULT_FADING_ENABLED = True
 DEFAULT_TW_BUFFER = 800000
@@ -48,13 +52,24 @@ class EmulatedLteChannel(EmulatedChannel):
     __package_names__ = ['lte-emulator']
     __app_name__ = 'lte_emulator'
 
+    nb_base_stations = Attribute(Integer, description='Number of nodes emulated by the AP',
+            default=8)
+
+    def __create__(self):
+        task = EmptyTask()
+        for group in self.groups:
+            ip4_assigns = group.iter_by_type_str("ipv4assignment")
+            for ip4_assign in ip4_assigns:
+                task = task | wait_resource_task(ip4_assign)
+
+        return task > super().__create__()
     #---------------------------------------------------------------------------
     # Attribute handlers
     #---------------------------------------------------------------------------
 
     async def _add_station(self, station):
         from vicn.resource.lxd.lxc_container    import LxcContainer
-        from vicn.resource.linux.veth_pair      import VethPair
+        from vicn.resource.linux.veth_pair_lxc  import VethPairLxc
         from vicn.resource.linux.tap_device     import TapChannel
 
         interfaces = list()
@@ -66,9 +81,9 @@ class EmulatedLteChannel(EmulatedChannel):
                 host = NetDevice(node = station.node,
                     device_name='vhh-' + station.name + '-' + self.name,
                     managed = False)
-                sta_if = VethPair(node = station,
+                sta_if = VethPairLxc(node = station,
                         name          = 'vh-' + station.name + '-' + self.name,
-                        device_name   = 'vh-' + station.name + '-' + self.name, 
+                        device_name   = 'vh-' + station.name + '-' + self.name,
                         host          = host,
                         owner = self)
                 bridged_sta = sta_if.host
@@ -104,31 +119,24 @@ class EmulatedLteChannel(EmulatedChannel):
             task = self.node.bridge._remove_interface(bridged_sta)
             await run_task(task, self._state.manager)
 
-            task = self.node.bridge._add_interface(bridged_sta,
-                    vlan = vlan)
+            task = self.node.bridge._add_interface(bridged_sta, vlan = vlan)
             await run_task(task, self._state.manager)
 
+        task = self.node.bridge._remove_interface(sta_tap)
+        await run_task(task, self._state.manager)
         task = self.node.bridge._add_interface(sta_tap, vlan = vlan)
         await run_task(task, self._state.manager)
 
     def _get_cmdline_params(self):
 
-        # IP have not been assign, use AddressManager for simplicity since it
-        # will remember the assignment
-        # NOTE: here the IP address passed to emulator program is hardcoded with
-        # a /24 mask(even if the associated IP with the station does not have a
-        # /24 mask). This is not a problem at all because the netmask passed to
-        # the emulator program has no impact on configuration in the emulator
-        # program. Indeed, the IP routing table in the emulator program are
-        # configured on a per address basis(one route per IP address) instead of
-        # on a per prefix basis(one route per prefix). This guarantees the IP
-        # routing will not change regardless of what netmask is. That is why we
-        # can always safely pass a hardcoded /24 mask to the emulator program.
-
         sta_list = list() # list of identifiers
         sta_macs = list() # list of macs
         sta_taps = list()
         sta_ips = list()
+
+        bs_ip_addr = self._ap_if.ip4_address
+        bs_ip = str(bs_ip_addr) + '/' + str(bs_ip_addr.prefix_len)
+
         for station in self.stations:
             if not station.managed:
                 interface = [i for i in station.interfaces if i.channel == self]
@@ -137,17 +145,15 @@ class EmulatedLteChannel(EmulatedChannel):
 
                 sta_list.append(interface.name)
                 sta_macs.append(interface.mac_address)
-                sta_ips.append(interface.ip4_address + '/24')
+                sta_ips.append(str(interface.ip4_address)+'/'+str(prefix_len))
             else:
                 identifier = self._sta_ifs[station]._state.uuid._uuid
                 sta_list.append(identifier)
 
                 mac = self._sta_ifs[station].mac_address
                 sta_macs.append(mac)
-
-                # Preallocate IP address
-                ip = AddressManager().get_ip(self._sta_ifs[station]) + '/24'
-                sta_ips.append(ip)
+                ip = self._sta_ifs[station].ip4_address
+                sta_ips.append(str(ip)+'/'+str(ip.prefix_len))
 
             tap = self._sta_taps[station].device_name
             sta_taps.append(tap)
@@ -178,8 +184,7 @@ class EmulatedLteChannel(EmulatedChannel):
             # Coma-separated list of stations' IP/netmask len
             'sta-ips'       : ','.join(sta_ips),
             # Base station IP/netmask len
-            'bs-ip'         : AddressManager().get_ip(self._ap_if) + '/' +
-                str(DEFAULT_NETMASK),
+            'bs-ip'         : bs_ip,
             'txBuffer'      : '800000',
             'isFading'      : 'true' if DEFAULT_FADING_ENABLED else 'false',
         }
diff --git a/vicn/resource/ns3/emulated_wifi_channel.py b/vicn/resource/ns3/emulated_wifi_channel.py
index 088d4444..d8838e47 100644
--- a/vicn/resource/ns3/emulated_wifi_channel.py
+++ b/vicn/resource/ns3/emulated_wifi_channel.py
@@ -38,7 +38,7 @@ class EmulatedWiFiChannel(EmulatedChannel):
 
     async def _add_station(self, station):
         from vicn.resource.lxd.lxc_container    import LxcContainer
-        from vicn.resource.linux.veth_pair      import VethPair
+        from vicn.resource.linux.veth_pair_lxc  import VethPairLxc
         from vicn.resource.linux.tap_device     import TapChannel
         from vicn.resource.linux.macvlan        import MacVlan
 
@@ -46,18 +46,21 @@ class EmulatedWiFiChannel(EmulatedChannel):
         if not station.managed:
             sta_if = None
         else:
+            # To connect a container to the EmulatedWifiChannel, we use a
+            # VethPairLxc connected to the bridge, that will be in the same VLAN as
+            # the station TAP entering the emulator
             if isinstance(station, LxcContainer):
                 host = NetDevice(node = station.node,
                     device_name='vhh-' + station.name + '-' + self.name,
                     managed = False)
-                sta_if = VethPair(node = station,
+                sta_if = VethPairLxc(node = station,
                         name          = 'vh-' + station.name + '-' + self.name,
-                        device_name   = 'vh-' + station.name + '-' + self.name, 
+                        device_name   = 'vh-' + station.name + '-' + self.name,
                         host          = host,
                         owner = self)
                 bridged_sta = sta_if.host
             else:
-                raise NotImplementedError 
+                raise NotImplementedError
 
         if sta_if:
             self._sta_ifs[station] = sta_if
@@ -65,7 +68,7 @@ class EmulatedWiFiChannel(EmulatedChannel):
             interfaces.append(sta_if)
             self._state.manager.commit_resource(sta_if)
 
-        sta_tap = TapChannel(node = self.node, 
+        sta_tap = TapChannel(node = self.node,
                 owner = self,
                 device_name = 'tap-' + station.name + '-' + self.name,
                 up = True,
@@ -81,23 +84,23 @@ class EmulatedWiFiChannel(EmulatedChannel):
 
         # Add interfaces to bridge
         # One vlan per station is needed to avoid broadcast loops
-        vlan = AddressManager().get('vlan', sta_tap) 
+        vlan = AddressManager().get('vlan', sta_tap)
         # sta_tap choosen because always there
         if sta_if:
             sta_if.set('channel', self)
 
             task = self.node.bridge._remove_interface(bridged_sta)
             await run_task(task, self._state.manager)
-            task = self.node.bridge._add_interface(bridged_sta,
-                    vlan = vlan)
+            task = self.node.bridge._add_interface(bridged_sta, vlan = vlan)
             await run_task(task, self._state.manager)
 
+        task = self.node.bridge._remove_interface(sta_tap)
+        await run_task(task, self._state.manager)
         task = self.node.bridge._add_interface(sta_tap, vlan = vlan)
         await run_task(task, self._state.manager)
 
 
     def _get_cmdline_params(self, ):
-
         # sta-macs and sta-list for unmanaged stations
         sta_list = list() # list of identifiers
         sta_macs = list() # list of macs
@@ -136,7 +139,7 @@ class EmulatedWiFiChannel(EmulatedChannel):
             # Y position of the Base Station
             'bs-y'          : 0, #self.ap.y,
             # Experiment ID
-            'experiment-id' : 'vicn', 
+            'experiment-id' : 'vicn',
             # Index of the base station
             'bs-name'       : self._ap_tap.device_name,
             # Base station MAC address