* GUI resource

* MemIf interface for VPP
* Better netmodel integration
* Draft documentation
* New tutorials
* Improved monitoring and error handling
* Refactored IP addresses and prefixes representation
* Improved image mgmt for LXD
* Various bugfixes and code refactoring

Change-Id: I90da6cf7b5716bc7deb6bf4e24d3f9f01b5a9b0f
Signed-off-by: Marcel Enguehard <mengueha+fdio@cisco.com>

6 files changed, 231 insertions, 34 deletions

diff --git a/vicn/resource/lxd/lxc_container.py b/vicn/resource/lxd/lxc_container.py
index 8c8b4816..5b1d4e3b 100644
--- a/vicn/resource/lxd/lxc_container.py
+++ b/vicn/resource/lxd/lxc_container.py
@@ -32,9 +32,11 @@ from vicn.core.exception            import ResourceNotFound
 from vicn.core.requirement          import Requirement
 from vicn.core.resource_mgr         import wait_resource_task
 from vicn.core.task                 import task, inline_task, BashTask, EmptyTask
+from vicn.core.task                 import inherit_parent
 from vicn.resource.linux.net_device import NetDevice
 from vicn.resource.node             import Node
 from vicn.resource.vpp.scripts      import APPARMOR_VPP_PROFILE
+from vicn.resource.lxd.lxc_image    import LxcImage
 from vicn.resource.lxd.lxd_profile  import LXD_PROFILE_DEFAULT_IFNAME
 
 log = logging.getLogger(__name__)
@@ -54,9 +56,20 @@ CMD_GET_IP6_FWD = 'sysctl -n net.ipv6.conf.all.forwarding'
 
 CMD_NETWORK_DHCP='dhclient {container.management_interface.device_name}'
 
+CMD_MOUNT_FOLDER='''
+lxc config device add {container.name} {device-name} disk source={host_path} path={container_path}
+sleep 1
+'''
+
 # Type: ContainerName
-ContainerName = String(max_size = 64, ascii = True,
-        forbidden = ('/', ',', ':'))
+# https://github.com/lxc/lxd/issues/1431
+# [...] all container names must be a valid hostname under the most
+#restrictive definition of this, that is, maximum 63 characters, may not contain
+#dots, may not start by a digit or dash, may not end by a dash and must be made
+#entirely of letters, digits or hyphens.
+# XXX better have a allowed property
+ContainerName = String.restrict(max_size = 63, ascii = True,
+        forbidden = ('/', ',', ':', '_'))
 
 class LxcContainer(Node):
     """
@@ -93,9 +106,9 @@ class LxcContainer(Node):
             ])
     profiles = Attribute(String, multiplicity = Multiplicity.OneToMany,
             default = ['vicn'])
-    image = Attribute(String, description = 'image', default = None)
-    is_image = Attribute(Bool, defaut = False)
-    pid = Attribute(Integer, description = 'PID of the container')
+    image = Attribute(LxcImage, description = 'image', default = None)
+    is_image = Attribute(Bool, default = False)
+    pid = Attribute(Integer, description = 'PID of the container', ro = True)
     ip6_forwarding = Attribute(Bool, default=True)
 
     #--------------------------------------------------------------------------
@@ -110,6 +123,7 @@ class LxcContainer(Node):
     # Resource lifecycle
     #--------------------------------------------------------------------------
 
+    @inherit_parent
     @inline_task
     def __initialize__(self):
         """
@@ -128,6 +142,7 @@ class LxcContainer(Node):
             if iface.get_type() == "dpdkdevice":
                 self.node.vpp_host.dpdk_devices.append(iface.pci_address)
 
+    @inherit_parent
     @task
     def __get__(self):
         client = self.node.lxd_hypervisor.client
@@ -136,6 +151,7 @@ class LxcContainer(Node):
         except pylxd.exceptions.NotFound:
             raise ResourceNotFound
 
+    @inherit_parent
     def __create__(self):
         """
         Make sure vpp_host is instanciated before starting the container.
@@ -153,6 +169,7 @@ class LxcContainer(Node):
     def _create_container(self):
         container = self._get_container_description()
         log.debug('Container description: {}'.format(container))
+        print('Container description: {}'.format(container))
         client = self.node.lxd_hypervisor.client
 
         self._container = client.containers.create(container, wait=True)
@@ -188,13 +205,13 @@ class LxcContainer(Node):
         # SOURCE
 
         image_names = [alias['name'] for alias in self.node.lxd_hypervisor.aliases]
-        image_exists = self.image is not None and self.image in image_names
+        image_exists = self.image.image is not None and self.image.image in image_names
 
         if image_exists:
             container['source'] = {
                 'type'      : 'image',
                 'mode'      : 'local',
-                'alias'     : self.image,
+                'alias'     : self.image.image,
             }
         else:
             container['source'] = {
@@ -234,6 +251,7 @@ class LxcContainer(Node):
         Method: Start the container
         """
         self._container.start(wait = True)
+        import time; time.sleep(1)
 
     @task
     def __method_stop__(self):
@@ -288,10 +306,35 @@ class LxcContainer(Node):
         """
 
         if not self._container:
-            log.error("Executing command on uninitialized container", self, command)
+            log.error("Executing command on uninitialized container {} {}".format(self, command))
             import os; os._exit(1)
 
-        ret = self._container.execute(shlex.split(command))
+        if 'vppctl_wrapper' in command:
+            vpp_log = '{}/vpp-{}.sh'.format(self._state.manager._base, self.name)
+            with open(vpp_log, 'a') as f:
+                print("lxc exec {} -- {}".format(self.name, command), file=f)
+
+        # XXX Workaround: pylxd 2.2.3 buggy  (w/ lxd 2.14) ?
+        # But this workaround is broken with lxd 2.15 and pylxd 2.2.4 works
+        # lxc exec freezes
+        #return self.node.execute('lxc exec {} -- {}'.format(self.name, command),
+        #        output = output, as_root = as_root)
+
+        print("lxc exec {} -- {}".format(self.name, command))
+        while True:
+            try:
+                ret = self._container.execute(shlex.split(command))
+                break
+            except pylxd.exceptions.NotFound:
+                print("=====> pylxd not found during {}".format(command))
+                time.sleep(1)
+            except pylxd.exceptions.ClientConnectionFailed:
+                print("=====> pylxd connection failed during {}".format(command))
+                time.sleep(1)
+            except requests.exceptions.SSLError:
+                print("=====> ssl error during {}".format(command))
+                time.sleep(1)
+
 
         # NOTE: pylxd documents the return value as a tuple, while it is in
         # fact a ContainerExecuteResult object
@@ -306,12 +349,8 @@ class LxcContainer(Node):
         return ReturnValue(*args)
 
     def _get_ip6_forwarding(self):
-        def parse(rv):
-            ret = {"ip6_forwarding" : False}
-            if rv.stdout == "1":
-                ret["ip6_forwarding"] = True
-            return ret
-        return BashTask(self, CMD_GET_IP6_FWD, parse=parse)
+        return BashTask(self, CMD_GET_IP6_FWD,
+            parse = lambda rv: {'ip6_forwarding' : rv.stdout == "1"})
 
     def _set_ip6_forwarding(self):
         cmd = CMD_SET_IP6_FWD if self.ip6_forwarding else CMD_UNSET_IP6_FWD
diff --git a/vicn/resource/lxd/lxc_image.py b/vicn/resource/lxd/lxc_image.py
index a3a03245..f630fe2f 100644
--- a/vicn/resource/lxd/lxc_image.py
+++ b/vicn/resource/lxd/lxc_image.py
@@ -24,7 +24,7 @@ from vicn.core.attribute                import Attribute, Multiplicity
 from vicn.core.exception                import ResourceNotFound
 from vicn.core.requirement              import Requirement
 from vicn.core.resource                 import Resource
-from vicn.core.task                     import task, inline_task
+from vicn.core.task                     import task, inline_task, inherit_parent
 from vicn.resource.linux.application    import LinuxApplication as Application
 from vicn.resource.node                 import Node
 
@@ -56,8 +56,11 @@ class LxcImage(Resource):
     # Resource lifecycle
     #---------------------------------------------------------------------------
 
+    @inherit_parent
     @task
     def __get__(self):
+        log.warning('Image test is currently disabled')
+        return
         aliases = [alias['name'] for images in self.node.lxd_hypervisor.client.images.all()
                          for alias in images.aliases]
         if not self.image in aliases:
@@ -69,6 +72,7 @@ class LxcImage(Resource):
         return
 
 
+    @inherit_parent
     @task
     def __create_DISABLED__(self):
         """
@@ -97,6 +101,7 @@ class LxcImage(Resource):
 
         tmp_container.delete()
 
+    @inherit_parent
     @task
     def __delete__(self):
         self.node.lxd_hypervisor.client.images.delete(self.name)
diff --git a/vicn/resource/lxd/lxd_certificate_store.py b/vicn/resource/lxd/lxd_certificate_store.py
new file mode 100644
index 00000000..97a8cada
--- /dev/null
+++ b/vicn/resource/lxd/lxd_certificate_store.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2017 Cisco and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from vicn.core.attribute                import Attribute, Multiplicity
+from vicn.core.resource                 import Resource
+from vicn.resource.linux.certificate    import Certificate
+from vicn.resource.node                 import Node
+
+PACKAGE = 'ca-certificates'
+
+CMD_ADD_CERTIFICATE = 'cp {certificate.cert} {store.PATH}'
+
+class LxdCertificateStore(Resource):
+    """
+    Resource: System-wide Certificate Store
+
+    This resource allows manipulation of the trusted certificates on the system.
+    Use with care.
+
+    See. vicn.resource.linux.certificate_store
+    """
+
+    PATH = '~/.config/lxc/servercerts/'
+
+    certificates = Attribute(Certificate, multiplicity = Multiplicity.OneToMany)
+    node = Attribute(Node, mandatory = True, requirements = [
+            #Requirement(PACKAGE, 'in', 'packages')
+        ])
+
+    def _add_certificate(self):
+        # Return a task that takes a certificate as parameter
+        PARAM = None
+        return BashTask(self.node, CMD_ADD_CERTIFICATE, {'store': self, 'certificate': PARAM})
diff --git a/vicn/resource/lxd/lxd_hypervisor.py b/vicn/resource/lxd/lxd_hypervisor.py
index bbdba7c6..fa63b96b 100644
--- a/vicn/resource/lxd/lxd_hypervisor.py
+++ b/vicn/resource/lxd/lxd_hypervisor.py
@@ -16,15 +16,6 @@
 # limitations under the License.
 #
 
-#-------------------------------------------------------------------------------
-# NOTES
-#-------------------------------------------------------------------------------
-#  - lxd >= 2.0.4 is required
-#      daemon/container: Remember the return code in the non wait-for-websocket
-#      case (Issue #2243)
-#  - Reference: https://github.com/lxc/lxd/tree/master/doc
-#-------------------------------------------------------------------------------
-
 import logging
 import os
 from pylxd                            import Client
@@ -34,7 +25,8 @@ from netmodel.model.type              import String, Integer
 from vicn.core.attribute              import Attribute, Multiplicity, Reference
 from vicn.core.exception              import ResourceNotFound
 from vicn.core.resource               import Resource
-from vicn.core.task                   import BashTask, task
+from vicn.core.task                   import EmptyTask, BashTask, task
+from vicn.core.task                   import inherit_parent, override_parent
 from vicn.resource.linux.application  import LinuxApplication as Application
 from vicn.resource.linux.service      import Service
 from vicn.resource.linux.certificate  import Certificate
@@ -74,10 +66,12 @@ CMD_LXD_NETWORK_SET = 'lxc network create {lxd_hypervisor.network} || true'
 class LxdInit(Application):
     __package_names__ = ['lxd', 'zfsutils-linux', 'lsof']
 
+    @inherit_parent
     def __get__(self):
         return BashTask(self.owner.node, CMD_LXD_CHECK_INIT,
                 {'lxd': self.owner})
 
+    @inherit_parent
     def __create__(self):
         cmd_params = {
             'storage-backend'       : self.owner.storage_backend,
@@ -107,12 +101,14 @@ class LxdInit(Application):
         # zfs-dkms in the host
         return BashTask(self.owner.node, cmd, as_root = True)
 
+    @inherit_parent
     def __delete__(self):
         raise NotImplementedError
 
 class LxdInstallCert(Resource):
     certificate = Attribute(Certificate, mandatory = True)
 
+    @inherit_parent
     @task
     def __get__(self):
         try:
@@ -126,6 +122,7 @@ class LxdInstallCert(Resource):
             raise ResourceNotFound
 
 
+    @inherit_parent
     @task
     def __create__(self):
         """
@@ -140,6 +137,8 @@ class LxdInstallCert(Resource):
 
 #------------------------------------------------------------------------------
 
+LxdStorageType = String.restrict(choices=('zfs'))
+
 class LxdHypervisor(Service):
     """
     Resource: LxdHypervisor
@@ -150,9 +149,8 @@ class LxdHypervisor(Service):
 
     lxd_port = Attribute(Integer, description = 'LXD REST API port',
             default = 8443)
-    storage_backend = Attribute(String, description = 'Storage backend',
-            default = 'zfs',
-            choices = ['zfs'])
+    storage_backend = Attribute(LxdStorageType, description = 'Storage backend',
+            default = 'zfs')
     storage_size = Attribute(Integer, description = 'Storage size',
             default = LXD_STORAGE_SIZE_DEFAULT) # GB
     zfs_pool = Attribute(String, description = 'ZFS pool',
@@ -190,6 +188,7 @@ class LxdHypervisor(Service):
     # Resource lifecycle
     #--------------------------------------------------------------------------
 
+    @inherit_parent
     def __subresources__(self):
         lxd_init = LxdInit(owner=self, node = self.node)
         lxd_local_cert = Certificate(node = Reference(self, 'node'),
@@ -208,6 +207,11 @@ class LxdHypervisor(Service):
 
         return (lxd_init | lxd_local_cert) > (lxd_vicn_profile | lxd_cert_install)
 
+    @override_parent
+    def __create__(self):
+        log.warning('Not restarting LXD')
+        return EmptyTask()
+
     #--------------------------------------------------------------------------
     # Private methods
     #--------------------------------------------------------------------------
diff --git a/vicn/resource/lxd/lxd_profile.py b/vicn/resource/lxd/lxd_profile.py
index db871671..e8e022d4 100644
--- a/vicn/resource/lxd/lxd_profile.py
+++ b/vicn/resource/lxd/lxd_profile.py
@@ -19,7 +19,7 @@
 from vicn.core.resource                 import Resource
 from netmodel.model.type                import String
 from vicn.core.attribute                import Attribute, Multiplicity
-from vicn.core.task                     import BashTask
+from vicn.core.task                     import BashTask, inherit_parent
 from vicn.core.exception              import ResourceNotFound
 
 CMD_LXD_PROFILE_CREATE = '''
@@ -41,18 +41,20 @@ LXD_PROFILE_DEFAULT_IFNAME = 'vicn_mgmt'
 
 class LxdProfile(Resource):
 
-    description = Attribute(String, descr="profile description", mandatory=True)
-    pool = Attribute(String, descr="ZFS pool used by the containers", mandatory=True)
-    network = Attribute(String, description='Network on which to attach', mandatory=True)
-    iface_name = Attribute(String, description='Default interface name',
+    description = Attribute(String, description = "profile description", mandatory=True)
+    pool = Attribute(String, description = "ZFS pool used by the containers", mandatory=True)
+    network = Attribute(String, description = 'Network on which to attach', mandatory=True)
+    iface_name = Attribute(String, description = 'Default interface name',
             default = LXD_PROFILE_DEFAULT_IFNAME)
     node = Attribute(Resource, mandatory=True)
 
+    @inherit_parent
     def __get__(self):
         def parse(rv):
             if not rv.stdout:
                 raise ResourceNotFound
         return BashTask(self.node, CMD_LXD_PROFILE_GET, {'profile':self}, parse=parse)
 
+    @inherit_parent
     def __create__(self):
         return BashTask(self.node, CMD_LXD_PROFILE_CREATE, {'profile':self})
diff --git a/vicn/resource/lxd/lxd_remote.py b/vicn/resource/lxd/lxd_remote.py
new file mode 100644
index 00000000..b5f8454a
--- /dev/null
+++ b/vicn/resource/lxd/lxd_remote.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2017 Cisco and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from vicn.core.resource         import Resource
+from netmodel.model.type        import String, Bool
+from vicn.core.attribute        import Attribute, Multiplicity
+from vicn.core.task             import BashTask, EmptyTask, inherit_parent
+from vicn.core.exception        import ResourceNotFound
+
+# From LXD 2.14
+#
+# lxc remote add [<remote>] <IP|FQDN|URL> [--accept-certificate] [--password=PASSWORD] [--public] [--protocol=PROTOCOL]
+# lxc remote remove <remote>
+# lxc remote list
+# lxc remote rename <old name> <new name>
+# lxc remote set-url <remote> <url>
+# lxc remote set-default <remote>
+# lxc remote get-default
+
+CMD_GET = 'lxc remote list | grep {remote.name}'
+
+CMD_CREATE  = 'lxc remote add {remote.name} {remote.url}{public}'
+CMD_CREATE += ' --protocol {remote.protocol}'
+
+CMD_DELETE = 'lxc remote remove {remote.name}'
+
+CMD_SET_URL = 'lxc remote set-url {remote.name} {remote.url}'
+
+CMD_GET_DEFAULT = 'lxc remote get-default'
+CMD_SET_DEFAULT = 'lxc remote set-default {remote.name}'
+
+LxdProtocol = String.restrict(choices = ('simplestreams', 'lxd'))
+
+class LxdRemote(Resource):
+    # name (inherited)
+    url = Attribute(String, mandatory = True)
+    protocol = Attribute(LxdProtocol, default='lxd')
+    public = Attribute(Bool, default = True)
+    static = Attribute(Bool, default = False)
+    default = Attribute(Bool, default = False)
+
+    # Used to identify the LXD instance
+    node = Attribute(Resource, mandatory=True)
+
+    @inherit_parent
+    def __get__(self):
+        def parse(rv):
+            if not rv.stdout:
+                raise ResourceNotFound
+            return {
+                'url': self.url,
+                'protocol': self.protocol,
+                'public': self.public,
+                'static': self.static,
+                'default': self.default,
+            }
+        return BashTask(self.node, CMD_GET, {'remote': self}, parse = parse)
+
+    @inherit_parent
+    def __create__(self):
+        public = ' --public' if self.public else ''
+        return BashTask(self.node, CMD_CREATE, {'remote': self, 'public': public})
+
+    @inherit_parent
+    def __delete__(self):
+        return BashTask(self.node, CMD_DELETE, {'remote': self})
+
+    def _get_url(self):
+        return None
+
+    def _set_url(self):
+        return BashTask(self.node, CMD_SET_URL, {'remote': self})
+
+    def _get_default(self):
+        def parse(rv):
+            return {'default': rv.stdout == self.name}
+        return BashTask(self.node, CMD_GET_DEFAULT, {'remote': self},
+                parse = parse)
+
+    def _set_default(self):
+        if self.default:
+            return BashTask(self.node, CMD_SET_DEFAULT, {'remote': self})
+        else:
+            return EmptyTask()