CSIT-1416 Remove installation of vpp from containers

Use parent system (Host, Container) installation of VPP. This will save
the internet bandwith by skip installing of prerequisites packages.
It will also skip dpkg install and simplify the process of initializing
VPP inside container.

Previosly initialization of VPP in container takes about 55s. With this
patch it is reduced to 2-3s.

This patch removes the bloated VOLUME creation between container
sidecars (a.k.a nested container) and fixes the hugepage allocation.


Change-Id: Ifa2be532edb77354657e1b84568bdc34993b00d0
Signed-off-by: Peter Mikus <pmikus@cisco.com>

2 files changed, 41 insertions, 32 deletions

diff --git a/resources/libraries/python/ContainerUtils.py b/resources/libraries/python/ContainerUtils.py
index fbfb890dfe..9c42a3c99c 100644
--- a/resources/libraries/python/ContainerUtils.py
+++ b/resources/libraries/python/ContainerUtils.py
@@ -135,19 +135,17 @@ class ContainerManager(object):
             self.engine.container = self.containers[container]
             self.engine.execute(command)
 
-    def install_vpp_in_all_containers(self):
-        """Install VPP into all containers."""
+    def start_vpp_in_all_containers(self):
+        """Start VPP in all containers."""
         for container in self.containers:
             self.engine.container = self.containers[container]
             # We need to install supervisor client/server system to control VPP
             # as a service
-            self.engine.execute('sleep 3; apt-get update')
             self.engine.install_supervisor()
-            self.engine.install_vpp()
-            self.engine.restart_vpp()
+            self.engine.start_vpp()
 
     def restart_vpp_in_all_containers(self):
-        """Restart VPP on all containers."""
+        """Restart VPP in all containers."""
         for container in self.containers:
             self.engine.container = self.containers[container]
             self.engine.restart_vpp()
@@ -395,7 +393,9 @@ class ContainerEngine(object):
 
     def install_supervisor(self):
         """Install supervisord inside a container."""
-        self.execute('apt-get install -y supervisor')
+        if isinstance(self, LXC):
+            self.execute('sleep 3; apt-get update')
+            self.execute('apt-get install -y supervisor')
         self.execute('echo "{config}" > {config_file} && '
                      'supervisord -c {config_file}'.
                      format(
@@ -415,36 +415,19 @@ class ContainerEngine(object):
                          'nodaemon=false\n\n',
                          config_file=SUPERVISOR_CONF))
 
-    def install_vpp(self):
-        """Install VPP inside a container."""
-        self.execute('ln -s /dev/null /etc/sysctl.d/80-vpp.conf')
-        # Workaround for install xenial vpp build on bionic ubuntu.
-        self.execute('apt-get install -y wget')
-        self.execute('deb=$(mktemp) && wget -O "${deb}" '
-                     'http://launchpadlibrarian.net/336117627/'
-                     'libmbedcrypto0_2.5.1-1ubuntu1_amd64.deb && '
-                     'dpkg -i "${deb}" && '
-                     'rm -f "${deb}"')
-        self.execute('deb=$(mktemp) && wget -O "${deb}" '
-                     'http://launchpadlibrarian.net/252876048/'
-                     'libboost-system1.58.0_1.58.0+dfsg-5ubuntu3_amd64.deb && '
-                     'dpkg -i "${deb}" && '
-                     'rm -f "${deb}"')
-        self.execute(
-            'dpkg -i --force-all '
-            '{guest_dir}/openvpp-testing/download_dir/*.deb'.
-            format(guest_dir=self.container.mnt[0].split(':')[1]))
-        self.execute('apt-get -f install -y')
-        self.execute('apt-get install -y ca-certificates')
+    def start_vpp(self):
+        """Start VPP inside a container."""
         self.execute('echo "{config}" >> {config_file}'.
                      format(
                          config='[program:vpp]\n'
                          'command=/usr/bin/vpp -c /etc/vpp/startup.conf\n'
+                         'autostart=false\n'
                          'autorestart=false\n'
                          'redirect_stderr=true\n'
                          'priority=1',
                          config_file=SUPERVISOR_CONF))
         self.execute('supervisorctl reload')
+        self.execute('supervisorctl start vpp')
 
     def restart_vpp(self):
         """Restart VPP service inside a container."""
@@ -608,7 +591,7 @@ class LXC(ContainerEngine):
                 return
 
         image = self.container.image if self.container.image else\
-            "-d ubuntu -r xenial -a amd64"
+            "-d ubuntu -r bionic -a amd64"
 
         cmd = 'lxc-create -t download --name {c.name} -- {image} '\
             '--no-validate'.format(c=self.container, image=image)
@@ -627,11 +610,14 @@ class LXC(ContainerEngine):
         if self.container.mnt:
             for mount in self.container.mnt:
                 host_dir, guest_dir = mount.split(':')
+                options = 'bind,create=dir' \
+                    if guest_dir.endswith('/') else 'bind,create=file'
                 entry = 'lxc.mount.entry = {host_dir} '\
                     '/var/lib/lxc/{c.name}/rootfs{guest_dir} none ' \
-                    'bind,create=dir 0 0'.format(c=self.container,
-                                                 host_dir=host_dir,
-                                                 guest_dir=guest_dir)
+                    '{options} 0 0'.format(c=self.container,
+                                           host_dir=host_dir,
+                                           guest_dir=guest_dir,
+                                           options=options)
                 ret, _, _ = self.container.ssh.exec_command_sudo(
                     "sh -c 'echo \"{e}\" >> /var/lib/lxc/{c.name}/config'".
                     format(e=entry, c=self.container))
diff --git a/resources/libraries/python/DUTSetup.py b/resources/libraries/python/DUTSetup.py
index c573985a9b..a20b2d7056 100644
--- a/resources/libraries/python/DUTSetup.py
+++ b/resources/libraries/python/DUTSetup.py
@@ -609,6 +609,9 @@ class DUTSetup(object):
             message = 'Failed to install VPP on host {host}!'.\
                 format(host=node['host'])
             if node['type'] == NodeType.DUT:
+                command = 'ln -s /dev/null /etc/sysctl.d/80-vpp.conf || true'
+                exec_cmd_no_error(node, command, sudo=True)
+
                 command = '. /etc/lsb-release; echo "${DISTRIB_ID}"'
                 stdout, _ = exec_cmd_no_error(node, command)
 
@@ -647,6 +650,26 @@ class DUTSetup(object):
         return True
 
     @staticmethod
+    def get_docker_mergeddir(node, uuid):
+        """Get Docker overlay for MergedDir diff.
+
+        :param node: DUT node.
+        :param uuid: Docker UUID.
+        :type node: dict
+        :type uuid: str
+        :returns: Docker container MergedDir.
+        :rtype: str
+        :raises RuntimeError: If getting output failed.
+        """
+        command = "docker inspect --format='"\
+            "{{{{.GraphDriver.Data.MergedDir}}}}' {uuid}".format(uuid=uuid)
+        message = 'Failed to get directory of {uuid} on host {host}'.\
+            format(uuid=uuid, host=node['host'])
+
+        stdout, _ = exec_cmd_no_error(node, command, sudo=True, message=message)
+        return stdout.strip()
+
+    @staticmethod
     def get_huge_page_size(node):
         """Get default size of huge pages in system.