CSIT-1389 Refactor current keywords for multichaining - CNF

- Add ability to create matrix of containers (chains x nodeness)
- Change default Docker image to csit-dut
- Use new resource mapping function

Change-Id: Iaac0f2efc99146fbaef0b33dc71b38770610023d
Signed-off-by: Peter Mikus <pmikus@cisco.com>

4 files changed, 107 insertions, 109 deletions

diff --git a/resources/libraries/python/ContainerUtils.py b/resources/libraries/python/ContainerUtils.py
index 478a9781a0..786a4013de 100644
--- a/resources/libraries/python/ContainerUtils.py
+++ b/resources/libraries/python/ContainerUtils.py
@@ -20,7 +20,6 @@ from collections import OrderedDict, Counter
 
 from resources.libraries.python.ssh import SSH
 from resources.libraries.python.constants import Constants
-from resources.libraries.python.CpuUtils import CpuUtils
 from resources.libraries.python.topology import Topology
 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
 
@@ -78,24 +77,6 @@ class ContainerManager(object):
         setattr(self.engine.container, 'env',
                 'MICROSERVICE_LABEL={label}'.format(label=kwargs['name']))
 
-        # Set cpuset.cpus cgroup
-        skip_cnt = kwargs['cpu_skip']
-        smt_used = CpuUtils.is_smt_enabled(kwargs['node']['cpuinfo'])
-        if not kwargs['cpu_shared']:
-            skip_cnt += kwargs['i'] * kwargs['cpu_count']
-        self.engine.container.cpuset_cpus = \
-            CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
-                                                cpu_node=kwargs['cpuset_mems'],
-                                                skip_cnt=skip_cnt,
-                                                cpu_cnt=1,
-                                                smt_used=False) \
-            + \
-            CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
-                                                cpu_node=kwargs['cpuset_mems'],
-                                                skip_cnt=skip_cnt+1,
-                                                cpu_cnt=kwargs['cpu_count']-1,
-                                                smt_used=smt_used)
-
         # Store container instance
         self.containers[kwargs['name']] = self.engine.container
 
@@ -326,19 +307,22 @@ class ContainerEngine(object):
         """Install VPP inside a container."""
         self.execute('ln -s /dev/null /etc/sysctl.d/80-vpp.conf')
         self.execute('apt-get update')
-        if self.container.install_dkms:
-            self.execute(
-                'apt-get install -y dkms && '
-                'dpkg -i --force-all '
-                '{guest_dir}/openvpp-testing/download_dir/*.deb'.
-                format(guest_dir=self.container.mnt[0].split(':')[1]))
-        else:
-            self.execute(
-                'for i in $(ls -I \"*dkms*\" '
-                '{guest_dir}/openvpp-testing/download_dir/); do '
-                'dpkg -i --force-all '
-                '{guest_dir}/openvpp-testing/download_dir/$i; done'.
-                format(guest_dir=self.container.mnt[0].split(':')[1]))
+        # Workaround for install xenial vpp build on bionic ubuntu.
+        self.execute('apt-get install -y wget')
+        self.execute('deb=$(mktemp) && wget -O "${deb}" '
+                     'http://launchpadlibrarian.net/336117627/'
+                     'libmbedcrypto0_2.5.1-1ubuntu1_amd64.deb && '
+                     'dpkg -i "${deb}" && '
+                     'rm -f "${deb}"')
+        self.execute('deb=$(mktemp) && wget -O "${deb}" '
+                     'http://launchpadlibrarian.net/252876048/'
+                     'libboost-system1.58.0_1.58.0+dfsg-5ubuntu3_amd64.deb && '
+                     'dpkg -i "${deb}" && '
+                     'rm -f "${deb}"')
+        self.execute(
+            'dpkg -i --force-all '
+            '{guest_dir}/openvpp-testing/download_dir/*.deb'.
+            format(guest_dir=self.container.mnt[0].split(':')[1]))
         self.execute('apt-get -f install -y')
         self.execute('apt-get install -y ca-certificates')
         self.execute('echo "{config}" >> {config_file}'.
@@ -685,10 +669,10 @@ class Docker(ContainerEngine):
             else:
                 return
 
-        image = self.container.image if self.container.image else\
-            "ubuntu:xenial-20180412"
+        if not self.container.image:
+            setattr(self.container, 'image', 'snergster/csit-sut:latest')
 
-        cmd = 'docker pull {image}'.format(image=image)
+        cmd = 'docker pull {image}'.format(image=self.container.image)
 
         ret, _, _ = self.container.ssh.exec_command_sudo(cmd, timeout=1800)
         if int(ret) != 0:
diff --git a/resources/libraries/robot/performance/performance_configuration.robot b/resources/libraries/robot/performance/performance_configuration.robot
index 17f2d5f13f..f8628d0734 100644
--- a/resources/libraries/robot/performance/performance_configuration.robot
+++ b/resources/libraries/robot/performance/performance_configuration.robot
@@ -983,8 +983,8 @@
 | | ${tg1_if2_mac}= | Get Interface MAC | ${tg} | ${tg_if2}
 | | ${dut1_if2_mac}= | Get Interface MAC | ${dut1} | ${dut1_if2}
 | | ${dut2_if1_mac}= | Get Interface MAC | ${dut2} | ${dut2_if1}
-| | ${sock1}= | Set Variable | memif-DUT1_VNF
-| | ${sock2}= | Set Variable | memif-DUT2_VNF
+| | ${sock1}= | Set Variable | memif-DUT1_CNF1
+| | ${sock2}= | Set Variable | memif-DUT2_CNF1
 | | Set up memif interfaces on DUT node | ${dut1} | ${sock1} | ${sock1}
 | | ... | ${1} | dut1-memif-1-if1 | dut1-memif-1-if2 | ${rxq_count_int}
 | | ... | ${rxq_count_int}
@@ -2821,7 +2821,7 @@
 | | ...
 | | ... | *Note:*
 | | ... | Socket paths for Memif are defined in following format:
-| | ... | - /tmp/memif-\${dut}_VNF\${number}-\${sid}
+| | ... | - /tmp/memif-\${dut}_CNF\${number}-\${sid}
 | | ...
 | | ... | KW uses test variable \${rxq_count_int} set by KW Add worker threads
 | | ... | and rxqueues to all DUTs
@@ -2834,8 +2834,8 @@
 | | [Arguments] | ${dut} | ${count}
 | | ...
 | | :FOR | ${number} | IN RANGE | 1 | ${count}+1
-| | | ${sock1}= | Set Variable | memif-${dut}_VNF
-| | | ${sock2}= | Set Variable | memif-${dut}_VNF
+| | | ${sock1}= | Set Variable | memif-${dut}_CNF1
+| | | ${sock2}= | Set Variable | memif-${dut}_CNF1
 | | | ${prev_index}= | Evaluate | ${number}-1
 | | | Set up memif interfaces on DUT node | ${nodes['${dut}']}
 | | | ... | ${sock1} | ${sock2} | ${number} | ${dut}-memif-${number}-if1
@@ -2880,7 +2880,7 @@
 | | ...
 | | ... | *Note:*
 | | ... | Socket paths for Memif are defined in following format:
-| | ... | - /tmp/memif-\${dut}_VNF\${number}-\${sid}
+| | ... | - /tmp/memif-\${dut}_CNF\${number}-\${sid}
 | | ...
 | | ... | *Example:*
 | | ...
@@ -2894,8 +2894,8 @@
 | | Add interface to bridge domain | ${nodes['${dut}']} | ${${dut}_if2}
 | | ... | ${bd_id2}
 | | :FOR | ${number} | IN RANGE | 1 | ${count}+1
-| | | ${sock1}= | Set Variable | memif-${dut}_VNF
-| | | ${sock2}= | Set Variable | memif-${dut}_VNF
+| | | ${sock1}= | Set Variable | memif-${dut}_CNF1
+| | | ${sock2}= | Set Variable | memif-${dut}_CNF1
 | | | Set up memif interfaces on DUT node | ${nodes['${dut}']}
 | | | ... | ${sock1} | ${sock2} | ${number} | ${dut}-memif-${number}-if1
 | | | ... | ${dut}-memif-${number}-if2 | ${rxq_count_int} | ${rxq_count_int}
@@ -2966,8 +2966,8 @@
 | | ... | ${dut1} | ${subif_index_1} | TAG_REWRITE_METHOD=${tag_rewrite}
 | | ...
 | | ${number}= | Set Variable | ${1}
-| | ${sock1}= | Set Variable | memif-DUT1_VNF
-| | ${sock2}= | Set Variable | memif-DUT1_VNF
+| | ${sock1}= | Set Variable | memif-DUT1_CNF1
+| | ${sock2}= | Set Variable | memif-DUT1_CNF1
 | | ${memif_if1_name}= | Set Variable | DUT1-memif-${number}-if1
 | | ${memif_if2_name}= | Set Variable | DUT1-memif-${number}-if2
 | | Set up memif interfaces on DUT node | ${dut1} | ${sock1} | ${sock2}
@@ -2978,9 +2978,9 @@
 | | Add interface to bridge domain | ${dut1} | ${${memif_if2_name}} | ${bd_id2}
 | | Add interface to bridge domain | ${dut1} | ${subif_index_1} | ${bd_id2}
 | | ${sock1}= | Run Keyword If | '${dut2_status}' == 'PASS'
-| | ... | Set Variable | memif-DUT2_VNF
+| | ... | Set Variable | memif-DUT2_CNF1
 | | ${sock2}= | Run Keyword If | '${dut2_status}' == 'PASS'
-| | ... | Set Variable | memif-DUT2_VNF
+| | ... | Set Variable | memif-DUT2_CNF1
 | | ${memif_if1_name}= | Run Keyword If | '${dut2_status}' == 'PASS'
 | | ... | Set Variable | DUT2-memif-${number}-if1
 | | ${memif_if2_name}= | Run Keyword If | '${dut2_status}' == 'PASS'
@@ -3015,7 +3015,7 @@
 | | ...
 | | ... | *Note:*
 | | ... | Socket paths for Memif are defined in following format:
-| | ... | - /tmp/memif-\${dut}_VNF\${number}-\${sid}
+| | ... | - /tmp/memif-\${dut}_CNF\${number}-\${sid}
 | | ...
 | | ... | *Example:*
 | | ...
@@ -3080,8 +3080,8 @@
 | | ${fib_table_1}= | Evaluate | ${fib_table_1} - ${1}
 | | ${ip_base_start}= | Set Variable | ${31}
 | | :FOR | ${number} | IN RANGE | 1 | ${count+${1}}
-| | | ${sock1}= | Set Variable | memif-${dut}_VNF
-| | | ${sock2}= | Set Variable | memif-${dut}_VNF
+| | | ${sock1}= | Set Variable | memif-${dut}_CNF1
+| | | ${sock2}= | Set Variable | memif-${dut}_CNF1
 | | | Set up memif interfaces on DUT node | ${nodes['${dut}']}
 | | | ... | ${sock1} | ${sock2} | ${number} | ${dut}-memif-${number}-if1
 | | | ... | ${dut}-memif-${number}-if2 | ${rxq_count_int} | ${rxq_count_int}
@@ -3155,7 +3155,7 @@
 | | ...
 | | ... | *Note:*
 | | ... | Socket paths for Memif are defined in following format:
-| | ... | - /tmp/memif-DUT1_VNF\${number}-\${sid}
+| | ... | - /tmp/memif-DUT1_CNF\${number}-\${sid}
 | | ...
 | | ... | KW uses test variable ${rxq_count_int} set by KW Add worker threads
 | | ... | and rxqueues to all DUTs
@@ -3168,7 +3168,7 @@
 | | ...
 | | ${duts}= | Get Matches | ${nodes} | DUT*
 | | :FOR | ${dut} | IN | @{duts}
-| | | ${sock}= | Set Variable | memif-${dut}_VNF
+| | | ${sock}= | Set Variable | memif-${dut}_CNF1
 | | | Set up single memif interface on DUT node | ${nodes['${dut}']} | ${sock}
 | | | ... | ${number} | ${dut}-memif-${number}-if1 | ${rxq_count_int}
 | | | ... | ${rxq_count_int}
@@ -3187,7 +3187,7 @@
 | | ...
 | | ... | *Note:*
 | | ... | Socket paths for Memif are defined in following format:
-| | ... | - /tmp/memif-DUT1_VNF\${number}-\${sid}
+| | ... | - /tmp/memif-DUT1_CNF\${number}-\${sid}
 | | ...
 | | ... | KW uses test variable ${rxq_count_int} set by KW Add worker threads
 | | ... | and rxqueues to all DUTs
@@ -3200,7 +3200,7 @@
 | | ...
 | | ${duts}= | Get Matches | ${nodes} | DUT*
 | | :FOR | ${dut} | IN | @{duts}
-| | | ${sock}= | Set Variable | memif-${dut}_VNF
+| | | ${sock}= | Set Variable | memif-${dut}_CNF1
 | | | Set up single memif interface on DUT node | ${nodes['${dut}']} | ${sock}
 | | | ... | ${number} | ${dut}-memif-${number}-if1 | ${rxq_count_int}
 | | | ... | ${rxq_count_int}
diff --git a/resources/libraries/robot/performance/performance_setup.robot b/resources/libraries/robot/performance/performance_setup.robot
index a06a0cfaec..a63a7df1c1 100644
--- a/resources/libraries/robot/performance/performance_setup.robot
+++ b/resources/libraries/robot/performance/performance_setup.robot
@@ -472,13 +472,23 @@
 | | [Documentation]
 | | ... | Suite preparation phase that starts containers
 | | ...
+| | ... | *Arguments:*
+| | ... | - chains: Total number of chains. Type: integer
+| | ... | - nodeness: Total number of nodes per chain. Type: integer
+| | ...
+| | ... | *Example:*
+| | ...
+| | ... | \| Set up performance topology with containers \| 1 \| 1 \|
+| | ...
+| | [Arguments] | ${chains}=${1} | ${nodeness}=${1}
+| | ...
 | | Set Suite Variable | @{container_groups} | @{EMPTY}
-| | Construct VNF containers on all DUTs
-| | Acquire all 'VNF' containers
-| | Create all 'VNF' containers
-| | Configure VPP in all 'VNF' containers
+| | Construct chains of containers on all DUTs | ${chains} | ${nodeness}
+| | Acquire all 'CNF' containers
+| | Create all 'CNF' containers
+| | Configure VPP in all 'CNF' containers
 | | Stop VPP service on all DUTs | ${nodes}
-| | Install VPP in all 'VNF' containers
+| | Install VPP in all 'CNF' containers
 | | Start VPP service on all DUTs | ${nodes}
 
 | Set up performance test suite with MEMIF
diff --git a/resources/libraries/robot/shared/container.robot b/resources/libraries/robot/shared/container.robot
index e7b101cb94..e4748dbf60 100644
--- a/resources/libraries/robot/shared/container.robot
+++ b/resources/libraries/robot/shared/container.robot
@@ -17,67 +17,71 @@
 | Library | resources.libraries.python.topology.Topology
 
 *** Keywords ***
-| Construct VNF containers on all DUTs
-| | [Documentation] | Construct 1..N VNF container(s) of specific technology on
-| | ... |  all DUT nodes.
+| Construct container on all DUTs
+| | [Documentation] | Construct 1 CNF of specific technology on all DUT nodes.
 | | ...
-| | ${group}= | Set Variable | VNF
-| | ${skip_cpus}= | Evaluate | ${vpp_cpus}+${system_cpus}
+| | ... | *Arguments:*
+| | ... | - chains: Total number of chains. Type: integer
+| | ... | - nodeness: Total number of nodes per chain. Type: integer
+| | ... | - chain_id: Chain ID. Type: integer
+| | ... | - node_id: Node ID. Type: integer
+| | ...
+| | ... | *Example:*
+| | ...
+| | ... | \| Construct container on all DUTs \| 1 \| 1 \| 1 \| 1 \|
+| | ...
+| | [Arguments] | ${chains}=${1} | ${nodeness}=${1} | ${chain_id}=${1}
+| | ... | ${node_id}=${1}
+| | ...
+| | ${group}= | Set Variable | CNF
 | | Import Library | resources.libraries.python.ContainerUtils.ContainerManager
 | | ... | engine=${container_engine} | WITH NAME | ${group}
 | | ${duts}= | Get Matches | ${nodes} | DUT*
 | | :FOR | ${dut} | IN | @{duts}
 | | | ${env}= | Create List | DEBIAN_FRONTEND=noninteractive
 | | | ${mnt}= | Create List | /tmp:/mnt/host | /dev/vfio:/dev/vfio
-| | | ${cpu_node}= | Get interfaces numa node | ${nodes['${dut}']}
-| | | ... | ${dut1_if1} | ${dut1_if2}
-| | | Run Keyword | ${group}.Construct containers
-| | | ... | name=${dut}_${group} | node=${nodes['${dut}']} | mnt=${mnt}
-| | | ... | image=${container_image} | cpu_count=${container_cpus}
-| | | ... | cpu_skip=${skip_cpus} | cpuset_mems=${cpu_node}
-| | | ... | cpu_shared=${False} | env=${env} | count=${container_count}
-| | | ... | install_dkms=${container_install_dkms}
-| | Append To List | ${container_groups} | ${group}
-
-| Construct ETCD containers on all DUTs
-| | [Documentation] | Construct Docker ETCD container on all DUTs.
-| | ...
-| | ${group}= | Set Variable | ETCD
-| | ${command}= | Set Variable
-| | ... | /usr/local/bin/etcd -advertise-client-urls http://0.0.0.0:2379 -listen-client-urls http://0.0.0.0:2379
-| | ${image}= | Set Variable | quay.io/coreos/etcd:v3.2.5
-| | ${publish}= | Create List | 2379:2379
-| | Import Library | resources.libraries.python.ContainerUtils.ContainerManager
-| | ... | engine=Docker | WITH NAME | ${group}
-| | ${duts}= | Get Matches | ${nodes} | DUT*
-| | :FOR | ${dut} | IN | @{duts}
-| | | ${cpu_node}= | Get interfaces numa node | ${nodes['${dut}']}
-| | | ... | ${dut1_if1} | ${dut1_if2}
+| | | ${nf_cpus}= | Create network function CPU list | ${dut}
+| | | ... | chains=${chains} | nodeness=${nodeness} | chain_id=${chain_id}
+| | | ... | node_id=${node_id} | auto_scale=${True}
 | | | Run Keyword | ${group}.Construct container
-| | | ... | name=${dut}_${group} | node=${nodes['${dut}']}
-| | | ... | image=${container_image} | cpu_count=${1} | cpu_skip=${0}
-| | | ... | cpuset_mems=${cpu_node} | cpu_shared=${True}
-| | | ... | publish=${publish} | command=${command}
+| | | ... | name=${dut}_${group}${chain_id}${node_id} | node=${nodes['${dut}']}
+| | | ... | mnt=${mnt} | env=${env} | cpuset_cpus=${nf_cpus}
 | | Append To List | ${container_groups} | ${group}
 
-| Construct Kafka containers on all DUTs
-| | [Documentation] | Construct Docker Kafka container on all DUTs.
+| Construct chain of containers on all DUTs
+| | [Documentation] | Construct 1 chain of 1..N CNFs on all DUT nodes.
 | | ...
-| | ${group}= | Set Variable | Kafka
-| | ${image}= | Set Variable | spotify/kafka
-| | ${publish}= | Create List | 2181:2181 | 9092:9092
-| | Import Library | resources.libraries.python.ContainerUtils.ContainerManager
-| | ... | engine=Docker | WITH NAME | ${group}
-| | ${duts}= | Get Matches | ${nodes} | DUT*
-| | :FOR | ${dut} | IN | @{duts}
-| | | ${env}= | Create List | ADVERTISED_HOST=172.17.0.1 | ADVERTISED_PORT=9092
-| | | ${cpu_node}= | Get interfaces numa node | ${nodes['${dut}']}
-| | | ... | ${dut1_if1} | ${dut1_if2}
-| | | Run Keyword | ${group}.Construct container
-| | | ... | name=${dut}_${group} | node=${nodes['${dut}']} | image=${image}
-| | | ... | cpu_count=${1} | cpu_skip=${0} | cpuset_mems=${cpu_node}
-| | | ... | cpu_shared=${True} | publish=${publish} | env=${env}
-| | Append To List | ${container_groups} | ${group}
+| | ... | *Arguments:*
+| | ... | - chains: Total number of chains. Type: integer
+| | ... | - nodeness: Total number of nodes per chain. Type: integer
+| | ... | - chain_id: Chain ID. Type: integer
+| | ...
+| | ... | *Example:*
+| | ...
+| | ... | \| Construct chain of containers on all DUTs \| 1 \| 1 \| 1 \|
+| | ...
+| | [Arguments] | ${chains} | ${nodeness} | ${chain_id}
+| | ...
+| | :FOR | ${node_id} | IN RANGE | 1 | ${nodeness}+1
+| | | Construct container on all DUTs | chains=${chains} | nodeness=${nodeness}
+| | | ... | chain_id=${chain_id} | node_id=${node_id}
+
+| Construct chains of containers on all DUTs
+| | [Documentation] | Construct 1..N chains of 1..N CNFs on all DUT nodes.
+| | ...
+| | ... | *Arguments:*
+| | ... | - chains: Total number of chains. Type: integer
+| | ... | - nodeness: Total number of nodes per chain. Type: integer
+| | ...
+| | ... | *Example:*
+| | ...
+| | ... | \| Construct chains of containers on all DUTs \| 1 \| 1 \|
+| | ...
+| | [Arguments] | ${chains}=${1} | ${nodeness}=${1}
+| | ...
+| | :FOR | ${chain_id} | IN RANGE | 1 | ${chains}+1
+| | | Construct chain of containers on all DUTs | chains=${chains}
+| | | ... | nodeness=${nodeness} | chain_id=${chain_id}
 
 | Acquire all '${group}' containers
 | | [Documentation] | Acquire all container(s) in specific container group on