CSIT-1065 High failure rate with K8s/Ligato orchestration

- Update Ligato Bootstrap for latest changes
- Update Ligato Test cases for latest vpp-agent changes
- Convert TC to new format
- Add Hyperthreading support

Change-Id: Id7c8513b01c66ee6c652b6294ddc50bc10be8f7d
Signed-off-by: Peter Mikus <pmikus@cisco.com>

5 files changed, 102 insertions, 15 deletions

diff --git a/resources/libraries/bash/k8s_setup.sh b/resources/libraries/bash/k8s_setup.sh
index f9f6c61ec7..4128512f21 100755
--- a/resources/libraries/bash/k8s_setup.sh
+++ b/resources/libraries/bash/k8s_setup.sh
@@ -29,7 +29,7 @@ do
         varname=$(echo "$line" | cut -d '=' -f 1)
         cfg[$varname]=$(echo "$line" | cut -d '=' -f 2-)
     fi
-done < ${script_dir}/../config/config
+done < ${SCRIPT_DIR}/config/config
 
 trap "k8s_utils.destroy" ERR
 
diff --git a/resources/libraries/python/KubernetesUtils.py b/resources/libraries/python/KubernetesUtils.py
index 3e45d3a60f..14cba480ac 100644
--- a/resources/libraries/python/KubernetesUtils.py
+++ b/resources/libraries/python/KubernetesUtils.py
@@ -423,12 +423,20 @@ class KubernetesUtils(object):
         :param kwargs: Key-value pairs used to create configuration.
         :param kwargs: dict
         """
+        smt_used = CpuUtils.is_smt_enabled(kwargs['node']['cpuinfo'])
+
         cpuset_cpus = \
             CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
                                                 cpu_node=kwargs['cpu_node'],
-                                                skip_cnt=kwargs['cpu_skip'],
-                                                cpu_cnt=kwargs['cpu_cnt'],
-                                                smt_used=kwargs['smt_used'])
+                                                skip_cnt=2,
+                                                cpu_cnt=kwargs['phy_cores'],
+                                                smt_used=smt_used)
+        cpuset_main = \
+            CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
+                                                cpu_node=kwargs['cpu_node'],
+                                                skip_cnt=1,
+                                                cpu_cnt=1,
+                                                smt_used=smt_used)
 
         # Create config instance
         vpp_config = VppConfigGenerator()
@@ -436,15 +444,18 @@ class KubernetesUtils(object):
         vpp_config.add_unix_cli_listen(value='0.0.0.0:5002')
         vpp_config.add_unix_nodaemon()
         vpp_config.add_dpdk_socketmem('1024,1024')
-        vpp_config.add_heapsize('3G')
+        vpp_config.add_heapsize('4G')
+        vpp_config.add_ip_heap_size('4G')
+        vpp_config.add_ip6_heap_size('4G')
         vpp_config.add_ip6_hash_buckets('2000000')
-        vpp_config.add_ip6_heap_size('3G')
-        if kwargs['framesize'] < 1522:
+        if not kwargs['jumbo']:
             vpp_config.add_dpdk_no_multi_seg()
-        vpp_config.add_dpdk_dev_default_rxq(kwargs['rxq'])
+        vpp_config.add_dpdk_no_tx_checksum_offload()
+        vpp_config.add_dpdk_dev_default_rxq(kwargs['rxq_count_int'])
         vpp_config.add_dpdk_dev(kwargs['if1'], kwargs['if2'])
+        vpp_config.add_dpdk_num_mbufs(kwargs['num_mbufs_int'])
         # We will pop first core from list to be main core
-        vpp_config.add_cpu_main_core(str(cpuset_cpus.pop(0)))
+        vpp_config.add_cpu_main_core(str(cpuset_main.pop(0)))
         # if this is not only core in list, the rest will be used as workers.
         if cpuset_cpus:
             corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
@@ -458,20 +469,21 @@ class KubernetesUtils(object):
         :param kwargs: Key-value pairs used to create configuration.
         :param kwargs: dict
         """
+        smt_used = CpuUtils.is_smt_enabled(kwargs['node']['cpuinfo'])
         skip_cnt = kwargs['cpu_skip'] + (kwargs['i'] - 1) * \
-            (kwargs['cpu_cnt'] - 1)
+            (kwargs['phy_cores'] - 1)
         cpuset_cpus = \
             CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
                                                 cpu_node=kwargs['cpu_node'],
                                                 skip_cnt=skip_cnt,
-                                                cpu_cnt=kwargs['cpu_cnt']-1,
-                                                smt_used=kwargs['smt_used'])
+                                                cpu_cnt=kwargs['phy_cores']-1,
+                                                smt_used=smt_used)
         cpuset_main = \
             CpuUtils.cpu_slice_of_list_per_node(node=kwargs['node'],
                                                 cpu_node=kwargs['cpu_node'],
                                                 skip_cnt=1,
                                                 cpu_cnt=1,
-                                                smt_used=kwargs['smt_used'])
+                                                smt_used=smt_used)
         # Create config instance
         vpp_config = VppConfigGenerator()
         vpp_config.set_node(kwargs['node'])
diff --git a/resources/libraries/robot/shared/default.robot b/resources/libraries/robot/shared/default.robot
index 842cb7faa0..d2eef2d407 100644
--- a/resources/libraries/robot/shared/default.robot
+++ b/resources/libraries/robot/shared/default.robot
@@ -183,6 +183,73 @@
 | | Set Test Variable | ${thr_count_int}
 | | Set Test Variable | ${rxq_count_int}
 
+| Create Kubernetes VSWITCH startup config on all DUTs
+| | [Documentation] | Create base startup configuration of VSWITCH in Kubernetes
+| | ... | deploy to all DUTs.
+| | ...
+| | ... | *Arguments:*
+| | ... | - ${jumbo} - Jumbo packet. Type: boolean
+| | ... | - ${phy_cores} - Physical cores. Type: integer
+| | ... | - ${rxq} - RX queues. Type: integer
+| | ...
+| | ... | *Example:*
+| | ...
+| | ... | \| Create Kubernetes VSWITCH startup config on all DUTs \| ${True} \
+| | ... | \| ${1} \| ${1}
+| | ...
+| | [Arguments] | ${phy_cores} | ${rx_queues}=${None} | ${jumbo}=${False}
+| | ...
+| | ${cpu_count_int} | Convert to Integer | ${phy_cores}
+| | ${thr_count_int} | Convert to Integer | ${phy_cores}
+| | ${num_mbufs_int} | Convert to Integer | 16384
+| | ${duts}= | Get Matches | ${nodes} | DUT*
+| | :FOR | ${dut} | IN | @{duts}
+| | | ${numa}= | Get interfaces numa node | ${nodes['${dut}']}
+| | | ... | ${${dut}_if1} | ${${dut}_if2}
+| | | ${smt_used}= | Is SMT enabled | ${nodes['${dut}']['cpuinfo']}
+| | | ${if1_pci}= | Get Interface PCI Addr | ${nodes['${dut}']} | ${${dut}_if1}
+| | | ${if2_pci}= | Get Interface PCI Addr | ${nodes['${dut}']} | ${${dut}_if2}
+| | | ${thr_count_int}= | Run keyword if | ${smt_used}
+| | | ... | Evaluate | int(${cpu_count_int}*2)
+| | | ... | ELSE | Set variable | ${thr_count_int}
+| | | ${rxq_count_int}= | Run keyword if | ${rx_queues}
+| | | ... | Set variable | ${rx_queues}
+| | | ... | ELSE | Evaluate | int(${thr_count_int}/2)
+| | | ${rxq_count_int}= | Run keyword if | ${rxq_count_int} == 0
+| | | ... | Set variable | ${1}
+| | | ... | ELSE | Set variable | ${rxq_count_int}
+| | | ${num_mbufs_int}= | Evaluate | int(${num_mbufs_int}*${rxq_count_int})
+| | | ${config}= | Run keyword | Create Kubernetes VSWITCH startup config
+| | | ... | node=${nodes['${dut}']} | phy_cores=${phy_cores}
+| | | ... | cpu_node=${numa} | jumbo=${jumbo} | rxq_count_int=${rxq_count_int}
+| | | ... | num_mbufs_int=${num_mbufs_int}
+| | | ... | filename=/tmp/vswitch.conf | if1=${if1_pci} | if2=${if2_pci}
+| | | Run keyword if | ${thr_count_int} > 1
+| | | ... | Set Tags | MTHREAD | ELSE | Set Tags | STHREAD
+| | | Set Tags | ${thr_count_int}T${cpu_count_int}C
+| | Set Test Variable | ${smt_used}
+| | Set Test Variable | ${thr_count_int}
+| | Set Test Variable | ${rxq_count_int}
+
+| Create Kubernetes VNF'${i}' startup config on all DUTs
+| | [Documentation] | Create base startup configuration of VNF in Kubernetes
+| | ... | deploy to all DUTs.
+| | ...
+| | ${i_int}= | Convert To Integer | ${i}
+| | ${cpu_skip}= | Evaluate | ${vswitch_cpus}+${system_cpus}
+| | ${dut1_numa}= | Get interfaces numa node | ${dut1}
+| | ... | ${dut1_if1} | ${dut1_if2}
+| | ${dut2_numa}= | Get interfaces numa node | ${dut2}
+| | ... | ${dut2_if1} | ${dut2_if2}
+| | ${config}= | Run keyword | Create Kubernetes VNF startup config
+| | ... | node=${dut1} | phy_cores=${vnf_cpus} | cpu_node=${dut1_numa}
+| | ... | cpu_skip=${cpu_skip} | filename=/tmp/vnf${i}.conf
+| | ... | i=${i_int}
+| | ${config}= | Run keyword | Create Kubernetes VNF startup config
+| | ... | node=${dut2} | phy_cores=${vnf_cpus} | cpu_node=${dut2_numa}
+| | ... | cpu_skip=${cpu_skip} | filename=/tmp/vnf${i}.conf
+| | ... | i=${i_int}
+
 | Add no multi seg to all DUTs
 | | [Documentation] | Add No Multi Seg to VPP startup configuration to all DUTs.
 | | ...
diff --git a/resources/templates/kubernetes/configmaps/vnf-agent-cfg.yaml b/resources/templates/kubernetes/configmaps/vnf-agent-cfg.yaml
index 77869d4111..17db0bf86f 100644
--- a/resources/templates/kubernetes/configmaps/vnf-agent-cfg.yaml
+++ b/resources/templates/kubernetes/configmaps/vnf-agent-cfg.yaml
@@ -12,4 +12,8 @@ data:
 
   kafka.conf: |
     addrs:
-      - "172.17.0.1:9092"
\ No newline at end of file
+      - "172.17.0.1:9092"
+
+  telemetry.conf: |
+    polling-interval: 30000000000
+    disabled: true
diff --git a/resources/templates/kubernetes/configmaps/vswitch-agent-cfg.yaml b/resources/templates/kubernetes/configmaps/vswitch-agent-cfg.yaml
index 5c97b4d64f..d22199ceee 100644
--- a/resources/templates/kubernetes/configmaps/vswitch-agent-cfg.yaml
+++ b/resources/templates/kubernetes/configmaps/vswitch-agent-cfg.yaml
@@ -12,4 +12,8 @@ data:
 
   kafka.conf: |
     addrs:
-      - "172.17.0.1:9092"
\ No newline at end of file
+      - "172.17.0.1:9092"
+
+  telemetry.conf: |
+    polling-interval: 30000000000
+    disabled: true