code and doc updates for vagrant VM dev/test environ setup

Change-Id: I0e4e4758b3abbfc2fe2e18200b85477293de218b
Signed-off-by: Naveen Joy <najoy@cisco.com>
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>

5 files changed, 246 insertions, 25 deletions

diff --git a/resources/job_specs/vpp_device/vpp-1n-vbox.md b/resources/job_specs/vpp_device/vpp-1n-vbox.md
new file mode 100644
index 0000000000..ef9b4c1e08
--- /dev/null
+++ b/resources/job_specs/vpp_device/vpp-1n-vbox.md
@@ -0,0 +1,209 @@
+# 1n-vbox
+## Replaced interface with virtual for vbox testing
+## ./container_memif
+### vbox-virtual
+#### dpdk-vfio-pci
+##### ethip4-l2xcbase-eth-2memif-1dcr
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4-l2xcbase-eth-2memif-1dcr
+##### ethip4-l2bdbasemaclrn-eth-2memif-1dcr
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4-l2bdbasemaclrn-eth-2memif-1dcr
+##### ethip4-ip4base-eth-2memif-1dcr
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4-ip4base-eth-2memif-1dcr
+## ./crypto/ethip4
+### vbox-virtual
+#### dpdk-vfio-pci
+##### ethip4ipsec1tnlsw-ip4base-int-aes128cbc-hmac512sha
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4ipsec1tnlsw-ip4base-int-aes128cbc-hmac512sha
+##### ethip4ipsec11tnlsw-ip4base-int-aes128cbc-hmac512sha
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4ipsec11tnlsw-ip4base-int-aes128cbc-hmac512sha
+##### ethip4ipsec1tnlsw-ip4base-policy-aes128cbc-hmac512sha
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4ipsec1tnlsw-ip4base-policy-aes128cbc-hmac512sha
+##### ethip4ipsec1tptsw-ip4base-policy-aes128cbc-hmac512sha
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4ipsec1tptsw-ip4base-policy-aes128cbc-hmac512sha
+## ./crypto/ethip6
+### vbox-virtual
+#### dpdk-vfio-pci
+##### ethip6ipsec1tnlsw-ip6base-policy-aes128cbc-hmac512sha
+0c AND 78b AND scapy AND virtual AND drv_vfio_pci AND ethip6ipsec1tnlsw-ip6base-policy-aes128cbc-hmac512sha
+##### ethip6ipsec1tptsw-ip6base-policy-aes128cbc-hmac512sha
+0c AND 78b AND scapy AND virtual AND drv_vfio_pci AND ethip6ipsec1tptsw-ip6base-policy-aes128cbc-hmac512sha
+## ./flow
+### intel-e810cq
+#### avf
+##### ethip4-flow-ip4-ipsec-ah
+0c AND 64b AND scapy AND e810cq AND drv_avf AND ethip4-flow-ip4-ipsec-ah
+##### ethip4-flow-ip4-ipsec-esp
+0c AND 64b AND scapy AND e810cq AND drv_avf AND ethip4-flow-ip4-ipsec-esp
+##### ethip4-flow-ip4-l2tpv3oip
+0c AND 64b AND scapy AND e810cq AND drv_avf AND ethip4-flow-ip4-l2tpv3oip
+##### ethip4-flow-ip4-ntuple-tcp
+0c AND 64b AND scapy AND e810cq AND drv_avf AND ethip4-flow-ip4-ntuple-tcp
+##### ethip4-flow-ip4-ntuple-udp
+0c AND 64b AND scapy AND e810cq AND drv_avf AND ethip4-flow-ip4-ntuple-udp
+##### ethip4-flow-ip4-tcp
+0c AND 64b AND scapy AND e810cq AND drv_avf AND ethip4-flow-ip4-tcp
+##### ethip4-flow-ip4-udp
+0c AND 64b AND scapy AND e810cq AND drv_avf AND ethip4-flow-ip4-udp
+##### ethip6-flow-ip6-ntuple-tcp
+0c AND 64b AND scapy AND e810cq AND drv_avf AND ethip6-flow-ip6-ntuple-tcp
+##### ethip6-flow-ip6-ntuple-udp
+0c AND 64b AND scapy AND e810cq AND drv_avf AND ethip6-flow-ip6-ntuple-udp
+##### ethip6-flow-ip6-tcp
+0c AND 64b AND scapy AND e810cq AND drv_avf AND ethip6-flow-ip6-tcp
+##### ethip6-flow-ip6-udp
+0c AND 64b AND scapy AND e810cq AND drv_avf AND ethip6-flow-ip6-udp
+#### dpdk-vfio-pci
+##### ethip4-flow-ip4-gtpu
+0c AND 64b AND scapy AND e810cq AND drv_vfio_pci AND ethip4-flow-ip4-gtpu
+##### ethip4-flow-ip4-ipsec-ah
+0c AND 64b AND scapy AND e810cq AND drv_vfio_pci AND ethip4-flow-ip4-ipsec-ah
+##### ethip4-flow-ip4-ipsec-esp
+0c AND 64b AND scapy AND e810cq AND drv_vfio_pci AND ethip4-flow-ip4-ipsec-esp
+##### ethip4-flow-ip4-l2tpv3oip
+0c AND 64b AND scapy AND e810cq AND drv_vfio_pci AND ethip4-flow-ip4-l2tpv3oip
+##### ethip4-flow-ip4-ntuple-tcp
+0c AND 64b AND scapy AND e810cq AND drv_vfio_pci AND ethip4-flow-ip4-ntuple-tcp
+##### ethip4-flow-ip4-ntuple-udp
+0c AND 64b AND scapy AND e810cq AND drv_vfio_pci AND ethip4-flow-ip4-ntuple-udp
+##### ethip4-flow-ip4-tcp
+0c AND 64b AND scapy AND e810cq AND drv_vfio_pci AND ethip4-flow-ip4-tcp
+##### ethip4-flow-ip4-udp
+0c AND 64b AND scapy AND e810cq AND drv_vfio_pci AND ethip4-flow-ip4-udp
+##### ethip6-flow-ip6-ntuple-tcp
+0c AND 64b AND scapy AND e810cq AND drv_vfio_pci AND ethip6-flow-ip6-ntuple-tcp
+##### ethip6-flow-ip6-ntuple-udp
+0c AND 64b AND scapy AND e810cq AND drv_vfio_pci AND ethip6-flow-ip6-ntuple-udp
+##### ethip6-flow-ip6-tcp
+0c AND 64b AND scapy AND e810cq AND drv_vfio_pci AND ethip6-flow-ip6-tcp
+##### ethip6-flow-ip6-udp
+0c AND 64b AND scapy AND e810cq AND drv_vfio_pci AND ethip6-flow-ip6-udp
+## ./interfaces
+### virtual
+#### dpdk-vfio-pci
+##### ethicmp4-ip4base-eth-1tap
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethicmp4-ip4base-eth-1tap
+##### ethicmp4-ip4base-eth-1tap-namespace
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethicmp4-ip4base-eth-1tap-namespace
+##### ethip4-l2bdbasemaclrn-eth-2tap
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4-l2bdbasemaclrn-eth-2tap
+## ./ip4
+### vbox-virtual
+#### dpdk-vfio-pci
+##### ethip4-ip4base-adlalwlistbase
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4-ip4base-adlalwlistbase
+##### ethip4-ip4base-adlblklistbase
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4-ip4base-adlblklistbase
+##### ethip4-ip4base
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4-ip4base
+##### ethip4-ip4base-iacldstbase
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4-ip4base-iacldstbase
+##### ethip4-ip4base-ipolicemarkbase
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4-ip4base-ipolicemarkbase
+##### ethip4tcp-nat44det
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4tcp-nat44det
+##### ethip4tcp-nat44ed
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4tcp-nat44ed
+##### ethip4udp-nat44det
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4udp-nat44det
+##### ethip4udp-nat44ed
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4udp-nat44ed
+## ./ip4_tunnel/lisp
+### vbox-virtual
+#### dpdk-vfio-pci
+##### ethip4lisp-ip4base
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4lisp-ip4base
+##### ethip4lispgpe-ip4base
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4lispgpe-ip4base
+##### ethip4lispgpe-ip6base
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4lispgpe-ip6base
+## ./ip4_tunnel
+### vbox-virtual
+#### dpdk-vfio-pci
+##### ethip4--ethip4udpgeneve-1tun-ip4base
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4--ethip4udpgeneve-1tun-ip4base
+##### ethip4vxlan-l2bdbasemaclrn
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4vxlan-l2bdbasemaclrn
+##### ethip4vxlan-l2xcbase
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4vxlan-l2xcbase
+## ./ip6
+### vbox-virtual
+#### dpdk-vfio-pci
+##### ethip6-ip6base-adlalwlistbase
+0c AND 78b AND scapy AND virtual AND drv_vfio_pci AND ethip6-ip6base-adlalwlistbase
+##### ethip6-ip6base-adlblklistbase
+0c AND 78b AND scapy AND virtual AND drv_vfio_pci AND ethip6-ip6base-adlblklistbase
+##### ethip6-ip6base-iacldstbase
+0c AND 78b AND scapy AND virtual AND drv_vfio_pci AND ethip6-ip6base-iacldstbase
+##### ethip6-ip6base-ipolicemarkbase
+0c AND 78b AND scapy AND virtual AND drv_vfio_pci AND ethip6-ip6base-ipolicemarkbase
+##### ethip6-ip6base
+0c AND 78b AND scapy AND virtual AND drv_vfio_pci AND ethip6-ip6base
+## ./ip6_tunnels/lisp
+### vbox-virtual
+#### dpdk-vfio-pci
+##### ethip6lispgpe-ip4base
+0c AND 78b AND scapy AND virtual AND drv_vfio_pci AND ethip6lispgpe-ip4base
+##### ethip6lispgpe-ip6base
+0c AND 78b AND scapy AND virtual AND drv_vfio_pci AND ethip6lispgpe-ip6base
+## ./l2bd
+### vbox-virtual
+#### avf
+##### ethip4-l2bdbasemaclrn
+0c AND 64b AND scapy AND virtual AND drv_avf AND ethip4-l2bdbasemaclrn
+#### dpdk-vfio-pci
+##### ethip4-l2bdbasemaclrn
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4-l2bdbasemaclrn
+##### ethip4-l2bdbasemaclrn-iacl1sf
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4-l2bdbasemaclrn-iacl1sf
+##### ethip4-l2bdbasemaclrn-iacl1sl
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4-l2bdbasemaclrn-iacl1sl
+##### ethip4-l2bdbasemaclrn-macip-iacl1sl
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4-l2bdbasemaclrn-macip-iacl1sl
+##### ethip4-l2bdbasemaclrn-oacl1sf
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4-l2bdbasemaclrn-oacl1sf
+##### ethip4-l2bdbasemaclrn-oacl1sl
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4-l2bdbasemaclrn-oacl1sl
+## ./l2patch
+### vbox-virtual
+#### dpdk-vfio-pci
+##### ethip4-l2patch
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4-l2patch
+## ./l2xc
+### vbox-virtual
+#### dpdk-vfio-pci
+##### ethip4-l2xcbase
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4-l2xcbase
+## ./srv6
+### vbox-virtual
+#### dpdk-vfio-pci
+##### ethip6ip6-ip6base-srv6enc1sid
+0c AND 78b AND scapy AND virtual AND drv_vfio_pci AND ethip6ip6-ip6base-srv6enc1sid
+##### ethip6srhip6-ip6base-srv6enc2sids
+0c AND 78b AND scapy AND virtual AND drv_vfio_pci AND ethip6srhip6-ip6base-srv6enc2sids
+##### ethip6srhip6-ip6base-srv6enc2sids-nodecaps
+0c AND 78b AND scapy AND virtual AND drv_vfio_pci AND ethip6srhip6-ip6base-srv6enc2sids-nodecaps
+##### ethip6srhip6-ip6base-srv6proxy-dyn
+0c AND 78b AND scapy AND virtual AND drv_vfio_pci AND ethip6srhip6-ip6base-srv6proxy-dyn
+##### ethip6srhip6-ip6base-srv6proxy-masq
+0c AND 78b AND scapy AND virtual AND drv_vfio_pci AND ethip6srhip6-ip6base-srv6proxy-masq
+##### ethip6srhip6-ip6base-srv6proxy-stat
+0c AND 78b AND scapy AND virtual AND drv_vfio_pci AND ethip6srhip6-ip6base-srv6proxy-stat
+## ./stats
+### vbox-virtual
+#### dpdk-vfio-pci
+##### ethip4-l2xcbase-stats
+0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4-l2xcbase-stats
+## ./vm/ethip4
+### vbox-virtual
+#### dpdk-vfio-pci
+##### ethip4-ip4base-eth-2vhost-1vm
+# 0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4-ip4base-eth-2vhost-1vm
+##### ethip4-l2bdbasemaclrn-eth-2vhost-1vm
+# 0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4-l2bdbasemaclrn-eth-2vhost-1vm
+##### ethip4-l2xcbase-eth-2vhost-1vm
+# 0c AND 64b AND scapy AND virtual AND drv_vfio_pci AND ethip4-l2xcbase-eth-2vhost-1vm
+## ./vm/ethip6
+### vbox-virtual
+#### dpdk-vfio-pci
+##### ethip6-ip6base-eth-2vhost-1vm
+# 0c AND 78b AND scapy AND virtual AND drv_vfio_pci AND ethip6-ip6base-eth-2vhost-1vm
diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh
index 78a99dae0d..9db0ac8dcb 100644
--- a/resources/libraries/bash/function/common.sh
+++ b/resources/libraries/bash/function/common.sh
@@ -311,31 +311,37 @@ function deactivate_docker_topology () {
     # Variables read:
     # - NODENESS - Node multiplicity of desired testbed.
     # - FLAVOR - Node flavor string, usually describing the processor.
+    # - CSIT_NO_CLEANUP - Variable to disable cleaning up the environment.
 
     set -exuo pipefail
 
-    case_text="${NODENESS}_${FLAVOR}"
-    case "${case_text}" in
-        "1n_skx" | "1n_tx2" | "1n_spr")
-            ssh="ssh root@172.17.0.1 -p 6022"
-            env_vars=$(env | grep CSIT_ | tr '\n' ' ' ) || die
-            # The "declare -f" output is long and boring.
-            set +x
-            ${ssh} "$(declare -f); deactivate_wrapper ${env_vars}" || {
-                die "Topology cleanup via shim-dcr failed!"
-            }
-            set -x
-            ;;
-        "1n_vbox")
-            enter_mutex || die
-            clean_environment || {
-                die "Topology cleanup locally failed!"
-            }
-            exit_mutex || die
-            ;;
-        *)
-            die "Unknown specification: ${case_text}!"
-    esac
+    if [[ ${CSIT_NO_CLEANUP:-0} -eq 0 ]]; then
+        case_text="${NODENESS}_${FLAVOR}"
+        case "${case_text}" in
+            "1n_skx" | "1n_tx2" | "1n_spr")
+                ssh="ssh root@172.17.0.1 -p 6022"
+                env_vars=$(env | grep CSIT_ | tr '\n' ' ' ) || die
+                # The "declare -f" output is long and boring.
+                set +x
+                ${ssh} "$(declare -f); deactivate_wrapper ${env_vars}" || {
+                    die "Topology cleanup via shim-dcr failed!"
+                }
+                set -x
+                ;;
+            "1n_vbox")
+                enter_mutex || die
+                clean_environment || {
+                    die "Topology cleanup locally failed!"
+                }
+                exit_mutex || die
+                ;;
+            *)
+                die "Unknown specification: ${case_text}!"
+        esac
+    else
+        echo "CSIT_NO_CLEANUP environment variable is set"
+        echo "Environment Cleanup Abandoned"
+    fi
 }
 
 
@@ -939,6 +945,7 @@ function select_tags () {
     awk_nics_sub_cmd+='gsub("nitro-50g","50ge1p1ENA");'
     awk_nics_sub_cmd+='gsub("nitro-100g","100ge1p1ENA");'
     awk_nics_sub_cmd+='gsub("nitro-200g","200ge1p1ENA");'
+    awk_nics_sub_cmd+='gsub("virtual","1ge1p82540em");'
     awk_nics_sub_cmd+='if ($9 =="drv_avf") drv="avf-";'
     awk_nics_sub_cmd+='else if ($9 =="drv_rdma_core") drv ="rdma-";'
     awk_nics_sub_cmd+='else if ($9 =="drv_mlx5_core") drv ="mlx5-";'
diff --git a/resources/libraries/bash/function/device.sh b/resources/libraries/bash/function/device.sh
index 334fd04bfa..9bf4a85c3b 100644
--- a/resources/libraries/bash/function/device.sh
+++ b/resources/libraries/bash/function/device.sh
@@ -705,7 +705,7 @@ function set_env_variables () {
     CSIT_TG_HOST="$(hostname --all-ip-addresses | awk '{print $1}')" || {
         die "Reading hostname IP address failed!"
     }
-    CSIT_TG_PORT="${DCR_PORTS[tg]#*:}"
+    CSIT_TG_PORT="${DCR_PORTS[tg]##*:}"
     CSIT_TG_UUID="${DCR_UUIDS[tg]}"
     CSIT_TG_ARCH="$(uname -i)" || {
         die "Reading machine architecture failed!"
@@ -713,7 +713,7 @@ function set_env_variables () {
     CSIT_DUT1_HOST="$(hostname --all-ip-addresses | awk '{print $1}')" || {
         die "Reading hostname IP address failed!"
     }
-    CSIT_DUT1_PORT="${DCR_PORTS[dut1]#*:}"
+    CSIT_DUT1_PORT="${DCR_PORTS[dut1]##*:}"
     CSIT_DUT1_UUID="${DCR_UUIDS[dut1]}"
     CSIT_DUT1_ARCH="$(uname -i)" || {
         die "Reading machine architecture failed!"
diff --git a/resources/libraries/python/Constants.py b/resources/libraries/python/Constants.py
index 01b64c949d..3240a0a33e 100644
--- a/resources/libraries/python/Constants.py
+++ b/resources/libraries/python/Constants.py
@@ -365,6 +365,7 @@ class Constants:
         u"Mellanox-CX556A": u"100ge2p1cx556a",
         u"Mellanox-CX6DX": u"100ge2p1cx6dx",
         u"Mellanox-CX7VEAT": u"200ge2p1cx7veat",
+        "virtual": "1ge1p82540em",
     }
 
     # Shortened lowercase NIC model name, useful for presentation.
@@ -382,6 +383,8 @@ class Constants:
         u"100ge2p1cx556a": u"cx556a",
         u"100ge2p1cx6dx": u"cx6dx",
         u"200ge2p1cx7veat": u"cx7veat",
+        "1ge1p82540em": "8250em",
+
     }
 
     # Not each driver is supported by each NIC.
@@ -399,6 +402,7 @@ class Constants:
         u"Mellanox-CX556A": [u"rdma-core", u"mlx5_core", u"af_xdp"],
         u"Mellanox-CX6DX": [u"rdma-core", u"mlx5_core", u"af_xdp"],
         u"Mellanox-CX7VEAT": [u"rdma-core", u"mlx5_core", u"af_xdp"],
+        "virtual": ["vfio-pci"],
     }
 
     # Each driver needs different plugin to work.
@@ -452,6 +456,7 @@ class Constants:
         u"Mellanox-CX556A": [u"mlx5_core"],
         u"Mellanox-CX6DX": [u"mlx5_core"],
         u"Mellanox-CX7VEAT": [u"mlx5_core"],
+        "virtual": ["vfio-pci"],
     }
 
     # Tags to differentiate tests for different NIC driver.
diff --git a/resources/libraries/python/SetupFramework.py b/resources/libraries/python/SetupFramework.py
index 3c381166c2..95ca8a7d51 100644
--- a/resources/libraries/python/SetupFramework.py
+++ b/resources/libraries/python/SetupFramework.py
@@ -106,7 +106,7 @@ def extract_tarball_at_node(tarball, node):
         node, cmd,
         message=f"Failed to extract {tarball} at node {node[u'type']} "
         f"host {node[u'host']}, port {node[u'port']}",
-        timeout=240, include_reason=True
+        timeout=600, include_reason=True
     )
     logger.console(
         f"Extracting tarball to {con.REMOTE_FW_DIR} on {node[u'type']} "