VPP install and verify in __init__.robot

Move VPP installation to separate test in test suite setup phase
to clearly indicate any issue with VPP installation.

Added test to check VPP responsiveness after installation.

Change-Id: Idc2c78152e23aa7301bb5dbf9b1b6f4b639c3e84
Signed-off-by: Andrej Kilvady <andrej.kilvady@pantheon.tech>

5 files changed, 212 insertions, 53 deletions

diff --git a/bootstrap.sh b/bootstrap.sh
index d9c41c5ffb..cfb5a8f320 100755
--- a/bootstrap.sh
+++ b/bootstrap.sh
@@ -265,6 +265,8 @@ for index in "${!VIRL_SERVER[@]}"; do
         ${VIRL_USERNAME}@${VIRL_SERVER[${index}]} \
         "start-testcase -vv --quota ${IP_QUOTA} --copy ${VIRL_TOPOLOGY} \
         --release ${VIRL_RELEASE} ${VPP_PKGS_FULL[@]}")
+        # TODO: remove param ${VPP_PKGS_FULL[@]} when start-testcase script is
+        # updated on all virl servers
     retval=$?
     if [ ${retval} -ne "0" ]; then
         echo "VIRL simulation start failed on ${VIRL_SERVER[${index}]}"
diff --git a/resources/libraries/python/DUTSetup.py b/resources/libraries/python/DUTSetup.py
index 4246a5a5d2..ca37d9e100 100644
--- a/resources/libraries/python/DUTSetup.py
+++ b/resources/libraries/python/DUTSetup.py
@@ -13,6 +13,8 @@
 
 """DUT setup library."""
 
+import os
+
 from robot.api import logger
 
 from resources.libraries.python.topology import NodeType, Topology
@@ -51,6 +53,12 @@ class DUTSetup(object):
         vat = VatExecutor()
         vat.execute_script("show_version_verbose.vat", node, json_out=False)
 
+        try:
+            vat.script_should_have_passed()
+        except AssertionError:
+            raise RuntimeError('Failed to get VPP version on host: {}'.
+                               format(node['host']))
+
     @staticmethod
     def show_vpp_version_on_all_duts(nodes):
         """Show VPP version verbose on all DUTs.
@@ -63,6 +71,22 @@ class DUTSetup(object):
                 DUTSetup.vpp_show_version_verbose(node)
 
     @staticmethod
+    def vpp_show_interfaces(node):
+        """Run "show interface" CLI command.
+
+        :param node: Node to run command on.
+        :type node: dict
+        """
+        vat = VatExecutor()
+        vat.execute_script("show_interface.vat", node, json_out=False)
+
+        try:
+            vat.script_should_have_passed()
+        except AssertionError:
+            raise RuntimeError('Failed to get VPP interfaces on host: {}'.
+                               format(node['host']))
+
+    @staticmethod
     def vpp_api_trace_save(node):
         """Run "api trace save" CLI command.
 
@@ -425,3 +449,104 @@ class DUTSetup(object):
         vat = VatExecutor()
         vat.execute_script("enable_dpdk_traces.vat", node, json_out=False)
         vat.execute_script("enable_vhost_user_traces.vat", node, json_out=False)
+
+    @staticmethod
+    def install_vpp_on_all_duts(nodes, vpp_pkg_dir, vpp_rpm_pkgs, vpp_deb_pkgs):
+        """Install VPP on all DUT nodes.
+
+        :param nodes: Nodes in the topology.
+        :param vpp_pkg_dir: Path to directory where VPP packages are stored.
+        :param vpp_rpm_pkgs: List of VPP rpm packages to be installed.
+        :param vpp_deb_pkgs: List of VPP deb packages to be installed.
+        :type nodes: dict
+        :type vpp_pkg_dir: str
+        :type vpp_rpm_pkgs: list
+        :type vpp_deb_pkgs: list
+        :raises: RuntimeError if failed to remove or install VPP
+        """
+
+        logger.debug("Installing VPP")
+
+        for node in nodes.values():
+            if node['type'] == NodeType.DUT:
+                logger.debug("Installing VPP on node {0}".format(node['host']))
+
+                ssh = SSH()
+                ssh.connect(node)
+
+                if os.path.isfile("/etc/redhat-release"):
+                    # workaroud - uninstall existing vpp installation until
+                    # start-testcase script is updated on all virl servers
+                    rpm_pkgs_remove = " ".join(vpp_rpm_pkgs)
+                    r_rcode, _, r_err = ssh.exec_command_sudo(
+                        "rpm -e {0}".format(rpm_pkgs_remove), timeout=90)
+                    if int(r_rcode) != 0:
+                        raise RuntimeError('Failed to remove previous VPP'
+                                           'installation on host {0}:\n{1}'
+                                           .format(node['host']), r_err)
+
+                    rpm_pkgs = "*.rpm ".join(str(vpp_pkg_dir + pkg)
+                                             for pkg in vpp_rpm_pkgs) + "*.rpm"
+                    ret_code, _, err = ssh.exec_command_sudo(
+                        "rpm -ivh {0}".format(rpm_pkgs), timeout=90)
+                    if int(ret_code) != 0:
+                        raise RuntimeError('Failed to install VPP on host {0}:'
+                                           '\n{1}'.format(node['host']), err)
+                    else:
+                        ssh.exec_command_sudo("rpm -qai vpp*")
+                        logger.info("VPP installed on node {0}".
+                                    format(node['host']))
+                else:
+                    # workaroud - uninstall existing vpp installation until
+                    # start-testcase script is updated on all virl servers
+                    deb_pkgs_remove = " ".join(vpp_deb_pkgs)
+                    r_rcode, _, r_err = ssh.exec_command_sudo(
+                        "dpkg --purge {0}".format(deb_pkgs_remove), timeout=90)
+                    if int(r_rcode) != 0:
+                        raise RuntimeError('Failed to remove previous VPP'
+                                           'installation on host {0}:\n{1}'
+                                           .format(node['host']), r_err)
+                    deb_pkgs = "*.deb ".join(str(vpp_pkg_dir + pkg)
+                                             for pkg in vpp_deb_pkgs) + "*.deb"
+                    ret_code, _, err = ssh.exec_command_sudo(
+                        "dpkg -i --force-all {0}".format(deb_pkgs), timeout=90)
+                    if int(ret_code) != 0:
+                        raise RuntimeError('Failed to install VPP on host {0}:'
+                                           '\n{1}'.format(node['host']), err)
+                    else:
+                        ssh.exec_command_sudo("dpkg -l | grep vpp")
+                        logger.info("VPP installed on node {0}".
+                                    format(node['host']))
+
+                ssh.disconnect(node)
+
+    @staticmethod
+    def verify_vpp_on_all_duts(nodes):
+        """Verify that VPP is installed on all DUT nodes.
+
+        :param nodes: Nodes in the topology.
+        :type nodes: dict
+        """
+
+        logger.debug("Verify VPP on all DUTs")
+
+        DUTSetup.start_vpp_service_on_all_duts(nodes)
+
+        for node in nodes.values():
+            if node['type'] == NodeType.DUT:
+                DUTSetup.verify_vpp_on_dut(node)
+
+    @staticmethod
+    def verify_vpp_on_dut(node):
+        """Verify that VPP is installed on DUT node.
+
+        :param node: DUT node.
+        :type node: dict
+        :raises: RuntimeError if failed to restart VPP, get VPP version or
+        get VPP interfaces
+        """
+
+        logger.debug("Verify VPP on node {0}".format(node['host']))
+
+        DUTSetup.vpp_show_version_verbose(node)
+        DUTSetup.vpp_show_interfaces(node)
diff --git a/resources/templates/vat/show_interface.vat b/resources/templates/vat/show_interface.vat
new file mode 100644
index 0000000000..474b8478ae
--- /dev/null
+++ b/resources/templates/vat/show_interface.vat
@@ -0,0 +1 @@
+exec show interface
diff --git a/resources/tools/virl/bin/start-testcase b/resources/tools/virl/bin/start-testcase
index f6c3cc32b8..ce47378188 100755
--- a/resources/tools/virl/bin/start-testcase
+++ b/resources/tools/virl/bin/start-testcase
@@ -32,6 +32,7 @@ import requests
 
 IPS_PER_SIMULATION = 5
 
+
 def indent(lines, amount, fillchar=' '):
     """Indent the string by amount of fill chars.
 
@@ -47,6 +48,7 @@ def indent(lines, amount, fillchar=' '):
     padding = amount * fillchar
     return padding + ('\n'+padding).join(lines.split('\n'))
 
+
 def print_to_stderr(msg, end='\n'):
     """Writes any text to stderr.
 
@@ -60,6 +62,7 @@ def print_to_stderr(msg, end='\n'):
     except ValueError:
         pass
 
+
 def get_assigned_interfaces(args, network="flat"):
     """Retrieve assigned interfaces in openstack network.
 
@@ -81,6 +84,7 @@ def get_assigned_interfaces(args, network="flat"):
                            "Status other than 200 HTTP OK:\n{}"
                            .format(req.content))
 
+
 def get_assigned_interfaces_count(args, network="flat"):
     """Count assigned interfaces in openstack network.
 
@@ -93,6 +97,7 @@ def get_assigned_interfaces_count(args, network="flat"):
     """
     return len(get_assigned_interfaces(args, network=network))
 
+
 def check_ip_addresses(args):
     """Check IP address availability.
 
@@ -101,8 +106,8 @@ def check_ip_addresses(args):
     :raises RuntimeError: If not enough free addresses available.
     """
     for i in range(args.wait_count):
-        if (args.quota - \
-            get_assigned_interfaces_count(args) >= IPS_PER_SIMULATION):
+        if (args.quota -
+                get_assigned_interfaces_count(args) >= IPS_PER_SIMULATION):
             break
         if args.verbosity >= 2:
             print_to_stderr("DEBUG: - Attempt {} out of {}, waiting for free "
@@ -112,6 +117,7 @@ def check_ip_addresses(args):
     else:
         raise RuntimeError("ERROR: Not enough IP addresses to run simulation")
 
+
 def check_virl_resources(args):
     """Check virl resources availability.
 
@@ -125,6 +131,8 @@ def check_virl_resources(args):
 # function executed in sequence. This should be broken down into multiple
 # functions.
 #
+
+
 def main():
     """ Main function."""
     #
@@ -189,6 +197,8 @@ def main():
     parser.add_argument("-q", "--quota",
                         help="VIRL quota for max number of allowed IPs",
                         type=int, default=74)
+    parser.add_argument("-si", "--skip-install", help="Skip VPP installation",
+                        action='store_true')
 
     args = parser.parse_args()
 
@@ -216,14 +226,15 @@ def main():
     #
     # Check if VPP package exists
     #
-    for package in args.packages:
-        if args.verbosity >= 2:
-            print_to_stderr("DEBUG: Checking if file {} exists"
-                            .format(package))
-        if not os.path.isfile(package):
-            print_to_stderr("ERROR: Debian package {} does not exist"
-                            .format(package))
-            sys.exit(1)
+    if not args.skip_install:
+        for package in args.packages:
+            if args.verbosity >= 2:
+                print_to_stderr("DEBUG: Checking if file {} exists"
+                                .format(package))
+            if not os.path.isfile(package):
+                print_to_stderr("ERROR: Required package {} does not exist"
+                                .format(package))
+                sys.exit(1)
 
     #
     # Start VIRL topology
@@ -496,46 +507,47 @@ def main():
     #
     # Upgrade VPP
     #
-    if args.verbosity >= 1:
-        print_to_stderr("DEBUG: Uprading VPP")
-
-    for key1 in nodeaddrs:
-        if not key1 == 'tg':
-            for key2 in nodeaddrs[key1]:
-                ipaddr = nodeaddrs[key1][key2]
-                if args.verbosity >= 2:
-                    print_to_stderr("DEBUG: Upgrading VPP on node {}"
-                                    .format(ipaddr))
-                paramiko.util.log_to_file(os.path.join(scratch_directory,
-                                                       "ssh.log"))
-                client = paramiko.SSHClient()
-                client.load_system_host_keys()
-                client.load_host_keys("/dev/null")
-                client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-                client.connect(ipaddr, username=args.ssh_user,
-                               key_filename=args.ssh_privkey)
-                if 'centos' in args.topology:
-                    if args.verbosity >= 1:
-                        print_to_stderr("DEBUG: Installing RPM packages")
-                    vpp_install_command = 'sudo rpm -ivh /scratch/vpp/*.rpm'
-                elif 'trusty' in args.topology or 'xenial' in args.topology:
-                    if args.verbosity >= 1:
-                        print_to_stderr("DEBUG: Installing DEB packages")
-                    vpp_install_command = 'sudo dpkg -i --force-all ' \
-                                          '/scratch/vpp/*.deb'
-                else:
-                    print_to_stderr("ERROR: Unsupported OS requested: {}"
-                                    .format(args.topology))
-                    vpp_install_command = ''
-                _, stdout, stderr = \
-                    client.exec_command(vpp_install_command)
-                c_stdout = stdout.read()
-                c_stderr = stderr.read()
-                if args.verbosity >= 2:
-                    print_to_stderr("DEBUG: Command output was:")
-                    print_to_stderr(c_stdout)
-                    print_to_stderr("DEBUG: Command stderr was:")
-                    print_to_stderr(c_stderr)
+    if not args.skip_install:
+        if args.verbosity >= 1:
+            print_to_stderr("DEBUG: Uprading VPP")
+
+        for key1 in nodeaddrs:
+            if not key1 == 'tg':
+                for key2 in nodeaddrs[key1]:
+                    ipaddr = nodeaddrs[key1][key2]
+                    if args.verbosity >= 2:
+                        print_to_stderr("DEBUG: Upgrading VPP on node {}"
+                                        .format(ipaddr))
+                    paramiko.util.log_to_file(os.path.join(scratch_directory,
+                                                           "ssh.log"))
+                    client = paramiko.SSHClient()
+                    client.load_system_host_keys()
+                    client.load_host_keys("/dev/null")
+                    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+                    client.connect(ipaddr, username=args.ssh_user,
+                                   key_filename=args.ssh_privkey)
+                    if 'centos' in args.topology:
+                        if args.verbosity >= 1:
+                            print_to_stderr("DEBUG: Installing RPM packages")
+                        vpp_install_command = 'sudo rpm -ivh /scratch/vpp/*.rpm'
+                    elif 'trusty' in args.topology or 'xenial' in args.topology:
+                        if args.verbosity >= 1:
+                            print_to_stderr("DEBUG: Installing DEB packages")
+                        vpp_install_command = 'sudo dpkg -i --force-all ' \
+                                              '/scratch/vpp/*.deb'
+                    else:
+                        print_to_stderr("ERROR: Unsupported OS requested: {}"
+                                         .format(args.topology))
+                        vpp_install_command = ''
+                    _, stdout, stderr = \
+                        client.exec_command(vpp_install_command)
+                    c_stdout = stdout.read()
+                    c_stderr = stderr.read()
+                    if args.verbosity >= 2:
+                        print_to_stderr("DEBUG: Command output was:")
+                        print_to_stderr(c_stdout)
+                        print_to_stderr("DEBUG: Command stderr was:")
+                        print_to_stderr(c_stderr)
 
     #
     # Write a file with timestamp to scratch directory. We can use this to track
diff --git a/tests/vpp/func/__init__.robot b/tests/vpp/func/__init__.robot
index 21693c3c86..9243257c0a 100644
--- a/tests/vpp/func/__init__.robot
+++ b/tests/vpp/func/__init__.robot
@@ -16,7 +16,26 @@
 | Resource | resources/libraries/robot/shared/interfaces.robot
 | Library | resources.libraries.python.SetupFramework
 | Library | resources.libraries.python.SetupFramework.CleanupFramework
-| Suite Setup | Run Keywords | Setup Framework | ${nodes}
-| ...         | AND          | Setup All DUTs | ${nodes}
-| ...         | AND          | Update All Interface Data On All Nodes | ${nodes}
+| Suite Setup | Run Keywords | Setup Functional Global Variables
+| ... | AND | Setup Framework | ${nodes}
+| ... | AND | Install Vpp On All Duts | ${nodes} | ${VPP_PKG_DIR}
+| ... | ${VPP_RPM_PKGS} | ${VPP_DEB_PKGS}
+| ... | AND | Verify Vpp On All Duts | ${nodes}
+| ... | AND | Setup All DUTs | ${nodes}
+| ... | AND | Update All Interface Data On All Nodes | ${nodes}
 | Suite Teardown | Cleanup Framework | ${nodes}
+
+*** Keywords ***
+| Setup Functional Global Variables
+| | [Documentation]
+| | ... | Setup suite Variables. Variables are used across functional testing.
+| | ...
+| | ... | _NOTE:_ This KW sets following suite variables:
+| | ... | - vpp_pkg_dir - Path to directory where VPP packages are stored.
+| | ...
+| | Set Global Variable | ${VPP_PKG_DIR} | /scratch/vpp/
+| | @{VPP_RPM_PKGS}= | Create List | vpp | vpp-devel | vpp-lib | vpp-plugins
+| | Set Global Variable | ${VPP_RPM_PKGS}
+| | @{VPP_DEB_PKGS}= | Create List | vpp | vpp-dbg | vpp-dev | vpp-lib
+| | ... | vpp-plugins
+| | Set Global Variable | ${VPP_DEB_PKGS}