HC Test: improve functional suite setup and teardown

Restart the whole stack (VPP+Honeycomb+-ODL) in between test suites.
Add generic suite setup and suite teardown keywords for all func suites
Re-enable NSH_SFC suite, cleanup test failures

Change-Id: I71025b4ed3a7c4912dcd7b62aedef6082357990c
Signed-off-by: selias <samelias@cisco.com>

2 files changed, 134 insertions, 77 deletions

diff --git a/resources/libraries/python/honeycomb/HcAPIKwInterfaces.py b/resources/libraries/python/honeycomb/HcAPIKwInterfaces.py
index 1e0b1a5d14..6f0bdda9fe 100644
--- a/resources/libraries/python/honeycomb/HcAPIKwInterfaces.py
+++ b/resources/libraries/python/honeycomb/HcAPIKwInterfaces.py
@@ -252,7 +252,7 @@ class InterfaceKeywords(object):
         return InterfaceKeywords._configure_interface(node, interface, new_data)
 
     @staticmethod
-    def set_interface_state(node, interface, state="up"):
+    def honeycomb_set_interface_state(node, interface, state="up"):
         """Set VPP interface state.
 
         The keyword changes the administration state of interface to up or down
@@ -310,7 +310,8 @@ class InterfaceKeywords(object):
         :rtype: bytearray
         """
 
-        return InterfaceKeywords.set_interface_state(node, interface, "up")
+        return InterfaceKeywords.honeycomb_set_interface_state(
+            node, interface, "up")
 
     @staticmethod
     def set_interface_down(node, interface):
@@ -324,7 +325,8 @@ class InterfaceKeywords(object):
         :rtype: bytearray
         """
 
-        return InterfaceKeywords.set_interface_state(node, interface, "down")
+        return InterfaceKeywords.honeycomb_set_interface_state(
+            node, interface, "down")
 
     @staticmethod
     def add_bridge_domain_to_interface(node, interface, bd_name,
@@ -796,7 +798,7 @@ class InterfaceKeywords(object):
             node, interface, path, value)
 
     @staticmethod
-    def create_vxlan_interface(node, interface, **kwargs):
+    def honeycomb_create_vxlan_interface(node, interface, **kwargs):
         """Create a new VxLAN interface.
 
         :param node: Honeycomb node.
@@ -860,7 +862,7 @@ class InterfaceKeywords(object):
         return resp
 
     @staticmethod
-    def configure_interface_vxlan(node, interface, **kwargs):
+    def honeycomb_configure_interface_vxlan(node, interface, **kwargs):
         """Configure VxLAN on the interface.
 
         The keyword configures VxLAN parameters on the given interface. The type
@@ -1052,7 +1054,8 @@ class InterfaceKeywords(object):
             node, interface, path, new_vhost_structure)
 
     @staticmethod
-    def create_sub_interface(node, super_interface, match, tags=None, **kwargs):
+    def honeycomb_create_sub_interface(node, super_interface, match, tags=None,
+                                       **kwargs):
         """Create a new sub-interface.
 
         :param node: Honeycomb node.
diff --git a/resources/libraries/python/honeycomb/HoneycombSetup.py b/resources/libraries/python/honeycomb/HoneycombSetup.py
index 5026cb4868..53130f405b 100644
--- a/resources/libraries/python/honeycomb/HoneycombSetup.py
+++ b/resources/libraries/python/honeycomb/HoneycombSetup.py
@@ -59,7 +59,7 @@ class HoneycombSetup(object):
 
         HoneycombSetup.print_environment(nodes)
 
-        logger.console("\nStarting Honeycomb service ...")
+        logger.console("\n(re)Starting Honeycomb service ...")
 
         cmd = "sudo service honeycomb start"
 
@@ -107,41 +107,6 @@ class HoneycombSetup(object):
                                  format(errors))
 
     @staticmethod
-    def restart_honeycomb_and_vpp_on_duts(*nodes):
-        """Restart the Honeycomb service on specified DUT nodes.
-
-        Use the keyword "Check Honeycomb Startup State" to check when Honeycomb
-        is fully restarted.
-        :param nodes: List of nodes to restart Honeycomb on.
-        :type nodes: list
-        :raises HoneycombError: If Honeycomb failed to restart.
-        """
-        logger.console("\nRestarting Honeycomb service ...")
-
-        cmd = "sudo service honeycomb restart "
-        errors = []
-
-        for node in nodes:
-            if node['type'] == NodeType.DUT:
-                ssh = SSH()
-                ssh.connect(node)
-                (ret_code, _, _) = ssh.exec_command_sudo(cmd)
-                if int(ret_code) != 0:
-                    errors.append(node['host'])
-                try:
-                    DUTSetup.setup_dut(node)
-                except Exception as err:
-                    logger.debug(err)
-                    errors.append(node['host'])
-                    continue
-                logger.info("Restart of Honeycomb and VPP on node {0} is "
-                            "in progress ...".format(node['host']))
-        if errors:
-            raise HoneycombError('Node(s) {0} failed to restart Honeycomb'
-                                 ' and/or VPP.'.
-                                 format(errors))
-
-    @staticmethod
     def check_honeycomb_startup_state(*nodes):
         """Check state of Honeycomb service during startup on specified nodes.
 
@@ -362,16 +327,19 @@ class HoneycombSetup(object):
                                  "node {0}, {1}".format(node, stderr))
 
     @staticmethod
-    def enable_module_features(node, *features):
-        """Configure Honeycomb to use VPP modules that are disabled by default.
+    def manage_honeycomb_features(node, feature, disable=False):
+        """Configure Honeycomb to use features that are disabled by default, or
+        disable previously enabled features.
 
         ..Note:: If the module is not enabled in VPP, Honeycomb will
         be unable to establish VPP connection.
 
         :param node: Honeycomb node.
-        :param features: Features to enable.
+        :param feature: Feature to enable.
+        :param disable: Disable the specified feature instead of enabling it.
         :type node: dict
-        :type features: string
+        :type feature: string
+        :type disable: bool
         :raises HoneycombError: If the configuration could not be changed.
          """
 
@@ -382,23 +350,24 @@ class HoneycombSetup(object):
         ssh = SSH()
         ssh.connect(node)
 
-        for feature in features:
-            if feature in disabled_features.keys():
-                # uncomment by replacing the entire line
-                find = replace = "{0}".format(disabled_features[feature])
-
-                argument = '"/{0}/c\\ {1}"'.format(find, replace)
-                path = "{0}/modules/*module-config"\
-                    .format(Const.REMOTE_HC_DIR)
-                command = "sed -i {0} {1}".format(argument, path)
-
-                (ret_code, _, stderr) = ssh.exec_command_sudo(command)
-                if ret_code != 0:
-                    raise HoneycombError("Failed to modify configuration on "
-                                         "node {0}, {1}".format(node, stderr))
-            else:
-                raise HoneycombError(
-                    "Unrecognized feature {0}.".format(feature))
+        if feature in disabled_features.keys():
+            # uncomment by replacing the entire line
+            find = replace = "{0}".format(disabled_features[feature])
+            if disable:
+                replace = "// {0}".format(find)
+
+            argument = '"/{0}/c\\ {1}"'.format(find, replace)
+            path = "{0}/modules/*module-config"\
+                .format(Const.REMOTE_HC_DIR)
+            command = "sed -i {0} {1}".format(argument, path)
+
+            (ret_code, _, stderr) = ssh.exec_command_sudo(command)
+            if ret_code != 0:
+                raise HoneycombError("Failed to modify configuration on "
+                                     "node {0}, {1}".format(node, stderr))
+        else:
+            raise HoneycombError(
+                "Unrecognized feature {0}.".format(feature))
 
     @staticmethod
     def copy_java_libraries(node):
@@ -438,37 +407,51 @@ class HoneycombSetup(object):
                                      "node {0}, {1}".format(node, stderr))
 
     @staticmethod
-    def configure_odl_client(node, odl_name):
-        """Start ODL client on the specified node.
+    def copy_odl_client(node, odl_name, src_path, dst_path):
+        """Copy ODL Client from source path to destination path.
 
-        Karaf should be located in /mnt/common, and VPP and Honeycomb should
-        already be running, otherwise the start will fail.
-        :param node: Node to start ODL client on.
+        :param node: Honeycomb node.
         :param odl_name: Name of ODL client version to use.
+        :param src_path: Source Path where to find ODl client.
+        :param dst_path: Destination path.
         :type node: dict
         :type odl_name: str
-        :raises HoneycombError: If Honeycomb fails to start.
+        :type src_path: str
+        :type dst_path: str
+        :raises HoneycombError: If the operation fails.
         """
 
-        logger.debug("Copying ODL Client to home dir.")
-
         ssh = SSH()
         ssh.connect(node)
 
-        cmd = "cp -r /mnt/common/*karaf_{name}* ~/karaf".format(name=odl_name)
+        cmd = "cp -r {src}/*karaf_{odl_name}* {dst}".format(
+            src=src_path, odl_name=odl_name, dst=dst_path)
 
-        (ret_code, _, _) = ssh.exec_command_sudo(cmd)
+        ret_code, _, _ = ssh.exec_command(cmd, timeout=30)
         if int(ret_code) != 0:
             raise HoneycombError(
                 "Failed to copy ODL client on node {0}".format(node["host"]))
 
-        logger.console("\nStarting ODL client ...")
+    @staticmethod
+    def setup_odl_client(node, path):
+        """Start ODL client on the specified node.
 
-        cmd = "~/*karaf*/bin/start"
+        Karaf should be located in the provided path, and VPP and Honeycomb
+        should already be running, otherwise the start will fail.
+        :param node: Node to start ODL client on.
+        :param path: Path to ODL client on node.
+        :type node: dict
+        :type path: str
+        :raises HoneycombError: If Honeycomb fails to start.
+        """
 
+        logger.console("\nStarting ODL client ...")
         ssh = SSH()
         ssh.connect(node)
-        (ret_code, _, _) = ssh.exec_command_sudo(cmd)
+
+        cmd = "{path}/*karaf*/bin/start clean".format(path=path)
+        ret_code, _, _ = ssh.exec_command_sudo(cmd)
+
         if int(ret_code) != 0:
             raise HoneycombError('Node {0} failed to start ODL.'.
                                  format(node['host']))
@@ -477,6 +460,31 @@ class HoneycombSetup(object):
                         "in progress ...".format(node['host']))
 
     @staticmethod
+    def install_odl_features(node, path, *features):
+        """Install required features on a running ODL client.
+
+        :param node: Honeycomb node.
+        :param path: Path to ODL client on node.
+        :param features: Optional, list of additional features to install.
+        :type node: dict
+        :type path: str
+        :type features: list
+        """
+
+        ssh = SSH()
+        ssh.connect(node)
+
+        cmd = "{path}/*karaf*/bin/client -u karaf feature:install " \
+              "odl-restconf-all odl-netconf-connector-all".format(path=path)
+        for feature in features:
+            cmd += " {0}".format(feature)
+
+        ret_code, _, stderr = ssh.exec_command_sudo(cmd, timeout=120)
+
+        if int(ret_code) != 0:
+            raise HoneycombError("Feature install did not succeed.")
+
+    @staticmethod
     def check_odl_startup_state(node):
         """Check the status of ODL client startup.
 
@@ -537,6 +545,52 @@ class HoneycombSetup(object):
         if status_code == HTTPCodes.OK:
             logger.info("ODL mount point configured successfully.")
         elif status_code == HTTPCodes.CONFLICT:
-            logger.warn("ODL mount point was already configured.")
+            logger.info("ODL mount point was already configured.")
         else:
             raise HoneycombError('Mount point configuration not successful')
+
+    @staticmethod
+    def stop_odl_client(node, path):
+        """Stop ODL client service on the specified node.
+
+        :param node: Node to start ODL client on.
+        :param path: Path to ODL client.
+        :type node: dict
+        :type path: str
+        :raises HoneycombError: If ODL client fails to stop.
+        """
+
+        ssh = SSH()
+        ssh.connect(node)
+
+        cmd = "{0}/*karaf*/bin/stop".format(path)
+
+        ssh = SSH()
+        ssh.connect(node)
+        ret_code, _, _ = ssh.exec_command_sudo(cmd)
+        if int(ret_code) != 0:
+            logger.warn("ODL Client refused to shut down.")
+            cmd = "pkill -f 'karaf'"
+            (ret_code, _, _) = ssh.exec_command_sudo(cmd)
+            if int(ret_code) != 0:
+                raise HoneycombError('Node {0} failed to stop ODL.'.
+                                     format(node['host']))
+
+        logger.info("ODL client service stopped.")
+
+    @staticmethod
+    def stop_vpp_service(node):
+        """Stop VPP service on the specified node.
+
+        :param node: VPP node.
+        :type node: dict
+        :raises RuntimeError: If VPP fails to stop.
+        """
+
+        ssh = SSH()
+        ssh.connect(node)
+        cmd = "service vpp stop"
+        ret_code, _, _ = ssh.exec_command_sudo(cmd)
+        if int(ret_code) != 0:
+            raise RuntimeError("Could not stop VPP service on node {0}".format(
+                node['host']))