HC Test: bugfixes for hc2vpp ODL jobs

 - add shutdown check for ODL client in teardown
 - stop VPP in teardown

 - add delay after HC startup
HC features load asynchronously, no way to check if they're all running.

Change-Id: I9d2c51b68ba51284d2d409a77d6cd9fd478e1725
Signed-off-by: selias <samelias@cisco.com>

2 files changed, 40 insertions, 4 deletions

diff --git a/resources/libraries/python/honeycomb/HoneycombSetup.py b/resources/libraries/python/honeycomb/HoneycombSetup.py
index a067b2f993..4c438ff55e 100644
--- a/resources/libraries/python/honeycomb/HoneycombSetup.py
+++ b/resources/libraries/python/honeycomb/HoneycombSetup.py
@@ -163,7 +163,11 @@ class HoneycombSetup(object):
                 except HTTPRequestError:
                     ssh = SSH()
                     ssh.connect(node)
-                    ssh.exec_command("tail -n 100 /var/log/syslog")
+                    ret_code, _, _ = ssh.exec_command_sudo(
+                        "tail -n 100 /var/log/syslog")
+                    if ret_code != 0:
+                        # It's probably Centos
+                        ssh.exec_command_sudo("tail -n 100 /var/log/messages")
                     raise
                 if status_code == HTTPCodes.OK:
                     logger.info("Honeycomb on node {0} is up and running".
@@ -554,6 +558,35 @@ class HoneycombSetup(object):
         return True
 
     @staticmethod
+    def check_odl_shutdown_state(node):
+        """Check the status of ODL client shutdown.
+
+        :param node: Honeycomb node.
+        :type node: dict
+        :returns: True when ODL is stopped.
+        :rtype: bool
+        :raises HoneycombError: When the response is not code 200: OK.
+        """
+
+        cmd = "pgrep -f karaf"
+        path = HcUtil.read_path_from_url_file(
+            "odl_client/odl_netconf_connector")
+
+        try:
+            status_code, _ = HTTPRequest.get(node, path, timeout=10,
+                                             enable_logging=False)
+            raise HoneycombError("ODL client is still running.")
+        except HTTPRequestError:
+            logger.debug("Connection refused, checking process state....")
+            ssh = SSH()
+            ssh.connect(node)
+            ret_code, _, _ = ssh.exec_command(cmd)
+            if ret_code == 0:
+                raise HoneycombError("ODL client is still running.")
+
+        return True
+
+    @staticmethod
     def mount_honeycomb_on_odl(node):
         """Tell ODL client to mount Honeycomb instance over netconf.
 
@@ -625,9 +658,7 @@ class HoneycombSetup(object):
         cmd = "service vpp stop"
         ret_code, _, _ = ssh.exec_command_sudo(cmd, timeout=80)
         if int(ret_code) != 0:
-            raise RuntimeError("Could not stop VPP service on node {0}".format(
-                node['host']))
-
+            logger.debug("VPP service refused to shut down.")
 
 class HoneycombStartupConfig(object):
     """Generator for Honeycomb startup configuration.
diff --git a/resources/libraries/robot/honeycomb/honeycomb.robot b/resources/libraries/robot/honeycomb/honeycomb.robot
index d0f9f0fbf6..1b08e3955f 100644
--- a/resources/libraries/robot/honeycomb/honeycomb.robot
+++ b/resources/libraries/robot/honeycomb/honeycomb.robot
@@ -41,6 +41,7 @@
 | | Start honeycomb on DUTs | @{duts}
 | | Wait until keyword succeeds | 4min | 16sec
 | | ... | Check honeycomb startup state | @{duts}
+| | Sleep | 5s | Make sure all modules are loaded and ready.
 
 | Stop Honeycomb service on DUTs
 | | [Documentation] | *Cleanup environment after honeycomb testing.*
@@ -247,8 +248,11 @@
 | | Run Keyword If | '${use_odl_client}' != '${NONE}'
 | | ... | Run Keywords
 | | ... | Stop ODL Client | ${node} | /tmp | AND
+| | ... | Wait until keyword succeeds | 2min | 15sec
+| | ... | Check ODL shutdown state | ${node} | AND
 | | ... | Set Global Variable | ${use_odl_client} | ${NONE}
 | | Stop Honeycomb service on DUTs | ${node}
+| | Stop VPP Service on DUT | ${node}
 
 | Enable Honeycomb Feature
 | | [Documentation] | Enable the specified feature in Honeycomb configuration.
@@ -320,3 +324,4 @@
 | | [Arguments] | ${node}
 | | Log Honeycomb and VPP process distribution on cores | ${node}
 | | Stop Honeycomb service on DUTs | ${node}
+| | Stop VPP Service on DUT | ${node}