CSIT-454: HC Test: Improve logging on test failure

 - set Honeycomb log level to TRACE before running Honeycomb tests
 - on a failed restconf request, print the last few lines of the log
 - show saved configuration during persistence suite restarts

Change-Id: I07ee37a5403cfca946fd7ef25a42861273248fbe
Signed-off-by: selias <samelias@cisco.com>

6 files changed, 95 insertions, 10 deletions

diff --git a/resources/libraries/python/honeycomb/HcPersistence.py b/resources/libraries/python/honeycomb/HcPersistence.py
index 3bbc52fa91..c76a276098 100644
--- a/resources/libraries/python/honeycomb/HcPersistence.py
+++ b/resources/libraries/python/honeycomb/HcPersistence.py
@@ -83,3 +83,22 @@ class HcPersistence(object):
         if ret_code != 0:
             raise HoneycombError("Failed to modify persistence file on node"
                                  " {0}, {1}".format(node, stderr))
+
+    @staticmethod
+    def log_persisted_configuration(node):
+        """Read contents of Honeycomb persistence files and print them to log.
+
+        :param node: Honeycomb node.
+        :type node: dict
+        """
+
+        commands = [
+            "cat {0}/config/data.json".format(Const.REMOTE_HC_PERSIST),
+            "cat {0}/context/data.json".format(Const.REMOTE_HC_PERSIST),
+        ]
+
+        ssh = SSH()
+        ssh.connect(node)
+        for command in commands:
+            (_, stdout, _) = ssh.exec_command_sudo(command)
+            logger.info(stdout)
diff --git a/resources/libraries/python/honeycomb/HoneycombSetup.py b/resources/libraries/python/honeycomb/HoneycombSetup.py
index 8a05ad7583..06b35db8a1 100644
--- a/resources/libraries/python/honeycomb/HoneycombSetup.py
+++ b/resources/libraries/python/honeycomb/HoneycombSetup.py
@@ -243,8 +243,7 @@ class HoneycombSetup(object):
                 "which java",
                 "java -version",
                 "dpkg --list | grep openjdk",
-                "ls -la /opt/honeycomb",
-                "ls -la /opt/honeycomb/v3po-karaf-1.0.0-SNAPSHOT")
+                "ls -la /opt/honeycomb")
 
         for node in nodes:
             if node['type'] == NodeType.DUT:
@@ -274,3 +273,26 @@ class HoneycombSetup(object):
             ssh.connect(node)
             ssh.exec_command_sudo(cmd)
 
+    @staticmethod
+    def configure_log_level(node, level):
+        """Set Honeycomb logging to the specified level.
+
+        :param node: Honeycomb node.
+        :param level: Log level (INFO, DEBUG, TRACE).
+        :type node: dict
+        :type level: str
+        """
+
+        find = 'logger name=\\"io.fd\\"'
+        replace = '<logger name=\\"io.fd\\" level=\\"{0}\\"/>'.format(level)
+
+        argument = '"/{0}/c\\ {1}"'.format(find, replace)
+        path = "{0}/config/logback.xml".format(Const.REMOTE_HC_DIR)
+        command = "sed -i {0} {1}".format(argument, path)
+
+        ssh = SSH()
+        ssh.connect(node)
+        (ret_code, _, stderr) = ssh.exec_command_sudo(command)
+        if ret_code != 0:
+            raise HoneycombError("Failed to modify configuration on "
+                                 "node {0}, {1}".format(node, stderr))
diff --git a/resources/libraries/python/honeycomb/HoneycombUtil.py b/resources/libraries/python/honeycomb/HoneycombUtil.py
index f2f012ea9b..22a7e7ef1e 100644
--- a/resources/libraries/python/honeycomb/HoneycombUtil.py
+++ b/resources/libraries/python/honeycomb/HoneycombUtil.py
@@ -27,7 +27,8 @@ from enum import Enum, unique
 
 from robot.api import logger
 
-from resources.libraries.python.HTTPRequest import HTTPRequest
+from resources.libraries.python.ssh import SSH
+from resources.libraries.python.HTTPRequest import HTTPRequest, HTTPCodes
 from resources.libraries.python.constants import Constants as Const
 
 
@@ -301,7 +302,10 @@ class HoneycombUtil(object):
         base_path = HoneycombUtil.read_path_from_url_file(url_file)
         path = base_path + path
         status_code, resp = HTTPRequest.get(node, path)
-        return status_code, loads(resp)
+        (status_node, response) = status_code, loads(resp)
+        if status_code != HTTPCodes.OK:
+            HoneycombUtil.read_log_tail(node)
+        return status_code, response
 
     @staticmethod
     def put_honeycomb_data(node, url_file, data, path="",
@@ -339,8 +343,12 @@ class HoneycombUtil(object):
         base_path = HoneycombUtil.read_path_from_url_file(url_file)
         path = base_path + path
         logger.trace(path)
-        return HTTPRequest.put(node=node, path=path, headers=header,
-                               payload=data)
+        (status_code, response) = HTTPRequest.put(
+            node=node, path=path, headers=header, payload=data)
+
+        if status_code not in (HTTPCodes.OK, HTTPCodes.ACCEPTED):
+            HoneycombUtil.read_log_tail(node)
+        return status_code, response
 
     @staticmethod
     def post_honeycomb_data(node, url_file, data=None,
@@ -375,8 +383,12 @@ class HoneycombUtil(object):
             data = dumps(data)
 
         path = HoneycombUtil.read_path_from_url_file(url_file)
-        return HTTPRequest.post(node=node, path=path, headers=header,
-                                payload=data, timeout=timeout)
+        (status_code, response) = HTTPRequest.post(
+            node=node, path=path, headers=header, payload=data, timeout=timeout)
+
+        if status_code not in (HTTPCodes.OK, HTTPCodes.ACCEPTED):
+            HoneycombUtil.read_log_tail(node)
+        return status_code, response
 
     @staticmethod
     def delete_honeycomb_data(node, url_file, path=""):
@@ -395,4 +407,33 @@ class HoneycombUtil(object):
 
         base_path = HoneycombUtil.read_path_from_url_file(url_file)
         path = base_path + path
-        return HTTPRequest.delete(node, path)
+        (status_code, response) = HTTPRequest.delete(node, path)
+
+        if status_code != HTTPCodes.OK:
+            HoneycombUtil.read_log_tail(node)
+        return status_code, response
+
+    @staticmethod
+    def read_log_tail(node, lines=120):
+        """Read  the last N lines of the Honeycomb log file and print them
+        to robot log.
+
+        :param node: Honeycomb node.
+        :param lines: Number of lines to read.
+        :type node: dict
+        :type lines: int
+        :return: Last N log lines.
+        :rtype: str
+        """
+
+        logger.trace(
+            "HTTP request failed, "
+            "obtaining last {0} lines of Honeycomb log...".format(lines))
+
+        ssh = SSH()
+        ssh.connect(node)
+        cmd = "tail -n {0} /var/log/honeycomb/honeycomb.log".format(lines)
+        # ssh also logs the reply on trace level
+        (_, stdout, _) = ssh.exec_command(cmd, timeout=30)
+
+        return stdout
diff --git a/resources/libraries/robot/honeycomb/interfaces.robot b/resources/libraries/robot/honeycomb/interfaces.robot
index a3c8a13b1b..9940966fb9 100644
--- a/resources/libraries/robot/honeycomb/interfaces.robot
+++ b/resources/libraries/robot/honeycomb/interfaces.robot
@@ -237,7 +237,7 @@
 
 | IPv4 neighbor from Honeycomb should be
 | | [Documentation] | Retrieves ipv4 neighbor list through Honeycomb\
-| | and compares with neighbor list supplied in argument.
+| | ... | and compares with neighbor list supplied in argument.
 | | ...
 | | ... | *Arguments:*
 | | ... | - node - information about a DUT node. Type: dictionary
diff --git a/resources/libraries/robot/honeycomb/persistence.robot b/resources/libraries/robot/honeycomb/persistence.robot
index 6d2cc1f2e3..84607a8991 100644
--- a/resources/libraries/robot/honeycomb/persistence.robot
+++ b/resources/libraries/robot/honeycomb/persistence.robot
@@ -37,6 +37,7 @@
 | | ... | \| Honeycomb is restarted \| ${nodes['DUT1']} \|
 | | [Arguments] | ${node}
 | | Stop Honeycomb service on DUTs | ${node}
+| | Log Persisted Configuration | ${node}
 | | Setup Honeycomb service on DUTs | ${node}
 
 | VPP is restarted
@@ -78,6 +79,7 @@
 | | ... | \| Honeycomb and VPP are restarted \| ${nodes['DUT1']} \|
 | | [Arguments] | ${node}
 | | Stop Honeycomb service on DUTs | ${node}
+| | Log Persisted Configuration | ${node}
 | | Setup DUT | ${node}
 | | Setup Honeycomb service on DUTs | ${node}
 
diff --git a/tests/func/honeycomb/__init__.robot b/tests/func/honeycomb/__init__.robot
index a38d53a464..6a553ac790 100644
--- a/tests/func/honeycomb/__init__.robot
+++ b/tests/func/honeycomb/__init__.robot
@@ -22,6 +22,7 @@
 | Suite Setup | Run Keywords | Setup All DUTs Before Test | AND
 | ... | Clear Persisted Honeycomb Configuration | ${node} | AND
 | ... | Configure Unsecured Access | ${node} | AND
+| ... | Configure Log Level | ${node} | TRACE | AND
 | ... | Setup Honeycomb Service On DUTs | ${node} | AND
 | ... | Set Global Variable | ${node}
 | Suite Teardown | Stop Honeycomb Service On DUTs | ${node}