CSIT-454: HC Test: Improve logging on test failure

 - set Honeycomb log level to TRACE before running Honeycomb tests
 - on a failed restconf request, print the last few lines of the log
 - show saved configuration during persistence suite restarts

Change-Id: I07ee37a5403cfca946fd7ef25a42861273248fbe
Signed-off-by: selias <samelias@cisco.com>

1 files changed, 48 insertions, 7 deletions

diff --git a/resources/libraries/python/honeycomb/HoneycombUtil.py b/resources/libraries/python/honeycomb/HoneycombUtil.py
index f2f012ea9b..22a7e7ef1e 100644
--- a/resources/libraries/python/honeycomb/HoneycombUtil.py
+++ b/resources/libraries/python/honeycomb/HoneycombUtil.py
@@ -27,7 +27,8 @@ from enum import Enum, unique
 
 from robot.api import logger
 
-from resources.libraries.python.HTTPRequest import HTTPRequest
+from resources.libraries.python.ssh import SSH
+from resources.libraries.python.HTTPRequest import HTTPRequest, HTTPCodes
 from resources.libraries.python.constants import Constants as Const
 
 
@@ -301,7 +302,10 @@ class HoneycombUtil(object):
         base_path = HoneycombUtil.read_path_from_url_file(url_file)
         path = base_path + path
         status_code, resp = HTTPRequest.get(node, path)
-        return status_code, loads(resp)
+        (status_node, response) = status_code, loads(resp)
+        if status_code != HTTPCodes.OK:
+            HoneycombUtil.read_log_tail(node)
+        return status_code, response
 
     @staticmethod
     def put_honeycomb_data(node, url_file, data, path="",
@@ -339,8 +343,12 @@ class HoneycombUtil(object):
         base_path = HoneycombUtil.read_path_from_url_file(url_file)
         path = base_path + path
         logger.trace(path)
-        return HTTPRequest.put(node=node, path=path, headers=header,
-                               payload=data)
+        (status_code, response) = HTTPRequest.put(
+            node=node, path=path, headers=header, payload=data)
+
+        if status_code not in (HTTPCodes.OK, HTTPCodes.ACCEPTED):
+            HoneycombUtil.read_log_tail(node)
+        return status_code, response
 
     @staticmethod
     def post_honeycomb_data(node, url_file, data=None,
@@ -375,8 +383,12 @@ class HoneycombUtil(object):
             data = dumps(data)
 
         path = HoneycombUtil.read_path_from_url_file(url_file)
-        return HTTPRequest.post(node=node, path=path, headers=header,
-                                payload=data, timeout=timeout)
+        (status_code, response) = HTTPRequest.post(
+            node=node, path=path, headers=header, payload=data, timeout=timeout)
+
+        if status_code not in (HTTPCodes.OK, HTTPCodes.ACCEPTED):
+            HoneycombUtil.read_log_tail(node)
+        return status_code, response
 
     @staticmethod
     def delete_honeycomb_data(node, url_file, path=""):
@@ -395,4 +407,33 @@ class HoneycombUtil(object):
 
         base_path = HoneycombUtil.read_path_from_url_file(url_file)
         path = base_path + path
-        return HTTPRequest.delete(node, path)
+        (status_code, response) = HTTPRequest.delete(node, path)
+
+        if status_code != HTTPCodes.OK:
+            HoneycombUtil.read_log_tail(node)
+        return status_code, response
+
+    @staticmethod
+    def read_log_tail(node, lines=120):
+        """Read  the last N lines of the Honeycomb log file and print them
+        to robot log.
+
+        :param node: Honeycomb node.
+        :param lines: Number of lines to read.
+        :type node: dict
+        :type lines: int
+        :return: Last N log lines.
+        :rtype: str
+        """
+
+        logger.trace(
+            "HTTP request failed, "
+            "obtaining last {0} lines of Honeycomb log...".format(lines))
+
+        ssh = SSH()
+        ssh.connect(node)
+        cmd = "tail -n {0} /var/log/honeycomb/honeycomb.log".format(lines)
+        # ssh also logs the reply on trace level
+        (_, stdout, _) = ssh.exec_command(cmd, timeout=30)
+
+        return stdout