HC tests: archive ODL logs (CSIT-1031)

Change-Id: I444a5d6cd32cb9ee68575dfdb5a208de2c4eb72a
Signed-off-by: Marek Gradzki <mgradzki@cisco.com>

6 files changed, 74 insertions, 9 deletions

diff --git a/bootstrap-hc2vpp-integration-odl.sh b/bootstrap-hc2vpp-integration-odl.sh
index 2f0444d3a9..e4d751d866 100755
--- a/bootstrap-hc2vpp-integration-odl.sh
+++ b/bootstrap-hc2vpp-integration-odl.sh
@@ -27,8 +27,8 @@ VIRL_PKEY=priv_key
 VIRL_SERVER_STATUS_FILE="status"
 VIRL_SERVER_EXPECTED_STATUS="PRODUCTION"
 
-JOB_ARCHIVE_ARTIFACTS=(log.html output.xml report.html honeycomb.log)
-LOG_ARCHIVE_ARTIFACTS=(log.html output.xml report.html honeycomb.log)
+JOB_ARCHIVE_ARTIFACTS=(log.html output.xml report.html honeycomb.log karaf.log)
+LOG_ARCHIVE_ARTIFACTS=(log.html output.xml report.html honeycomb.log karaf.log)
 JOB_ARCHIVE_DIR="archive"
 LOG_ARCHIVE_DIR="$WORKSPACE/archives"
 mkdir -p ${JOB_ARCHIVE_DIR}
@@ -248,6 +248,10 @@ RETURN_STATUS=$?
 scp ${SSH_OPTIONS} \
     ${VIRL_USERNAME}@${VIRL_SERVER}:/scratch/${VIRL_SID}/honeycomb.log . || true
 
+# Get ODL karaf log file from virl host
+scp ${SSH_OPTIONS} \
+    ${VIRL_USERNAME}@${VIRL_SERVER}:/scratch/${VIRL_SID}/karaf.log . || true
+
 # Archive JOB artifacts in jenkins
 for i in ${JOB_ARCHIVE_ARTIFACTS[@]}; do
     cp $( readlink -f ${i} | tr '\n' ' ' ) ${JOB_ARCHIVE_DIR}/
diff --git a/bootstrap-hc2vpp-verify-odl.sh b/bootstrap-hc2vpp-verify-odl.sh
index 6f0acd6ad9..ed28463f42 100755
--- a/bootstrap-hc2vpp-verify-odl.sh
+++ b/bootstrap-hc2vpp-verify-odl.sh
@@ -27,8 +27,8 @@ VIRL_PKEY=priv_key
 VIRL_SERVER_STATUS_FILE="status"
 VIRL_SERVER_EXPECTED_STATUS="PRODUCTION"
 
-JOB_ARCHIVE_ARTIFACTS=(log.html output.xml report.html honeycomb.log)
-LOG_ARCHIVE_ARTIFACTS=(log.html output.xml report.html honeycomb.log)
+JOB_ARCHIVE_ARTIFACTS=(log.html output.xml report.html honeycomb.log karaf.log)
+LOG_ARCHIVE_ARTIFACTS=(log.html output.xml report.html honeycomb.log karaf.log)
 JOB_ARCHIVE_DIR="archive"
 LOG_ARCHIVE_DIR="$WORKSPACE/archives"
 mkdir -p ${JOB_ARCHIVE_DIR}
@@ -247,6 +247,10 @@ RETURN_STATUS=$?
 scp ${SSH_OPTIONS} \
     ${VIRL_USERNAME}@${VIRL_SERVER}:/scratch/${VIRL_SID}/honeycomb.log . || true
 
+# Get ODL karaf log file from virl host
+scp ${SSH_OPTIONS} \
+    ${VIRL_USERNAME}@${VIRL_SERVER}:/scratch/${VIRL_SID}/karaf.log . || true
+
 # Archive JOB artifacts in jenkins
 for i in ${JOB_ARCHIVE_ARTIFACTS[@]}; do
     cp $( readlink -f ${i} | tr '\n' ' ' ) ${JOB_ARCHIVE_DIR}/
diff --git a/resources/libraries/python/honeycomb/HoneycombUtil.py b/resources/libraries/python/honeycomb/HoneycombUtil.py
index 76bb5b3878..b7cde82219 100644
--- a/resources/libraries/python/honeycomb/HoneycombUtil.py
+++ b/resources/libraries/python/honeycomb/HoneycombUtil.py
@@ -429,6 +429,31 @@ class HoneycombUtil(object):
                 hc_log=Const.REMOTE_HC_LOG))
 
     @staticmethod
+    def append_odl_log(node, odl_name, suite_name):
+        """Append ODL karaf log for the current test suite to the full log.
+
+        :param node: Honeycomb node.
+        :param odl_name: Name of ODL client version to use.
+        :param suite_name: Name of the current test suite. ${SUITE_NAME}
+            variable in robotframework.
+        :type node: dict
+        :type odl_name: str
+        :type suite_name: str
+        """
+
+        ssh = SSH()
+        ssh.connect(node)
+
+        ssh.exec_command(
+            "echo '{separator}' >> /tmp/karaf.log".format(separator="="*80))
+        ssh.exec_command(
+            "echo 'Log for suite: {suite}' >> /tmp/karaf.log".format(
+                suite=suite_name))
+        ssh.exec_command(
+            "cat /tmp/karaf_{odl_name}/data/log/karaf.log >> /tmp/karaf.log"
+            .format(odl_name=odl_name))
+
+    @staticmethod
     def clear_honeycomb_log(node):
         """Delete the Honeycomb log file for the current test suite.
 
@@ -463,3 +488,17 @@ class HoneycombUtil(object):
                 get=True,
                 timeout=60)
             ssh.exec_command("rm /tmp/honeycomb.log")
+
+    @staticmethod
+    def archive_odl_log(node):
+        """Copy ODL karaf log file from DUT node to VIRL for archiving.
+
+        :param node: Honeycomb node.
+        :type node: dict
+        """
+
+        ssh = SSH()
+        ssh.connect(node)
+
+        cmd = "cp /tmp/karaf.log /scratch/"
+        ssh.exec_command_sudo(cmd, timeout=60)
diff --git a/resources/libraries/robot/honeycomb/honeycomb.robot b/resources/libraries/robot/honeycomb/honeycomb.robot
index 88de12b84c..0a1462c0cf 100644
--- a/resources/libraries/robot/honeycomb/honeycomb.robot
+++ b/resources/libraries/robot/honeycomb/honeycomb.robot
@@ -134,8 +134,8 @@
 | | Configure Honeycomb service on DUTs | ${node}
 | | Check honeycomb startup state | ${node} | timeout=120
 
-| Archive Honeycomb log file
-| | [Documentation] | Copy honeycomb.log file from Honeycomb node\
+| Archive Honeycomb logs
+| | [Documentation] | Copies log files from Honeycomb node\
 | | ... | to test executor.
 | | ...
 | | ... | *Arguments:*
@@ -144,10 +144,11 @@
 | | ...
 | | ... | *Example:*
 | | ...
-| | ... | \| Archive Honeycomb log file \| ${nudes['DUT1']} \|
+| | ... | \| Archive Honeycomb logs \| ${nudes['DUT1']} \|
 | | ...
 | | [Arguments] | ${node} | ${perf}=${False}
 | | Archive Honeycomb log | ${node} | ${perf}
+| | Archive ODL log | ${node}
 
 | Configure ODL Client Service On DUT
 | | [Documentation] | Configure and start ODL client, then repeatedly check if
@@ -249,6 +250,7 @@
 | | ${use_odl_client}= | Get Variable Value | ${HC_ODL}
 | | Run Keyword If | '${use_odl_client}' != '${NONE}'
 | | ... | Run Keywords
+| | ... | Append suite to ODL log file | ${node} | ${use_odl_client} | AND
 | | ... | Stop ODL Client | ${node} | /tmp | AND
 | | ... | Wait until keyword succeeds | 3min | 15sec
 | | ... | Check ODL shutdown state | ${node} | AND
@@ -332,6 +334,22 @@
 | | [Arguments] | ${node}
 | | Append Honeycomb log | ${node} | ${SUITE_NAME}
 
+| Append suite to ODL log file
+| | [Documentation] | Add the contents of ODL karaf.log for the current suite\
+| | ... | to the full log which will be archived.
+| | ...
+| | ... | *Arguments:*
+| | ... | - node - information about a DUT node. Type: dictionary
+| | ... | - odl_name - name of ODL client version. Type: string
+| | ...
+| | ... | *Example:*
+| | ...
+| | ... | \| Append suite to ODL karaf log file \| ${nodes['DUT1']} \
+| | ... | \| Oxygen \|
+| | ...
+| | [Arguments] | ${node} | ${odl_name}
+| | Append ODL log | ${node} | ${odl_name} | ${SUITE_NAME}
+
 | Generate Honeycomb startup configuration for ODL test
 | | [Documentation] | Create HC startup configuration and apply to config
 | | ... | file on DUT. Requires Honeycomb restart to take effect.
diff --git a/tests/honeycomb/func/__init__.robot b/tests/honeycomb/func/__init__.robot
index cb25ee909d..89c5dc90a2 100644
--- a/tests/honeycomb/func/__init__.robot
+++ b/tests/honeycomb/func/__init__.robot
@@ -32,5 +32,5 @@
 | ... | Stop Honeycomb service on DUTs | ${node} | AND
 | ... | Clear Honeycomb Log | ${node}
 | ...
-| Suite Teardown | Run Keywords | Archive Honeycomb log file | ${node} | AND
+| Suite Teardown | Run Keywords | Archive Honeycomb logs | ${node} | AND
 | ... | Cleanup Framework | ${nodes}
diff --git a/tests/honeycomb/perf/__init__.robot b/tests/honeycomb/perf/__init__.robot
index a5f5c2dc33..1a4e814b7c 100644
--- a/tests/honeycomb/perf/__init__.robot
+++ b/tests/honeycomb/perf/__init__.robot
@@ -26,7 +26,7 @@
 | Suite Setup | Setup suite for Honeycomb performance tests
 | Suite Teardown | Run Keywords
 | ... | Stop VPP Service on DUT | ${node}
-| ... | AND | Archive Honeycomb Log File | ${node} | perf=${True}
+| ... | AND | Archive Honeycomb logs | ${node} | perf=${True}
 | ... | AND | Stop honeycomb service on DUTs | ${node}
 
 *** Keywords ***