aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorpmikus <pmikus@cisco.com>2016-05-03 07:21:15 +0200
committerMiroslav Miklus <mmiklus@cisco.com>2016-05-19 07:48:13 +0000
commita881153db68401e040f37262d60b5d5e3cc486ac (patch)
tree90cfdf9b7604534f1aad314df5242df49059351d
parent180d17939d123c04bf142cedf02daa325e3f4fa6 (diff)
Parse robot output.xml for performance reporting
- JIRA: CSIT-58 - parse robot framework output.xml file - find performance related data - write formatted json to specified file - copy archive artifact to directory Change-Id: I47e45bcb68c06044a23192cb1fca46f43782941e Signed-off-by: pmikus <pmikus@cisco.com> Signed-off-by: Peter Mikus <pmikus@cisco.com>
-rwxr-xr-xbootstrap-verify-perf.sh17
-rwxr-xr-xresources/tools/robot_output_parser.py161
2 files changed, 178 insertions, 0 deletions
diff --git a/bootstrap-verify-perf.sh b/bootstrap-verify-perf.sh
index de478e5f11..16dd536456 100755
--- a/bootstrap-verify-perf.sh
+++ b/bootstrap-verify-perf.sh
@@ -25,6 +25,8 @@ INSTALLATION_DIR="/tmp/install_dir"
PYBOT_ARGS="--noncritical MULTI_THREAD"
+ARCHIVE_ARTIFACTS=(log.html output.xml report.html output_perf_data.json)
+
# If we run this script from CSIT jobs we want to use stable vpp version
if [[ ${JOB_NAME} == csit-* ]] ;
then
@@ -160,3 +162,18 @@ case "$TEST_TAG" in
tests/
esac
+# Pybot output post-processing
+python ${CUR_DIR}/resources/tools/robot_output_parser.py \
+ -i ${CUR_DIR}/output.xml \
+ -o ${CUR_DIR}/output_perf_data.json \
+ -v ${VPP_STABLE_VER}
+if [ ! $? -eq 0 ]; then
+ echo "Parsing ${CUR_DIR}/output.xml failed"
+fi
+
+# Archive artifacts
+mkdir archive
+for i in ${ARCHIVE_ARTIFACTS[@]}; do
+ cp $( readlink -f ${i} | tr '\n' ' ' ) archive/
+done
+
diff --git a/resources/tools/robot_output_parser.py b/resources/tools/robot_output_parser.py
new file mode 100755
index 0000000000..28a9f268ef
--- /dev/null
+++ b/resources/tools/robot_output_parser.py
@@ -0,0 +1,161 @@
+#!/usr/bin/python
+
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Script parses the data taken by robot framework (output.xml) and dumps
+intereted values into JSON output file."""
+
+import json
+import re
+import sys, getopt
+
+from robot.api import ExecutionResult, ResultVisitor
+
+
+class ExecutionTestChecker(ResultVisitor):
+ """Iterates through test cases."""
+
+ def __init__(self, vDeviceVersion):
+ self.vDeviceVersion = vDeviceVersion
+ self.out = []
+
+ def visit_test(self, test):
+ """Overloaded function. Called when test is found to process data.
+
+ :param test: Test to process.
+ :type test: ExecutionTestChecker
+ """
+
+ test_id = test.longname
+ test_status = 'Failed'
+ framesize = ''
+ throughput = ''
+ throughput_units = ''
+ workers_per_nic = ''
+ workers = ''
+
+ if any("PERFTEST" in tag for tag in test.tags):
+ if test.status == 'PASS':
+ test_status = 'Passed'
+ if any("PERFTEST_LONG" in tag for tag in test.tags):
+ throughput = test.message.split(' ')[1]
+ throughput_units = test.message.split(' ')[2]
+ elif any("PERFTEST_SHORT" in tag for tag in test.tags):
+ for keyword in test.keywords:
+ for assign in keyword.assign:
+ if assign == '${rate}':
+ temp = re.findall(r"(\d*\.\d+|\d+)([A-Za-z]*)",
+ keyword.args[0])
+ throughput = temp[0][0]
+ throughput_units = temp[0][1]
+
+ for keyword in test.keywords:
+ for assign in keyword.assign:
+ if assign == '${framesize}':
+ framesize = keyword.args[0]
+ if 'worker threads' in keyword.name:
+ workers = keyword.name.split('\'')[1]
+ workers_per_nic = keyword.name.split('\'')[3]
+
+ self.out.append({'testCase': {
+ 'testId': test_id,
+ 'testStatus': test_status,
+ 'workerThreads': workers,
+ 'workerThreadsPerNic': workers_per_nic,
+ 'testTags': [tag for tag in test.tags],
+ 'l2FrameSize': {'value': framesize,
+ 'units': 'Bytes'},
+ 'throughput': {'value': throughput,
+ 'units': throughput_units},
+ 'vDevice': {'version': self.vDeviceVersion}}})
+
+
+def parse_tests(xml_file, vDeviceVersion):
+ """Parser result of robot output file and return.
+
+ :param xml_file: Output.xml from robot run.
+ :param vDeviceVersion: vDevice version.
+ :type xml_file: file
+ :type vDeviceVersion: str
+
+ :return: JSON formatted output.
+ :rtype: dict
+ """
+
+ result = ExecutionResult(xml_file)
+ checker = ExecutionTestChecker(vDeviceVersion)
+ result.visit(checker)
+
+ return checker.out
+
+
+def print_help():
+ """Print help on stdout."""
+
+ print "args: [-h] -i <input_log_file> -o <output_json_file>" + \
+ " -v <vpp_version>"
+
+
+def print_error(msg):
+ """Print error message on stderr.
+
+ :param msg: Error message to print.
+ :type msg: str
+ :return: nothing
+ """
+
+ sys.stderr.write(msg+'\n')
+
+
+def main(argv):
+ """Main function."""
+
+ _log_file = None
+ _json_file = None
+ _vpp = None
+
+ try:
+ opts, _ = getopt.getopt(argv, "hi:o:v:", ["help"])
+ except getopt.GetoptError:
+ print_help()
+ sys.exit(1)
+
+ for opt, arg in opts:
+ if opt in ('-h', "--help"):
+ print_help()
+ sys.exit()
+ elif opt == '-i':
+ _log_file = arg
+ elif opt == '-o':
+ _json_file = arg
+ elif opt == '-v':
+ _vpp = arg
+
+ if _log_file is None or _json_file is None or _vpp is None:
+ print_help()
+ sys.exit(1)
+
+ try:
+ with open(_log_file, 'r') as input_file:
+ with open(_json_file, 'w') as output_file:
+ out = parse_tests(input_file, _vpp)
+ json.dump(out, fp=output_file, sort_keys=True,
+ indent=4, separators=(',', ': '))
+ except IOError as ex_error:
+ print_error(str(ex_error))
+ sys.exit(1)
+
+
+if __name__ == "__main__":
+ main(sys.argv[1:])