CSIT-1124: Support multi-sample tests

 + Store parsed MRR results as AvgStdevMetadata
 + Modify tables and plots to use AvgStdevMetadata

Change-Id: I29bb1e492a664544e63a180055f66bb0eecfb957
Signed-off-by: Tibor Frank <tifrank@cisco.com>

1 files changed, 46 insertions, 26 deletions

diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py
index cf13237774..91c7747752 100644
--- a/resources/tools/presentation/input_data_parser.py
+++ b/resources/tools/presentation/input_data_parser.py
@@ -16,6 +16,7 @@
 - extract data from output.xml files generated by Jenkins jobs and store in
   pandas' Series,
 - provide access to the data.
+- filter the data using tags,
 """
 
 import multiprocessing
@@ -29,6 +30,7 @@ from robot import errors
 from collections import OrderedDict
 from string import replace
 from os import remove
+from jumpavg.AvgStdevMetadataFactory import AvgStdevMetadataFactory
 
 from input_data_files import download_and_unzip_data_file
 from utils import Worker
@@ -42,18 +44,21 @@ class ExecutionChecker(ResultVisitor):
     Performance tests:
 
     {
-        "metadata": {  # Optional
-            "version": "VPP version",
+        "metadata": {
+            "generated": "Timestamp",
+            "version": "SUT version",
             "job": "Jenkins job name",
             "build": "Information about the build"
         },
         "suites": {
-            "Suite name 1": {
+            "Suite long name 1": {
+                "name": Suite name,
                 "doc": "Suite 1 documentation",
                 "parent": "Suite 1 parent",
                 "level": "Level of the suite in the suite hierarchy"
             }
-            "Suite name N": {
+            "Suite long name N": {
+                "name": Suite name,
                 "doc": "Suite N documentation",
                 "parent": "Suite 2 parent",
                 "level": "Level of the suite in the suite hierarchy"
@@ -66,12 +71,12 @@ class ExecutionChecker(ResultVisitor):
                 "doc": "Test documentation"
                 "msg": "Test message"
                 "tags": ["tag 1", "tag 2", "tag n"],
-                "type": "PDR" | "NDR",
-                "throughput": {
+                "type": "PDR" | "NDR" | "TCP" | "MRR" | "BMRR",
+                "throughput": {  # Only type: "PDR" | "NDR"
                     "value": int,
                     "unit": "pps" | "bps" | "percentage"
                 },
-                "latency": {
+                "latency": {  # Only type: "PDR" | "NDR"
                     "direction1": {
                         "100": {
                             "min": int,
@@ -107,9 +112,15 @@ class ExecutionChecker(ResultVisitor):
                         }
                     }
                 },
-                "lossTolerance": "lossTolerance",  # Only for PDR
-                "vat-history": "DUT1 and DUT2 VAT History"
+                "result": {  # Only type: "TCP"
+                    "value": int,
+                    "unit": "cps" | "rps"
                 },
+                "result": {  # Only type: "MRR" | "BMRR"
+                    "receive-rate": AvgStdevMetadata,
+                },
+                "lossTolerance": "lossTolerance",  # Only type: "PDR"
+                "vat-history": "DUT1 and DUT2 VAT History"
                 "show-run": "Show Run"
             },
             "ID" {
@@ -118,8 +129,8 @@ class ExecutionChecker(ResultVisitor):
         }
     }
 
-    Functional tests:
 
+    Functional tests:
 
     {
         "metadata": {  # Optional
@@ -162,7 +173,7 @@ class ExecutionChecker(ResultVisitor):
     REGEX_RATE = re.compile(r'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)\s(\w+)')
 
     REGEX_LAT_NDR = re.compile(r'^[\D\d]*'
-                               r'LAT_\d+%NDR:\s\[\'(-?\d+\/-?\d+/-?\d+)\','
+                               r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
                                r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
                                r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
                                r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
@@ -186,6 +197,8 @@ class ExecutionChecker(ResultVisitor):
     REGEX_MRR = re.compile(r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
                            r'tx\s(\d*),\srx\s(\d*)')
 
+    REGEX_BMRR = re.compile(r'Maximum Receive Rate Results \[(.*)\]')
+
     def __init__(self, metadata):
         """Initialisation.
 
@@ -467,7 +480,8 @@ class ExecutionChecker(ResultVisitor):
         test_result["status"] = test.status
         if test.status == "PASS" and ("NDRPDRDISC" in tags or
                                       "TCP" in tags or
-                                      "MRR" in tags):
+                                      "MRR" in tags or
+                                      "BMRR" in tags):
             if "NDRDISC" in tags:
                 test_type = "NDR"
             elif "PDRDISC" in tags:
@@ -476,6 +490,8 @@ class ExecutionChecker(ResultVisitor):
                 test_type = "TCP"
             elif "MRR" in tags:
                 test_type = "MRR"
+            elif "FRMOBL" in tags or "BMRR" in tags:
+                test_type = "BMRR"
             else:
                 return
 
@@ -509,15 +525,20 @@ class ExecutionChecker(ResultVisitor):
                 test_result["result"]["value"] = int(groups.group(2))
                 test_result["result"]["unit"] = groups.group(1)
 
-            elif test_type in ("MRR", ):
-                groups = re.search(self.REGEX_MRR, test.message)
+            elif test_type in ("MRR", "BMRR"):
                 test_result["result"] = dict()
-                test_result["result"]["duration"] = int(groups.group(1))
-                test_result["result"]["tx"] = int(groups.group(2))
-                test_result["result"]["rx"] = int(groups.group(3))
-                test_result["result"]["throughput"] = int(
-                    test_result["result"]["rx"] /
-                    test_result["result"]["duration"])
+                groups = re.search(self.REGEX_BMRR, test.message)
+                if groups is not None:
+                    items_str = groups.group(1)
+                    items_float = [float(item.strip()) for item
+                                   in items_str.split(",")]
+                    test_result["result"]["receive-rate"] = \
+                        AvgStdevMetadataFactory.from_data(items_float)
+                else:
+                    groups = re.search(self.REGEX_MRR, test.message)
+                    test_result["result"]["receive-rate"] = \
+                        AvgStdevMetadataFactory.from_data([
+                            float(groups.group(3)) / float(groups.group(1)), ])
 
         self._test_ID = test.longname.lower()
         self._data["tests"][self._test_ID] = test_result
@@ -728,12 +749,11 @@ class InputData(object):
     - job name
       - build number
         - metadata
-          - job
-          - build
-          - vpp version
+          (as described in ExecutionChecker documentation)
         - suites
+          (as described in ExecutionChecker documentation)
         - tests
-          - ID: test data (as described in ExecutionChecker documentation)
+          (as described in ExecutionChecker documentation)
     """
 
     def __init__(self, spec):
@@ -1026,13 +1046,13 @@ class InputData(object):
 
         - job 1
           - build 1
-            - test (suite) 1 ID:
+            - test (or suite) 1 ID:
               - param 1
               - param 2
               ...
               - param n
             ...
-            - test (suite) n ID:
+            - test (or suite) n ID:
             ...
           ...
           - build n