Add support for HDRhistogram

+ Enable hdrh in trex server.
+ Append hdrh coded output after min/avg/max/.
+ Read (not show nor decode) hdrh value in PAL.
+ Also, remove old ndrpdrdisc code.

Change-Id: I99d99f10386a621772b5419ca1f36080fa15aca7
Signed-off-by: Vratko Polak <vrpolak@cisco.com>

5 files changed, 65 insertions, 151 deletions

diff --git a/resources/libraries/python/TrafficGenerator.py b/resources/libraries/python/TrafficGenerator.py
index b6d28a4ca6..49c19b19d8 100644
--- a/resources/libraries/python/TrafficGenerator.py
+++ b/resources/libraries/python/TrafficGenerator.py
@@ -336,10 +336,10 @@ class TrafficGenerator(AbstractMeasurer):
 
             # Start TRex.
             cmd = ("sh -c 'cd {dir}/scripts/ && "
-                   "nohup ./t-rex-64 {mode} -i -c 7 > "
+                   "nohup ./t-rex-64 --hdrh{mode} -i -c 7 > "
                    "/tmp/trex.log 2>&1 &' > /dev/null"
                    .format(dir=Constants.TREX_INSTALL_DIR,
-                           mode='--astf' if osi_layer == 'L7' else ''))
+                           mode=' --astf' if osi_layer == 'L7' else ''))
             try:
                 exec_cmd_no_error(self._node, cmd, sudo=True)
             except RuntimeError:
diff --git a/resources/libraries/robot/performance/performance_utils.robot b/resources/libraries/robot/performance/performance_utils.robot
index 60d51f6dfe..f917a0c0cd 100644
--- a/resources/libraries/robot/performance/performance_utils.robot
+++ b/resources/libraries/robot/performance/performance_utils.robot
@@ -229,7 +229,7 @@
 | | Set Test Message | ${\n}${text}: ${rate_total} pps, | append=yes
 | | Set Test Message | ${bandwidth_total} Gbps (initial) | append=yes
 | | Return From Keyword If | not """${latency}"""
-| | Set Test Message | ${\n}LATENCY usec [min/avg/max] per stream: ${latency}
+| | Set Test Message | ${\n}LATENCY [min/avg/max/hdrh] per stream: ${latency}
 | | ... | append=yes
 
 | Display result of NDRPDR search
diff --git a/resources/tools/presentation/generator_plots.py b/resources/tools/presentation/generator_plots.py
index 1179a0a240..d43663f48c 100644
--- a/resources/tools/presentation/generator_plots.py
+++ b/resources/tools/presentation/generator_plots.py
@@ -296,7 +296,7 @@ def plot_latency_error_bars_name(plot, input_data):
             for test in build:
                 try:
                     logging.debug("test['latency']: {0}\n".
-                                 format(test["latency"]))
+                                  format(test["latency"]))
                 except ValueError as err:
                     logging.warning(repr(err))
                 if y_tmp_vals.get(test["parent"], None) is None:
diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py
index 6ab7e4d76a..7e2abe6dbf 100644
--- a/resources/tools/presentation/input_data_parser.py
+++ b/resources/tools/presentation/input_data_parser.py
@@ -19,6 +19,7 @@
 - filter the data using tags,
 """
 
+import copy
 import re
 import resource
 import pandas as pd
@@ -97,24 +98,28 @@ class ExecutionChecker(ResultVisitor):
                         "direction1": {
                             "min": float,
                             "avg": float,
-                            "max": float
+                            "max": float,
+                            "hdrh": str
                         },
                         "direction2": {
                             "min": float,
                             "avg": float,
-                            "max": float
+                            "max": float,
+                            "hdrh": str
                         }
                     },
                     "PDR": {
                         "direction1": {
                             "min": float,
                             "avg": float,
-                            "max": float
+                            "max": float,
+                            "hdrh": str
                         },
                         "direction2": {
                             "min": float,
                             "avg": float,
-                            "max": float
+                            "max": float,
+                            "hdrh": str
                         }
                     }
                 }
@@ -146,60 +151,6 @@ class ExecutionChecker(ResultVisitor):
                 }
             }
 
-            # TODO: Remove when definitely no NDRPDRDISC tests are used:
-            # NDRPDRDISC tests:
-            "ID": {
-                "name": "Test name",
-                "parent": "Name of the parent of the test",
-                "doc": "Test documentation",
-                "msg": "Test message",
-                "tags": ["tag 1", "tag 2", "tag n"],
-                "type": "PDR" | "NDR",
-                "status": "PASS" | "FAIL",
-                "throughput": {  # Only type: "PDR" | "NDR"
-                    "value": int,
-                    "unit": "pps" | "bps" | "percentage"
-                },
-                "latency": {  # Only type: "PDR" | "NDR"
-                    "direction1": {
-                        "100": {
-                            "min": int,
-                            "avg": int,
-                            "max": int
-                        },
-                        "50": {  # Only for NDR
-                            "min": int,
-                            "avg": int,
-                            "max": int
-                        },
-                        "10": {  # Only for NDR
-                            "min": int,
-                            "avg": int,
-                            "max": int
-                        }
-                    },
-                    "direction2": {
-                        "100": {
-                            "min": int,
-                            "avg": int,
-                            "max": int
-                        },
-                        "50": {  # Only for NDR
-                            "min": int,
-                            "avg": int,
-                            "max": int
-                        },
-                        "10": {  # Only for NDR
-                            "min": int,
-                            "avg": int,
-                            "max": int
-                        }
-                    }
-                },
-                "lossTolerance": "lossTolerance",  # Only type: "PDR"
-                "conf-history": "DUT1 and DUT2 VAT History"
-                "show-run": "Show Run"
-            },
             "ID" {
                 # next test
             }
@@ -258,19 +209,6 @@ class ExecutionChecker(ResultVisitor):
                                    r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
                                    r'PDR_UPPER:\s(\d+.\d+)')
 
-    # TODO: Remove when definitely no NDRPDRDISC tests are used:
-    REGEX_LAT_NDR = re.compile(r'^[\D\d]*'
-                               r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
-                               r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
-                               r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
-                               r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
-                               r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
-                               r'\s\'(-?\d+/-?\d+/-?\d+)\'\]')
-
-    REGEX_LAT_PDR = re.compile(r'^[\D\d]*'
-                               r'LAT_\d+%PDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
-                               r'\s\'(-?\d+/-?\d+/-?\d+)\'\][\D\d]*')
-
     REGEX_NDRPDR_LAT = re.compile(r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
                                   r'LATENCY.*\[\'(.*)\', \'(.*)\'\]')
 
@@ -554,53 +492,6 @@ class ExecutionChecker(ResultVisitor):
                 except KeyError:
                     pass
 
-    # TODO: Remove when definitely no NDRPDRDISC tests are used:
-    def _get_latency(self, msg, test_type):
-        """Get the latency data from the test message.
-
-        :param msg: Message to be parsed.
-        :param test_type: Type of the test - NDR or PDR.
-        :type msg: str
-        :type test_type: str
-        :returns: Latencies parsed from the message.
-        :rtype: dict
-        """
-
-        if test_type == "NDR":
-            groups = re.search(self.REGEX_LAT_NDR, msg)
-            groups_range = range(1, 7)
-        elif test_type == "PDR":
-            groups = re.search(self.REGEX_LAT_PDR, msg)
-            groups_range = range(1, 3)
-        else:
-            return {}
-
-        latencies = list()
-        for idx in groups_range:
-            try:
-                lat = [int(item) for item in str(groups.group(idx)).split('/')]
-            except (AttributeError, ValueError):
-                lat = [-1, -1, -1]
-            latencies.append(lat)
-
-        keys = ("min", "avg", "max")
-        latency = {
-            "direction1": {
-            },
-            "direction2": {
-            }
-        }
-
-        latency["direction1"]["100"] = dict(zip(keys, latencies[0]))
-        latency["direction2"]["100"] = dict(zip(keys, latencies[1]))
-        if test_type == "NDR":
-            latency["direction1"]["50"] = dict(zip(keys, latencies[2]))
-            latency["direction2"]["50"] = dict(zip(keys, latencies[3]))
-            latency["direction1"]["10"] = dict(zip(keys, latencies[4]))
-            latency["direction2"]["10"] = dict(zip(keys, latencies[5]))
-
-        return latency
-
     def _get_ndrpdr_throughput(self, msg):
         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
         message.
@@ -665,31 +556,52 @@ class ExecutionChecker(ResultVisitor):
         :returns: Parsed data as a dict and the status (PASS/FAIL).
         :rtype: tuple(dict, str)
         """
-
+        latency_default = {"min": -1.0, "avg": -1.0, "max": -1.0, "hdrh": ""}
         latency = {
             "NDR": {
-                "direction1": {"min": -1.0, "avg": -1.0, "max": -1.0},
-                "direction2": {"min": -1.0, "avg": -1.0, "max": -1.0}
+                "direction1": copy.copy(latency_default),
+                "direction2": copy.copy(latency_default)
             },
             "PDR": {
-                "direction1": {"min": -1.0, "avg": -1.0, "max": -1.0},
-                "direction2": {"min": -1.0, "avg": -1.0, "max": -1.0}
+                "direction1": copy.copy(latency_default),
+                "direction2": copy.copy(latency_default)
             }
         }
         status = "FAIL"
         groups = re.search(self.REGEX_NDRPDR_LAT, msg)
 
+        def process_latency(in_str):
+            """Return object with parsed latency values.
+
+            TODO: Define class for the return type.
+
+            :param in_str: Input string, min/avg/max/hdrh format.
+            :type in_str: str
+            :returns: Dict with corresponding keys, except hdrh float values.
+            :rtype dict:
+            :throws IndexError: If in_str does not have enough substrings.
+            :throws ValueError: If a substring does not convert to float.
+            """
+            in_list = in_str.split('/')
+
+            rval = {
+                "min": float(in_list[0]),
+                "avg": float(in_list[1]),
+                "max": float(in_list[2]),
+                "hdrh": ""
+            }
+
+            if len(in_list) == 4:
+                rval["hdrh"] = str(in_list[3])
+
+            return rval
+
         if groups is not None:
-            keys = ("min", "avg", "max")
             try:
-                latency["NDR"]["direction1"] = dict(
-                    zip(keys, [float(l) for l in groups.group(1).split('/')]))
-                latency["NDR"]["direction2"] = dict(
-                    zip(keys, [float(l) for l in groups.group(2).split('/')]))
-                latency["PDR"]["direction1"] = dict(
-                    zip(keys, [float(l) for l in groups.group(3).split('/')]))
-                latency["PDR"]["direction2"] = dict(
-                    zip(keys, [float(l) for l in groups.group(4).split('/')]))
+                latency["NDR"]["direction1"] = process_latency(groups.group(1))
+                latency["NDR"]["direction2"] = process_latency(groups.group(2))
+                latency["PDR"]["direction1"] = process_latency(groups.group(3))
+                latency["PDR"]["direction2"] = process_latency(groups.group(4))
                 status = "PASS"
             except (IndexError, ValueError):
                 pass
diff --git a/resources/tools/trex/trex_stateless_profile.py b/resources/tools/trex/trex_stateless_profile.py
index 15a0225911..b888bcdea9 100755
--- a/resources/tools/trex/trex_stateless_profile.py
+++ b/resources/tools/trex/trex_stateless_profile.py
@@ -27,17 +27,19 @@ sys.path.insert(0, "/opt/trex-core-2.61/scripts/automation/"
 from trex.stl.api import *
 
 
-def fmt_latency(lat_min, lat_avg, lat_max):
+def fmt_latency(lat_min, lat_avg, lat_max, hdrh):
     """Return formatted, rounded latency.
 
     :param lat_min: Min latency
     :param lat_avg: Average latency
     :param lat_max: Max latency
-    :type lat_min: string
-    :type lat_avg: string
-    :type lat_max: string
-    :return: Formatted and rounded output "min/avg/max"
-    :rtype: string
+    :param hdrh: Base64 encoded compressed HDRHistogram object.
+    :type lat_min: str
+    :type lat_avg: str
+    :type lat_max: str
+    :type hdrh: str
+    :return: Formatted and rounded output (hdrh unchanged) "min/avg/max/hdrh".
+    :rtype: str
     """
     try:
         t_min = int(round(float(lat_min)))
@@ -52,7 +54,7 @@ def fmt_latency(lat_min, lat_avg, lat_max):
     except ValueError:
         t_max = int(-1)
 
-    return "/".join(str(tmp) for tmp in (t_min, t_avg, t_max))
+    return "/".join(str(tmp) for tmp in (t_min, t_avg, t_max, hdrh))
 
 
 def simple_burst(profile_file, duration, framesize, rate, warmup_time, port_0,
@@ -100,8 +102,8 @@ def simple_burst(profile_file, duration, framesize, rate, warmup_time, port_0,
     total_sent = 0
     lost_a = 0
     lost_b = 0
-    lat_a = "-1/-1/-1"
-    lat_b = "-1/-1/-1"
+    lat_a = "-1/-1/-1/"
+    lat_b = "-1/-1/-1/"
 
     # Read the profile:
     try:
@@ -214,15 +216,15 @@ def simple_burst(profile_file, duration, framesize, rate, warmup_time, port_0,
             # Stats index is not a port number, but "pgid".
             # TODO: Find out what "pgid" means.
             if latency:
+                lat_obj = stats["latency"][0]["latency"]
                 lat_a = fmt_latency(
-                    str(stats["latency"][0]["latency"]["total_min"]),
-                    str(stats["latency"][0]["latency"]["average"]),
-                    str(stats["latency"][0]["latency"]["total_max"]))
+                    str(lat_obj["total_min"]), str(lat_obj["average"]),
+                    str(lat_obj["total_max"]), str(lat_obj["hdrh"]))
                 if traffic_directions > 1:
+                    lat_obj = stats["latency"][1]["latency"]
                     lat_b = fmt_latency(
-                        str(stats["latency"][1]["latency"]["total_min"]),
-                        str(stats["latency"][1]["latency"]["average"]),
-                        str(stats["latency"][1]["latency"]["total_max"]))
+                        str(lat_obj["total_min"]), str(lat_obj["average"]),
+                        str(lat_obj["total_max"]), str(lat_obj["hdrh"]))
 
             if traffic_directions > 1:
                 total_sent = stats[0]["opackets"] + stats[1]["opackets"]