Introduce reconfig suites, for dot1q+ip4+vxlan

Ticket: CSIT-1551

+ Config copied from other suites.
+ Layer keywords prepared for additional chains.
+ Renamed "create" argument to "vlan_per_chain".
+ TrafficGenerator result extraction improvements.
 + Also unified TG type and subtype checking.
+ Throughput estimated as PDR for zero PLR.
+ Actual Qemu start is skipped.
+ Parse xstats instead of stats in trex stop scipt.
 - Because stats are unconditionally cleared on each script connect.
+ Remember start values to get better ReceiveMeasurementResult at stop.
+ Expose loss count and time as appropriate test message.
+ Autogen support.
+ Multiple chain amounts and core densities.
 - Only 1 additional chain.
+ Proper TAGs. Documented.
 + Clarified min framesize tags are for TG-DUT.
  + Added a TODO concerning overhead values.
 + Added 118B tag.
+ Robot performance keywords improvements.
 + Added missing return value documentation.
 + Frame size as required test variable, not an explicit argument.
 + MLRsearch keyword to store result (not into test message).
  - TODO: De-duplicate: main keyword to measure, minor to set test message.
   - But implementation uses the fact default arguments are different.
 + Keywords for starting and stopping background traffic.
  - No checks whether start-stop pair is complete.
+ Multiple minor indentation and logging improvements.
- Missing / TODOs:
 - Support also vlan_per_chain=True?
 - Support dot1q without vxlan?
 - Support also deletion?
 - How to package config steps so -reconf can be generated from -ndrpdr?

Change-Id: If0270697f9ffa837c34ef802917220a53f156703
Signed-off-by: Vratko Polak <vrpolak@cisco.com>

2 files changed, 165 insertions, 74 deletions

diff --git a/resources/libraries/python/TrafficGenerator.py b/resources/libraries/python/TrafficGenerator.py
index cc968f8e09..2e9b787321 100644
--- a/resources/libraries/python/TrafficGenerator.py
+++ b/resources/libraries/python/TrafficGenerator.py
@@ -13,6 +13,8 @@
 
 """Performance testing traffic generator library."""
 
+import time
+
 from robot.api import logger
 from robot.libraries.BuiltIn import BuiltIn
 
@@ -30,6 +32,31 @@ from .PLRsearch.PLRsearch import PLRsearch
 __all__ = ['TGDropRateSearchImpl', 'TrafficGenerator', 'OptimizedSearch']
 
 
+def check_subtype(node):
+    """Return supported subtype of given node, or raise an exception.
+
+    Currently only one subtype is supported,
+    but we want our code to be ready for other ones.
+
+    :param node: Topology node to check. Can be None.
+    :type node: dict or NoneType
+    :returns: Subtype detected.
+    :rtype: NodeSubTypeTG
+    :raises RuntimeError: If node is not supported, message explains how.
+    """
+    if node.get('type') is None:
+        raise RuntimeError('Node type is not defined')
+    elif node['type'] != NodeType.TG:
+        raise RuntimeError('Node type is {typ!r}, not a TG'.format(
+            typ=node['type']))
+    elif node.get('subtype') is None:
+        raise RuntimeError('TG subtype is not defined')
+    elif node['subtype'] == NodeSubTypeTG.TREX:
+        return NodeSubTypeTG.TREX
+    raise RuntimeError('TG subtype {sub!r} is not supported'.format(
+        sub=node['subtype']))
+
+
 class TGDropRateSearchImpl(DropRateSearch):
     """Drop Rate Search implementation."""
 
@@ -62,34 +89,23 @@ class TGDropRateSearchImpl(DropRateSearch):
         # to be able to use trex_stl-*()
         tg_instance = BuiltIn().get_library_instance(
             'resources.libraries.python.TrafficGenerator')
-
-        if tg_instance.node['subtype'] is None:
-            raise RuntimeError('TG subtype not defined')
-        elif tg_instance.node['subtype'] == NodeSubTypeTG.TREX:
+        subtype = check_subtype(tg_instance.node)
+        if subtype == NodeSubTypeTG.TREX:
             unit_rate = str(rate) + self.get_rate_type_str()
             if skip_warmup:
-                tg_instance.trex_stl_start_remote_exec(self.get_duration(),
-                                                       unit_rate, frame_size,
-                                                       traffic_profile,
-                                                       warmup_time=0.0)
+                tg_instance.trex_stl_start_remote_exec(
+                    self.get_duration(), unit_rate, frame_size, traffic_profile,
+                    warmup_time=0.0)
             else:
-                tg_instance.trex_stl_start_remote_exec(self.get_duration(),
-                                                       unit_rate, frame_size,
-                                                       traffic_profile)
+                tg_instance.trex_stl_start_remote_exec(
+                    self.get_duration(), unit_rate, frame_size, traffic_profile)
             loss = tg_instance.get_loss()
             sent = tg_instance.get_sent()
             if self.loss_acceptance_type_is_percentage():
                 loss = (float(loss) / float(sent)) * 100
-
-            logger.trace("comparing: {} < {} {}".format(loss,
-                                                        loss_acceptance,
-                                                        loss_acceptance_type))
-            if float(loss) > float(loss_acceptance):
-                return False
-            else:
-                return True
-        else:
-            raise NotImplementedError("TG subtype not supported")
+            logger.trace("comparing: {los} < {acc} {typ}".format(
+                los=loss, acc=loss_acceptance, typ=loss_acceptance_type))
+            return float(loss) <= float(loss_acceptance)
 
     def get_latency(self):
         """Returns min/avg/max latency.
@@ -114,15 +130,20 @@ class TrafficGenerator(AbstractMeasurer):
     ROBOT_LIBRARY_SCOPE = 'TEST SUITE'
 
     def __init__(self):
+        # TODO: Number of fields will be reduced with CSIT-1378.
+        self._node = None
+        # T-REX interface order mapping
+        self._ifaces_reordered = False
+        # Result holding fields, to be removed.
         self._result = None
         self._loss = None
         self._sent = None
         self._latency = None
         self._received = None
-        self._node = None
-        # T-REX interface order mapping
-        self._ifaces_reordered = False
-        # Parameters not given by measure().
+        # Measurement input fields, needed for async stop result.
+        self._start_time = None
+        self._rate = None
+        # Other input parameters, not knowable from measure() signature.
         self.frame_size = None
         self.traffic_profile = None
         self.warmup_time = None
@@ -199,11 +220,10 @@ class TrafficGenerator(AbstractMeasurer):
         :returns: nothing
         :raises RuntimeError: In case of issue during initialization.
         """
-        if tg_node['type'] != NodeType.TG:
-            raise RuntimeError('Node type is not a TG')
-        self._node = tg_node
+        subtype = check_subtype(tg_node)
+        if subtype == NodeSubTypeTG.TREX:
+            self._node = tg_node
 
-        if self._node['subtype'] == NodeSubTypeTG.TREX:
             ssh = SSH()
             ssh.connect(self._node)
 
@@ -293,9 +313,7 @@ class TrafficGenerator(AbstractMeasurer):
         :type osi_layer: str
         :raises RuntimeError: If node subtype is not a TREX or startup failed.
         """
-        if self._node['subtype'] != NodeSubTypeTG.TREX:
-            raise RuntimeError('Node subtype is not a TREX!')
-
+        # No need to check subtype, we know it is TREX.
         for _ in range(0, 3):
             # Kill TRex only if it is already running.
             cmd = "sh -c 'pgrep t-rex && pkill t-rex && sleep 3 || true'"
@@ -357,8 +375,7 @@ class TrafficGenerator(AbstractMeasurer):
         :rtype: bool
         :raises RuntimeError: If node type is not a TG.
         """
-        if node['type'] != NodeType.TG:
-            raise RuntimeError('Node type is not a TG')
+        # No need to check subtype, we know it is TREX.
 
         ssh = SSH()
         ssh.connect(node)
@@ -375,9 +392,8 @@ class TrafficGenerator(AbstractMeasurer):
         :raises RuntimeError: If node type is not a TG,
             or if TRex teardown fails.
         """
-        if node['type'] != NodeType.TG:
-            raise RuntimeError('Node type is not a TG')
-        if node['subtype'] == NodeSubTypeTG.TREX:
+        subtype = check_subtype(node)
+        if subtype == NodeSubTypeTG.TREX:
             ssh = SSH()
             ssh.connect(node)
             (ret, _, _) = ssh.exec_command(
@@ -385,25 +401,49 @@ class TrafficGenerator(AbstractMeasurer):
             if int(ret) != 0:
                 raise RuntimeError('pkill t-rex failed')
 
-    @staticmethod
-    def trex_stl_stop_remote_exec(node):
+    def _parse_traffic_results(self, stdout):
+        """Parse stdout of scripts into fieds of self.
+
+        Block of code to reuse, by sync start, or stop after async.
+        TODO: Is the output TG subtype dependent?
+
+        :param stdout: Text containing the standard output.
+        :type stdout: str
+        """
+        # last line from console output
+        line = stdout.splitlines()[-1]
+        self._result = line
+        logger.info('TrafficGen result: {0}'.format(self._result))
+        self._received = self._result.split(', ')[1].split('=')[1]
+        self._sent = self._result.split(', ')[2].split('=')[1]
+        self._loss = self._result.split(', ')[3].split('=')[1]
+        self._latency = []
+        self._latency.append(self._result.split(', ')[4].split('=')[1])
+        self._latency.append(self._result.split(', ')[5].split('=')[1])
+
+    def trex_stl_stop_remote_exec(self, node):
         """Execute script on remote node over ssh to stop running traffic.
 
+        Internal state is updated with results.
+
         :param node: TRex generator node.
         :type node: dict
         :returns: Nothing
         :raises RuntimeError: If stop traffic script fails.
         """
+        # No need to check subtype, we know it is TREX.
         ssh = SSH()
         ssh.connect(node)
 
-        (ret, _, _) = ssh.exec_command(
+        (ret, stdout, _) = ssh.exec_command(
             "sh -c '{}/resources/tools/trex/"
             "trex_stateless_stop.py'".format(Constants.REMOTE_FW_DIR))
 
         if int(ret) != 0:
             raise RuntimeError('TRex stateless runtime error')
 
+        self._parse_traffic_results(stdout)
+
     def trex_stl_start_remote_exec(
             self, duration, rate, frame_size, traffic_profile, async_call=False,
             latency=True, warmup_time=5.0, unidirection=False, tx_port=0,
@@ -435,6 +475,7 @@ class TrafficGenerator(AbstractMeasurer):
         :type rx_port: int
         :raises RuntimeError: In case of TG driver issue.
         """
+        # No need to check subtype, we know it is TREX.
         ssh = SSH()
         ssh.connect(self._node)
         reorder = self._ifaces_reordered  # Just to make the next line fit.
@@ -471,21 +512,16 @@ class TrafficGenerator(AbstractMeasurer):
             raise RuntimeError('TRex stateless runtime error')
         elif async_call:
             #no result
+            self._start_time = time.time()
+            self._rate = float(rate[:-3]) if "pps" in rate else rate
             self._received = None
             self._sent = None
             self._loss = None
             self._latency = None
         else:
-            # last line from console output
-            line = stdout.splitlines()[-1]
-            self._result = line
-            logger.info('TrafficGen result: {0}'.format(self._result))
-            self._received = self._result.split(', ')[1].split('=')[1]
-            self._sent = self._result.split(', ')[2].split('=')[1]
-            self._loss = self._result.split(', ')[3].split('=')[1]
-            self._latency = []
-            self._latency.append(self._result.split(', ')[4].split('=')[1])
-            self._latency.append(self._result.split(', ')[5].split('=')[1])
+            self._parse_traffic_results(stdout)
+            self._start_time = None
+            self._rate = None
 
     def stop_traffic_on_tg(self):
         """Stop all traffic on TG.
@@ -493,9 +529,8 @@ class TrafficGenerator(AbstractMeasurer):
         :returns: Nothing
         :raises RuntimeError: If TG is not set.
         """
-        if self._node is None:
-            raise RuntimeError("TG is not set")
-        if self._node['subtype'] == NodeSubTypeTG.TREX:
+        subtype = check_subtype(self._node)
+        if subtype == NodeSubTypeTG.TREX:
             self.trex_stl_stop_remote_exec(self._node)
 
     def send_traffic_on_tg(
@@ -547,22 +582,11 @@ class TrafficGenerator(AbstractMeasurer):
             or if subtype is not specified.
         :raises NotImplementedError: If TG is not supported.
         """
-
-        node = self._node
-        if node is None:
-            raise RuntimeError("TG is not set")
-
-        if node['type'] != NodeType.TG:
-            raise RuntimeError('Node type is not a TG')
-
-        if node['subtype'] is None:
-            raise RuntimeError('TG subtype not defined')
-        elif node['subtype'] == NodeSubTypeTG.TREX:
+        subtype = check_subtype(self._node)
+        if subtype == NodeSubTypeTG.TREX:
             self.trex_stl_start_remote_exec(
                 duration, rate, frame_size, traffic_profile, async_call,
                 latency, warmup_time, unidirection, tx_port, rx_port)
-        else:
-            raise NotImplementedError("TG subtype not supported")
 
         return self._result
 
@@ -629,6 +653,36 @@ class TrafficGenerator(AbstractMeasurer):
         self.traffic_profile = str(traffic_profile)
         self.warmup_time = float(warmup_time)
 
+    def get_measurement_result(self, duration=None, transmit_rate=None):
+        """Return the result of last measurement as ReceiveRateMeasurement.
+
+        Separate function, as measurements can end either by time
+        or by explicit call, this is the common block at the end.
+
+        TODO: Fail on running or already reported measurement.
+
+        :param duration: Measurement duration [s] if known beforehand.
+            For explicitly stopped measurement it is estimated.
+        :param transmit_rate: Target aggregate transmit rate [pps].
+            If not given, computed assuming it was bidirectional.
+        :type duration: float or NoneType
+        :type transmit_rate: float or NoneType
+        :returns: Structure containing the result of the measurement.
+        :rtype: ReceiveRateMeasurement
+        """
+        if duration is None:
+            duration = time.time() - self._start_time
+            self._start_time = None
+        if transmit_rate is None:
+            # Assuming bi-directional traffic here.
+            transmit_rate = self._rate * 2.0
+        transmit_count = int(self.get_sent())
+        loss_count = int(self.get_loss())
+        measurement = ReceiveRateMeasurement(
+            duration, transmit_rate, transmit_count, loss_count)
+        measurement.latency = self.get_latency_int()
+        return measurement
+
     def measure(self, duration, transmit_rate):
         """Run bi-directional measurement, parse and return results.
 
@@ -644,17 +698,12 @@ class TrafficGenerator(AbstractMeasurer):
         """
         duration = float(duration)
         transmit_rate = float(transmit_rate)
-        # Trex needs target Tr per stream, but reports aggregate Tx and Dx.
+        # TG needs target Tr per stream, but reports aggregate Tx and Dx.
         unit_rate = str(transmit_rate / 2.0) + "pps"
         self.send_traffic_on_tg(
             duration, unit_rate, self.frame_size, self.traffic_profile,
             warmup_time=self.warmup_time, latency=True)
-        transmit_count = int(self.get_sent())
-        loss_count = int(self.get_loss())
-        measurement = ReceiveRateMeasurement(
-            duration, transmit_rate, transmit_count, loss_count)
-        measurement.latency = self.get_latency_int()
-        return measurement
+        return self.get_measurement_result(duration, transmit_rate)
 
 
 class OptimizedSearch(object):
diff --git a/resources/libraries/python/autogen/Regenerator.py b/resources/libraries/python/autogen/Regenerator.py
index a9f3ef433d..778ced76c9 100644
--- a/resources/libraries/python/autogen/Regenerator.py
+++ b/resources/libraries/python/autogen/Regenerator.py
@@ -207,8 +207,48 @@ def write_default_files(in_filename, in_prolog, kwargs_list):
                     testcase, iface, suite_id, file_out, kwargs_list)
 
 
+def write_reconf_files(in_filename, in_prolog, kwargs_list):
+    """Using given filename and prolog, write all generated reconf suites.
+
+    Use this for suite type reconf, as its local template
+    is incompatible with mrr/ndrpdr/soak local template,
+    while test cases are compatible.
+
+    :param in_filename: Template filename to derive real filenames from.
+    :param in_prolog: Template content to derive real content from.
+    :param kwargs_list: List of kwargs for add_testcase.
+    :type in_filename: str
+    :type in_prolog: str
+    :type kwargs_list: list of dict
+    """
+    _, suite_id = get_iface_and_suite_id(in_filename)
+    testcase = Testcase.default(suite_id)
+    for nic_name in Constants.NIC_NAME_TO_CODE:
+        out_filename = replace_defensively(
+            in_filename, "10ge2p1x710",
+            Constants.NIC_NAME_TO_CODE[nic_name], 1,
+            "File name should contain NIC code once.", in_filename)
+        out_prolog = replace_defensively(
+            in_prolog, "Intel-X710", nic_name, 2,
+            "NIC name should appear twice (tag and variable).",
+            in_filename)
+        if out_prolog.count("HW_") == 2:
+            # TODO CSIT-1481: Crypto HW should be read
+            # from topology file instead.
+            if nic_name in Constants.NIC_NAME_TO_CRYPTO_HW.keys():
+                out_prolog = replace_defensively(
+                    out_prolog, "HW_DH895xcc",
+                    Constants.NIC_NAME_TO_CRYPTO_HW[nic_name], 1,
+                    "HW crypto name should appear.", in_filename)
+        iface, suite_id = get_iface_and_suite_id(out_filename)
+        with open(out_filename, "w") as file_out:
+            file_out.write(out_prolog)
+            add_default_testcases(
+                testcase, iface, suite_id, file_out, kwargs_list)
+
+
 def write_tcp_files(in_filename, in_prolog, kwargs_list):
-    """Using given filename and prolog, write all generated suites.
+    """Using given filename and prolog, write all generated tcp suites.
 
     :param in_filename: Template filename to derive real filenames from.
     :param in_prolog: Template content to derive real content from.
@@ -293,6 +333,8 @@ class Regenerator(object):
                     file_in.read().partition("*** Test Cases ***")[:-1])
             if in_filename.endswith("-ndrpdr.robot"):
                 write_default_files(in_filename, in_prolog, default_kwargs_list)
+            elif in_filename.endswith("-reconf.robot"):
+                write_reconf_files(in_filename, in_prolog, default_kwargs_list)
             elif in_filename[-10:] in ("-cps.robot", "-rps.robot"):
                 write_tcp_files(in_filename, in_prolog, tcp_kwargs_list)
             else: