feat(MLRsearch): MLRsearch v7

Replaces MLRv2, suitable for "big bang" upgrade across CSIT.
PyPI metadata updated only partially (full edits will come separately).
Pylint wants less complexity, but the differences are only minor.

+ Use the same (new CSIT) defaults everywhere, also in Python library.
+ Update also PLRsearch to use the new result class.
+ Make upper bound optional in UTI.
+ Fix ASTF approximate duration detection.
+ Do not keep approximated_receive_rate (for MRR) in result structure.

Change-Id: I03406f32d5c93f56b527cb3f93791b61955dfd74
Signed-off-by: Vratko Polak <vrpolak@cisco.com>

1 files changed, 130 insertions, 112 deletions

diff --git a/resources/libraries/python/TrafficGenerator.py b/resources/libraries/python/TrafficGenerator.py
index e036448f77..7d6fdead64 100644
--- a/resources/libraries/python/TrafficGenerator.py
+++ b/resources/libraries/python/TrafficGenerator.py
@@ -16,14 +16,17 @@
 import math
 import time
 
+from typing import Callable, List, Optional, Union
+
 from robot.api import logger
 from robot.libraries.BuiltIn import BuiltIn
 
 from .Constants import Constants
 from .DropRateSearch import DropRateSearch
-from .MLRsearch.AbstractMeasurer import AbstractMeasurer
-from .MLRsearch.MultipleLossRatioSearch import MultipleLossRatioSearch
-from .MLRsearch.ReceiveRateMeasurement import ReceiveRateMeasurement
+from .MLRsearch import (
+    AbstractMeasurer, Config, MeasurementResult,
+    MultipleLossRatioSearch, SearchGoal, TrimmedStat,
+)
 from .PLRsearch.PLRsearch import PLRsearch
 from .OptionString import OptionString
 from .ssh import exec_cmd_no_error, exec_cmd
@@ -513,7 +516,7 @@ class TrafficGenerator(AbstractMeasurer):
         """Stop all traffic on TG.
 
         :returns: Structure containing the result of the measurement.
-        :rtype: ReceiveRateMeasurement
+        :rtype: MeasurementResult
         :raises ValueError: If TG traffic profile is not supported.
         """
         subtype = check_subtype(self._node)
@@ -533,7 +536,7 @@ class TrafficGenerator(AbstractMeasurer):
         """Compute duration for profile driver.
 
         The final result is influenced by transaction scale and duration limit.
-        It is assumed a higher level function has already set those to self.
+        It is assumed a higher level function has already set those on self.
         The duration argument is the target value from search point of view,
         before the overrides are applied here.
 
@@ -853,7 +856,7 @@ class TrafficGenerator(AbstractMeasurer):
         :type state_timeout: float
         :type ramp_up_only: bool
         :returns: TG results.
-        :rtype: ReceiveRateMeasurement or None
+        :rtype: MeasurementResult or None
         :raises ValueError: If TG traffic profile is not supported.
         """
         self.set_rate_provider_defaults(
@@ -900,7 +903,7 @@ class TrafficGenerator(AbstractMeasurer):
         :type rate: float
         :type async_call: bool
         :returns: TG results.
-        :rtype: ReceiveRateMeasurement or None
+        :rtype: MeasurementResult or None
         :raises ValueError: If TG traffic profile is not supported.
         """
         subtype = check_subtype(self._node)
@@ -951,7 +954,7 @@ class TrafficGenerator(AbstractMeasurer):
         :type async_call: bool
         :type ramp_up_only: bool
         :returns: TG results.
-        :rtype: ReceiveRateMeasurement or None
+        :rtype: MeasurementResult or None
         :raises ValueError: If TG traffic profile is not supported.
         """
         complete = False
@@ -1006,7 +1009,7 @@ class TrafficGenerator(AbstractMeasurer):
         trial_end = time.monotonic()
         if self.ramp_up_rate:
             # Optimization: No loss acts as a good ramp-up, if it was complete.
-            if complete and result is not None and result.loss_count == 0:
+            if complete and result is not None and result.loss_ratio == 0.0:
                 logger.debug(u"Good trial acts as a ramp-up")
                 self.ramp_up_start = trial_start
                 self.ramp_up_stop = trial_end
@@ -1178,19 +1181,20 @@ class TrafficGenerator(AbstractMeasurer):
                         int(self._result.get(u"server_tcp_rx_bytes", 0))
 
     def _get_measurement_result(self):
-        """Return the result of last measurement as ReceiveRateMeasurement.
+        """Return the result of last measurement as MeasurementResult.
 
         Separate function, as measurements can end either by time
         or by explicit call, this is the common block at the end.
 
-        The target_tr field of ReceiveRateMeasurement is in
+        The intended_load field of MeasurementResult is in
         transactions per second. Transmit count and loss count units
         depend on the transaction type. Usually they are in transactions
         per second, or aggregated packets per second.
 
         :returns: Structure containing the result of the measurement.
-        :rtype: ReceiveRateMeasurement
+        :rtype: MeasurementResult
         """
+        duration_with_overheads = time.monotonic() - self._start_time
         try:
             # Client duration seems to include a setup period
             # where TRex does not send any packets yet.
@@ -1230,7 +1234,7 @@ class TrafficGenerator(AbstractMeasurer):
             expected_attempt_count = max(expected_attempt_count, self._sent)
             unsent = expected_attempt_count - self._sent
             pass_count = self._received
-            fail_count = expected_attempt_count - pass_count
+            loss_count = self._loss
         elif self.transaction_type == u"udp_cps":
             if not self.transaction_scale:
                 raise RuntimeError(u"Add support for no-limit udp_cps.")
@@ -1239,7 +1243,7 @@ class TrafficGenerator(AbstractMeasurer):
             expected_attempt_count = self.transaction_scale
             unsent = expected_attempt_count - partial_attempt_count
             pass_count = self._l7_data[u"client"][u"received"]
-            fail_count = expected_attempt_count - pass_count
+            loss_count = partial_attempt_count - pass_count
         elif self.transaction_type == u"tcp_cps":
             if not self.transaction_scale:
                 raise RuntimeError(u"Add support for no-limit tcp_cps.")
@@ -1252,14 +1256,14 @@ class TrafficGenerator(AbstractMeasurer):
             # but we are testing NAT session so client/connects counts that
             # (half connections from TCP point of view).
             pass_count = self._l7_data[u"client"][u"tcp"][u"connects"]
-            fail_count = expected_attempt_count - pass_count
+            loss_count = partial_attempt_count - pass_count
         elif self.transaction_type == u"udp_pps":
             if not self.transaction_scale:
                 raise RuntimeError(u"Add support for no-limit udp_pps.")
             partial_attempt_count = self._sent
             expected_attempt_count = self.transaction_scale * self.ppta
             unsent = expected_attempt_count - self._sent
-            fail_count = self._loss + unsent
+            loss_count = self._loss
         elif self.transaction_type == u"tcp_pps":
             if not self.transaction_scale:
                 raise RuntimeError(u"Add support for no-limit tcp_pps.")
@@ -1273,29 +1277,30 @@ class TrafficGenerator(AbstractMeasurer):
             # Probability of retransmissions exactly cancelling
             # packets unsent due to duration stretching is quite low.
             unsent = abs(expected_attempt_count - self._sent)
-            fail_count = self._loss + unsent
+            loss_count = self._loss
         else:
             raise RuntimeError(f"Unknown parsing {self.transaction_type!r}")
         if unsent and isinstance(self._approximated_duration, float):
             # Do not report unsent for "manual".
             logger.debug(f"Unsent packets/transactions: {unsent}")
-        if fail_count < 0 and not self.negative_loss:
-            fail_count = 0
-        measurement = ReceiveRateMeasurement(
-            duration=target_duration,
-            target_tr=transmit_rate,
-            transmit_count=expected_attempt_count,
-            loss_count=fail_count,
-            approximated_duration=approximated_duration,
-            partial_transmit_count=partial_attempt_count,
+        if loss_count < 0 and not self.negative_loss:
+            loss_count = 0
+        measurement = MeasurementResult(
+            intended_duration=target_duration,
+            intended_load=transmit_rate,
+            offered_count=partial_attempt_count,
+            loss_count=loss_count,
+            offered_duration=approximated_duration,
+            duration_with_overheads=duration_with_overheads,
+            intended_count=expected_attempt_count,
         )
         measurement.latency = self.get_latency_int()
         return measurement
 
-    def measure(self, duration, transmit_rate):
+    def measure(self, intended_duration, intended_load):
         """Run trial measurement, parse and return results.
 
-        The input rate is for transactions. Stateles bidirectional traffic
+        The intended load is for transactions. Stateles bidirectional traffic
         is understood as sequence of (asynchronous) transactions,
         two packets each.
 
@@ -1303,33 +1308,32 @@ class TrafficGenerator(AbstractMeasurer):
         the count either transactions or packets (aggregated over directions).
 
         Optionally, this method sleeps if measurement finished before
-        the time specified as duration.
+        the time specified as intended_duration (PLRsearch needs time for math).
 
-        :param duration: Trial duration [s].
-        :param transmit_rate: Target rate in transactions per second.
-        :type duration: float
-        :type transmit_rate: float
+        :param intended_duration: Trial duration [s].
+        :param intended_load: Target rate in transactions per second.
+        :type intended_duration: float
+        :type intended_load: float
         :returns: Structure containing the result of the measurement.
-        :rtype: ReceiveRateMeasurement
+        :rtype: MeasurementResult
         :raises RuntimeError: If TG is not set or if node is not TG
             or if subtype is not specified.
         :raises NotImplementedError: If TG is not supported.
         """
-        duration = float(duration)
+        intended_duration = float(intended_duration)
         time_start = time.monotonic()
-        time_stop = time_start + duration
+        time_stop = time_start + intended_duration
         if self.resetter:
             self.resetter()
         result = self._send_traffic_on_tg_with_ramp_up(
-            duration=duration,
-            rate=transmit_rate,
+            duration=intended_duration,
+            rate=intended_load,
             async_call=False,
         )
         logger.debug(f"trial measurement result: {result!r}")
         # In PLRsearch, computation needs the specified time to complete.
         if self.sleep_till_duration:
-            sleeptime = time_stop - time.monotonic()
-            if sleeptime > 0.0:
+            while (sleeptime := time_stop - time.monotonic()) > 0.0:
                 time.sleep(sleeptime)
         return result
 
@@ -1399,7 +1403,7 @@ class TrafficGenerator(AbstractMeasurer):
         self.frame_size = frame_size
         self.traffic_profile = str(traffic_profile)
         self.resetter = resetter
-        self.ppta = ppta
+        self.ppta = int(ppta)
         self.traffic_directions = int(traffic_directions)
         self.transaction_duration = float(transaction_duration)
         self.transaction_scale = int(transaction_scale)
@@ -1421,29 +1425,30 @@ class OptimizedSearch:
     """
 
     @staticmethod
-    def perform_optimized_ndrpdr_search(
-            frame_size,
-            traffic_profile,
-            minimum_transmit_rate,
-            maximum_transmit_rate,
-            packet_loss_ratio=0.005,
-            final_relative_width=0.005,
-            final_trial_duration=30.0,
-            initial_trial_duration=1.0,
-            number_of_intermediate_phases=2,
-            timeout=1200.0,
-            ppta=1,
-            resetter=None,
-            traffic_directions=2,
-            transaction_duration=0.0,
-            transaction_scale=0,
-            transaction_type=u"packet",
-            use_latency=False,
-            ramp_up_rate=None,
-            ramp_up_duration=None,
-            state_timeout=240.0,
-            expansion_coefficient=4.0,
-    ):
+    def perform_mlr_search(
+        frame_size: Union[int, str],
+        traffic_profile: str,
+        min_load: float,
+        max_load: float,
+        loss_ratio: float = 0.005,
+        relative_width: float = 0.005,
+        initial_trial_duration: float = 1.0,
+        final_trial_duration: float = 1.0,
+        duration_sum: float = 20.0,
+        expansion_coefficient: int = 2,
+        preceding_targets: int = 2,
+        search_duration_max: float = 1200.0,
+        ppta: int = 1,
+        resetter: Optional[Callable[[], None]] = None,
+        traffic_directions: int = 2,
+        transaction_duration: float = 0.0,
+        transaction_scale: int = 0,
+        transaction_type: str = "packet",
+        use_latency: bool = False,
+        ramp_up_rate: float = 0.0,
+        ramp_up_duration: float = 0.0,
+        state_timeout: float = 240.0,
+    ) -> List[TrimmedStat]:
         """Setup initialized TG, perform optimized search, return intervals.
 
         If transaction_scale is nonzero, all init and non-init trial durations
@@ -1455,18 +1460,20 @@ class OptimizedSearch:
         :param frame_size: Frame size identifier or value [B].
         :param traffic_profile: Module name as a traffic profile identifier.
             See GPL/traffic_profiles/trex for implemented modules.
-        :param minimum_transmit_rate: Minimal load in transactions per second.
-        :param maximum_transmit_rate: Maximal load in transactions per second.
-        :param packet_loss_ratio: Ratio of packets lost, for PDR [1].
-        :param final_relative_width: Final lower bound transmit rate
+        :param min_load: Minimal load in transactions per second.
+        :param max_load: Maximal load in transactions per second.
+        :param loss_ratio: Ratio of packets lost, for PDR [1].
+        :param relative_width: Final lower bound intended load
             cannot be more distant that this multiple of upper bound [1].
-        :param final_trial_duration: Trial duration for the final phase [s].
         :param initial_trial_duration: Trial duration for the initial phase
             and also for the first intermediate phase [s].
-        :param number_of_intermediate_phases: Number of intermediate phases
+        :param final_trial_duration: Trial duration for the final phase [s].
+        :param duration_sum: Max sum of duration for deciding [s].
+        :param expansion_coefficient: In external search multiply width by this.
+        :param preceding_targets: Number of intermediate phases
             to perform before the final phase [1].
-        :param timeout: The search will fail itself when not finished
-            before this overall time [s].
+        :param search_duration_max: The search will fail itself
+            when not finished before this overall time [s].
         :param ppta: Packets per transaction, aggregated over directions.
             Needed for udp_pps which does not have a good transaction counter,
             so we need to compute expected number of packets.
@@ -1485,17 +1492,18 @@ class OptimizedSearch:
         :param ramp_up_rate: Rate to use in ramp-up trials [pps].
         :param ramp_up_duration: Duration of ramp-up trials [s].
         :param state_timeout: Time of life of DUT state [s].
-        :param expansion_coefficient: In external search multiply width by this.
         :type frame_size: str or int
         :type traffic_profile: str
-        :type minimum_transmit_rate: float
-        :type maximum_transmit_rate: float
-        :type packet_loss_ratio: float
-        :type final_relative_width: float
-        :type final_trial_duration: float
+        :type min_load: float
+        :type max_load: float
+        :type loss_ratio: float
+        :type relative_width: float
         :type initial_trial_duration: float
-        :type number_of_intermediate_phases: int
-        :type timeout: float
+        :type final_trial_duration: float
+        :type duration_sum: float
+        :type expansion_coefficient: int
+        :type preceding_targets: int
+        :type search_duration_max: float
         :type ppta: int
         :type resetter: Optional[Callable[[], None]]
         :type traffic_directions: int
@@ -1506,11 +1514,11 @@ class OptimizedSearch:
         :type ramp_up_rate: float
         :type ramp_up_duration: float
         :type state_timeout: float
-        :type expansion_coefficient: float
         :returns: Structure containing narrowed down NDR and PDR intervals
             and their measurements.
-        :rtype: List[Receiverateinterval]
-        :raises RuntimeError: If total duration is larger than timeout.
+        :rtype: List[TrimmedStat]
+        :raises RuntimeError: If search duration exceeds search_duration_max
+            or if min load becomes an upper bound for any search goal.
         """
         # we need instance of TrafficGenerator instantiated by Robot Framework
         # to be able to use trex_stl-*()
@@ -1521,8 +1529,9 @@ class OptimizedSearch:
         if transaction_scale:
             initial_trial_duration = 1.0
             final_trial_duration = 1.0
-            number_of_intermediate_phases = 0
-            timeout += transaction_scale * 3e-4
+            preceding_targets = 1
+            # TODO: Move the value to Constants.py?
+            search_duration_max += transaction_scale * 3e-4
         tg_instance.set_rate_provider_defaults(
             frame_size=frame_size,
             traffic_profile=traffic_profile,
@@ -1538,34 +1547,43 @@ class OptimizedSearch:
             ramp_up_duration=ramp_up_duration,
             state_timeout=state_timeout,
         )
-        algorithm = MultipleLossRatioSearch(
-            measurer=tg_instance,
-            final_trial_duration=final_trial_duration,
-            final_relative_width=final_relative_width,
-            number_of_intermediate_phases=number_of_intermediate_phases,
-            initial_trial_duration=initial_trial_duration,
-            timeout=timeout,
-            debug=logger.debug,
-            expansion_coefficient=expansion_coefficient,
-        )
-        if packet_loss_ratio:
-            packet_loss_ratios = [0.0, packet_loss_ratio]
+        if loss_ratio:
+            loss_ratios = [0.0, loss_ratio]
+            exceed_ratio = 0.5
         else:
             # Happens in reconf tests.
-            packet_loss_ratios = [packet_loss_ratio]
-        results = algorithm.narrow_down_intervals(
-            min_rate=minimum_transmit_rate,
-            max_rate=maximum_transmit_rate,
-            packet_loss_ratios=packet_loss_ratios,
-        )
-        return results
+            loss_ratios = [0.0]
+            exceed_ratio = 0.0
+        goals = [
+            SearchGoal(
+                loss_ratio=loss_ratio,
+                exceed_ratio=exceed_ratio,
+                relative_width=relative_width,
+                initial_trial_duration=initial_trial_duration,
+                final_trial_duration=final_trial_duration,
+                duration_sum=duration_sum,
+                preceding_targets=preceding_targets,
+                expansion_coefficient=expansion_coefficient,
+                fail_fast=True,
+            )
+            for loss_ratio in loss_ratios
+        ]
+        config = Config()
+        config.goals = goals
+        config.min_load = min_load
+        config.max_load = max_load
+        config.search_duration_max = search_duration_max
+        config.warmup_duration = 1.0
+        algorithm = MultipleLossRatioSearch(config)
+        results = algorithm.search(measurer=tg_instance, debug=logger.debug)
+        return [results[goal] for goal in goals]
 
     @staticmethod
     def perform_soak_search(
             frame_size,
             traffic_profile,
-            minimum_transmit_rate,
-            maximum_transmit_rate,
+            min_load,
+            max_load,
             plr_target=1e-7,
             tdpt=0.1,
             initial_count=50,
@@ -1587,8 +1605,8 @@ class OptimizedSearch:
         :param frame_size: Frame size identifier or value [B].
         :param traffic_profile: Module name as a traffic profile identifier.
             See GPL/traffic_profiles/trex for implemented modules.
-        :param minimum_transmit_rate: Minimal load in transactions per second.
-        :param maximum_transmit_rate: Maximal load in transactions per second.
+        :param min_load: Minimal load in transactions per second.
+        :param max_load: Maximal load in transactions per second.
         :param plr_target: Ratio of packets lost to achieve [1].
         :param tdpt: Trial duration per trial.
             The algorithm linearly increases trial duration with trial number,
@@ -1622,8 +1640,8 @@ class OptimizedSearch:
         :param state_timeout: Time of life of DUT state [s].
         :type frame_size: str or int
         :type traffic_profile: str
-        :type minimum_transmit_rate: float
-        :type maximum_transmit_rate: float
+        :type min_load: float
+        :type max_load: float
         :type plr_target: float
         :type initial_count: int
         :type timeout: float
@@ -1672,7 +1690,7 @@ class OptimizedSearch:
             trace_enabled=trace_enabled,
         )
         result = algorithm.search(
-            min_rate=minimum_transmit_rate,
-            max_rate=maximum_transmit_rate,
+            min_rate=min_load,
+            max_rate=max_load,
         )
         return result