aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorVratko Polak <vrpolak@cisco.com>2023-10-24 15:44:53 +0200
committerVratko Polak <vrpolak@cisco.com>2023-10-25 10:21:05 +0000
commit8993ddb4f38f2754ae3af1c61e69a2e747f32a67 (patch)
treebb8759980251c44f1a395197e9fe506d1fa77ef7
parentee28e8ae476c6b0c098cd3895c586316feb4bdb9 (diff)
feat(MLRsearch): use goal result as in draft05
No effect on NDRPDR results, just different result packaging between the MLRsearch library and the rest of CSIT. - PyPI metadata still to be updated in a separate Change. Change-Id: I547134da189d1d7761594e92f36cc7c1c232ee32 Signed-off-by: Vratko Polak <vrpolak@cisco.com>
-rw-r--r--resources/libraries/python/MLRsearch/__init__.py4
-rw-r--r--resources/libraries/python/MLRsearch/goal_result.py72
-rw-r--r--resources/libraries/python/MLRsearch/multiple_loss_ratio_search.py8
-rw-r--r--resources/libraries/python/MLRsearch/target_stat.py1
-rw-r--r--resources/libraries/python/MLRsearch/trimmed_stat.py25
-rw-r--r--resources/libraries/python/TrafficGenerator.py13
-rw-r--r--resources/libraries/robot/performance/performance_display.robot4
-rw-r--r--resources/libraries/robot/performance/performance_utils.robot2
8 files changed, 89 insertions, 40 deletions
diff --git a/resources/libraries/python/MLRsearch/__init__.py b/resources/libraries/python/MLRsearch/__init__.py
index 349a9a96b3..09ce7e6719 100644
--- a/resources/libraries/python/MLRsearch/__init__.py
+++ b/resources/libraries/python/MLRsearch/__init__.py
@@ -23,8 +23,8 @@ __init__ file for Python package "MLRsearch".
# Import user-facing (API) stuff, so users do not need to know submodules.
from .config import Config
-from .search_goal import SearchGoal
+from .goal_result import GoalResult
from .multiple_loss_ratio_search import MultipleLossRatioSearch
from .pep3140 import Pep3140Dict
+from .search_goal import SearchGoal
from .trial_measurement import AbstractMeasurer, MeasurementResult
-from .trimmed_stat import TrimmedStat
diff --git a/resources/libraries/python/MLRsearch/goal_result.py b/resources/libraries/python/MLRsearch/goal_result.py
new file mode 100644
index 0000000000..91dccec0bb
--- /dev/null
+++ b/resources/libraries/python/MLRsearch/goal_result.py
@@ -0,0 +1,72 @@
+# Copyright (c) 2023 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module defining GoalResult class."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional
+
+from .discrete_load import DiscreteLoad
+from .relevant_bounds import RelevantBounds
+from .trimmed_stat import TrimmedStat
+
+
+@dataclass
+class GoalResult:
+ """Composite to be mapped for each search goal at the end of the search.
+
+ The values are stored as trimmed stats,
+ the conditional throughput is returned as a discrete loads.
+ Thus, users interested only in float values have to convert explicitly.
+
+ Irregular goal results are supported as instances with a bound missing.
+ """
+
+ relevant_lower_bound: Optional[TrimmedStat]
+ """The relevant lower bound for the search goal."""
+ relevant_upper_bound: Optional[TrimmedStat]
+ """The relevant lower upper for the search goal."""
+
+ @staticmethod
+ def from_bounds(bounds: RelevantBounds) -> GoalResult:
+ """Factory, so that the call site can be shorter.
+
+ :param bounds: The relevant bounds as found in measurement database.
+ :type bounds: RelevantBounds
+ :returns: Newly created instance based on the bounds.
+ :rtype: GoalResult
+ """
+ return GoalResult(
+ relevant_lower_bound=bounds.clo,
+ relevant_upper_bound=bounds.chi,
+ )
+
+ @property
+ def conditional_throughput(self) -> Optional[DiscreteLoad]:
+ """Compute conditional throughput from the relevant lower bound.
+
+ If the relevant lower bound is missing, None is returned.
+
+ The conditional throughput has the same semantics as load,
+ so if load is unidirectional and user wants bidirectional
+ throughput, the manager has to compensate.
+
+ :return: Conditional throughput at the relevant lower bound.
+ :rtype: Optional[DiscreteLoad]
+ """
+ if not (rlb := self.relevant_lower_bound):
+ return None
+ stat = next(iter(rlb.target_to_stat.values()))
+ return rlb * (1.0 - stat.pessimistic_loss_ratio)
diff --git a/resources/libraries/python/MLRsearch/multiple_loss_ratio_search.py b/resources/libraries/python/MLRsearch/multiple_loss_ratio_search.py
index 9f7be4fcd1..4d3ff7c4cb 100644
--- a/resources/libraries/python/MLRsearch/multiple_loss_ratio_search.py
+++ b/resources/libraries/python/MLRsearch/multiple_loss_ratio_search.py
@@ -25,6 +25,7 @@ from .dataclass import secondary_field
from .discrete_load import DiscreteLoad
from .discrete_result import DiscreteResult
from .expander import GlobalWidth
+from .goal_result import GoalResult
from .limit_handler import LimitHandler
from .load_rounding import LoadRounding
from .measurement_database import MeasurementDatabase
@@ -33,7 +34,6 @@ from .search_goal import SearchGoal
from .selector import Selector
from .target_scaling import TargetScaling
from .trial_measurement import AbstractMeasurer
-from .trimmed_stat import TrimmedStat
@dataclass
@@ -126,7 +126,7 @@ class MultipleLossRatioSearch:
self,
measurer: AbstractMeasurer,
debug: Optional[Callable[[str], None]] = None,
- ) -> Pep3140Dict[SearchGoal, Optional[TrimmedStat]]:
+ ) -> Pep3140Dict[SearchGoal, GoalResult]:
"""Perform initial trials, create state object, proceed with main loop.
Stateful arguments (measurer and debug) are stored.
@@ -139,7 +139,7 @@ class MultipleLossRatioSearch:
:returns: Structure containing conditional throughputs and other stats,
one for each search goal. If a value is None it means there is
no lower bound (min load turned out to be an upper bound).
- :rtype: Pep3140Dict[SearchGoal, Optional[TrimmedStat]]
+ :rtype: Pep3140Dict[SearchGoal, GoalResult]
:raises RuntimeError: If total duration is larger than timeout,
or if min load becomes an upper bound for a search goal
that has fail fast true.
@@ -168,7 +168,7 @@ class MultipleLossRatioSearch:
for goal in self.config.goals:
target = self.scaling.goal_to_final_target[goal]
bounds = self.database.get_relevant_bounds(target=target)
- ret_dict[goal] = bounds.clo
+ ret_dict[goal] = GoalResult.from_bounds(bounds=bounds)
return ret_dict
def measure(self, duration: float, load: DiscreteLoad) -> DiscreteResult:
diff --git a/resources/libraries/python/MLRsearch/target_stat.py b/resources/libraries/python/MLRsearch/target_stat.py
index e558139af7..18e1ff4161 100644
--- a/resources/libraries/python/MLRsearch/target_stat.py
+++ b/resources/libraries/python/MLRsearch/target_stat.py
@@ -117,6 +117,7 @@ class TargetStat:
pessimistic = (effective_dursum - self.good_long) <= limit_dursum
return optimistic, pessimistic
+ @property
def pessimistic_loss_ratio(self) -> float:
"""Return the loss ratio for conditional throughput computation.
diff --git a/resources/libraries/python/MLRsearch/trimmed_stat.py b/resources/libraries/python/MLRsearch/trimmed_stat.py
index 088e8beaf8..74918d78b0 100644
--- a/resources/libraries/python/MLRsearch/trimmed_stat.py
+++ b/resources/libraries/python/MLRsearch/trimmed_stat.py
@@ -16,9 +16,7 @@
from __future__ import annotations
from dataclasses import dataclass
-from typing import Optional
-from .discrete_load import DiscreteLoad
from .load_stats import LoadStats
from .target_spec import TargetSpec
@@ -52,26 +50,3 @@ class TrimmedStat(LoadStats):
int_load=stats.int_load,
target_to_stat={target: stats.target_to_stat[target]},
)
-
- @property
- def conditional_throughput(self) -> Optional[DiscreteLoad]:
- """Compute conditional throughput from the load.
-
- The conditional throughput has the same semantics as load,
- so if load is unicirectional and user wants bidirectional
- throughput, the manager has to compensate.
-
- This only works correctly if the self load is a lower bound
- for the self target, but this method does not check that.
- Its should not matter, as MLRsearch controller only returns
- the relevant lower bounds to the manager.
-
- :return: Conditional throughput assuming self is a relevant lower bound.
- :rtype: Optional[DiscreteLoad]
- :raises RuntimeError: If target is unclear or load is spurious.
- """
- target = list(self.target_to_stat.keys())[0]
- stat = self.target_to_stat[target]
- loss_ratio = stat.pessimistic_loss_ratio()
- ret = self * (1.0 - loss_ratio)
- return ret
diff --git a/resources/libraries/python/TrafficGenerator.py b/resources/libraries/python/TrafficGenerator.py
index 9faa3c49f6..f28c77856e 100644
--- a/resources/libraries/python/TrafficGenerator.py
+++ b/resources/libraries/python/TrafficGenerator.py
@@ -24,8 +24,8 @@ from robot.libraries.BuiltIn import BuiltIn
from .Constants import Constants
from .DropRateSearch import DropRateSearch
from .MLRsearch import (
- AbstractMeasurer, Config, MeasurementResult,
- MultipleLossRatioSearch, SearchGoal, TrimmedStat,
+ AbstractMeasurer, Config, GoalResult, MeasurementResult,
+ MultipleLossRatioSearch, SearchGoal,
)
from .PLRsearch.PLRsearch import PLRsearch
from .OptionString import OptionString
@@ -1448,7 +1448,7 @@ class OptimizedSearch:
ramp_up_rate: float = 0.0,
ramp_up_duration: float = 0.0,
state_timeout: float = 240.0,
- ) -> List[TrimmedStat]:
+ ) -> List[GoalResult]:
"""Setup initialized TG, perform optimized search, return intervals.
If transaction_scale is nonzero, all init and non-init trial durations
@@ -1514,9 +1514,10 @@ class OptimizedSearch:
:type ramp_up_rate: float
:type ramp_up_duration: float
:type state_timeout: float
- :returns: Structure containing narrowed down NDR and PDR intervals
- and their measurements.
- :rtype: List[TrimmedStat]
+ :returns: Goal result (based on unidirectional tps) for each goal.
+ The result contains both the offered load for stat trial,
+ and the conditional throughput for display.
+ :rtype: List[GoalResult]
:raises RuntimeError: If search duration exceeds search_duration_max
or if min load becomes an upper bound for any search goal.
"""
diff --git a/resources/libraries/robot/performance/performance_display.robot b/resources/libraries/robot/performance/performance_display.robot
index 791a45db62..6ee493a17e 100644
--- a/resources/libraries/robot/performance/performance_display.robot
+++ b/resources/libraries/robot/performance/performance_display.robot
@@ -92,8 +92,8 @@
| |
| | [Arguments] | ${result}
| |
-| | ${ndr} = | Convert To Number | ${result[0]}
-| | ${pdr} = | Convert To Number | ${result[1]}
+| | ${ndr} = | Convert To Number | ${result[0].relevant_lower_bound}
+| | ${pdr} = | Convert To Number | ${result[1].relevant_lower_bound}
| | Display single bound | NDR | ${result[0].conditional_throughput}
| | Display single bound | PDR | ${result[1].conditional_throughput}
| | Return From Keyword | ${ndr} | ${pdr}
diff --git a/resources/libraries/robot/performance/performance_utils.robot b/resources/libraries/robot/performance/performance_utils.robot
index fd03583ac9..4211486a19 100644
--- a/resources/libraries/robot/performance/performance_utils.robot
+++ b/resources/libraries/robot/performance/performance_utils.robot
@@ -278,7 +278,7 @@
| | ... | use_latency=${use_latency}
| | ... | ramp_up_duration=${ramp_up_duration}
| | ... | ramp_up_rate=${ramp_up_rate}
-| | ${ret} = | Convert To Number | ${result[0]}
+| | ${ret} = | Convert To Number | ${result[0].relevant_lower_bound}
| | Return From Keyword | ${ret}
| Measure and show latency at specified rate