diff options
Diffstat (limited to 'resources')
18 files changed, 2050 insertions, 1093 deletions
diff --git a/resources/api/vpp/supported_crcs.yaml b/resources/api/vpp/supported_crcs.yaml index 21b14904f3..cbe38bce29 100644 --- a/resources/api/vpp/supported_crcs.yaml +++ b/resources/api/vpp/supported_crcs.yaml @@ -88,6 +88,7 @@ det44_plugin_enable_disable: '0x617b6bf8' # perf det44_plugin_enable_disable_reply: '0xe8d4e804' # perf det44_session_dump: '0xe45a3af7' # perf teardown + # TODO: Which test to run to verify det44_* messages? # dhcp_proxy_dump / details # honeycomb gpe_enable_disable: '0xc264d7bf' # dev gpe_enable_disable_reply: '0xe8d4e804' # dev @@ -196,8 +197,7 @@ nat_show_config_reply: '0x7903ef06' # dev teardown nat_worker_details: '0x84bf06fc' # dev teardown nat_worker_dump: '0x51077d14' # dev teardown - # 6x^ tc01-64B-1c-ethip4udp-snat44det-h1-p1-s1-mrr - # ^ nat44NOTscaleNOTsrc_user_1 + # TODO: Which test to run to verify nat* messages? nsim_configure: '0x16ed400f' # perf nsim_configure_reply: '0xe8d4e804' # perf nsim_output_feature_enable_disable: '0x3865946c' # perf diff --git a/resources/libraries/python/Constants.py b/resources/libraries/python/Constants.py index 3775d98377..faf861d89d 100644 --- a/resources/libraries/python/Constants.py +++ b/resources/libraries/python/Constants.py @@ -232,6 +232,9 @@ class Constants: # Duration of one trial in MRR test. PERF_TRIAL_DURATION = get_float_from_env(u"PERF_TRIAL_DURATION", 1.0) + # Whether to use latency streams in main search trials. + PERF_USE_LATENCY = get_pessimistic_bool_from_env(u"PERF_USE_LATENCY") + # Duration of one latency-specific trial in NDRPDR test. PERF_TRIAL_LATENCY_DURATION = get_float_from_env( u"PERF_TRIAL_LATENCY_DURATION", 5.0) diff --git a/resources/libraries/python/MLRsearch/AbstractMeasurer.py b/resources/libraries/python/MLRsearch/AbstractMeasurer.py index 622b8fdba6..82116f2e43 100644 --- a/resources/libraries/python/MLRsearch/AbstractMeasurer.py +++ b/resources/libraries/python/MLRsearch/AbstractMeasurer.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2020 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -24,7 +24,7 @@ class AbstractMeasurer(metaclass=ABCMeta): """Perform trial measurement and return the result. :param duration: Trial duration [s]. - :param transmit_rate: Target transmit rate [pps]. + :param transmit_rate: Target transmit rate [tps]. :type duration: float :type transmit_rate: float :returns: Structure containing the result of the measurement. diff --git a/resources/libraries/python/MLRsearch/AbstractSearchAlgorithm.py b/resources/libraries/python/MLRsearch/AbstractSearchAlgorithm.py index f4f2d3f096..f2bf04e1b1 100644 --- a/resources/libraries/python/MLRsearch/AbstractSearchAlgorithm.py +++ b/resources/libraries/python/MLRsearch/AbstractSearchAlgorithm.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2020 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -30,16 +30,19 @@ class AbstractSearchAlgorithm(metaclass=ABCMeta): @abstractmethod def narrow_down_ndr_and_pdr( - self, fail_rate, line_rate, packet_loss_ratio): + self, min_rate, max_rate, packet_loss_ratio): """Perform measurements to narrow down intervals, return them. This will be renamed when custom loss ratio lists are supported. - :param fail_rate: Minimal target transmit rate [pps]. - :param line_rate: Maximal target transmit rate [pps]. + :param min_rate: Minimal target transmit rate [tps]. + Usually, tests are set to fail if search reaches this or below. + :param max_rate: Maximal target transmit rate [tps]. + Usually computed from line rate and various other limits, + to prevent failures or duration stretching in Traffic Generator. :param packet_loss_ratio: Fraction of packets lost, for PDR [1]. - :type fail_rate: float - :type line_rate: float + :type min_rate: float + :type max_rate: float :type packet_loss_ratio: float :returns: Structure containing narrowed down intervals and their measurements. diff --git a/resources/libraries/python/MLRsearch/MultipleLossRatioSearch.py b/resources/libraries/python/MLRsearch/MultipleLossRatioSearch.py index 2db65ef755..87dc784cbc 100644 --- a/resources/libraries/python/MLRsearch/MultipleLossRatioSearch.py +++ b/resources/libraries/python/MLRsearch/MultipleLossRatioSearch.py @@ -157,7 +157,7 @@ class MultipleLossRatioSearch(AbstractSearchAlgorithm): :returns: The relative width of double logarithmic size. :rtype: float """ - return 1.999 * relative_width - relative_width * relative_width + return 1.99999 * relative_width - relative_width * relative_width # The number should be 2.0, but we want to avoid rounding errors, # and ensure half of double is not larger than the original value. @@ -255,52 +255,50 @@ class MultipleLossRatioSearch(AbstractSearchAlgorithm): 1.0 - MultipleLossRatioSearch.half_relative_width(relative_width) ) - def narrow_down_ndr_and_pdr( - self, minimum_transmit_rate, maximum_transmit_rate, - packet_loss_ratio): + def narrow_down_ndr_and_pdr(self, min_rate, max_rate, packet_loss_ratio): """Perform initial phase, create state object, proceed with next phases. - :param minimum_transmit_rate: Minimal target transmit rate [pps]. - :param maximum_transmit_rate: Maximal target transmit rate [pps]. + :param min_rate: Minimal target transmit rate [tps]. + :param max_rate: Maximal target transmit rate [tps]. :param packet_loss_ratio: Fraction of packets lost, for PDR [1]. - :type minimum_transmit_rate: float - :type maximum_transmit_rate: float + :type min_rate: float + :type max_rate: float :type packet_loss_ratio: float :returns: Structure containing narrowed down intervals and their measurements. :rtype: NdrPdrResult.NdrPdrResult :raises RuntimeError: If total duration is larger than timeout. """ - minimum_transmit_rate = float(minimum_transmit_rate) - maximum_transmit_rate = float(maximum_transmit_rate) + minimum_transmit_rate = float(min_rate) + maximum_transmit_rate = float(max_rate) packet_loss_ratio = float(packet_loss_ratio) - line_measurement = self.measurer.measure( + max_measurement = self.measurer.measure( self.initial_trial_duration, maximum_transmit_rate) initial_width_goal = self.final_relative_width for _ in range(self.number_of_intermediate_phases): initial_width_goal = self.double_relative_width(initial_width_goal) max_lo = maximum_transmit_rate * (1.0 - initial_width_goal) - mrr = max( - minimum_transmit_rate, min(max_lo, line_measurement.receive_rate) - ) + mrr = max(minimum_transmit_rate, min( + max_lo, max_measurement.relative_receive_rate + )) mrr_measurement = self.measurer.measure( self.initial_trial_duration, mrr ) # Attempt to get narrower width. if mrr_measurement.loss_fraction > 0.0: max2_lo = mrr * (1.0 - initial_width_goal) - mrr2 = min(max2_lo, mrr_measurement.receive_rate) + mrr2 = min(max2_lo, mrr_measurement.relative_receive_rate) else: mrr2 = mrr / (1.0 - initial_width_goal) if minimum_transmit_rate < mrr2 < maximum_transmit_rate: - line_measurement = mrr_measurement + max_measurement = mrr_measurement mrr_measurement = self.measurer.measure( self.initial_trial_duration, mrr2) if mrr2 > mrr: - line_measurement, mrr_measurement = \ - (mrr_measurement, line_measurement) + max_measurement, mrr_measurement = \ + (mrr_measurement, max_measurement) starting_interval = ReceiveRateInterval( - mrr_measurement, line_measurement) + mrr_measurement, max_measurement) starting_result = NdrPdrResult(starting_interval, starting_interval) state = self.ProgressState( starting_result, self.number_of_intermediate_phases, diff --git a/resources/libraries/python/MLRsearch/ReceiveRateMeasurement.py b/resources/libraries/python/MLRsearch/ReceiveRateMeasurement.py index 31a6f8202e..c732e66026 100644 --- a/resources/libraries/python/MLRsearch/ReceiveRateMeasurement.py +++ b/resources/libraries/python/MLRsearch/ReceiveRateMeasurement.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2020 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -17,18 +17,39 @@ class ReceiveRateMeasurement: """Structure defining the result of single Rr measurement.""" - def __init__(self, duration, target_tr, transmit_count, loss_count): + def __init__( + self, duration, target_tr, transmit_count, loss_count, + approximated_duration=0.0, partial_transmit_count=0): """Constructor, normalize primary and compute secondary quantities. + If approximated_duration is nonzero, it is stored. + If approximated_duration is zero, duration value is stored. + Either way, additional secondary quantities are computed + from the store value. + + If there is zero transmit_count, fractions are set to zero. + + In some cases, traffic generator does not attempt all the needed + transactions. In that case, nonzero partial_transmit_count + holds (an estimate of) count of the actually attempted transactions. + This is used to populate some secondary quantities. + + TODO: Use None instead of zero? + :param duration: Measurement duration [s]. :param target_tr: Target transmit rate [pps]. If bidirectional traffic is measured, this is bidirectional rate. :param transmit_count: Number of packets transmitted [1]. :param loss_count: Number of packets transmitted but not received [1]. + :param approximated_duration: Estimate of the actual time of the trial. + :param partial_transmit_count: Estimate count of actually attempted + transactions. :type duration: float :type target_tr: float :type transmit_count: int :type loss_count: int + :type approximated_duration: float + :type partial_transmit_count: int """ self.duration = float(duration) self.target_tr = float(target_tr) @@ -38,8 +59,41 @@ class ReceiveRateMeasurement: self.transmit_rate = transmit_count / self.duration self.loss_rate = loss_count / self.duration self.receive_rate = self.receive_count / self.duration - self.loss_fraction = float(self.loss_count) / self.transmit_count - # TODO: Do we want to store also the real time (duration + overhead)? + self.loss_fraction = ( + float(self.loss_count) / self.transmit_count + if self.transmit_count > 0 else 1.0 + ) + self.receive_fraction = ( + float(self.receive_count) / self.transmit_count + if self.transmit_count > 0 else 0.0 + ) + self.approximated_duration = ( + float(approximated_duration) if approximated_duration + else self.duration + ) + self.approximated_receive_rate = ( + self.receive_count / self.approximated_duration + if self.approximated_duration > 0.0 else 0.0 + ) + # If the traffic generator is unreliable and sends less packets, + # the absolute receive rate might be too low for next target. + self.partial_transmit_count = ( + int(partial_transmit_count) if partial_transmit_count + else self.transmit_count + ) + self.partial_receive_fraction = ( + float(self.receive_count) / self.partial_transmit_count + if self.partial_transmit_count > 0 else 0.0 + ) + self.partial_receive_rate = ( + self.target_tr * self.partial_receive_fraction + ) + # We use relative packet ratios in order to support cases + # where target_tr is in transactions per second, + # but there are multiple packets per transaction. + self.relative_receive_rate = ( + self.target_tr * self.receive_count / self.transmit_count + ) def __str__(self): """Return string reporting input and loss fraction.""" @@ -51,4 +105,6 @@ class ReceiveRateMeasurement: return f"ReceiveRateMeasurement(duration={self.duration!r}," \ f"target_tr={self.target_tr!r}," \ f"transmit_count={self.transmit_count!r}," \ - f"loss_count={self.loss_count!r})" + f"loss_count={self.loss_count!r}," \ + f"approximated_duration={self.approximated_duration!r}," \ + f"partial_transmit_count={self.partial_transmit_count!r})" diff --git a/resources/libraries/python/NATUtil.py b/resources/libraries/python/NATUtil.py index a9f760768e..422feaedb8 100644 --- a/resources/libraries/python/NATUtil.py +++ b/resources/libraries/python/NATUtil.py @@ -159,6 +159,10 @@ class NATUtil: flag=u"NAT_IS_NONE"): """Set NAT44 address range. + The return value is a callable (zero argument Python function) + which can be used to reset NAT state, so repeated trial measurements + hit the same slow path. + :param node: DUT node. :param start_ip: IP range start. :param end_ip: IP range end. @@ -169,6 +173,8 @@ class NATUtil: :type end_ip: str :type vrf_id: int :type flag: str + :returns: Resetter of the NAT state. + :rtype: Callable[[], None] """ cmd = u"nat44_add_del_address_range" err_msg = f"Failed to set NAT44 address range on host {node[u'host']}" @@ -183,6 +189,18 @@ class NATUtil: with PapiSocketExecutor(node) as papi_exec: papi_exec.add(cmd, **args_in).get_reply(err_msg) + # A closure, accessing the variables above. + def resetter(): + """Delete and re-add the NAT range setting.""" + with PapiSocketExecutor(node) as papi_exec: + args_in[u"is_add"] = False + papi_exec.add(cmd, **args_in) + args_in[u"is_add"] = True + papi_exec.add(cmd, **args_in) + papi_exec.get_replies(err_msg) + + return resetter + @staticmethod def show_nat_config(node): """Show the NAT configuration. @@ -345,6 +363,10 @@ class NATUtil: def set_det44_mapping(node, ip_in, subnet_in, ip_out, subnet_out): """Set DET44 mapping. + The return value is a callable (zero argument Python function) + which can be used to reset NAT state, so repeated trial measurements + hit the same slow path. + :param node: DUT node. :param ip_in: Inside IP. :param subnet_in: Inside IP subnet. @@ -369,6 +391,18 @@ class NATUtil: with PapiSocketExecutor(node) as papi_exec: papi_exec.add(cmd, **args_in).get_reply(err_msg) + # A closure, accessing the variables above. + def resetter(): + """Delete and re-add the deterministic NAT mapping.""" + with PapiSocketExecutor(node) as papi_exec: + args_in[u"is_add"] = False + papi_exec.add(cmd, **args_in) + args_in[u"is_add"] = True + papi_exec.add(cmd, **args_in) + papi_exec.get_replies(err_msg) + + return resetter + @staticmethod def get_det44_mapping(node): """Get DET44 mapping data. @@ -391,14 +425,12 @@ class NATUtil: def get_det44_sessions_number(node): """Get number of established DET44 sessions from actual DET44 mapping data. - :param node: DUT node. :type node: dict :returns: Number of established DET44 sessions. :rtype: int """ det44_data = NATUtil.get_det44_mapping(node) - return det44_data.get(u"ses_num", 0) @staticmethod diff --git a/resources/libraries/python/PLRsearch/PLRsearch.py b/resources/libraries/python/PLRsearch/PLRsearch.py index ec58fbd10f..226b482d76 100644 --- a/resources/libraries/python/PLRsearch/PLRsearch.py +++ b/resources/libraries/python/PLRsearch/PLRsearch.py @@ -54,7 +54,7 @@ class PLRsearch: def __init__( self, measurer, trial_duration_per_trial, packet_loss_ratio_target, - trial_number_offset=0, timeout=1800.0, trace_enabled=False): + trial_number_offset=0, timeout=7200.0, trace_enabled=False): """Store rate measurer and additional parameters. The measurer must never report negative loss count. @@ -205,7 +205,7 @@ class PLRsearch: if (trial_number - self.trial_number_offset) <= 1: next_load = max_rate elif (trial_number - self.trial_number_offset) <= 3: - next_load = (measurement.receive_rate / ( + next_load = (measurement.relative_receive_rate / ( 1.0 - self.packet_loss_ratio_target)) else: next_load = (avg1 + avg2) / 2.0 @@ -439,7 +439,7 @@ class PLRsearch: :param lfit_func: Fitting function, typically lfit_spread or lfit_erf. :param trial_result_list: List of trial measurement results. :param mrr: The mrr parameter for the fitting function. - :param spread: The spread parameter for the fittinmg function. + :param spread: The spread parameter for the fitting function. :type trace: function (str, object) -> None :type lfit_func: Function from 3 floats to float. :type trial_result_list: list of MLRsearch.ReceiveRateMeasurement @@ -455,20 +455,21 @@ class PLRsearch: trace(u"for tr", result.target_tr) trace(u"lc", result.loss_count) trace(u"d", result.duration) - log_avg_loss_per_second = lfit_func( + # _rel_ values use units of target_tr (transactions per second). + log_avg_rel_loss_per_second = lfit_func( trace, result.target_tr, mrr, spread ) - log_avg_loss_per_trial = ( - log_avg_loss_per_second + math.log(result.duration) + # _abs_ values use units of loss count (maybe packets). + # There can be multiple packets per transaction. + log_avg_abs_loss_per_trial = log_avg_rel_loss_per_second + math.log( + result.transmit_count / result.target_tr ) - # Poisson probability computation works nice for logarithms. - log_trial_likelihood = ( - result.loss_count * log_avg_loss_per_trial - - math.exp(log_avg_loss_per_trial) - ) - log_trial_likelihood -= math.lgamma(1 + result.loss_count) + # Geometric probability computation for logarithms. + log_trial_likelihood = log_plus(0.0, -log_avg_abs_loss_per_trial) + log_trial_likelihood *= -result.loss_count + log_trial_likelihood -= log_plus(0.0, +log_avg_abs_loss_per_trial) log_likelihood += log_trial_likelihood - trace(u"avg_loss_per_trial", math.exp(log_avg_loss_per_trial)) + trace(u"avg_loss_per_trial", math.exp(log_avg_abs_loss_per_trial)) trace(u"log_trial_likelihood", log_trial_likelihood) return log_likelihood diff --git a/resources/libraries/python/TrafficGenerator.py b/resources/libraries/python/TrafficGenerator.py index f44ff870bd..80248add03 100644 --- a/resources/libraries/python/TrafficGenerator.py +++ b/resources/libraries/python/TrafficGenerator.py @@ -67,7 +67,7 @@ class TGDropRateSearchImpl(DropRateSearch): def measure_loss( self, rate, frame_size, loss_acceptance, loss_acceptance_type, - traffic_profile, skip_warmup=False): + traffic_profile): """Runs the traffic and evaluate the measured results. :param rate: Offered traffic load. @@ -76,13 +76,11 @@ class TGDropRateSearchImpl(DropRateSearch): :param loss_acceptance_type: Type of permitted loss. :param traffic_profile: Module name as a traffic profile identifier. See GPL/traffic_profiles/trex for implemented modules. - :param skip_warmup: Start TRex without warmup traffic if true. :type rate: float :type frame_size: str :type loss_acceptance: float :type loss_acceptance_type: LossAcceptanceType :type traffic_profile: str - :type skip_warmup: bool :returns: Drop threshold exceeded? (True/False) :rtype: bool :raises NotImplementedError: If TG is not supported. @@ -96,15 +94,9 @@ class TGDropRateSearchImpl(DropRateSearch): subtype = check_subtype(tg_instance.node) if subtype == NodeSubTypeTG.TREX: unit_rate = str(rate) + self.get_rate_type_str() - if skip_warmup: - tg_instance.trex_stl_start_remote_exec( - self.get_duration(), unit_rate, frame_size, traffic_profile, - warmup_time=0.0 - ) - else: - tg_instance.trex_stl_start_remote_exec( - self.get_duration(), unit_rate, frame_size, traffic_profile - ) + tg_instance.trex_stl_start_remote_exec( + self.get_duration(), unit_rate, frame_size, traffic_profile + ) loss = tg_instance.get_loss() sent = tg_instance.get_sent() if self.loss_acceptance_type_is_percentage(): @@ -145,6 +137,8 @@ class TrafficGenerator(AbstractMeasurer): ROBOT_LIBRARY_SCOPE = u"TEST SUITE" def __init__(self): + # TODO: Separate into few dataclasses/dicts. + # Pylint dislikes large unstructured state, and it is right. self._node = None self._mode = None # TG interface order mapping @@ -160,14 +154,23 @@ class TrafficGenerator(AbstractMeasurer): self._l7_data = None # Measurement input fields, needed for async stop result. self._start_time = None + self._stop_time = None self._rate = None + self._target_duration = None + self._duration = None # Other input parameters, not knowable from measure() signature. self.frame_size = None self.traffic_profile = None - self.warmup_time = None self.traffic_directions = None self.negative_loss = None self.use_latency = None + self.ppta = None + self.resetter = None + self.transaction_scale = None + self.transaction_duration = None + self.sleep_till_duration = None + self.transaction_type = None + self.duration_limit = None # Transient data needed for async measurements. self._xstats = (None, None) # TODO: Rename "xstats" to something opaque, so T-Rex is not privileged? @@ -454,108 +457,6 @@ class TrafficGenerator(AbstractMeasurer): message=u"T-Rex kill failed!" ) - def _parse_traffic_results(self, stdout): - """Parse stdout of scripts into fields of self. - - Block of code to reuse, by sync start, or stop after async. - - :param stdout: Text containing the standard output. - :type stdout: str - """ - subtype = check_subtype(self._node) - if subtype == NodeSubTypeTG.TREX: - # Last line from console output - line = stdout.splitlines()[-1] - results = line.split(u",") - if results[-1] in (u" ", u""): - results.pop(-1) - self._result = dict() - for result in results: - key, value = result.split(u"=", maxsplit=1) - self._result[key.strip()] = value - logger.info(f"TrafficGen results:\n{self._result}") - self._received = self._result.get(u"total_received") - self._sent = self._result.get(u"total_sent") - self._loss = self._result.get(u"frame_loss") - self._approximated_duration = \ - self._result.get(u"approximated_duration") - self._approximated_rate = self._result.get(u"approximated_rate") - self._latency = list() - self._latency.append(self._result.get(u"latency_stream_0(usec)")) - self._latency.append(self._result.get(u"latency_stream_1(usec)")) - if self._mode == TrexMode.ASTF: - self._l7_data = dict() - self._l7_data[u"client"] = dict() - self._l7_data[u"client"][u"active_flows"] = \ - self._result.get(u"client_active_flows") - self._l7_data[u"client"][u"established_flows"] = \ - self._result.get(u"client_established_flows") - self._l7_data[u"client"][u"err_rx_throttled"] = \ - self._result.get(u"client_err_rx_throttled") - self._l7_data[u"client"][u"err_c_nf_throttled"] = \ - self._result.get(u"client_err_nf_throttled") - self._l7_data[u"client"][u"err_flow_overflow"] = \ - self._result.get(u"client_err_flow_overflow") - self._l7_data[u"server"] = dict() - self._l7_data[u"server"][u"active_flows"] = \ - self._result.get(u"server_active_flows") - self._l7_data[u"server"][u"established_flows"] = \ - self._result.get(u"server_established_flows") - self._l7_data[u"server"][u"err_rx_throttled"] = \ - self._result.get(u"client_err_rx_throttled") - if u"udp" in self.traffic_profile: - self._l7_data[u"client"][u"udp"] = dict() - self._l7_data[u"client"][u"udp"][u"established_flows"] = \ - self._result.get(u"client_udp_connects") - self._l7_data[u"client"][u"udp"][u"closed_flows"] = \ - self._result.get(u"client_udp_closed") - self._l7_data[u"client"][u"udp"][u"tx_bytes"] = \ - self._result.get(u"client_udp_tx_bytes") - self._l7_data[u"client"][u"udp"][u"rx_bytes"] = \ - self._result.get(u"client_udp_rx_bytes") - self._l7_data[u"client"][u"udp"][u"tx_packets"] = \ - self._result.get(u"client_udp_tx_packets") - self._l7_data[u"client"][u"udp"][u"rx_packets"] = \ - self._result.get(u"client_udp_rx_packets") - self._l7_data[u"client"][u"udp"][u"keep_drops"] = \ - self._result.get(u"client_udp_keep_drops") - self._l7_data[u"server"][u"udp"] = dict() - self._l7_data[u"server"][u"udp"][u"accepted_flows"] = \ - self._result.get(u"server_udp_accepts") - self._l7_data[u"server"][u"udp"][u"closed_flows"] = \ - self._result.get(u"server_udp_closed") - self._l7_data[u"server"][u"udp"][u"tx_bytes"] = \ - self._result.get(u"server_udp_tx_bytes") - self._l7_data[u"server"][u"udp"][u"rx_bytes"] = \ - self._result.get(u"server_udp_rx_bytes") - self._l7_data[u"server"][u"udp"][u"tx_packets"] = \ - self._result.get(u"server_udp_tx_packets") - self._l7_data[u"server"][u"udp"][u"rx_packets"] = \ - self._result.get(u"server_udp_rx_packets") - elif u"tcp" in self.traffic_profile: - self._l7_data[u"client"][u"tcp"] = dict() - self._l7_data[u"client"][u"tcp"][u"initiated_flows"] = \ - self._result.get(u"client_tcp_connect_inits") - self._l7_data[u"client"][u"tcp"][u"established_flows"] = \ - self._result.get(u"client_tcp_connects") - self._l7_data[u"client"][u"tcp"][u"closed_flows"] = \ - self._result.get(u"client_tcp_closed") - self._l7_data[u"client"][u"tcp"][u"tx_bytes"] = \ - self._result.get(u"client_tcp_tx_bytes") - self._l7_data[u"client"][u"tcp"][u"rx_bytes"] = \ - self._result.get(u"client_tcp_rx_bytes") - self._l7_data[u"server"][u"tcp"] = dict() - self._l7_data[u"server"][u"tcp"][u"accepted_flows"] = \ - self._result.get(u"server_tcp_accepts") - self._l7_data[u"server"][u"tcp"][u"established_flows"] = \ - self._result.get(u"server_tcp_connects") - self._l7_data[u"server"][u"tcp"][u"closed_flows"] = \ - self._result.get(u"server_tcp_closed") - self._l7_data[u"server"][u"tcp"][u"tx_bytes"] = \ - self._result.get(u"server_tcp_tx_bytes") - self._l7_data[u"server"][u"tcp"][u"rx_bytes"] = \ - self._result.get(u"server_tcp_rx_bytes") - def trex_astf_stop_remote_exec(self, node): """Execute T-Rex ASTF script on remote node over ssh to stop running traffic. @@ -612,59 +513,77 @@ class TrafficGenerator(AbstractMeasurer): :raises ValueError: If TG traffic profile is not supported. """ subtype = check_subtype(self._node) - if subtype == NodeSubTypeTG.TREX: - if u"trex-astf" in self.traffic_profile: - self.trex_astf_stop_remote_exec(self._node) - elif u"trex-stl" in self.traffic_profile: - self.trex_stl_stop_remote_exec(self._node) - else: - raise ValueError(u"Unsupported T-Rex traffic profile!") + if subtype != NodeSubTypeTG.TREX: + raise ValueError(f"Unsupported TG subtype: {subtype!r}") + if u"trex-astf" in self.traffic_profile: + self.trex_astf_stop_remote_exec(self._node) + elif u"trex-stl" in self.traffic_profile: + self.trex_stl_stop_remote_exec(self._node) + else: + raise ValueError(u"Unsupported T-Rex traffic profile!") + self._stop_time = time.monotonic() return self.get_measurement_result() def trex_astf_start_remote_exec( - self, duration, mult, frame_size, traffic_profile, async_call=False, - latency=True, warmup_time=5.0, traffic_directions=2, tx_port=0, - rx_port=1): + self, duration, multiplier, async_call=False): """Execute T-Rex ASTF script on remote node over ssh to start running traffic. In sync mode, measurement results are stored internally. In async mode, initial data including xstats are stored internally. - :param duration: Time expresed in seconds for how long to send traffic. - :param mult: Traffic rate expressed with units (pps, %) - :param frame_size: L2 frame size to send (without padding and IPG). - :param traffic_profile: Module name as a traffic profile identifier. - See GPL/traffic_profiles/trex for implemented modules. + This method contains the logic to compute duration as maximum time + if transaction_scale is nonzero. + The transaction_scale argument defines (limits) how many transactions + will be started in total. As that amount of transaction can take + considerable time (sometimes due to explicit delays in the profile), + the real time a trial needs to finish is computed here. For now, + in that case the duration argument is ignored, assuming it comes + from ASTF-unaware search algorithm. The overall time a single + transaction needs is given in parameter transaction_duration, + it includes both explicit delays and implicit time it takes + to transfer data (or whatever the transaction does). + + Currently it is observed TRex does not start the ASTF traffic + immediately, an ad-hoc constant is added to the computed duration + to compensate for that. + + If transaction_scale is zero, duration is not recomputed. + It is assumed the subsequent result parsing gets the real duration + if the traffic stops sooner for any reason. + + Currently, it is assumed traffic profile defines a single transaction. + To avoid heavy logic here, the input rate is expected to be in + transactions per second, as that directly translates to TRex multiplier, + (assuming the profile does not override the default cps value of one). + + :param duration: Time expressed in seconds for how long to send traffic. + :param multiplier: Traffic rate in transactions per second. :param async_call: If enabled then don't wait for all incoming traffic. - :param latency: With latency measurement. - :param warmup_time: Warmup time period. - :param traffic_directions: Traffic is bi- (2) or uni- (1) directional. - Default: 2 - :param tx_port: Traffic generator transmit port for first flow. - Default: 0 - :param rx_port: Traffic generator receive port for first flow. - Default: 1 :type duration: float - :type mult: int - :type frame_size: str - :type traffic_profile: str + :type multiplier: int :type async_call: bool - :type latency: bool - :type warmup_time: float - :type traffic_directions: int - :type tx_port: int - :type rx_port: int :raises RuntimeError: In case of T-Rex driver issue. """ self.check_mode(TrexMode.ASTF) - p_0, p_1 = (rx_port, tx_port) if self._ifaces_reordered \ - else (tx_port, rx_port) + p_0, p_1 = (1, 0) if self._ifaces_reordered else (0, 1) if not isinstance(duration, (float, int)): duration = float(duration) - if not isinstance(warmup_time, (float, int)): - warmup_time = float(warmup_time) + + # Duration logic. + computed_duration = duration + if duration > 0.0: + if self.transaction_scale: + computed_duration = self.transaction_scale / multiplier + # Log the computed duration, + # so we can compare with what telemetry suggests + # the real duration was. + logger.debug(f"Expected duration {computed_duration}") + computed_duration += 0.1115 + # Else keep -1. + if self.duration_limit: + computed_duration = min(computed_duration, self.duration_limit) command_line = OptionString().add(u"python3") dirname = f"{Constants.REMOTE_FW_DIR}/GPL/tools/trex" @@ -672,31 +591,31 @@ class TrafficGenerator(AbstractMeasurer): command_line.change_prefix(u"--") dirname = f"{Constants.REMOTE_FW_DIR}/GPL/traffic_profiles/trex" command_line.add_with_value( - u"profile", f"'{dirname}/{traffic_profile}.py'" + u"profile", f"'{dirname}/{self.traffic_profile}.py'" ) - command_line.add_with_value(u"duration", f"{duration!r}") - command_line.add_with_value(u"frame_size", frame_size) - command_line.add_with_value(u"mult", int(mult)) - command_line.add_with_value(u"warmup_time", f"{warmup_time!r}") + command_line.add_with_value(u"duration", f"{computed_duration!r}") + command_line.add_with_value(u"frame_size", self.frame_size) + command_line.add_with_value(u"multiplier", multiplier) command_line.add_with_value(u"port_0", p_0) command_line.add_with_value(u"port_1", p_1) - command_line.add_with_value(u"traffic_directions", traffic_directions) + command_line.add_with_value( + u"traffic_directions", self.traffic_directions + ) command_line.add_if(u"async_start", async_call) - command_line.add_if(u"latency", latency) + command_line.add_if(u"latency", self.use_latency) command_line.add_if(u"force", Constants.TREX_SEND_FORCE) + self._start_time = time.monotonic() + self._rate = multiplier stdout, _ = exec_cmd_no_error( - self._node, command_line, - timeout=int(duration) + 600 if u"tcp" in self.traffic_profile - else 60, + self._node, command_line, timeout=computed_duration + 10.0, message=u"T-Rex ASTF runtime error!" ) - self.traffic_directions = traffic_directions if async_call: # no result - self._start_time = time.time() - self._rate = float(mult) + self._target_duration = None + self._duration = None self._received = None self._sent = None self._loss = None @@ -706,24 +625,28 @@ class TrafficGenerator(AbstractMeasurer): self._l7_data[u"client"] = dict() self._l7_data[u"client"][u"active_flows"] = None self._l7_data[u"client"][u"established_flows"] = None + self._l7_data[u"client"][u"traffic_duration"] = None self._l7_data[u"server"] = dict() self._l7_data[u"server"][u"active_flows"] = None self._l7_data[u"server"][u"established_flows"] = None + self._l7_data[u"server"][u"traffic_duration"] = None if u"udp" in self.traffic_profile: self._l7_data[u"client"][u"udp"] = dict() - self._l7_data[u"client"][u"udp"][u"established_flows"] = None + self._l7_data[u"client"][u"udp"][u"connects"] = None self._l7_data[u"client"][u"udp"][u"closed_flows"] = None + self._l7_data[u"client"][u"udp"][u"err_cwf"] = None self._l7_data[u"server"][u"udp"] = dict() self._l7_data[u"server"][u"udp"][u"accepted_flows"] = None self._l7_data[u"server"][u"udp"][u"closed_flows"] = None elif u"tcp" in self.traffic_profile: self._l7_data[u"client"][u"tcp"] = dict() self._l7_data[u"client"][u"tcp"][u"initiated_flows"] = None - self._l7_data[u"client"][u"tcp"][u"established_flows"] = None + self._l7_data[u"client"][u"tcp"][u"connects"] = None self._l7_data[u"client"][u"tcp"][u"closed_flows"] = None + self._l7_data[u"client"][u"tcp"][u"connattempt"] = None self._l7_data[u"server"][u"tcp"] = dict() self._l7_data[u"server"][u"tcp"][u"accepted_flows"] = None - self._l7_data[u"server"][u"tcp"][u"established_flows"] = None + self._l7_data[u"server"][u"tcp"][u"connects"] = None self._l7_data[u"server"][u"tcp"][u"closed_flows"] = None else: logger.warn(u"Unsupported T-Rex ASTF traffic profile!") @@ -736,53 +659,36 @@ class TrafficGenerator(AbstractMeasurer): break self._xstats = tuple(xstats) else: + self._target_duration = duration + self._duration = computed_duration self._parse_traffic_results(stdout) - self._start_time = None - self._rate = None - def trex_stl_start_remote_exec( - self, duration, rate, frame_size, traffic_profile, async_call=False, - latency=False, warmup_time=5.0, traffic_directions=2, tx_port=0, - rx_port=1): + def trex_stl_start_remote_exec(self, duration, rate, async_call=False): """Execute T-Rex STL script on remote node over ssh to start running traffic. In sync mode, measurement results are stored internally. In async mode, initial data including xstats are stored internally. + Mode-unaware code (e.g. in search algorithms) works with transactions. + To keep the logic simple, multiplier is set to that value. + As bidirectional traffic profiles send packets in both directions, + they are treated as transactions with two packets (one per direction). + :param duration: Time expressed in seconds for how long to send traffic. - :param rate: Traffic rate expressed with units (pps, %) - :param frame_size: L2 frame size to send (without padding and IPG). - :param traffic_profile: Module name as a traffic profile identifier. - See GPL/traffic_profiles/trex for implemented modules. + :param rate: Traffic rate in transactions per second. :param async_call: If enabled then don't wait for all incoming traffic. - :param latency: With latency measurement. - :param warmup_time: Warmup time period. - :param traffic_directions: Traffic is bi- (2) or uni- (1) directional. - Default: 2 - :param tx_port: Traffic generator transmit port for first flow. - Default: 0 - :param rx_port: Traffic generator receive port for first flow. - Default: 1 :type duration: float :type rate: str - :type frame_size: str - :type traffic_profile: str :type async_call: bool - :type latency: bool - :type warmup_time: float - :type traffic_directions: int - :type tx_port: int - :type rx_port: int :raises RuntimeError: In case of T-Rex driver issue. """ self.check_mode(TrexMode.STL) - p_0, p_1 = (rx_port, tx_port) if self._ifaces_reordered \ - else (tx_port, rx_port) + p_0, p_1 = (1, 0) if self._ifaces_reordered else (0, 1) if not isinstance(duration, (float, int)): duration = float(duration) - if not isinstance(warmup_time, (float, int)): - warmup_time = float(warmup_time) + if self.duration_limit: + duration = min(duration, self.duration_limit) command_line = OptionString().add(u"python3") dirname = f"{Constants.REMOTE_FW_DIR}/GPL/tools/trex" @@ -790,29 +696,32 @@ class TrafficGenerator(AbstractMeasurer): command_line.change_prefix(u"--") dirname = f"{Constants.REMOTE_FW_DIR}/GPL/traffic_profiles/trex" command_line.add_with_value( - u"profile", f"'{dirname}/{traffic_profile}.py'" + u"profile", f"'{dirname}/{self.traffic_profile}.py'" ) command_line.add_with_value(u"duration", f"{duration!r}") - command_line.add_with_value(u"frame_size", frame_size) + command_line.add_with_value(u"frame_size", self.frame_size) command_line.add_with_value(u"rate", f"{rate!r}") - command_line.add_with_value(u"warmup_time", f"{warmup_time!r}") command_line.add_with_value(u"port_0", p_0) command_line.add_with_value(u"port_1", p_1) - command_line.add_with_value(u"traffic_directions", traffic_directions) + command_line.add_with_value( + u"traffic_directions", self.traffic_directions + ) command_line.add_if(u"async_start", async_call) - command_line.add_if(u"latency", latency) + command_line.add_if(u"latency", self.use_latency) command_line.add_if(u"force", Constants.TREX_SEND_FORCE) + # TODO: This is ugly. Handle parsing better. + self._start_time = time.monotonic() + self._rate = float(rate[:-3]) if u"pps" in rate else float(rate) stdout, _ = exec_cmd_no_error( self._node, command_line, timeout=int(duration) + 60, message=u"T-Rex STL runtime error" ) - self.traffic_directions = traffic_directions if async_call: # no result - self._start_time = time.time() - self._rate = float(rate[:-3]) if u"pps" in rate else float(rate) + self._target_duration = None + self._duration = None self._received = None self._sent = None self._loss = None @@ -828,14 +737,25 @@ class TrafficGenerator(AbstractMeasurer): break self._xstats = tuple(xstats) else: + self._target_duration = duration + self._duration = duration self._parse_traffic_results(stdout) - self._start_time = None - self._rate = None def send_traffic_on_tg( - self, duration, rate, frame_size, traffic_profile, warmup_time=5, - async_call=False, latency=False, traffic_directions=2, tx_port=0, - rx_port=1): + self, + duration, + rate, + frame_size, + traffic_profile, + async_call=False, + ppta=1, + traffic_directions=2, + transaction_duration=0.0, + transaction_scale=0, + transaction_type=u"packet", + duration_limit=0.0, + use_latency=False, + ): """Send traffic from all configured interfaces on TG. In async mode, xstats is stored internally, @@ -843,61 +763,102 @@ class TrafficGenerator(AbstractMeasurer): In both modes, stdout is returned, but _parse_traffic_results only works in sync output. - Note that bidirectional traffic also contains flows - transmitted from rx_port and received in tx_port. - But some tests use asymmetric traffic, so those arguments are relevant. - - Also note that traffic generator uses DPDK driver which might + Note that traffic generator uses DPDK driver which might reorder port numbers based on wiring and PCI numbering. This method handles that, so argument values are invariant, but you can see swapped valued in debug logs. + When transaction_scale is specified, the duration value is ignored + and the needed time is computed. For cases where this results in + to too long measurement (e.g. teardown trial with small rate), + duration_limit is applied (of non-zero), so the trial is stopped sooner. + + Bidirectional STL profiles are treated as transactions with two packets. + :param duration: Duration of test traffic generation in seconds. - :param rate: Traffic rate. - - T-Rex stateless mode => Offered load per interface in pps, - - T-Rex advanced stateful mode => multiplier of profile CPS. + :param rate: Traffic rate in transactions per second. :param frame_size: Frame size (L2) in Bytes. :param traffic_profile: Module name as a traffic profile identifier. See GPL/traffic_profiles/trex for implemented modules. - :param warmup_time: Warmup phase in seconds. :param async_call: Async mode. - :param latency: With latency measurement. + :param ppta: Packets per transaction, aggregated over directions. + Needed for udp_pps which does not have a good transaction counter, + so we need to compute expected number of packets. + Default: 1. :param traffic_directions: Traffic is bi- (2) or uni- (1) directional. Default: 2 - :param tx_port: Traffic generator transmit port for first flow. - Default: 0 - :param rx_port: Traffic generator receive port for first flow. - Default: 1 + :param transaction_duration: Total expected time to close transaction. + :param transaction_scale: Number of transactions to perform. + 0 (default) means unlimited. + :param transaction_type: An identifier specifying which counters + and formulas to use when computing attempted and failed + transactions. Default: "packet". + :param duration_limit: Zero or maximum limit for computed (or given) + duration. + :param use_latency: Whether to measure latency during the trial. + Default: False. :type duration: float :type rate: float :type frame_size: str :type traffic_profile: str - :type warmup_time: float :type async_call: bool - :type latency: bool + :type ppta: int :type traffic_directions: int - :type tx_port: int - :type rx_port: int + :type transaction_duration: float + :type transaction_scale: int + :type transaction_type: str + :type duration_limit: float + :type use_latency: bool + :returns: TG results. + :rtype: str + :raises ValueError: If TG traffic profile is not supported. + """ + self.set_rate_provider_defaults( + frame_size=frame_size, + traffic_profile=traffic_profile, + ppta=ppta, + traffic_directions=traffic_directions, + transaction_duration=transaction_duration, + transaction_scale=transaction_scale, + transaction_type=transaction_type, + duration_limit=duration_limit, + use_latency=use_latency, + ) + self._send_traffic_on_tg_internal(duration, rate, async_call) + + def _send_traffic_on_tg_internal(self, duration, rate, async_call=False): + """Send traffic from all configured interfaces on TG. + + This is an internal function, it assumes set_rate_provider_defaults + has been called to remember most values. + The reason why need to remember various values is that + the traffic can be asynchronous, and parsing needs those values. + The reason why this is is a separate function from the one + which calls set_rate_provider_defaults is that some search algorithms + need to specify their own values, and we do not want the measure call + to overwrite them with defaults. + + :param duration: Duration of test traffic generation in seconds. + :param rate: Traffic rate in transactions per second. + :param async_call: Async mode. + :type duration: float + :type rate: float + :type async_call: bool :returns: TG results. :rtype: str :raises ValueError: If TG traffic profile is not supported. """ subtype = check_subtype(self._node) if subtype == NodeSubTypeTG.TREX: - if self.traffic_profile != str(traffic_profile): - self.traffic_profile = str(traffic_profile) if u"trex-astf" in self.traffic_profile: self.trex_astf_start_remote_exec( - duration, int(rate), frame_size, self.traffic_profile, - async_call, latency, warmup_time, traffic_directions, - tx_port, rx_port + duration, float(rate), async_call ) elif u"trex-stl" in self.traffic_profile: unit_rate_str = str(rate) + u"pps" + # TODO: Suport transaction_scale et al? self.trex_stl_start_remote_exec( - duration, unit_rate_str, frame_size, self.traffic_profile, - async_call, latency, warmup_time, traffic_directions, - tx_port, rx_port + duration, unit_rate_str, async_call ) else: raise ValueError(u"Unsupported T-Rex traffic profile!") @@ -918,6 +879,8 @@ class TrafficGenerator(AbstractMeasurer): def fail_if_no_traffic_forwarded(self): """Fail if no traffic forwarded. + TODO: Check number of passed transactions instead. + :returns: nothing :raises Exception: If no traffic forwarded. """ @@ -952,74 +915,234 @@ class TrafficGenerator(AbstractMeasurer): f"Traffic loss {loss} above loss acceptance: {loss_acceptance}" ) - def set_rate_provider_defaults( - self, frame_size, traffic_profile, warmup_time=0.0, - traffic_directions=2, negative_loss=True, latency=False): - """Store values accessed by measure(). + def _parse_traffic_results(self, stdout): + """Parse stdout of scripts into fields of self. - :param frame_size: Frame size identifier or value [B]. - :param traffic_profile: Module name as a traffic profile identifier. - See GPL/traffic_profiles/trex for implemented modules. - :param warmup_time: Traffic duration before measurement starts [s]. - :param traffic_directions: Traffic is bi- (2) or uni- (1) directional. - Default: 2 - :param negative_loss: If false, negative loss is reported as zero loss. - :param latency: Whether to measure latency during the trial. - Default: False. - :type frame_size: str or int - :type traffic_profile: str - :type warmup_time: float - :type traffic_directions: int - :type negative_loss: bool - :type latency: bool + Block of code to reuse, by sync start, or stop after async. + + :param stdout: Text containing the standard output. + :type stdout: str """ - self.frame_size = frame_size - self.traffic_profile = str(traffic_profile) - self.warmup_time = float(warmup_time) - self.traffic_directions = traffic_directions - self.negative_loss = negative_loss - self.use_latency = latency + subtype = check_subtype(self._node) + if subtype == NodeSubTypeTG.TREX: + # Last line from console output + line = stdout.splitlines()[-1] + results = line.split(u";") + if results[-1] in (u" ", u""): + results.pop(-1) + self._result = dict() + for result in results: + key, value = result.split(u"=", maxsplit=1) + self._result[key.strip()] = value + logger.info(f"TrafficGen results:\n{self._result}") + self._received = int(self._result.get(u"total_received"), 0) + self._sent = int(self._result.get(u"total_sent", 0)) + self._loss = int(self._result.get(u"frame_loss", 0)) + self._approximated_duration = \ + self._result.get(u"approximated_duration", 0.0) + if u"manual" not in str(self._approximated_duration): + self._approximated_duration = float(self._approximated_duration) + self._latency = list() + self._latency.append(self._result.get(u"latency_stream_0(usec)")) + self._latency.append(self._result.get(u"latency_stream_1(usec)")) + if self._mode == TrexMode.ASTF: + self._l7_data = dict() + self._l7_data[u"client"] = dict() + self._l7_data[u"client"][u"sent"] = \ + int(self._result.get(u"client_sent", 0)) + self._l7_data[u"client"][u"received"] = \ + int(self._result.get(u"client_received", 0)) + self._l7_data[u"client"][u"active_flows"] = \ + int(self._result.get(u"client_active_flows", 0)) + self._l7_data[u"client"][u"established_flows"] = \ + int(self._result.get(u"client_established_flows", 0)) + self._l7_data[u"client"][u"traffic_duration"] = \ + float(self._result.get(u"client_traffic_duration", 0.0)) + self._l7_data[u"client"][u"err_rx_throttled"] = \ + int(self._result.get(u"client_err_rx_throttled", 0)) + self._l7_data[u"client"][u"err_c_nf_throttled"] = \ + int(self._result.get(u"client_err_nf_throttled", 0)) + self._l7_data[u"client"][u"err_flow_overflow"] = \ + int(self._result.get(u"client_err_flow_overflow", 0)) + self._l7_data[u"server"] = dict() + self._l7_data[u"server"][u"active_flows"] = \ + int(self._result.get(u"server_active_flows", 0)) + self._l7_data[u"server"][u"established_flows"] = \ + int(self._result.get(u"server_established_flows", 0)) + self._l7_data[u"server"][u"traffic_duration"] = \ + float(self._result.get(u"server_traffic_duration", 0.0)) + self._l7_data[u"server"][u"err_rx_throttled"] = \ + int(self._result.get(u"client_err_rx_throttled", 0)) + if u"udp" in self.traffic_profile: + self._l7_data[u"client"][u"udp"] = dict() + self._l7_data[u"client"][u"udp"][u"connects"] = \ + int(self._result.get(u"client_udp_connects", 0)) + self._l7_data[u"client"][u"udp"][u"closed_flows"] = \ + int(self._result.get(u"client_udp_closed", 0)) + self._l7_data[u"client"][u"udp"][u"tx_bytes"] = \ + int(self._result.get(u"client_udp_tx_bytes", 0)) + self._l7_data[u"client"][u"udp"][u"rx_bytes"] = \ + int(self._result.get(u"client_udp_rx_bytes", 0)) + self._l7_data[u"client"][u"udp"][u"tx_packets"] = \ + int(self._result.get(u"client_udp_tx_packets", 0)) + self._l7_data[u"client"][u"udp"][u"rx_packets"] = \ + int(self._result.get(u"client_udp_rx_packets", 0)) + self._l7_data[u"client"][u"udp"][u"keep_drops"] = \ + int(self._result.get(u"client_udp_keep_drops", 0)) + self._l7_data[u"client"][u"udp"][u"err_cwf"] = \ + int(self._result.get(u"client_err_cwf", 0)) + self._l7_data[u"server"][u"udp"] = dict() + self._l7_data[u"server"][u"udp"][u"accepted_flows"] = \ + int(self._result.get(u"server_udp_accepts", 0)) + self._l7_data[u"server"][u"udp"][u"closed_flows"] = \ + int(self._result.get(u"server_udp_closed", 0)) + self._l7_data[u"server"][u"udp"][u"tx_bytes"] = \ + int(self._result.get(u"server_udp_tx_bytes", 0)) + self._l7_data[u"server"][u"udp"][u"rx_bytes"] = \ + int(self._result.get(u"server_udp_rx_bytes", 0)) + self._l7_data[u"server"][u"udp"][u"tx_packets"] = \ + int(self._result.get(u"server_udp_tx_packets", 0)) + self._l7_data[u"server"][u"udp"][u"rx_packets"] = \ + int(self._result.get(u"server_udp_rx_packets", 0)) + elif u"tcp" in self.traffic_profile: + self._l7_data[u"client"][u"tcp"] = dict() + self._l7_data[u"client"][u"tcp"][u"initiated_flows"] = \ + int(self._result.get(u"client_tcp_connect_inits", 0)) + self._l7_data[u"client"][u"tcp"][u"connects"] = \ + int(self._result.get(u"client_tcp_connects", 0)) + self._l7_data[u"client"][u"tcp"][u"closed_flows"] = \ + int(self._result.get(u"client_tcp_closed", 0)) + self._l7_data[u"client"][u"tcp"][u"connattempt"] = \ + int(self._result.get(u"client_tcp_connattempt", 0)) + self._l7_data[u"client"][u"tcp"][u"tx_bytes"] = \ + int(self._result.get(u"client_tcp_tx_bytes", 0)) + self._l7_data[u"client"][u"tcp"][u"rx_bytes"] = \ + int(self._result.get(u"client_tcp_rx_bytes", 0)) + self._l7_data[u"server"][u"tcp"] = dict() + self._l7_data[u"server"][u"tcp"][u"accepted_flows"] = \ + int(self._result.get(u"server_tcp_accepts", 0)) + self._l7_data[u"server"][u"tcp"][u"connects"] = \ + int(self._result.get(u"server_tcp_connects", 0)) + self._l7_data[u"server"][u"tcp"][u"closed_flows"] = \ + int(self._result.get(u"server_tcp_closed", 0)) + self._l7_data[u"server"][u"tcp"][u"tx_bytes"] = \ + int(self._result.get(u"server_tcp_tx_bytes", 0)) + self._l7_data[u"server"][u"tcp"][u"rx_bytes"] = \ + int(self._result.get(u"server_tcp_rx_bytes", 0)) - def get_measurement_result(self, duration=None, transmit_rate=None): + def get_measurement_result(self): """Return the result of last measurement as ReceiveRateMeasurement. Separate function, as measurements can end either by time or by explicit call, this is the common block at the end. + The target_tr field of ReceiveRateMeasurement is in + transactions per second. Transmit count and loss count units + depend on the transaction type. Usually they are in transactions + per second, or aggregate packets per second. + TODO: Fail on running or already reported measurement. - :param duration: Measurement duration [s] if known beforehand. - For explicitly stopped measurement it is estimated. - :param transmit_rate: Target aggregate transmit rate [pps]. - If not given, computed assuming it was bidirectional. - :type duration: float or NoneType - :type transmit_rate: float or NoneType :returns: Structure containing the result of the measurement. :rtype: ReceiveRateMeasurement """ - if duration is None: - duration = time.time() - self._start_time - self._start_time = None - if transmit_rate is None: - transmit_rate = self._rate * self.traffic_directions - transmit_count = int(self.get_sent()) - loss_count = int(self.get_loss()) - if loss_count < 0 and not self.negative_loss: - loss_count = 0 + try: + # Client duration seems to include a setup period + # where TRex does not send any packets yet. + # Server duration does not include it. + server_data = self._l7_data[u"server"] + approximated_duration = float(server_data[u"traffic_duration"]) + except (KeyError, AttributeError, ValueError, TypeError): + approximated_duration = None + try: + if not approximated_duration: + approximated_duration = float(self._approximated_duration) + except ValueError: # "manual" + approximated_duration = None + if not approximated_duration: + if self._duration and self._duration > 0: + # Known recomputed or target duration. + approximated_duration = self._duration + else: + # It was an explicit stop. + if not self._stop_time: + raise RuntimeError(u"Unable to determine duration.") + approximated_duration = self._stop_time - self._start_time + target_duration = self._target_duration + if not target_duration: + target_duration = approximated_duration + transmit_rate = self._rate + if self.transaction_type == u"packet": + partial_attempt_count = self._sent + expected_attempt_count = self._sent + fail_count = self._loss + elif self.transaction_type == u"udp_cps": + if not self.transaction_scale: + raise RuntimeError(u"Add support for no-limit udp_cps.") + partial_attempt_count = self._l7_data[u"client"][u"sent"] + # We do not care whether TG is slow, it should have attempted all. + expected_attempt_count = self.transaction_scale + pass_count = self._l7_data[u"client"][u"received"] + fail_count = expected_attempt_count - pass_count + elif self.transaction_type == u"tcp_cps": + if not self.transaction_scale: + raise RuntimeError(u"Add support for no-limit tcp_cps.") + ctca = self._l7_data[u"client"][u"tcp"][u"connattempt"] + partial_attempt_count = ctca + # We do not care whether TG is slow, it should have attempted all. + expected_attempt_count = self.transaction_scale + # TODO: Is there a better packet-based counter? + pass_count = self._l7_data[u"server"][u"tcp"][u"connects"] + fail_count = expected_attempt_count - pass_count + elif self.transaction_type == u"udp_pps": + if not self.transaction_scale: + raise RuntimeError(u"Add support for no-limit udp_pps.") + partial_attempt_count = self._sent + expected_attempt_count = self.transaction_scale * self.ppta + fail_count = self._loss + (expected_attempt_count - self._sent) + elif self.transaction_type == u"tcp_pps": + if not self.transaction_scale: + raise RuntimeError(u"Add support for no-limit tcp_pps.") + partial_attempt_count = self._sent + expected_attempt_count = self.transaction_scale * self.ppta + # One loss-like scenario happens when TRex receives all packets + # on L2 level, but is not fast enough to process them all + # at L7 level, which leads to retransmissions. + # Those manifest as opackets larger than expected. + # A simple workaround is to add absolute difference. + # Probability of retransmissions exactly cancelling + # packets unsent due to duration stretching is quite low. + fail_count = self._loss + abs(expected_attempt_count - self._sent) + else: + raise RuntimeError(f"Unknown parsing {self.transaction_type!r}") + if fail_count < 0 and not self.negative_loss: + fail_count = 0 measurement = ReceiveRateMeasurement( - duration, transmit_rate, transmit_count, loss_count + duration=target_duration, + target_tr=transmit_rate, + transmit_count=expected_attempt_count, + loss_count=fail_count, + approximated_duration=approximated_duration, + partial_transmit_count=partial_attempt_count, ) measurement.latency = self.get_latency_int() return measurement def measure(self, duration, transmit_rate): - """Run trial measurement, parse and return aggregate results. + """Run trial measurement, parse and return results. - Aggregate means sum over traffic directions. + The input rate is for transactions. Stateles bidirectional traffic + is understood as sequence of (asynchronous) transactions, + two packets each. + + The result units depend on test type, generally + the count either transactions or packets (aggregated over directions). + + Optionally, this method sleeps if measurement finished before + the time specified as duration. :param duration: Trial duration [s]. - :param transmit_rate: Target aggregate transmit rate [pps] / Connections - per second (CPS) for UDP/TCP flows. + :param transmit_rate: Target rate in transactions per second. :type duration: float :type transmit_rate: float :returns: Structure containing the result of the measurement. @@ -1029,18 +1152,93 @@ class TrafficGenerator(AbstractMeasurer): :raises NotImplementedError: If TG is not supported. """ duration = float(duration) - # TG needs target Tr per stream, but reports aggregate Tx and Dx. - unit_rate_int = transmit_rate / float(self.traffic_directions) - self.send_traffic_on_tg( - duration, - unit_rate_int, - self.frame_size, - self.traffic_profile, - warmup_time=self.warmup_time, - latency=self.use_latency, - traffic_directions=self.traffic_directions + time_start = time.monotonic() + time_stop = time_start + duration + if self.resetter: + self.resetter() + self._send_traffic_on_tg_internal( + duration=duration, + rate=transmit_rate, + async_call=False, ) - return self.get_measurement_result(duration, transmit_rate) + result = self.get_measurement_result() + logger.debug(f"trial measurement result: {result!r}") + # In PLRsearch, computation needs the specified time to complete. + if self.sleep_till_duration: + sleeptime = time_stop - time.monotonic() + if sleeptime > 0.0: + # TODO: Sometimes we have time to do additional trials here, + # adapt PLRsearch to accept all the results. + time.sleep(sleeptime) + return result + + def set_rate_provider_defaults( + self, + frame_size, + traffic_profile, + ppta=1, + resetter=None, + traffic_directions=2, + transaction_duration=0.0, + transaction_scale=0, + transaction_type=u"packet", + duration_limit=0.0, + negative_loss=True, + sleep_till_duration=False, + use_latency=False, + ): + """Store values accessed by measure(). + + :param frame_size: Frame size identifier or value [B]. + :param traffic_profile: Module name as a traffic profile identifier. + See GPL/traffic_profiles/trex for implemented modules. + :param ppta: Packets per transaction, aggregated over directions. + Needed for udp_pps which does not have a good transaction counter, + so we need to compute expected number of packets. + Default: 1. + :param resetter: Callable to reset DUT state for repeated trials. + :param traffic_directions: Traffic from packet counting point of view + is bi- (2) or uni- (1) directional. + Default: 2 + :param transaction_duration: Total expected time to close transaction. + :param transaction_scale: Number of transactions to perform. + 0 (default) means unlimited. + :param transaction_type: An identifier specifying which counters + and formulas to use when computing attempted and failed + transactions. Default: "packet". + TODO: Does this also specify parsing for the measured duration? + :param duration_limit: Zero or maximum limit for computed (or given) + duration. + :param negative_loss: If false, negative loss is reported as zero loss. + :param sleep_till_duration: If true and measurement returned faster, + sleep until it matches duration. Needed for PLRsearch. + :param use_latency: Whether to measure latency during the trial. + Default: False. + :type frame_size: str or int + :type traffic_profile: str + :type ppta: int + :type resetter: Optional[Callable[[], None]] + :type traffic_directions: int + :type transaction_duration: float + :type transaction_scale: int + :type transaction_type: str + :type duration_limit: float + :type negative_loss: bool + :type sleep_till_duration: bool + :type use_latency: bool + """ + self.frame_size = frame_size + self.traffic_profile = str(traffic_profile) + self.resetter = resetter + self.ppta = ppta + self.traffic_directions = int(traffic_directions) + self.transaction_duration = float(transaction_duration) + self.transaction_scale = int(transaction_scale) + self.transaction_type = str(transaction_type) + self.duration_limit = float(duration_limit) + self.negative_loss = bool(negative_loss) + self.sleep_till_duration = bool(sleep_till_duration) + self.use_latency = bool(use_latency) class OptimizedSearch: @@ -1052,20 +1250,38 @@ class OptimizedSearch: @staticmethod def perform_optimized_ndrpdr_search( - frame_size, traffic_profile, minimum_transmit_rate, - maximum_transmit_rate, packet_loss_ratio=0.005, - final_relative_width=0.005, final_trial_duration=30.0, - initial_trial_duration=1.0, number_of_intermediate_phases=2, - timeout=720.0, doublings=1, traffic_directions=2, latency=False): + frame_size, + traffic_profile, + minimum_transmit_rate, + maximum_transmit_rate, + packet_loss_ratio=0.005, + final_relative_width=0.005, + final_trial_duration=30.0, + initial_trial_duration=1.0, + number_of_intermediate_phases=2, + timeout=720.0, + doublings=1, + ppta=1, + resetter=None, + traffic_directions=2, + transaction_duration=0.0, + transaction_scale=0, + transaction_type=u"packet", + use_latency=False, + ): """Setup initialized TG, perform optimized search, return intervals. + If transaction_scale is nonzero, all non-init trial durations + are set to 2.0 (as they do not affect the real trial duration) + and zero intermediate phases are used. + The initial phase still uses 1.0 seconds, to force remeasurement. + That makes initial phase act as a warmup. + :param frame_size: Frame size identifier or value [B]. :param traffic_profile: Module name as a traffic profile identifier. See GPL/traffic_profiles/trex for implemented modules. - :param minimum_transmit_rate: Minimal uni-directional - target transmit rate [pps]. - :param maximum_transmit_rate: Maximal uni-directional - target transmit rate [pps]. + :param minimum_transmit_rate: Minimal load in transactions per second. + :param maximum_transmit_rate: Maximal load in transactions per second. :param packet_loss_ratio: Fraction of packets lost, for PDR [1]. :param final_relative_width: Final lower bound transmit rate cannot be more distant that this multiple of upper bound [1]. @@ -1079,9 +1295,20 @@ class OptimizedSearch: :param doublings: How many doublings to do in external search step. Default 1 is suitable for fairly stable tests, less stable tests might get better overal duration with 2 or more. + :param ppta: Packets per transaction, aggregated over directions. + Needed for udp_pps which does not have a good transaction counter, + so we need to compute expected number of packets. + Default: 1. + :param resetter: Callable to reset DUT state for repeated trials. :param traffic_directions: Traffic is bi- (2) or uni- (1) directional. Default: 2 - :param latency: Whether to measure latency during the trial. + :param transaction_duration: Total expected time to close transaction. + :param transaction_scale: Number of transactions to perform. + 0 (default) means unlimited. + :param transaction_type: An identifier specifying which counters + and formulas to use when computing attempted and failed + transactions. Default: "packet". + :param use_latency: Whether to measure latency during the trial. Default: False. :type frame_size: str or int :type traffic_profile: str @@ -1094,53 +1321,85 @@ class OptimizedSearch: :type number_of_intermediate_phases: int :type timeout: float :type doublings: int + :type ppta: int + :type resetter: Optional[Callable[[], None]] :type traffic_directions: int - :type latency: bool + :type transaction_duration: float + :type transaction_scale: int + :type transaction_type: str + :type use_latency: bool :returns: Structure containing narrowed down NDR and PDR intervals and their measurements. :rtype: NdrPdrResult :raises RuntimeError: If total duration is larger than timeout. """ - minimum_transmit_rate *= traffic_directions - maximum_transmit_rate *= traffic_directions # we need instance of TrafficGenerator instantiated by Robot Framework # to be able to use trex_stl-*() tg_instance = BuiltIn().get_library_instance( u"resources.libraries.python.TrafficGenerator" ) + # Overrides for fixed transaction amount. + # TODO: Move to robot code? We have two call sites, so this saves space, + # even though this is surprising for log readers. + if transaction_scale: + initial_trial_duration = 1.0 + final_trial_duration = 2.0 + number_of_intermediate_phases = 0 + timeout = 3600.0 tg_instance.set_rate_provider_defaults( - frame_size, - traffic_profile, + frame_size=frame_size, + traffic_profile=traffic_profile, + sleep_till_duration=False, + ppta=ppta, + resetter=resetter, traffic_directions=traffic_directions, - latency=latency + transaction_duration=transaction_duration, + transaction_scale=transaction_scale, + transaction_type=transaction_type, + use_latency=use_latency, ) algorithm = MultipleLossRatioSearch( - measurer=tg_instance, final_trial_duration=final_trial_duration, + measurer=tg_instance, + final_trial_duration=final_trial_duration, final_relative_width=final_relative_width, number_of_intermediate_phases=number_of_intermediate_phases, - initial_trial_duration=initial_trial_duration, timeout=timeout, - doublings=doublings + initial_trial_duration=initial_trial_duration, + timeout=timeout, + doublings=doublings, ) result = algorithm.narrow_down_ndr_and_pdr( - minimum_transmit_rate, maximum_transmit_rate, packet_loss_ratio + min_rate=minimum_transmit_rate, + max_rate=maximum_transmit_rate, + packet_loss_ratio=packet_loss_ratio, ) return result @staticmethod def perform_soak_search( - frame_size, traffic_profile, minimum_transmit_rate, - maximum_transmit_rate, plr_target=1e-7, tdpt=0.1, - initial_count=50, timeout=1800.0, trace_enabled=False, - traffic_directions=2, latency=False): + frame_size, + traffic_profile, + minimum_transmit_rate, + maximum_transmit_rate, + plr_target=1e-7, + tdpt=0.1, + initial_count=50, + timeout=7200.0, + ppta=1, + resetter=None, + trace_enabled=False, + traffic_directions=2, + transaction_duration=0.0, + transaction_scale=0, + transaction_type=u"packet", + use_latency=False, + ): """Setup initialized TG, perform soak search, return avg and stdev. :param frame_size: Frame size identifier or value [B]. :param traffic_profile: Module name as a traffic profile identifier. See GPL/traffic_profiles/trex for implemented modules. - :param minimum_transmit_rate: Minimal uni-directional - target transmit rate [pps]. - :param maximum_transmit_rate: Maximal uni-directional - target transmit rate [pps]. + :param minimum_transmit_rate: Minimal load in transactions per second. + :param maximum_transmit_rate: Maximal load in transactions per second. :param plr_target: Fraction of packets lost to achieve [1]. :param tdpt: Trial duration per trial. The algorithm linearly increases trial duration with trial number, @@ -1150,10 +1409,24 @@ class OptimizedSearch: This is needed because initial "search" phase of integrator takes significant time even without any trial results. :param timeout: The search will stop after this overall time [s]. + :param ppta: Packets per transaction, aggregated over directions. + Needed for udp_pps which does not have a good transaction counter, + so we need to compute expected number of packets. + Default: 1. + :param resetter: Callable to reset DUT state for repeated trials. :param trace_enabled: True if trace enabled else False. + This is very verbose tracing on numeric computations, + do not use in production. + Default: False :param traffic_directions: Traffic is bi- (2) or uni- (1) directional. Default: 2 - :param latency: Whether to measure latency during the trial. + :param transaction_duration: Total expected time to close transaction. + :param transaction_scale: Number of transactions to perform. + 0 (default) means unlimited. + :param transaction_type: An identifier specifying which counters + and formulas to use when computing attempted and failed + transactions. Default: "packet". + :param use_latency: Whether to measure latency during the trial. Default: False. :type frame_size: str or int :type traffic_profile: str @@ -1162,29 +1435,48 @@ class OptimizedSearch: :type plr_target: float :type initial_count: int :type timeout: float + :type ppta: int + :type resetter: Optional[Callable[[], None]] :type trace_enabled: bool :type traffic_directions: int - :type latency: bool + :type transaction_duration: float + :type transaction_scale: int + :type transaction_type: str + :type use_latency: bool :returns: Average and stdev of estimated aggregate rate giving PLR. :rtype: 2-tuple of float """ - minimum_transmit_rate *= traffic_directions - maximum_transmit_rate *= traffic_directions tg_instance = BuiltIn().get_library_instance( u"resources.libraries.python.TrafficGenerator" ) + # Overrides for fixed transaction amount. + # TODO: Move to robot code? We have a single call site + # but MLRsearch has two and we want the two to be used similarly. + if transaction_scale: + timeout = 7200.0 tg_instance.set_rate_provider_defaults( - frame_size, - traffic_profile, - traffic_directions=traffic_directions, + frame_size=frame_size, + traffic_profile=traffic_profile, negative_loss=False, - latency=latency + sleep_till_duration=True, + ppta=ppta, + resetter=resetter, + traffic_directions=traffic_directions, + transaction_duration=transaction_duration, + transaction_scale=transaction_scale, + transaction_type=transaction_type, + use_latency=use_latency, ) algorithm = PLRsearch( - measurer=tg_instance, trial_duration_per_trial=tdpt, + measurer=tg_instance, + trial_duration_per_trial=tdpt, packet_loss_ratio_target=plr_target, - trial_number_offset=initial_count, timeout=timeout, - trace_enabled=trace_enabled + trial_number_offset=initial_count, + timeout=timeout, + trace_enabled=trace_enabled, + ) + result = algorithm.search( + min_rate=minimum_transmit_rate, + max_rate=maximum_transmit_rate, ) - result = algorithm.search(minimum_transmit_rate, maximum_transmit_rate) return result diff --git a/resources/libraries/python/autogen/Regenerator.py b/resources/libraries/python/autogen/Regenerator.py index e6474e566d..76501d6ad0 100644 --- a/resources/libraries/python/autogen/Regenerator.py +++ b/resources/libraries/python/autogen/Regenerator.py @@ -160,6 +160,9 @@ def add_default_testcases(testcase, iface, suite_id, file_out, tc_kwargs_list): emit = False if kwargs[u"frame_size"] not in MIN_FRAME_SIZE_VALUES: emit = False + if u"-cps-" in suite_id or u"-pps-" in suite_id: + if kwargs[u"frame_size"] not in MIN_FRAME_SIZE_VALUES: + emit = False if emit: file_out.write(testcase.generate(**kwargs)) diff --git a/resources/libraries/robot/ip/nat.robot b/resources/libraries/robot/ip/nat.robot index dee04f0b61..75b8cce7b6 100644 --- a/resources/libraries/robot/ip/nat.robot +++ b/resources/libraries/robot/ip/nat.robot @@ -40,13 +40,26 @@ | Initialize NAT44 endpoint-dependent mode in circular topology | | [Documentation] | Initialization of NAT44 endpoint-dependent mode on DUT1 | | +| | ... | This keyword also sets a test variable \${resetter} +| | ... | to hold a callable which resets VPP state. +| | ... | Keywords performing search will call it to get consistent trials. +| | ... | Tests which do not wish to reset NAT state should use ramp-up, +| | ... | so the resetter is not set if \${ramp_up_duration} variable +| | ... | is (defined and) nonzero. +| | +| | ... | *Test variables read:* +| | ... | - n_sessions - Expected number of opened sessions. +| | | | ${max_sessions}= | Compute Max Translations Per Thread | | ... | ${n_sessions} | ${thr_count_int} | | Enable NAT44 Plugin | ${dut1} | mode=${nat_mode} | sessions=${max_sessions} | | Configure inside and outside interfaces | | ... | ${dut1} | ${DUT1_${int}1}[0] | ${DUT1_${int}2}[0] -| | Set NAT44 Address Range +| | ${resetter} = | Set NAT44 Address Range | | ... | ${dut1} | ${out_net} | ${out_net_end} +| | ${ramp_up_duration} = | Get Ramp Up Duration +| | Return From Keyword If | ${ramp_up_duration} +| | Set Test Variable | \${resetter} # TODO: Remove when 'ip4.Initialize IPv4 forwarding in circular topology' KW # adapted to use IP values from variables @@ -184,6 +197,13 @@ | Configure deterministic mode for NAT44 | | [Documentation] | Set deterministic behaviour of NAT44 (DET44). | | +| | ... | This keyword also sets a test variable \${resetter} +| | ... | to hold a callable which resets VPP state. +| | ... | Keywords performing search will call it to get consistent trials. +| | ... | Tests which do not wish to reset NAT state should use ramp-up, +| | ... | so the resetter is not set if \${ramp_up_duration} variable +| | ... | is (defined and) nonzero. +| | | | ... | *Arguments:* | | ... | - node - DUT node to set deterministic mode for NAT44 on. | | ... | Type: dictionary @@ -199,8 +219,11 @@ | | | | [Arguments] | ${node} | ${ip_in} | ${subnet_in} | ${ip_out} | ${subnet_out} | | -| | Set DET44 Mapping +| | ${resetter} = | Set DET44 Mapping | | ... | ${node} | ${ip_in} | ${subnet_in} | ${ip_out} | ${subnet_out} +| | ${ramp_up_duration} = | Get Ramp Up Duration +| | Return From Keyword If | ${ramp_up_duration} +| | Set Test Variable | \${resetter} | Initialize NAT44 deterministic mode in circular topology | | [Documentation] | Initialization of NAT44 deterministic mode (DET44) diff --git a/resources/libraries/robot/performance/performance_actions.robot b/resources/libraries/robot/performance/performance_actions.robot new file mode 100644 index 0000000000..2cf954a5b3 --- /dev/null +++ b/resources/libraries/robot/performance/performance_actions.robot @@ -0,0 +1,92 @@ +# Copyright (c) 2020 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +| Variables | resources/libraries/python/Constants.py +| Resource | resources/libraries/robot/performance/performance_utils.robot +| +| Documentation +| ... | Performance suite keywords - Actions related to performance tests. + +*** Keywords *** +| Additional Statistics Action For bash-perf-stat +| | [Documentation] +| | ... | Additional Statistics Action for bash command "perf stat". +| | +| | Run Keyword If | ${extended_debug}==${True} +| | ... | Perf Stat On All DUTs | ${nodes} | cpu_list=${cpu_alloc_str} + +| Additional Statistics Action For clear-show-runtime-with-traffic +| | [Documentation] +| | ... | Additional Statistics Action for clear and show runtime counters with +| | ... | running traffic. +| | +| | ... | See documentation of the called keyword for required test variables. +| | +| | Clear and show runtime counters with running traffic + +| Additional Statistics Action For noop +| | [Documentation] +| | ... | Additional Statistics Action for no operation. +| | +| | No operation + +| Additional Statistics Action For vpp-clear-runtime +| | [Documentation] +| | ... | Additional Statistics Action for clear VPP runtime. +| | +| | VPP Clear Runtime On All DUTs | ${nodes} + +| Additional Statistics Action For vpp-clear-stats +| | [Documentation] +| | ... | Additional Statistics Action for clear VPP statistics. +| | +| | Clear Statistics On All DUTs | ${nodes} + +| Additional Statistics Action For vpp-enable-elog +| | [Documentation] +| | ... | Additional Statistics Action for enable VPP elog trace. +| | +| | VPP Enable Elog Traces On All DUTs | ${nodes} + +| Additional Statistics Action For vpp-enable-packettrace +| | [Documentation] +| | ... | Additional Statistics Action for enable VPP packet trace. +| | +| | Run Keyword If | ${extended_debug}==${True} +| | ... | VPP Enable Traces On All DUTs | ${nodes} | fail_on_error=${False} + +| Additional Statistics Action For vpp-show-elog +| | [Documentation] +| | ... | Additional Statistics Action for show VPP elog trace. +| | +| | Show Event Logger On All DUTs | ${nodes} + +| Additional Statistics Action For vpp-show-packettrace +| | [Documentation] +| | ... | Additional Statistics Action for show VPP packet trace. +| | +| | Run Keyword If | ${extended_debug}==${True} +| | ... | Show Packet Trace On All Duts | ${nodes} | maximum=${100} + +| Additional Statistics Action For vpp-show-runtime +| | [Documentation] +| | ... | Additional Statistics Action for show VPP runtime. +| | +| | VPP Show Runtime On All DUTs | ${nodes} + +| Additional Statistics Action For vpp-show-stats +| | [Documentation] +| | ... | Additional Statistics Action for show VPP statistics. +| | +| | Show Statistics On All DUTs | ${nodes} diff --git a/resources/libraries/robot/performance/performance_display.robot b/resources/libraries/robot/performance/performance_display.robot new file mode 100644 index 0000000000..e8dfdbbfbd --- /dev/null +++ b/resources/libraries/robot/performance/performance_display.robot @@ -0,0 +1,240 @@ +# Copyright (c) 2020 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +| Documentation +| ... | Performance suite keywords - Displaying results as test messages. +| ... | This includes checks to fail test. + +*** Keywords *** +| Check NDRPDR interval validity +| | [Documentation] +| | ... | Extract loss ratio of lower bound of the interval. +| | ... | Fail if it does not reach the allowed value. +| | +| | ... | *Arguments:* +| | ... | - interval - Measured interval. Type: ReceiveRateInterval +| | ... | - packet_loss_ratio - Accepted loss (0.0 for NDR). Type: float +| | +| | ... | *Example:* +| | +| | ... | \| Check NDRPDR interval validity \| \${result.pdr_interval} \ +| | ... | \| \${0.005} \| +| | +| | [Arguments] | ${interval} | ${packet_loss_ratio}=${0.0} +| | +| | ${lower_bound} = | Set Variable | ${interval.measured_low} +| | ${lower_bound_lf} = | Set Variable | ${lower_bound.loss_fraction} +| | Return From Keyword If | ${lower_bound_lf} <= ${packet_loss_ratio} +| | Set Test Variable | \${rate_for_teardown} | ${lower_bound_lf} +| | ${message}= | Catenate | SEPARATOR=${SPACE} +| | ... | Minimal rate loss fraction ${lower_bound_lf} +| | ... | does not reach target ${packet_loss_ratio}. +| | ${message_zero} = | Set Variable | Zero packets forwarded! +| | ${message_other} = | Set Variable | ${lower_bound.loss_count} packets lost. +| | ${message} = | Set Variable If | ${lower_bound_lf} >= 1.0 +| | ... | ${message}${\n}${message_zero} | ${message}${\n}${message_other} +| | Fail | ${message} + +| Display Reconfig Test Message +| | [Documentation] +| | ... | Display the number of packets lost (bidirectionally) +| | ... | due to reconfiguration under traffic. +| | +| | ... | *Arguments:* +| | ... | - result - Result of bidirectional measurtement. +| | ... | Type: ReceiveRateMeasurement +| | +| | ... | *Example:* +| | +| | ... | \| Display Reconfig Test Message \| \${result} \| +| | +| | [Arguments] | ${result} +| | +| | ${ppta} = | Get Packets Per Transaction Aggregated +| | ${packet_rate} = | Evaluate | ${result.target_tr} * ${ppta} +| | ${packet_loss} = | Set Variable | ${result.loss_count} +| | ${time_loss} = | Evaluate | ${packet_loss} / ${packet_rate} +| | Set Test Message | Packets lost due to reconfig: ${packet_loss} +| | Set Test Message | ${\n}Implied time lost: ${time_loss} | append=yes + +| Display result of NDRPDR search +| | [Documentation] +| | ... | Display result of NDR+PDR search, both quantities, both bounds, +| | ... | aggregate in units given by trasaction type, e.g. by default +| | ... | in packet per seconds and Gbps total bandwidth +| | ... | (for initial packet size). +| | ... | +| | ... | The bound to display is encoded as target rate, it is assumed +| | ... | it is in transactions per second. Bidirectional traffic +| | ... | transaction is understood as having 2 packets, for this purpose. +| | ... | +| | ... | Througput is calculated as: +| | ... | Sum of measured rate over streams +| | ... | Bandwidth is calculated as: +| | ... | (Throughput * (L2 Frame Size + IPG) * 8) +| | ... | If the results contain latency data, display them for lower bounds. +| | +| | ... | *Test (or broader scope) variables read:* +| | ... | - frame_size_num - L2 Frame Size [B]. Type: integer or float +| | ... | - transaction_type - String identifier to determine how to count +| | ... | transactions. Default is "packet". +| | ... | *Arguments:* +| | ... | - result - Measured result data. Aggregate rate, tps or pps. +| | ... | Type: NdrPdrResult +| | +| | ... | *Example:* +| | +| | ... | \| Display result of NDRPDR search \| \${result} \| +| | +| | [Arguments] | ${result} +| | +| | Display single bound | NDR_LOWER +| | ... | ${result.ndr_interval.measured_low.target_tr} +| | ... | ${result.ndr_interval.measured_low.latency} +| | Display single bound | NDR_UPPER +| | ... | ${result.ndr_interval.measured_high.target_tr} +| | Display single bound | PDR_LOWER +| | ... | ${result.pdr_interval.measured_low.target_tr} +| | ... | ${result.pdr_interval.measured_low.latency} +| | Display single bound | PDR_UPPER +| | ... | ${result.pdr_interval.measured_high.target_tr} + +| Display result of soak search +| | [Documentation] +| | ... | Display result of soak search, avg+-stdev, as upper/lower bounds. +| | ... | See Display single bound for units used. +| | +| | ... | *Test (or broader scope) variables read:* +| | ... | - frame_size - L2 Frame Size [B] or IMIX string. Type: integer or +| | ... | string +| | ... | - transaction_type - String identifier to determine how to count +| | ... | transactions. Default is "packet". +| | ... | *Arguments:* +| | ... | - avg - Estimated average critical load [pps]. Type: float +| | ... | - stdev - Standard deviation of critical load [pps]. Type: float +| | +| | ... | *Returns:* +| | ... | - Lower and upper bound of critical load [pps]. Type: 2-tuple of float +| | +| | ... | *Example:* +| | +| | ... | \| Display result of soak search \| \${100000} \| \${100} \| +| | +| | [Arguments] | ${avg} | ${stdev} +| | +| | ${avg} = | Convert To Number | ${avg} +| | ${stdev} = | Convert To Number | ${stdev} +| | ${lower} = | Evaluate | ${avg} - ${stdev} +| | ${upper} = | Evaluate | ${avg} + ${stdev} +| | Display single bound | PLRsearch lower bound | ${lower} +| | Display single bound | PLRsearch upper bound | ${upper} +| | Return From Keyword | ${lower} | ${upper} + +| Display single bound +| | [Documentation] +| | ... | Compute and display one bound of NDR+PDR (or soak) search result. +| | ... | If the latency string is present, it is displayed as well. +| | ... | +| | ... | The bound to display is given as target transfer rate, it is assumed +| | ... | it is in transactions per second. Bidirectional traffic +| | ... | transaction is understood as having 2 packets, for this purpose. +| | ... | +| | ... | Pps values are aggregate in packet per seconds, +| | ... | and Gbps total bandwidth (for initial packet size). +| | ... | +| | ... | Througput is calculated as: +| | ... | Sum of measured rate over streams +| | ... | Bandwidth is calculated as: +| | ... | (Throughput * (L2 Frame Size + IPG) * 8) +| | ... | If the results contain latency data, display them for lower bounds. +| | +| | ... | *Test (or broader scope) variables read:* +| | ... | - transaction_type - String identifier to determine how to count +| | ... | transactions. Default is "packet". +| | ... | *Arguments:* +| | ... | - text - Flavor text describing which bound is this. Type: string +| | ... | - tps - Transaction rate [tps]. Type: float +| | ... | - latency - Latency data to display if non-empty. Type: string +| | +| | ... | *Example:* +| | +| | ... | \| Display single bound \| NDR lower bound \| \${12345.67} \ +| | ... | \| latency=\${EMPTY} \| +| | +| | [Arguments] | ${text} | ${tps} | ${latency}=${EMPTY} +| | +| | ${transaction_type} = | Get Transaction Type +| | Run Keyword And Return If | """_cps""" in """${transaction_type}""" +| | ... | Display single cps bound | ${text} | ${tps} | ${latency} +| | Display single pps bound | ${text} | ${tps} | ${latency} + +| Display single cps bound +| | [Documentation] +| | ... | Display one bound of NDR+PDR search for CPS tests. +| | ... | The bounds are expressed as transactions per second. +| | ... | If the latency string is present, it is displayed as well. +| | +| | ... | *Arguments:* +| | ... | - text - Flavor text describing which bound is this. Type: string +| | ... | - tps - Transaction rate [tps]. Type: float +| | ... | - latency - Latency data to display if non-empty. Type: string +| | +| | ... | *Example:* +| | +| | ... | \| Display single cps bound \| NDR lower bound \| \${12345.67} \ +| | ... | \| latency=\${EMPTY} \| +| | +| | [Arguments] | ${text} | ${tps} | ${latency}=${EMPTY} +| | +| | Set Test Message | ${\n}${text}: ${tps} CPS | append=yes +| | Return From Keyword If | not """${latency}""" +| | Set Test Message | ${\n}LATENCY [min/avg/max/hdrh] per stream: ${latency} +| | ... | append=yes + +| Display single pps bound +| | [Documentation] +| | ... | Display one pps bound of NDR+PDR search, +| | ... | aggregate in packet per seconds and Gbps total bandwidth +| | ... | (for initial packet size). +| | ... | +| | ... | The bound to display is given as target transfer rate, it is assumed +| | ... | it is in transactions per second. Bidirectional traffic +| | ... | transaction is understood as having 2 packets, for this purpose. +| | ... | +| | ... | Througput is calculated as: +| | ... | Sum of measured rates over streams +| | ... | Bandwidth is calculated as: +| | ... | (Throughput * (L2 Frame Size + IPG) * 8) +| | ... | If the latency string is present, it is displayed as well. +| | +| | ... | *Arguments:* +| | ... | - text - Flavor text describing which bound is this. Type: string +| | ... | - tps - Transaction rate [tps]. Type: float +| | ... | - latency - Latency data to display if non-empty. Type: string +| | +| | ... | *Example:* +| | +| | ... | \| Display single pps bound \| NDR lower bound \| \${12345.67} \ +| | ... | \| latency=\${EMPTY} \| +| | +| | [Arguments] | ${text} | ${tps} | ${latency}=${EMPTY} +| | +| | ${ppta} = | Get Packets Per Transaction Aggregated +| | ${pps} = | Evaluate | ${tps} * ${ppta} +| | ${bandwidth} = | Evaluate | ${pps} * (${avg_frame_size}+20)*8 / 1e9 +| | Set Test Message | ${\n}${text}: ${pps} pps, | append=yes +| | Set Test Message | ${bandwidth} Gbps (initial) | append=yes +| | Return From Keyword If | not """${latency}""" +| | Set Test Message | ${\n}LATENCY [min/avg/max/hdrh] per stream: ${latency} +| | ... | append=yes diff --git a/resources/libraries/robot/performance/performance_limits.robot b/resources/libraries/robot/performance/performance_limits.robot deleted file mode 100644 index 332ed9368b..0000000000 --- a/resources/libraries/robot/performance/performance_limits.robot +++ /dev/null @@ -1,153 +0,0 @@ -# Copyright (c) 2020 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -*** Settings *** -| Documentation | Performance suite keywords - Handling NIC and TG limits. -| Library | Collections -| Variables | ${CURDIR}/../../python/Constants.py - -*** Keywords *** -| Get Average Frame Size -| | [Documentation] -| | ... | Framesize can be either integer in case of a single packet -| | ... | in stream, or set of packets in case of IMIX type or simmilar. -| | -| | ... | *Arguments:* -| | ... | - frame_size - Framesize. Type: integer or string -| | ... | - overhead - Overhead in bytes; default value: ${0}. Type: integer -| | -| | ... | *Returns:* -| | ... | Average frame size including overhead. Type: float -| | -| | ... | *Example:* -| | -| | ... | \| Get Average Frame Size \| IMIX_v4_1 \| -| | -| | [Arguments] | ${frame_size} | ${overhead}=${0} -| | -| | ${frame_size} = | Run Keyword If | '${frame_size}' == 'IMIX_v4_1' -| | ... | Set Variable | ${353.83333} -| | ... | ELSE -| | ... | Convert To Number | ${frame_size} -| | ${avg_frame_size} = | Evaluate | ${frame_size} + ${overhead} -| | Return From Keyword | ${avg_frame_size} - -| Get Maximum Frame Size -| | [Documentation] -| | ... | Framesize can be either integer in case of a single packet -| | ... | in stream, or set of packets in case of IMIX type or simmilar. -| | -| | ... | *Arguments:* -| | ... | - frame_size - Framesize. Type: integer or string -| | ... | - overhead - Overhead in bytes; default value: ${0}. Type: integer -| | -| | ... | *Returns:* -| | ... | Maximum frame size including overhead. Type: float -| | -| | ... | *Example:* -| | -| | ... | \| Get Maximum Frame Size \| IMIX_v4_1 \| -| | -| | [Arguments] | ${frame_size} | ${overhead}=${0} -| | -| | ${frame_size} = | Run Keyword If | '${frame_size}' == 'IMIX_v4_1' -| | ... | Set Variable | ${1518} -| | ... | ELSE -| | ... | Convert To Number | ${frame_size} -| | ${max_frame_size} = | Evaluate | ${frame_size} + ${overhead} -| | Return From Keyword | ${max_frame_size} - -| Set Max Rate And Jumbo -| | [Documentation] -| | ... | Input framesize can be either integer in case of a single packet -| | ... | in stream, or IMIX string defining mix of packets. -| | ... | For jumbo frames detection, the maximal packet size is relevant. -| | ... | For maximal transmit rate, the average packet size is relevant. -| | ... | In both cases, encapsulation overhead (if any) has effect. -| | ... | The maximal rate is computed from NIC name. -| | ... | The implementation works by mapping from exact -| | ... | whitelisted NIC names. -| | ... | The mapping is hardcoded in nic_limits.yaml -| | ... | TODO: Make the mapping from NIC names case insensistive. -| | -| | ... | TODO: Make pps limit also definable per NIC. -| | -| | ... | This keyword computes maximal unidirectional transmit rate -| | ... | and jumbo boolean (some suites need that for configuration decisions). -| | ... | To streamline suite autogeneration, both input and output values -| | ... | are communicated as test (or broader scope) variables, -| | ... | instead of explicit arguments and return values. -| | -| | ... | *Test (or broader scope) variables read:* -| | ... | - nic_name - Name of bottleneck NIC. Type: string -| | ... | - overhead - Overhead in bytes; default value: 0. Type: integer -| | ... | - frame_size - L2 Frame Size [B] or IMIX string. Type: integer or -| | ... | string -| | -| | ... | *Test variables set:* -| | ... | - max_rate - Calculated unidirectional maximal transmit rate [pps]. -| | ... | Type: float -| | ... | - jumbo - Jumbo boolean, true if jumbo packet support has to be -| | ... | enabled. Type: boolean -| | -| | ... | *Example:* -| | -| | ... | \| Set Test Variable \| \${frame_size} \| IMIX_v4_1 \| -| | ... | \| Set Max Rate And Jumbo \| -| | -| | # Negative overhead is possible, if DUT-DUT traffic is less encapsulated -| | # than TG-DUT traffic. -| | # TODO: Re-check overhead values in suites with both traffics encapsulated. -| | # TODO: Improve layered setup to detect encap/decap and update overhead. -| | ${overhead} = | Set Variable If | ${overhead} >= 0 | ${overhead} | ${0} -| | ${pps_limit} = | Get From Dictionary -| | ... | ${NIC_NAME_TO_PPS_LIMIT} | ${nic_name} -| | ${bps_limit} = | Get From Dictionary -| | ... | ${NIC_NAME_TO_BPS_LIMIT} | ${nic_name} -| | # swo := size_with_overhead -| | ${avg_swo} = | Get Average Frame Size | ${frame_size} | ${overhead} -| | ${rate} = | Evaluate | ${bps_limit} / ((${avg_swo} + 20.0) * 8) -| | ${max_rate} = | Set Variable If | ${rate} > ${pps_limit} -| | ... | ${pps_limit} | ${rate} -| | Set Test Variable | \${max_rate} -| | Set Jumbo - -| Set Jumbo -| | [Documentation] -| | ... | For jumbo frames detection, the maximal packet size is relevant, -| | ... | encapsulation overhead (if any) has effect. -| | -| | ... | This keyword computes jumbo boolean (some suites need that for -| | ... | configuration decisions). -| | ... | To streamline suite autogeneration, both input and output values -| | ... | are communicated as test (or broader scope) variables, -| | ... | instead of explicit arguments and return values. -| | -| | ... | *Test (or broader scope) variables read:* -| | ... | - overhead - Overhead in bytes; default value: 0. Type: integer -| | ... | - frame_size - L2 Frame Size [B] or IMIX string. Type: integer or -| | ... | string -| | -| | ... | *Test variables set:* -| | ... | - jumbo - Jumbo boolean, true if jumbo packet support has to be -| | ... | enabled. Type: boolean -| | -| | ... | *Example:* -| | -| | ... | \| Set Jumnbo \| -| | -| | ${overhead} = | Set Variable If | ${overhead} >= 0 | ${overhead} | ${0} -| | ${max_swo} = | Get Maximum Frame Size | ${frame_size} | ${overhead} -| | ${jumbo} = | Set Variable If | ${max_swo} < 1522 -| | ... | ${False} | ${True} -| | Set Test Variable | \${jumbo} diff --git a/resources/libraries/robot/performance/performance_utils.robot b/resources/libraries/robot/performance/performance_utils.robot index 69402dcd41..5cf6dba84c 100644 --- a/resources/libraries/robot/performance/performance_utils.robot +++ b/resources/libraries/robot/performance/performance_utils.robot @@ -22,20 +22,120 @@ | Library | resources.libraries.python.TrafficGenerator.TGDropRateSearchImpl | Library | resources.libraries.python.Trace | Variables | resources/libraries/python/Constants.py +| Resource | resources/libraries/robot/performance/performance_actions.robot +| Resource | resources/libraries/robot/performance/performance_display.robot +| Resource | resources/libraries/robot/performance/performance_vars.robot | | Documentation | ... | Performance suite keywords - utilities to find and verify NDR and PDR. +| ... | See performance_vars.robot for values accessed via there. *** Variables *** -| ${trial_duration}= | ${PERF_TRIAL_DURATION} -| ${trial_multiplicity}= | ${PERF_TRIAL_MULTIPLICITY} -| ${extended_debug}= | ${EXTENDED_DEBUG} | # Variable holding multiplicator of main heap size. By default it is set to 1 | # that means the main heap size will be set to 2G. Some tests may require more | # memory for IP FIB (e.g. nat44det tests with 4M or 16M sessions). | ${heap_size_mult}= | ${1} *** Keywords *** +| Clear and show runtime counters with running traffic +| | [Documentation] +| | ... | Start traffic at specified rate then clear runtime counters on all +| | ... | DUTs. Wait for specified amount of time and capture runtime counters +| | ... | on all DUTs. Finally stop traffic. +| | +| | ... | TODO: Support resetter if this is not the first trial-ish action? +| | +| | ... | *Example:* +| | +| | ... | \| Clear and show runtime counters with running traffic \| +| | +| | ${ppta} = | Get Packets Per Transaction Aggregated +| | ${runtime_duration} = | Get Runtime Duration +| | ${runtime_rate} = | Get Runtime Rate +| | ${traffic_directions} = | Get Traffic Directions +| | ${transaction_duration} = | Get Transaction Duration +| | ${transaction_scale} = | Get Transaction Scale +| | ${transaction_type} = | Get Transaction Type +| | ${use_latency} = | Get Use Latency +| | # Duration of -1 means we will stop traffic manually. +| | Send traffic on tg +| | ... | duration=${-1} +| | ... | rate=${runtime_rate} +| | ... | frame_size=${frame_size} +| | ... | traffic_profile=${traffic_profile} +| | ... | async_call=${True} +| | ... | ppta=${ppta} +| | ... | use_latency=${use_latency} +| | ... | traffic_directions=${traffic_directions} +| | ... | transaction_duration=${transaction_duration} +| | ... | transaction_scale=${transaction_scale} +| | ... | transaction_type=${transaction_type} +| | ... | duration_limit=${0.0} +| | FOR | ${action} | IN | @{pre_run_stats} +| | | Run Keyword | Additional Statistics Action For ${action} +| | END +| | Sleep | ${runtime_duration} +| | FOR | ${action} | IN | @{post_run_stats} +| | | Run Keyword | Additional Statistics Action For ${action} +| | END +| | Stop traffic on tg + +| Find critical load using PLRsearch +| | [Documentation] +| | ... | Find boundaries for troughput (of hardcoded target loss ratio) +| | ... | using PLRsearch algorithm. +| | ... | Display results as formatted test message. +| | ... | Fail if computed lower bound is 110% of the minimal rate or less. +| | ... | Input rates are understood as uni-directional, +| | ... | reported result contains aggregate rates. +| | ... | Currently, the min_rate value is hardcoded to match test teardowns. +| | ... | Call \${resetter} (if defined) to reset DUT state before each trial. +| | +| | ... | *Test (or broader scope) variables read:* +| | ... | - traffic_profile - Name of module defining traffc for measurements. +| | ... | Type: string +| | ... | - frame_size - L2 Frame Size [B] or IMIX string. Type: integer or +| | ... | string +| | ... | - max_rate - Calculated unidirectional maximal transmit rate [pps]. +| | ... | Type: float +| | +| | ... | *Example:* +| | +| | ... | \| Find critical load using PLR search \| +| | +| | # Get values via performance_vars. +| | ${max_rate} = | Get Max Rate +| | ${min_rate} = | Get Min Rate +| | ${ppta} = | Get Packets Per Transaction Aggregated +| | ${resetter} = | Get Resetter +| | ${traffic_directions} = | Get Traffic Directions +| | ${transaction_duration} = | Get Transaction Duration +| | ${transaction_scale} = | Get Transaction Scale +| | ${transaction_type} = | Get Transaction Type +| | ${use_latency} = | Get Use Latency +| | ${average} | ${stdev} = | Perform soak search +| | ... | frame_size=${frame_size} +| | ... | traffic_profile=${traffic_profile} +| | ... | minimum_transmit_rate=${min_rate} +| | ... | maximum_transmit_rate=${max_rate} +| | ... | plr_target=${1e-7} +| | ... | tdpt=${0.1} +| | ... | initial_count=${50} +| | ... | ppta=${ppta} +| | ... | resetter=${resetter} +| | ... | timeout=${1800.0} +| | ... | trace_enabled=${False} +| | ... | traffic_directions=${traffic_directions} +| | ... | transaction_scale=${transaction_scale} +| | ... | transaction_duration=${transaction_duration} +| | ... | transaction_type=${transaction_type} +| | ... | use_latency=${use_latency} +| | ${lower} | ${upper} = | Display result of soak search +| | ... | ${average} | ${stdev} +| | Set Test Variable | \${rate for teardown} | ${lower} +| | Should Not Be True | 1.1 * ${min_rate} > ${lower} +| | ... | Lower bound ${lower} too small for unidirectional minimum ${min_rate}. + | Find NDR and PDR intervals using optimized search | | [Documentation] | | ... | Find boundaries for RFC2544 compatible NDR and PDR values @@ -45,386 +145,243 @@ | | ... | Input rates are understood as uni-directional, | | ... | reported result contains aggregate rates. | | ... | Additional latency measurements are performed for smaller loads, -| | ... | their results are also displayed. +| | ... | even if latency stream is disabled in search. Their results +| | ... | are also displayed. | | ... | Finally, two measurements for runtime stats are done (not displayed). | | ... | Currently, the min_rate value is hardcoded to 90kpps, | | ... | allowing measurement at 10% of the discovered rate | | ... | without breaking latency streams. +| | ... | Call \${resetter} (if defined) to reset DUT state before each trial. | | | | ... | *Test (or broader scope) variables read:* | | ... | - traffic_profile - Name of module defining traffc for measurements. | | ... | Type: string | | ... | - frame_size - L2 Frame Size [B] or IMIX string. Type: integer or | | ... | string -| | ... | - max_rate - Calculated unidirectional maximal transmit rate [pps]. +| | ... | - max_rate - Calculated maximal transmit rate [tps]. | | ... | Type: float -| | -| | ... | *Arguments:* -| | ... | - packet_loss_ratio - Accepted loss during search. Type: float -| | ... | - final_relative_width - Maximal width multiple of upper. Type: float -| | ... | - final_trial_duration - Duration of final trials [s]. Type: float -| | ... | - initial_trial_duration - Duration of initial trials [s]. Type: float -| | ... | - intermediate phases - Number of intermediate phases [1]. -| | ... | Type: integer -| | ... | - timeout - Fail if search duration is longer [s]. Type: float -| | ... | - doublings - How many doublings to do when expanding [1]. -| | ... | Type: integer -| | ... | - traffic_directions - Bi- (2) or uni- (1) directional traffic. -| | ... | Type: integer -| | ... | - latency_duration - Duration for latency-specific trials. Type: float -| | ... | - latency - False to disable latency measurement; default value: True. -| | ... | Type: boolean +| | ... | - resetter - Callable to reset DUT state before each trial. +| | ... | - transaction_scale - Number of ASTF transaction (zero if unlimited). +| | ... | - transaction_type - String identifier to determine how to count +| | ... | transactions. Default is "packet". +| | ... | - disable_latency - If true, skip anything related to latency. +| | ... | Useful if transaction_scale is high and TPS is low. Default: false. | | | | ... | *Example:* | | -| | ... | \| Find NDR and PDR intervals using optimized search \| \${0.005} \ -| | ... | \| \${0.005} \| \${30.0} \| \${1.0} \| \${2} \| \${600.0} \| \${2} \ -| | ... | \| \${2} \| ${5.0} \| -| | -| | [Arguments] | ${packet_loss_ratio}=${0.005} -| | ... | ${final_relative_width}=${0.005} | ${final_trial_duration}=${30.0} -| | ... | ${initial_trial_duration}=${1.0} -| | ... | ${number_of_intermediate_phases}=${2} | ${timeout}=${720.0} -| | ... | ${doublings}=${2} | ${traffic_directions}=${2} -| | ... | ${latency_duration}=${PERF_TRIAL_LATENCY_DURATION} -| | ... | ${latency}=${True} -| | -| | # Latency measurements will need more than 9000 pps. -| | ${result} = | Perform optimized ndrpdr search | ${frame_size} -| | ... | ${traffic_profile} | ${9001} | ${max_rate} -| | ... | ${packet_loss_ratio} | ${final_relative_width} -| | ... | ${final_trial_duration} | ${initial_trial_duration} -| | ... | ${number_of_intermediate_phases} | timeout=${timeout} -| | ... | doublings=${doublings} | traffic_directions=${traffic_directions} -| | ... | latency=${latency} +| | ... | \| Find NDR and PDR intervals using optimized search \| +| | +| | # Get values via performance_vars. +| | ${disable_latency} = | Get Disable Latency +| | ${max_rate} = | Get Max Rate +| | ${min_rate} = | Get Min Rate +| | # \${packet_loss_ratio} is used twice so it is worth a variable. +| | ${packet_loss_ratio} = | Get Packet Loss Ratio +| | ${ppta} = | Get Packets Per Transaction Aggregated +| | ${resetter} = | Get Resetter +| | ${traffic_directions} = | Get Traffic Directions +| | ${transaction_duration} = | Get Transaction Duration +| | ${transaction_scale} = | Get Transaction Scale +| | ${transaction_type} = | Get Transaction Type +| | ${use_latency} = | Get Use Latency +| | ${result} = | Perform optimized ndrpdr search +| | ... | frame_size=${frame_size} +| | ... | traffic_profile=${traffic_profile} +| | ... | minimum_transmit_rate=${min_rate} +| | ... | maximum_transmit_rate=${max_rate} +| | ... | packet_loss_ratio=${packet_loss_ratio} +| | ... | final_relative_width=${0.005} +| | ... | final_trial_duration=${30.0} +| | ... | initial_trial_duration=${1.0} +| | ... | number_of_intermediate_phases=${2} +| | ... | timeout=${720.0} +| | ... | doublings=${2} +| | ... | ppta=${ppta} +| | ... | resetter=${resetter} +| | ... | traffic_directions=${traffic_directions} +| | ... | transaction_duration=${transaction_duration} +| | ... | transaction_scale=${transaction_scale} +| | ... | transaction_type=${transaction_type} +| | ... | use_latency=${use_latency} | | Display result of NDRPDR search | ${result} | | Check NDRPDR interval validity | ${result.pdr_interval} | | ... | ${packet_loss_ratio} | | Check NDRPDR interval validity | ${result.ndr_interval} -| | ${pdr_sum}= | Set Variable | ${result.pdr_interval.measured_low.target_tr} -| | ${pdr_per_stream}= | Evaluate | ${pdr_sum} / float(${traffic_directions}) -| | ${ndr_sum}= | Set Variable | ${result.ndr_interval.measured_low.target_tr} -| | ${ndr_per_stream}= | Evaluate | ${ndr_sum} / float(${traffic_directions}) -| | ${rate}= | Evaluate | 0.9 * ${pdr_per_stream} -| | Run Keyword If | ${latency} -| | ... | Measure and show latency at specified rate | Latency at 90% PDR: -| | ... | ${latency_duration} | ${rate} | ${framesize} -| | ... | ${traffic_profile} | ${traffic_directions} -| | ${rate}= | Evaluate | 0.5 * ${pdr_per_stream} -| | Run Keyword If | ${latency} -| | ... | Measure and show latency at specified rate | Latency at 50% PDR: -| | ... | ${latency_duration} | ${rate} | ${framesize} -| | ... | ${traffic_profile} | ${traffic_directions} -| | ${rate}= | Evaluate | 0.1 * ${pdr_per_stream} -| | Run Keyword If | ${latency} -| | ... | Measure and show latency at specified rate | Latency at 10% PDR: -| | ... | ${latency_duration} | ${rate} | ${framesize} -| | ... | ${traffic_profile} | ${traffic_directions} -| | Run Keyword If | ${latency} -| | ... | Measure and show latency at specified rate | Latency at 0% PDR: -| | ... | ${latency_duration} | ${0} | ${framesize} -| | ... | ${traffic_profile} | ${traffic_directions} -| | # Finally, trials with runtime and other stats. +| | ${pdr} = | Set Variable | ${result.pdr_interval.measured_low.target_tr} +| | ${ndr} = | Set Variable | ${result.ndr_interval.measured_low.target_tr} | | # We expect NDR and PDR to have different-looking stats. | | Send traffic at specified rate -| | ... | ${1.0} | ${pdr_per_stream} | ${framesize} | ${traffic_profile} -| | ... | traffic_directions=${traffic_directions} -| | Send traffic at specified rate -| | ... | ${1.0} | ${ndr_per_stream} | ${framesize} | ${traffic_profile} -| | ... | traffic_directions=${traffic_directions} +| | ... | rate=${pdr} +| | ... | trial_duration=${1.0} +| | ... | trial_multiplicity=${1} +| | ... | use_latency=${use_latency} +| | ... | duration_limit=${1.0} +| | Run Keyword If | ${ndr} != ${pdr} +| | ... | Send traffic at specified rate +| | ... | rate=${ndr} +| | ... | trial_duration=${1.0} +| | ... | trial_multiplicity=${1} +| | ... | use_latency=${use_latency} +| | ... | duration_limit=${1.0} +| | Return From Keyword If | ${disable_latency} +| | ${rate} = | Evaluate | 0.9 * ${pdr} +| | Measure and show latency at specified rate | Latency at 90% PDR: | ${rate} +| | ${rate} = | Evaluate | 0.5 * ${pdr} +| | Measure and show latency at specified rate | Latency at 50% PDR: | ${rate} +| | ${rate} = | Evaluate | 0.1 * ${pdr} +| | Measure and show latency at specified rate | Latency at 10% PDR: | ${rate} +| | Measure and show latency at specified rate | Latency at 0% PDR: | ${0.0} | Find Throughput Using MLRsearch | | [Documentation] -| | ... | Find and return lower bound PDR (zero PLR by default) +| | ... | Find and return lower bound NDR (zero PLR) | | ... | aggregate throughput using MLRsearch algorithm. | | ... | Input rates are understood as uni-directional. | | ... | Currently, the min_rate value is hardcoded to match test teardowns. +| | ... | Call \${resetter} (if defined) to reset DUT state before each trial. | | | | ... | *Test (or broader scope) variables read:* | | ... | - traffic_profile - Name of module defining traffc for measurements. | | ... | Type: string | | ... | - frame_size - L2 Frame Size [B] or IMIX string. Type: integer or | | ... | string -| | ... | - max_rate - Calculated unidirectional maximal transmit rate [pps]. +| | ... | - max_rate - Calculated maximal transmit rate [tps]. | | ... | Type: float -| | -| | ... | *Arguments:* -| | ... | - packet_loss_ratio - Accepted loss during search. Type: float -| | ... | - final_relative_width - Maximal width multiple of upper. Type: float -| | ... | - final_trial_duration - Duration of final trials [s]. Type: float -| | ... | - initial_trial_duration - Duration of initial trials [s]. Type: float -| | ... | - intermediate phases - Number of intermediate phases [1]. -| | ... | Type: integer -| | ... | - timeout - Fail if search duration is longer [s]. Type: float -| | ... | - doublings - How many doublings to do when expanding [1]. -| | ... | Type: integer -| | ... | - traffic_directions - Bi- (2) or uni- (1) directional traffic. -| | ... | Type: integer -| | ... | - latency - True to enable latency measurement; default value: False. -| | ... | Type: boolean +| | ... | - resetter - Callable to reset DUT state before each trial. +| | ... | - transaction_scale - Number of ASTF transaction (zero if unlimited). +| | ... | - transaction_type - String identifier to determine how to count +| | ... | transactions. Default is "packet". | | | | ... | *Returns:* | | ... | - Lower bound for bi-directional throughput at given PLR. Type: float | | | | ... | *Example:* | | -| | ... | \| \${throughpt}= \| Find Throughput Using MLRsearch \| \${0} \ -| | ... | \| \${0.001} \| \${10.0}\| \${1.0} \| \${1} \| \${720.0} \| \${2} \ -| | ... | \| \${2} \| -| | -| | [Arguments] | ${packet_loss_ratio}=${0.0} -| | ... | ${final_relative_width}=${0.001} | ${final_trial_duration}=${10.0} -| | ... | ${initial_trial_duration}=${1.0} -| | ... | ${number_of_intermediate_phases}=${1} | ${timeout}=${720.0} -| | ... | ${doublings}=${2} | ${traffic_directions}=${2} | ${latency}=${False} -| | -| | ${result} = | Perform optimized ndrpdr search | ${frame_size} -| | ... | ${traffic_profile} | ${10000} | ${max_rate} -| | ... | ${packet_loss_ratio} | ${final_relative_width} -| | ... | ${final_trial_duration} | ${initial_trial_duration} -| | ... | ${number_of_intermediate_phases} | timeout=${timeout} -| | ... | doublings=${doublings} | traffic_directions=${traffic_directions} -| | ... | latency=${latency} +| | ... | \| \${throughpt}= \| Find Throughput Using MLRsearch \| +| | +| | ${max_rate} = | Get Max Rate +| | ${min_rate} = | Get Min Rate +| | ${ppta} = | Get Packets Per Transaction Aggregated +| | ${resetter} = | Get Resetter +| | ${traffic_directions} = | Get Traffic Directions +| | ${transaction_duration} = | Get Transaction Duration +| | ${transaction_scale} = | Get Transaction Scale +| | ${transaction_type} = | Get Transaction Type +| | ${use_latency} = | Get Use Latency +| | ${result} = | Perform optimized ndrpdr search +| | ... | frame_size=${frame_size} +| | ... | traffic_profile=${traffic_profile} +| | ... | minimum_transmit_rate=${min_rate} +| | ... | maximum_transmit_rate=${max_rate} +| | ... | packet_loss_ratio=${0.0} +| | ... | final_relative_width=${0.001} +| | ... | final_trial_duration=${10.0} +| | ... | initial_trial_duration=${1.0} +| | ... | number_of_intermediate_phases=${1} +| | ... | timeout=${720} +| | ... | doublings=${2} +| | ... | ppta=${ppta} +| | ... | resetter=${resetter} +| | ... | traffic_directions=${traffic_directions} +| | ... | transaction_duration=${transaction_duration} +| | ... | transaction_scale=${transaction_scale} +| | ... | transaction_type=${transaction_type} +| | ... | use_latency=${use_latency} | | Check NDRPDR interval validity | ${result.pdr_interval} -| | ... | ${packet_loss_ratio} +| | ... | ${0.0} | | Return From Keyword | ${result.pdr_interval.measured_low.target_tr} -| Find critical load using PLRsearch -| | [Documentation] -| | ... | Find boundaries for troughput (of given target loss ratio) -| | ... | using PLRsearch algorithm. -| | ... | Display results as formatted test message. -| | ... | Fail if computed lower bound 110% of the minimal rate or less. -| | ... | Input rates are understood as uni-directional, -| | ... | reported result contains aggregate rates. -| | ... | Currently, the min_rate value is hardcoded to match test teardowns. -| | -| | ... | *Test (or broader scope) variables read:* -| | ... | - traffic_profile - Name of module defining traffc for measurements. -| | ... | Type: string -| | ... | - frame_size - L2 Frame Size [B] or IMIX string. Type: integer or -| | ... | string -| | ... | - max_rate - Calculated unidirectional maximal transmit rate [pps]. -| | ... | Type: float -| | -| | ... | *Arguments:* -| | ... | - packet_loss_ratio - Accepted loss during search. Type: float -| | ... | - timeout - Stop when search duration is longer [s]. Type: float -| | ... | - traffic_directions - Bi- (2) or uni- (1) directional traffic. -| | ... | Type: integer -| | ... | - latency - True to enable latency measurement; default value: False. -| | ... | Type: boolean -| | -| | ... | *Example:* -| | -| | ... | \| Find critical load using PLR search \| \${1e-7} \| \${120} \ -| | ... | \| \${2} \| -| | -| | [Arguments] | ${packet_loss_ratio}=${1e-7} | ${timeout}=${1800.0} -| | ... | ${traffic_directions}=${2} | ${latency}=${False} -| | -| | ${min_rate} = | Set Variable | ${10000} -| | ${average} | ${stdev} = | Perform soak search | ${frame_size} -| | ... | ${traffic_profile} | ${min_rate} | ${max_rate} -| | ... | ${packet_loss_ratio} | timeout=${timeout} -| | ... | traffic_directions=${traffic_directions} | latency=${latency} -| | ${lower} | ${upper} = | Display result of soak search -| | ... | ${average} | ${stdev} -| | Should Not Be True | 1.1 * ${traffic_directions} * ${min_rate} > ${lower} -| | ... | Lower bound ${lower} too small for unidirectional minimum ${min_rate}. - -| Display single bound -| | [Documentation] -| | ... | Display one bound of NDR+PDR search, -| | ... | in packet per seconds (total and per stream) -| | ... | and Gbps total bandwidth (for initial packet size). -| | ... | Througput is calculated as: -| | ... | Sum of measured rates over streams -| | ... | Bandwidth is calculated as: -| | ... | (Throughput * (L2 Frame Size + IPG) * 8) -| | ... | The given result should contain latency data as well. -| | -| | ... | *Arguments:* -| | ... | - text - Flavor text describing which bound is this. Type: string -| | ... | - rate_total - Total (not per stream) measured Tr [pps]. Type: float -| | ... | - frame_size - L2 Frame Size [B]. Type: integer -| | ... | - latency - Latency data to display if non-empty. Type: string -| | -| | ... | *Example:* -| | -| | ... | \| Display single bound \| NDR lower bound \| \${12345.67} \ -| | ... | \| \${64} \| latency=\${EMPTY} \| -| | -| | [Arguments] | ${text} | ${rate_total} | ${frame_size} | ${latency}=${EMPTY} -| | -| | ${bandwidth_total} = | Evaluate | ${rate_total} * (${frame_size}+20)*8 / 1e9 -| | Set Test Message | ${\n}${text}: ${rate_total} pps, | append=yes -| | Set Test Message | ${bandwidth_total} Gbps (initial) | append=yes -| | Return From Keyword If | not """${latency}""" -| | Set Test Message | ${\n}LATENCY [min/avg/max/hdrh] per stream: ${latency} -| | ... | append=yes - -| Display Reconfig Test Message -| | [Documentation] -| | ... | Display the number of packets lost (bidirectionally) -| | ... | due to reconfiguration under traffic. -| | -| | ... | *Arguments:* -| | ... | - result - Result of bidirectional measurtement. -| | ... | Type: ReceiveRateMeasurement -| | -| | ... | *Example:* -| | -| | ... | \| Display Reconfig Test Message \| \${result} \| -| | -| | [Arguments] | ${result} -| | -| | Set Test Message | Packets lost due to reconfig: ${result.loss_count} -| | ${time_lost} = | Evaluate | ${result.loss_count} / ${result.target_tr} -| | Set Test Message | ${\n}Implied time lost: ${time_lost} | append=yes - -| Display result of NDRPDR search -| | [Documentation] -| | ... | Display result of NDR+PDR search, both quantities, both bounds, -| | ... | aggregate in packet per seconds -| | ... | and Gbps total bandwidth (for initial packet size). -| | ... | Througput is calculated as: -| | ... | Sum of measured rate over streams -| | ... | Bandwidth is calculated as: -| | ... | (Throughput * (L2 Frame Size + IPG) * 8) -| | ... | The given result should contain latency data as well. -| | -| | ... | *Test (or broader scope) variables read:* -| | ... | - frame_size - L2 Frame Size [B] or IMIX string. Type: integer or -| | ... | string -| | ... | *Arguments:* -| | ... | - result - Measured result data per stream [pps]. Type: NdrPdrResult -| | -| | ... | *Example:* -| | -| | ... | \| Display result of NDRPDR search \| \${result} \| -| | -| | [Arguments] | ${result} -| | -| | ${frame_size} = | Get Average Frame Size | ${frame_size} -| | Display single bound | NDR_LOWER -| | ... | ${result.ndr_interval.measured_low.target_tr} | ${frame_size} -| | ... | ${result.ndr_interval.measured_low.latency} -| | Display single bound | NDR_UPPER -| | ... | ${result.ndr_interval.measured_high.target_tr} | ${frame_size} -| | Display single bound | PDR_LOWER -| | ... | ${result.pdr_interval.measured_low.target_tr} | ${frame_size} -| | ... | ${result.pdr_interval.measured_low.latency} -| | Display single bound | PDR_UPPER -| | ... | ${result.pdr_interval.measured_high.target_tr} | ${frame_size} - -| Display result of soak search -| | [Documentation] -| | ... | Display result of soak search, avg+-stdev, as upper/lower bounds, -| | ... | in aggregate packets per seconds -| | ... | and Gbps total bandwidth (for initial packet size). -| | ... | Througput is calculated as: -| | ... | Sum of measured rates over streams -| | ... | Bandwidth is calculated as: -| | ... | (Throughput * (L2 Frame Size + IPG) * 8) -| | -| | ... | *Test (or broader scope) variables read:* -| | ... | - frame_size - L2 Frame Size [B] or IMIX string. Type: integer or -| | ... | string -| | ... | *Arguments:* -| | ... | - avg - Estimated average critical load [pps]. Type: float -| | ... | - stdev - Standard deviation of critical load [pps]. Type: float -| | -| | ... | *Returns:* -| | ... | - Lower and upper bound of critical load [pps]. Type: 2-tuple of float -| | -| | ... | *Example:* -| | -| | ... | \| Display result of soak search \| \${100000} \| \${100} \| -| | -| | [Arguments] | ${avg} | ${stdev} -| | -| | ${frame_size} = | Get Average Frame Size | ${frame_size} -| | ${avg} = | Convert To Number | ${avg} -| | ${stdev} = | Convert To Number | ${stdev} -| | ${lower} = | Evaluate | ${avg} - ${stdev} -| | ${upper} = | Evaluate | ${avg} + ${stdev} -| | Display single bound | PLRsearch lower bound | ${lower} | ${frame_size} -| | Display single bound | PLRsearch upper bound | ${upper} | ${frame_size} -| | Return From Keyword | ${lower} | ${upper} - -| Check NDRPDR interval validity +| Measure and show latency at specified rate | | [Documentation] -| | ... | Extract loss ratio of lower bound of the interval. -| | ... | Fail if it does not reach the allowed value. +| | ... | Send traffic at specified rate, single trial. +| | ... | Extract latency information and append it to text message. +| | ... | The rate argument is int, so should not include "pps". +| | ... | If the given rate is too low, a safe value is used instead. +| | ... | Call \${resetter} (if defined) to reset DUT state before each trial. | | | | ... | *Arguments:* -| | ... | - interval - Measured interval. Type: ReceiveRateInterval -| | ... | - packet_loss_ratio - Accepted loss (0.0 for NDR). Type: float +| | ... | - message_prefix - Preface to test message addition. Type: string +| | ... | - rate - Rate [tps] for sending packets. +| | ... | Type: float | | | | ... | *Example:* | | -| | ... | \| Check NDRPDR interval validity \| \${result.pdr_interval} \ -| | ... | \| \${0.005} \| -| | -| | [Arguments] | ${interval} | ${packet_loss_ratio}=${0.0} -| | -| | ${lower_bound} = | Set Variable | ${interval.measured_low} -| | ${lower_bound_lf} = | Set Variable | ${lower_bound.loss_fraction} -| | Return From Keyword If | ${lower_bound_lf} <= ${packet_loss_ratio} -| | ${message}= | Catenate | SEPARATOR=${SPACE} -| | ... | Minimal rate loss fraction ${lower_bound_lf} -| | ... | does not reach target ${packet_loss_ratio}. -| | ${message_zero} = | Set Variable | Zero packets forwarded! -| | ${message_other} = | Set Variable | ${lower_bound.loss_count} packets lost. -| | ${message} = | Set Variable If | ${lower_bound_lf} >= 1.0 -| | ... | ${message}${\n}${message_zero} | ${message}${\n}${message_other} -| | Fail | ${message} +| | ... | \| Measure and show latency at specified rate \| Latency at 90% NDR \ +| | ... | \| ${10000000} \| +| | +| | [Arguments] | ${message_prefix} | ${rate} +| | +| | ${min_rate} = | Get Min Rate +| | ${ppta} = | Get Packets Per Transaction Aggregated +| | ${real_rate} = | Evaluate | max(${rate}, ${min_rate}) +| | ${traffic_directions} = | Get Traffic Directions +| | ${transaction_duration} = | Get Transaction Duration +| | ${transaction_scale} = | Get Transaction Scale +| | ${transaction_type} = | Get Transaction Type +| | Call Resetter +| | Send traffic on tg +| | ... | duration=${PERF_TRIAL_LATENCY_DURATION} +| | ... | rate=${real_rate} +| | ... | frame_size=${frame_size} +| | ... | traffic_profile=${traffic_profile} +| | ... | async_call=${False} +| | ... | duration_limit=${PERF_TRIAL_LATENCY_DURATION} +| | ... | ppta=${ppta} +| | ... | traffic_directions=${traffic_directions} +| | ... | transaction_duration=${transaction_duration} +| | ... | transaction_scale=${transaction_scale} +| | ... | transaction_type=${transaction_type} +| | ... | use_latency=${True} +| | ${latency} = | Get Latency Int +| | Set Test Message | ${\n}${message_prefix} ${latency} | append=${True} -| Traffic should pass with maximum rate +| Send ramp-up traffic | | [Documentation] -| | ... | Send traffic at maximum rate. +| | ... | Do nothing unless positive ramp-up duration is specified. +| | ... | Else perform one trial with appropriate rate and duration. +| | ... | This is useful for tests that set DUT state via traffic. +| | ... | Rate has to bee low enough so packets are not lost, +| | ... | Duration has to be long enough to set all the state. +| | ... | The trial results are discarded. | | | | ... | *Test (or broader scope) variables read:* | | ... | - traffic_profile - Name of module defining traffic for measurements. | | ... | Type: string | | ... | - frame_size - L2 Frame Size [B] or IMIX string. Type: integer or | | ... | string -| | ... | - max_rate - Calculated unidirectional maximal transmit rate [pps]. +| | ... | - ramp_up_duration - Suitable traffic duration [s]. | | ... | Type: float -| | -| | ... | *Arguments:* -| | ... | - trial_duration - Duration of single trial [s]. Type: float -| | ... | - fail_no_traffic - Whether to fail on zero receive count; -| | ... | default value: True. Type: boolean -| | ... | - trial_multiplicity - How many trials in this measurement. -| | ... | Type: integer -| | ... | - traffic_directions - Bi- (2) or uni- (1) directional traffic; -| | ... | default value: 2. Type: integer -| | ... | - tx_port - TX port of TG; default value: 0. Type: integer -| | ... | - rx_port - RX port of TG; default value: 1. Type: integer -| | ... | - latency - True to enable latency measurement; default value: False. -| | ... | Type: boolean +| | ... | - ramp_up_rate - Suitable transmit rate [tps]. +| | ... | Type: float +| | ... | - transaction_type - String identifier to determine how to count +| | ... | transactions. Default is "packet". | | | | ... | *Example:* | | -| | ... | \| Traffic should pass with maximum rate \| \${1} \| \${False} \ -| | ... | \| \${10.0} \| \${2} \| \${0} \| \${1} \| \${True} \| -| | -| | [Arguments] | ${trial_duration}=${trial_duration} -| | ... | ${fail_no_traffic}=${True} -| | ... | ${trial_multiplicity}=${trial_multiplicity} -| | ... | ${traffic_directions}=${2} | ${tx_port}=${0} | ${rx_port}=${1} -| | ... | ${latency}=${False} -| | -| | ${results}= | Send traffic at specified rate -| | ... | ${trial_duration} | ${max_rate} | ${frame_size} -| | ... | ${traffic_profile} | ${trial_multiplicity} -| | ... | ${traffic_directions} | ${tx_port} | ${rx_port} | latency=${latency} -| | Set Test Message | ${\n}Maximum Receive Rate trial results -| | Set Test Message | in packets per second: ${results} -| | ... | append=yes -| | Run Keyword If | ${fail_no_traffic} | Fail if no traffic forwarded +| | ... | \| Send ramp-up traffic \| +| | +| | ${ramp_up_duration} = | Get Ramp Up Duration +| | Run Keyword Unless | ${ramp_up_duration} > 0.0 | Return From Keyword +| | ${ramp_up_rate} = | Get Ramp Up Rate +| | ${ppta} = | Get Packets Per Transaction Aggregated +| | ${traffic_directions} = | Get Traffic Directions +| | ${transaction_duration} = | Get Transaction Duration +| | ${transaction_scale} = | Get Transaction Scale +| | ${transaction_type} = | Get Transaction Type +| | ${use_latency} = | Get Use Latency +| | Send traffic on tg +| | ... | duration=${ramp_up_duration} +| | ... | rate=${ramp_up_rate} +| | ... | frame_size=${frame_size} +| | ... | traffic_profile=${traffic_profile} +| | ... | async_call=${False} +| | ... | duration_limit=${0.0} +| | ... | ppta=${ppta} +| | ... | use_latency=${use_latency} +| | ... | traffic_directions=${traffic_directions} +| | ... | transaction_duration=${transaction_duration} +| | ... | transaction_scale=${transaction_scale} +| | ... | transaction_type=${transaction_type} | Send traffic at specified rate | | [Documentation] @@ -432,178 +389,62 @@ | | ... | Then send traffic at specified rate, possibly multiple trials. | | ... | Show various DUT stats, optionally also packet trace. | | ... | Return list of measured receive rates. +| | ... | Call \${resetter} (if defined) to reset DUT state before each trial. | | | | ... | *Arguments:* | | ... | - trial_duration - Duration of single trial [s]. Type: float -| | ... | - rate - Target aggregate transmit rate [pps] / Connections per second -| | ... | (CPS) for UDP/TCP flows. Type: float -| | ... | - frame_size - L2 Frame Size [B]. Type: integer or string -| | ... | - traffic_profile - Name of module defining traffc for measurements. +| | ... | - rate - Target transmit rate [tps]. Type: float | | ... | Type: string | | ... | - trial_multiplicity - How many trials in this measurement. -| | ... | Type: integer -| | ... | - traffic_directions - Bi- (2) or uni- (1) directional traffic. -| | ... | Type: integer -| | ... | - tx_port - TX port of TG; default value: 0. Type: integer -| | ... | - rx_port - RX port of TG; default value: 1. Type: integer -| | ... | - extended_debug - True to enable extended debug. | | ... | Type: boolean -| | ... | - latency - True to enable latency measurement; default value: False. +| | ... | - use_latency - Use latency stream in search; default value: False. | | ... | Type: boolean +| | ... | - duration_limit - Hard limit for trial duration, overriding duration +| | ... | computed from transaction_scale. Default 0.0 means no limit. | | | | ... | *Example:* | | | | ... | \| Send traffic at specified rate \| \${1.0} \| ${4000000.0} \ -| | ... | \| \${64} \| 3-node-IPv4 \| \${10} \| \${2} \| \${0} \| \${1} \ -| | ... | \| ${False} \| ${True} \| +| | ... | \| \${10} \| ${False} \| ${1.0} \| | | -| | [Arguments] | ${trial_duration} | ${rate} | ${frame_size} -| | ... | ${traffic_profile} | ${trial_multiplicity}=${trial_multiplicity} -| | ... | ${traffic_directions}=${2} | ${tx_port}=${0} | ${rx_port}=${1} -| | ... | ${extended_debug}=${extended_debug} | ${latency}=${False} -| | -| | Set Test Variable | ${extended_debug} -| | # Following setting of test variables is needed for some pre_stats actions. -| | Set Test Variable | ${rate} -| | Set Test Variable | ${traffic_directions} -| | Set Test Variable | ${tx_port} -| | Set Test Variable | ${rx_port} +| | [Arguments] | ${trial_duration} | ${rate} | ${trial_multiplicity} +| | ... | ${use_latency}=${False} | ${duration_limit}=${0.0} | | +| | ${ppta} = | Get Packets Per Transaction Aggregated +| | ${traffic_directions} = | Get Traffic Directions +| | ${transaction_duration} = | Get Transaction Duration +| | ${transaction_scale} = | Get Transaction Scale +| | ${transaction_type} = | Get Transaction Type +| | Set Test Variable | \${rate_for_teardown} | ${rate} | | FOR | ${action} | IN | @{pre_stats} | | | Run Keyword | Additional Statistics Action For ${action} | | END | | ${results} = | Create List | | FOR | ${i} | IN RANGE | ${trial_multiplicity} -| | | # The following line is skipping some default arguments, -| | | # that is why subsequent arguments have to be named. +| | | Call Resetter | | | Send traffic on tg -| | | ... | ${trial_duration} | ${rate} | ${frame_size} | ${traffic_profile} -| | | ... | warmup_time=${0} | traffic_directions=${traffic_directions} -| | | ... | tx_port=${tx_port} | rx_port=${rx_port} | latency=${latency} -| | | ${rx} = | Get Received -| | | ${rr} = | Evaluate | ${rx} / ${trial_duration} -| | | Append To List | ${results} | ${rr} +| | | ... | duration=${trial_duration} +| | | ... | rate=${rate} +| | | ... | frame_size=${frame_size} +| | | ... | traffic_profile=${traffic_profile} +| | | ... | async_call=${False} +| | | ... | duration_limit=${duration_limit} +| | | ... | ppta=${ppta} +| | | ... | traffic_directions=${traffic_directions} +| | | ... | transaction_duration=${transaction_duration} +| | | ... | transaction_scale=${transaction_scale} +| | | ... | transaction_type=${transaction_type} +| | | ... | use_latency=${use_latency} +| | | ${result}= | Get Measurement Result +| | | # Out of several quantities for aborted traffic (duration stretching), +| | | # the approximated receive rate is the best estimate we have. +| | | Append To List | ${results} | ${result.approximated_receive_rate} | | END | | FOR | ${action} | IN | @{post_stats} | | | Run Keyword | Additional Statistics Action For ${action} | | END | | Return From Keyword | ${results} -| Measure and show latency at specified rate -| | [Documentation] -| | ... | Send traffic at specified rate, single trial. -| | ... | Extract latency information and append it to text message. -| | ... | The rate argument is int, so should not include "pps". -| | ... | If the given rate is too low, a safe value is used instead. -| | -| | ... | *Arguments:* -| | ... | - message_prefix - Preface to test message addition. Type: string -| | ... | - trial_duration - Duration of single trial [s]. Type: float -| | ... | - rate - Rate [pps] for sending packets in case of T-Rex stateless -| | ... | mode or multiplier of profile CPS in case of T-Rex astf mode. -| | ... | Type: float -| | ... | - frame_size - L2 Frame Size [B]. Type: integer or string -| | ... | - traffic_profile - Name of module defining traffic for measurements. -| | ... | Type: string -| | ... | - traffic_directions - Bi- (2) or uni- (1) directional traffic. -| | ... | Type: integer -| | ... | - tx_port - TX port of TG; default value: 0. Type: integer -| | ... | - rx_port - RX port of TG; default value: 1. Type: integer -| | ... | - safe_rate - To apply if rate is below this, as latency pps is fixed. -| | ... | In pps. Type: integer. -| | -| | ... | *Example:* -| | -| | ... | \| Measure and show latency at specified rate \| Latency at 90% NDR \ -| | ... | \| \${1.0} \| ${10000000} \| \${64} \| 3-node-IPv4 \| \${2} \ -| | ... | \| \${0} \| \${1} \| ${9500} \| -| | -| | [Arguments] | ${message_prefix} | ${trial_duration} | ${rate} -| | ... | ${frame_size} | ${traffic_profile} | ${traffic_directions}=${2} -| | ... | ${tx_port}=${0} | ${rx_port}=${1} | ${safe_rate}=${9001} -| | -| | ${real_rate} = | Evaluate | max(${rate}, ${safe_rate}) -| | # The following line is skipping some default arguments, -| | # that is why subsequent arguments have to be named. -| | Send traffic on tg | ${trial_duration} | ${real_rate} | ${frame_size} -| | ... | ${traffic_profile} | warmup_time=${0} -| | ... | traffic_directions=${traffic_directions} | tx_port=${tx_port} -| | ... | rx_port=${rx_port} | latency=${True} -| | ${latency} = | Get Latency Int -| | Set Test Message | ${\n}${message_prefix} ${latency} | append=${True} - -| Clear and show runtime counters with running traffic -| | [Documentation] -| | ... | Start traffic at specified rate then clear runtime counters on all -| | ... | DUTs. Wait for specified amount of time and capture runtime counters -| | ... | on all DUTs. Finally stop traffic -| | -| | ... | *Arguments:* -| | ... | - duration - Duration of traffic run [s]. Type: integer -| | ... | - rate - Rate [pps] for sending packets in case of T-Rex stateless -| | ... | mode or multiplier of profile CPS in case of T-Rex astf mode. -| | ... | Type: float -| | ... | - frame_size - L2 Frame Size [B] or IMIX_v4_1. Type: integer or string -| | ... | - traffic_profile - Name of module defining traffc for measurements. -| | ... | Type: string -| | ... | - traffic_directions - Bi- (2) or uni- (1) directional traffic. -| | ... | Type: integer -| | ... | - tx_port - TX port of TG; default value: 0. Type: integer -| | ... | - rx_port - RX port of TG, default value: 1. Type: integer -| | -| | ... | *Example:* -| | -| | ... | \| Clear and show runtime counters with running traffic \| \${10} \ -| | ... | \| ${4000000.0} \| \${64} \| 3-node-IPv4 \| \${2} \| \${0} \| \${1} \| -| | -| | [Arguments] | ${duration} | ${rate} | ${frame_size} | ${traffic_profile} -| | ... | ${traffic_directions}=${2} | ${tx_port}=${0} | ${rx_port}=${1} -| | -| | # Duration of -1 means we will stop traffic manually. -| | Send traffic on tg | ${-1} | ${rate} | ${frame_size} | ${traffic_profile} -| | ... | warmup_time=${0} | async_call=${True} | latency=${False} -| | ... | traffic_directions=${traffic_directions} | tx_port=${tx_port} -| | ... | rx_port=${rx_port} -| | FOR | ${action} | IN | @{pre_run_stats} -| | | Run Keyword | Additional Statistics Action For ${action} -| | END -| | Sleep | ${duration} -| | FOR | ${action} | IN | @{post_run_stats} -| | | Run Keyword | Additional Statistics Action For ${action} -| | END -| | Stop traffic on tg - -| Send ramp-up traffic -| | [Documentation] -| | ... | Start ramp-up traffic at specified rate for defined duration. -| | -| | ... | *Arguments:* -| | ... | - duration - Duration of traffic run [s]. Type: integer -| | ... | - rate - Rate [pps] for sending packets in case of T-Rex stateless -| | ... | mode or multiplier of profile CPS in case of T-Rex astf mode. -| | ... | Type: float -| | ... | - frame_size - L2 Frame Size [B] or IMIX_v4_1. Type: integer or string -| | ... | - traffic_profile - Name of module defining traffc for measurements. -| | ... | Type: string -| | ... | - traffic_directions - Bi- (2) or uni- (1) directional traffic. -| | ... | Type: integer -| | ... | - tx_port - TX port of TG; default value: 0. Type: integer -| | ... | - rx_port - RX port of TG, default value: 1. Type: integer -| | -| | ... | *Example:* -| | -| | ... | \| Send ramp-up traffic \| \${10} \| ${400000.0} \| ${64} \ -| | ... | \| ${2} \| ${0} \| ${1} \| -| | -| | [Arguments] | ${duration}=${ramp_up_duration} | ${rate}=${ramp_up_rate} -| | ... | ${frame_size}=${frame_size} | ${traffic_profile}=${traffic_profile} -| | ... | ${traffic_directions}=${2} | ${tx_port}=${0} | ${rx_port}=${1} -| | -| | Send traffic on tg -| | ... | ${duration} | ${rate} | ${frame_size} | ${traffic_profile} -| | ... | warmup_time=${0} | traffic_directions=${traffic_directions} -| | ... | tx_port=${tx_port} | rx_port=${rx_port} | latency=${False} - | Start Traffic on Background | | [Documentation] | | ... | Start traffic at specified rate then return control to Robot. @@ -616,27 +457,36 @@ | | ... | - frame_size - L2 Frame Size [B] or IMIX string. Type: integer or | | ... | string | | ... | *Arguments:* -| | ... | - rate - Rate [pps] for sending packets in case of T-Rex stateless -| | ... | mode or multiplier of profile CPS in case of T-Rex astf mode. +| | ... | - rate - Rate [tps] for sending packets. | | ... | Type: float -| | ... | - traffic_directions - Bi- (2) or uni- (1) directional traffic. -| | ... | Type: integer -| | ... | - tx_port - TX port of TG; default value: 0. Type: integer -| | ... | - rx_port - RX port of TG; default value: 1. Type: integer | | | | ... | *Example:* | | -| | ... | \| Start Traffic on Background \| ${4000000.0} \| \${2} \| \${0} \ -| | ... | \| \${1} \| +| | ... | \| Start Traffic on Background \| ${4000000.0} \| | | -| | [Arguments] | ${rate} | ${traffic_directions}=${2} | ${tx_port}=${0} -| | ... | ${rx_port}=${1} +| | [Arguments] | ${rate} | | +| | ${ppta} = | Get Packets Per Transaction Aggregated +| | ${traffic_directions} = | Get Traffic Directions +| | ${transaction_duration} = | Get Transaction Duration +| | ${transaction_scale} = | Get Transaction Scale +| | ${transaction_type} = | Get Transaction Type +| | ${use_latency} = | Get Use Latency +| | Call Resetter | | # Duration of -1 means we will stop traffic manually. -| | Send traffic on tg | ${-1} | ${rate} | ${frame_size} | ${traffic_profile} -| | ... | warmup_time=${0} | async_call=${True} | latency=${False} -| | ... | traffic_directions=${traffic_directions} | tx_port=${tx_port} -| | ... | rx_port=${rx_port} +| | Send traffic on tg +| | ... | duration=${-1} +| | ... | rate=${rate} +| | ... | frame_size=${frame_size} +| | ... | traffic_profile=${traffic_profile} +| | ... | async_call=${True} +| | ... | duration_limit=${0.0} +| | ... | ppta=${ppta} +| | ... | traffic_directions=${traffic_directions} +| | ... | transaction_duration=${transaction_duration} +| | ... | transaction_scale=${transaction_scale} +| | ... | transaction_type=${transaction_type} +| | ... | use_latency=${use_latency} | Stop Running Traffic | | [Documentation] @@ -653,75 +503,41 @@ | | ${result}= | Stop traffic on tg | | Return From Keyword | ${result} -| Additional Statistics Action For vpp-clear-stats -| | [Documentation] -| | ... | Additional Statistics Action for clear VPP statistics. -| | -| | Clear Statistics On All DUTs | ${nodes} - -| Additional Statistics Action For vpp-show-stats -| | [Documentation] -| | ... | Additional Statistics Action for show VPP statistics. -| | -| | Show Statistics On All DUTs | ${nodes} - -| Additional Statistics Action For vpp-clear-runtime -| | [Documentation] -| | ... | Additional Statistics Action for clear VPP runtime. -| | -| | VPP Clear Runtime On All DUTs | ${nodes} - -| Additional Statistics Action For vpp-show-runtime -| | [Documentation] -| | ... | Additional Statistics Action for show VPP runtime. -| | -| | VPP Show Runtime On All DUTs | ${nodes} - -| Additional Statistics Action For vpp-enable-packettrace -| | [Documentation] -| | ... | Additional Statistics Action for enable VPP packet trace. -| | -| | Run Keyword If | ${extended_debug}==${True} -| | ... | VPP Enable Traces On All DUTs | ${nodes} | fail_on_error=${False} - -| Additional Statistics Action For vpp-show-packettrace -| | [Documentation] -| | ... | Additional Statistics Action for show VPP packet trace. -| | -| | Run Keyword If | ${extended_debug}==${True} -| | ... | Show Packet Trace On All Duts | ${nodes} | maximum=${100} - -| Additional Statistics Action For vpp-enable-elog -| | [Documentation] -| | ... | Additional Statistics Action for enable VPP elog trace. -| | -| | VPP Enable Elog Traces On All DUTs | ${nodes} - -| Additional Statistics Action For vpp-show-elog -| | [Documentation] -| | ... | Additional Statistics Action for show VPP elog trace. -| | -| | Show Event Logger On All DUTs | ${nodes} - -| Additional Statistics Action For bash-perf-stat +| Traffic should pass with maximum rate | | [Documentation] -| | ... | Additional Statistics Action for bash command "perf stat". +| | ... | Send traffic at maximum rate. +| | ... | Call \${resetter} (if defined) to reset DUT state before each trial. +| | ... | Fail if no packets were forwarded. | | -| | Run Keyword If | ${extended_debug}==${True} -| | ... | Perf Stat On All DUTs | ${nodes} | cpu_list=${cpu_alloc_str} - -| Additional Statistics Action For clear-show-runtime-with-traffic -| | [Documentation] -| | ... | Additional Statistics Action for clear and show runtime counters with -| | ... | running traffic. +| | ... | *Test (or broader scope) variables read:* +| | ... | - traffic_profile - Name of module defining traffic for measurements. +| | ... | Type: string +| | ... | - frame_size - L2 Frame Size [B] or IMIX string. Type: integer or +| | ... | string +| | ... | - max_rate - Calculated maximal transmit rate [tps]. +| | ... | Type: float +| | ... | - transaction_type - String identifier to determine how to count +| | ... | transactions. Default is "packet". | | -| | Clear and show runtime counters with running traffic -| | ... | ${trial_duration} | ${rate} -| | ... | ${frame_size} | ${traffic_profile} | ${traffic_directions} -| | ... | ${tx_port} | ${rx_port} - -| Additional Statistics Action For noop -| | [Documentation] -| | ... | Additional Statistics Action for no operation. +| | ... | *Example:* | | -| | No operation +| | ... | \| Traffic should pass with maximum rate \| +| | +| | ${max_rate} = | Get Max Rate +| | ${transaction_type} = | Get Transaction Type +| | ${trial_duration} = | Get Mrr Trial Duration +| | ${trial_multiplicity} = | Get Mrr Trial Multiplicity +| | ${use_latency} = | Get Use Latency +| | # The following also sets \${rate_for_teardown} +| | ${results} = | Send traffic at specified rate +| | ... | rate=${max_rate} +| | ... | trial_duration=${trial_duration} +| | ... | trial_multiplicity=${trial_multiplicity} +| | ... | use_latency=${use_latency} +| | ... | duration_limit=${0.0} +| | ${unit} = | Set Variable If | """_cps""" in """${transaction_type}""" +| | ... | estimated connections per second | packets per second +| | Set Test Message | ${\n}Maximum Receive Rate trial results +| | Set Test Message | in ${unit}: ${results} +| | ... | append=yes +| | Fail if no traffic forwarded diff --git a/resources/libraries/robot/performance/performance_vars.robot b/resources/libraries/robot/performance/performance_vars.robot new file mode 100644 index 0000000000..130237ed82 --- /dev/null +++ b/resources/libraries/robot/performance/performance_vars.robot @@ -0,0 +1,522 @@ +# Copyright (c) 2020 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +| Documentation | Performance suite keywords - Handling of various values +| ... | to allow autodetection, computation and overriding by suite variables. +| Library | Collections +| Variables | ${CURDIR}/../../python/Constants.py + +*** Variables *** +| ${extended_debug}= | ${EXTENDED_DEBUG} + +*** Keywords *** +| Get Disable Latency +| | [Documentation] +| | ... | If Get Use Latency returns true, return false. +| | ... | Otherwise return value of \${disable_latency} variable, +| | ... | or \${False} if not defined. +| | +| | ... | The return value controls whether latency trials in NDRPDR tests +| | ... | are executed. For example, ASTF tests do not support latency +| | ... | measurements yet, so executing the trials just wastes time. +| | ... | Return type: bool. +| | +| | ... | *Example:* +| | +| | ... | \| \${disable_latency} = \| Get Disable Latency \| +| | +| | ${use_latency} = | Get Use Latency +| | Return From Keyword If | ${use_latency} | ${False} +| | ${disable_latency} = | Get Variable Value | \${disable_latency} | ${False} +| | Return From Keyword | ${disable_latency} + +| Get Max Rate +| | [Documentation] +| | ... | Return value of \${max_rate} variable, +| | ... | fail if it is not defined. +| | ... | Call this just before calling a Python keyword, +| | ... | as those have restricted access to Robot variables. +| | +| | ... | The return value controls the maximal unidirectional packet rate. +| | ... | The value is also usable for minimal TPS value for ASTF tests. +| | ... | Return type: float. +| | +| | ... | *Example:* +| | +| | ... | \| \${max_rate} = \| Get Max Rate \| +| | +| | ${max_rate} = | Get Variable Value | \${max_rate} | ${0.0} +| | Return From Keyword If | ${max_rate} | ${max_rate} +| | Fail | \${max_rate} is not defined. Call Set Max Rate And Jumbo keyword. + +| Get Min Rate +| | [Documentation] +| | ... | Return a hardcoded value. This is an abstraction, useful in case +| | ... | we start allowing various other overrides or computations. +| | ... | Call this just before calling a Python keyword, +| | ... | as those have restricted access to Robot variables. +| | +| | ... | The return value controls the minimal unidirectional packet rate. +| | ... | The value is also usable for minimal TPS value for ASTF tests. +| | ... | The current value is the smallest one permitted +| | ... | by STL profiles with latency streams. +| | ... | Return type: float. +| | +| | ... | *Example:* +| | +| | ... | \| \${min_rate} = \| Get Min Rate \| +| | +| | Return From Keyword | ${9001.0} + +| Get Mrr Trial Duration +| | [Documentation] +| | ... | Return value from Constants. This is an abstraction, useful in case +| | ... | we start allowing various other overrides or computations. +| | ... | Call this just before calling a Python keyword, +| | ... | as those have restricted access to Robot variables. +| | +| | ... | The return value controls the duration of main trial measurement +| | ... | for MRR type tests. +| | ... | Return type: float. +| | +| | ... | *Example:* +| | +| | ... | \| \${mrr_trial_duration} = \| Get Mrr Trial Duration \| +| | +| | Return From Keyword | ${PERF_TRIAL_DURATION} + +| Get Mrr Trial Multiplicity +| | [Documentation] +| | ... | Return value from Constants. This is an abstraction, useful in case +| | ... | we start allowing various other overrides or computations. +| | ... | Call this just before calling a Python keyword, +| | ... | as those have restricted access to Robot variables. +| | +| | ... | The return value controls the number of main trial measurement +| | ... | for (B)MRR type tests. +| | ... | Return type: integer. +| | +| | ... | *Example:* +| | +| | ... | \| \${mrr_trial_multiplicity} = \| Get Mrr Trial Multiplicity \| +| | +| | Return From Keyword | ${PERF_TRIAL_MULTIPLICITY} + +| Get Packet Loss Ratio +| | [Documentation] +| | ... | Return a hardcoded value. This is an abstraction, useful in case +| | ... | we start allowing various other overrides or computations. +| | ... | Call this just before calling a Python keyword, +| | ... | as those have restricted access to Robot variables. +| | +| | ... | The return value controls the default packet loss ration for PDR +| | ... | in NDRPDR tests. Some other usages of MLRsearch (e.g. reconf tests) +| | ... | may use a different value. +| | ... | Return type: float. +| | +| | ... | *Example:* +| | +| | ... | \| \${packet_loss_ratio} = \| Get Packet Loss Ratio \| +| | +| | Return From Keyword | ${0.005} + +| Get Packets Per Transaction Aggregated +| | [Documentation] +| | ... | Return value of \${packets_per_transaction_aggregated}; +| | ... | if not defined, assume traffic is symmetric and compute +| | ... | from unidirectional values. +| | +| | ... | The return value is used when reporting PPS values from TPS found +| | ... | by some search (e.g. NDRPDR). +| | ... | Return type: integer. +| | +| | ... | *Example:* +| | +| | ... | \| \${ppta} = \| Get Packets Per Transaction Aggregated \| +| | +| | ${ppta} = | Get Variable Value | \${packets_per_transaction_aggregated} +| | ... | ${0} +| | Return From Keyword If | "${ppta}" != "0" | ${ppta} +| | # TODO: Insert TCP computation from packet size here. +| | ${pptad} = | Get Packets Per Transaction And Direction +| | ${traffic_directions} = | Get Traffic Directions +| | # We do not support ASTF profiles with multiple transactions, yet. +| | ${ppta} = | Evaluate | ${pptad} * ${traffic_directions} +| | Return From Keyword | ${ppta} + +| Get Packets Per Transaction And Direction +| | [Documentation] +| | ... | Return value of \${packets_per_transaction_and_direction}, +| | ... | or ${1} if not defined. +| | +| | ... | The return value is used when computing max rate (TPS), +| | ... | so for asymmetric transaction use the more numerous direction. +| | ... | Return type: integer. +| | +| | ... | *Example:* +| | +| | ... | \| \${pptad} = \| Get Packets Per Transaction And Direction \| +| | +| | ${pptad} = | Get Variable Value | \${packets_per_transaction_and_direction} +| | ... | ${1} +| | Return From Keyword | ${pptad} + +| Get Ramp Up Duration +| | [Documentation] +| | ... | Return value of \${ramp_up_duration}, +| | ... | or ${0.0} if not defined. +| | +| | ... | The return value determines the required duration of ramp-up phase. +| | ... | Typically used to prepare a specific state on DUT. +| | ... | If the value is zero, ramp-up phase is skipped. +| | ... | Return type: float. +| | +| | ... | *Example:* +| | +| | ... | \| \${ramp_up_duration} = \| Get Ramp Up Duration \| +| | +| | ${ramp_up_duration} = | Get Variable Value | \${ramp_up_duration} | ${0.0} +| | Return From Keyword | ${ramp_up_duration} + +| Get Ramp Up Rate +| | [Documentation] +| | ... | Return value of \${ramp_up_rate}, +| | ... | if not defined return \${max_rate}. +| | +| | ... | The return value determines the rate for ramp-up phase. +| | ... | Typically used to limit the rate when max rate +| | ... | would lose packets in the ramp up phase, thus not setting +| | ... | the DUT state correctly. +| | ... | Return type: float. +| | +| | ... | *Example:* +| | +| | ... | \| \${ramp_up_rate} = \| Get Ramp Up Rate \| +| | +| | ${ramp_up_rate} = | Get Variable Value | \${ramp_up_rate} | ${0.0} +| | Return From Keyword If | ${ramp_up_rate} | ${ramp_up_rate} +| | Run Keyword And Return | Get Max Rate + +| Get Rate For Teardown +| | [Documentation] +| | ... | Return value of \${rate_for_teardown}, +| | ... | if not defined (or zero) return the min rate. +| | +| | ... | The return value determines the rate for teardown trial, +| | ... | that is executed if a perf test fails. +| | ... | The \${rate_for_teardown} is usually not defined in suite, +| | ... | but search keywords set it in places where failure can occur, +| | ... | so the trial is done at the rate interesting for the failure. +| | ... | Return type: float. +| | +| | ... | *Example:* +| | +| | ... | \| \${rate_for_teardown} = \| Get Rate For Teardown \| +| | +| | ${rate_for_teardown} = | Get Variable Value | \${rate_for_teardown} | ${0.0} +| | Return From Keyword If | ${rate_for_teardown} | ${rate_for_teardown} +| | Run Keyword And Return | Get Min Rate + +| Get Resetter +| | [Documentation] +| | ... | Return value of \${resetter} variable, +| | ... | or \${None} if not defined. +| | +| | ... | If not \${None}, the returned value is callable. +| | ... | Its use is to reset DUT to initial conditions, +| | ... | for example to remove NAT sessions created in the previous trial. +| | +| | ... | *Example:* +| | +| | ... | \| \${resetter} = \| Get Resetter \| +| | +| | ${resetter} = | Get Variable Value | \${resetter} | ${None} +| | Return From Keyword | ${resetter} + +| Get Runtime Duration +| | [Documentation] +| | ... | Return value of \${runtime_duration} variable, +| | ... | if not defined return ${1.0}. +| | +| | ... | The return value controls the duration of runtime trial, +| | ... | which also acts as a warmup. Usually one second is enough, +| | ... | but some suites need longer time to set up state on DUT. +| | ... | Return type: float. +| | +| | ... | *Example:* +| | +| | ... | \| \${runtime_duration} = \| Get Runtime Duration \| +| | +| | ${runtime_duration} = | Get Variable Value | \${runtime_duration} | ${1.0} +| | Return From Keyword | ${runtime_duration} + +| Get Runtime Rate +| | [Documentation] +| | ... | Return value of \${runtime_rate} variable, +| | ... | if not defined return the max rate. +| | +| | ... | The return value controls the rate (TPS unidir) of runtime trial, +| | ... | which also acts as a warmup. No plans to ever use a different rate, +| | ... | but keywords look better if access to such values is uniform. +| | ... | Return type: float. +| | +| | ... | *Example:* +| | +| | ... | \| \${runtime_rate} = \| Get Runtime Rate \| +| | +| | ${runtime_rate} = | Get Variable Value | \${runtime_rate} | ${0.0} +| | Return From Keyword If | ${runtime_rate} | ${runtime_rate} +| | Run Keyword And Return | Get Max Rate + +| Get Traffic Directions +| | [Documentation] +| | ... | Return value of \${traffic_directions}, +| | ... | or ${2} if not defined. +| | +| | ... | The return value used when parsing for measurement results. +| | ... | This needs to be known already in profile driver, +| | ... | as bidirectional parsing may fail on unidirectional traffic. +| | ... | Return type: integer. +| | +| | ... | *Example:* +| | +| | ... | \| \${traffic_directions} = \| Get Traffic Directions \| +| | +| | ${traffic_directions} = | Get Variable Value | \${traffic_directions} | ${2} +| | Return From Keyword | ${traffic_directions} + +| Get Transaction Duration +| | [Documentation] +| | ... | Return value of \${transaction_duration} variable, +| | ... | or \${0.0} if not defined. +| | +| | ... | The return value is the expected duration of single (ASTF) transaction +| | ... | if it is not negligible for overall trial duration computation. +| | ... | Most tests use very short transactions (without explicit delays), +| | ... | so the zero default works (and suite saves one line +| | ... | of Variables table). +| | ... | Return type: float. +| | +| | ... | *Example:* +| | +| | ... | \| \${transaction_duration} = \| Get Transaction Duration \| +| | +| | ${transaction_duration} = | Get Variable Value | \${transaction_duration} +| | ... | ${0.0} +| | Return From Keyword | ${transaction_duration} + +| Get Transaction Scale +| | [Documentation] +| | ... | Return value of \${transaction_scale} variable, +| | ... | or \${0} if not defined. +| | +| | ... | Zero return value means the number of transactions is not limited, +| | ... | which is true for most STL TRex profiles (transaction is a packet). +| | ... | Nonzero return value means the number of transactions is fixed, +| | ... | for example in stateful NAT scale tests. +| | ... | Return type: integer. +| | +| | ... | *Example:* +| | +| | ... | \| \${transaction_scale} = \| Get Transaction Scale \| +| | +| | ${transaction_scale} = | Get Variable Value | \${transaction_scale} | ${0} +| | Return From Keyword | ${transaction_scale} + +| Get Transaction Type +| | [Documentation] +| | ... | Return value of \${transaction_type} variable, +| | ... | or "packet" if not defined. +| | +| | ... | The return value describes the type of transaction +| | ... | the test is executed. For example "packet" means a transaction +| | ... | is just a single packet. For more sophisticated transactions, +| | ... | the logic to determine the number of passed transactions +| | ... | is different from merely counting the packets received from DUT. +| | ... | Return type: string. +| | +| | ... | *Example:* +| | +| | ... | \| \${transaction_type} = \| Get Transaction Type \| +| | +| | ${transaction_type} = | Get Variable Value | \${transaction_type} | packet +| | Return From Keyword | ${transaction_type} + +| Get Use Latency +| | [Documentation] +| | ... | Return value of \${use_latency} variable, +| | ... | if not defined return the value from Constants. +| | +| | ... | The return value controls whether latency streams are active +| | ... | during the main search. +| | ... | Return type: bool. +| | +| | ... | *Example:* +| | +| | ... | \| \${use_latency} = \| Get Use Latency \| +| | +| | ${use_latency} = | Get Variable Value | ${use_latency} | ${PERF_USE_LATENCY} +| | Return From Keyword | ${use_latency} + +| Set Jumbo +| | [Documentation] +| | ... | For jumbo frames detection, the maximal packet size is relevant, +| | ... | encapsulation overhead (if any) has effect. +| | +| | ... | This keyword computes jumbo boolean (some suites need that for +| | ... | configuration decisions). +| | ... | To streamline suite autogeneration, both input and output values +| | ... | are communicated as test (or broader scope) variables, +| | ... | instead of explicit arguments and return values. +| | +| | ... | *Test (or broader scope) variables read:* +| | ... | - overhead - Overhead in bytes; default value: 0. Type: integer +| | ... | - frame_size - L2 Frame Size [B] or IMIX string. Type: integer or +| | ... | string +| | +| | ... | *Test variables set:* +| | ... | - jumbo - Jumbo boolean, true if jumbo packet support has to be +| | ... | enabled. Type: boolean +| | +| | ... | *Example:* +| | +| | ... | \| Set Jumbo \| +| | +| | # Already called by Set Max Rate And Jumbo, but some suites (e.g. device) +| | # are calling this directly. +| | Set Numeric Frame Sizes +| | ${jumbo} = | Set Variable If | ${max_frame_size} < 1522 +| | ... | ${False} | ${True} +| | Set Test Variable | \${jumbo} + +| Set Max Rate And Jumbo +| | [Documentation] +| | ... | Input framesize can be either integer in case of a single packet +| | ... | in stream, or IMIX string defining mix of packets. +| | ... | For jumbo frames detection, the maximal packet size is relevant. +| | ... | For maximal transmit rate, the average packet size is relevant. +| | ... | In both cases, encapsulation overhead (if any) has effect. +| | ... | The maximal rate is computed from NIC name. +| | ... | The implementation works by mapping from exact +| | ... | whitelisted NIC names. +| | ... | The mapping is hardcoded in nic_limits.yaml +| | ... | TODO: Make the mapping from NIC names case insensistive. +| | +| | ... | This keyword computes maximal unidirectional transmit rate +| | ... | and jumbo boolean (some suites need that for configuration decisions). +| | ... | To streamline suite autogeneration, both input and output values +| | ... | are communicated as test (or broader scope) variables, +| | ... | instead of explicit arguments and return values. +| | +| | ... | If this keyword detects the test is interested in (unidirectional) +| | ... | transactons per second maximal rate (tps), that is returned (not pps). +| | +| | ... | *Test (or broader scope) variables read:* +| | ... | - nic_name - Name of bottleneck NIC. Type: string +| | ... | - overhead - Overhead in bytes; default value: 0. Type: integer +| | ... | - frame_size - L2 Frame Size [B] or IMIX string. Type: integer or +| | ... | string +| | ... | - packets_per_transaction_and_direction - Pps-tps conversion. +| | ... | Optional, default 1. +| | +| | ... | *Test variables set:* +| | ... | - max_rate - Calculated unidirectional maximal transmit rate [pps]. +| | ... | This never exceeds bandwidth on TG-DUT nor DUT-DUT links. +| | ... | Type: float +| | ... | - jumbo - Jumbo boolean, true if jumbo packet support has to be +| | ... | enabled. Type: boolean +| | ... | avg_frame_size - Average frame size including overhead. Type: float +| | ... | max_frame_size - Maximal frame size including overhead. Type: float +| | +| | ... | *Example:* +| | +| | ... | \| Set Max Rate And Jumbo \| +| | +| | # TODO: Re-check overhead values in suites with both traffics encapsulated. +| | # TODO: Improve layered setup to detect encap/decap and update overhead. +| | ${pps_limit} = | Get From Dictionary +| | ... | ${NIC_NAME_TO_PPS_LIMIT} | ${nic_name} +| | ${bps_limit} = | Get From Dictionary +| | ... | ${NIC_NAME_TO_BPS_LIMIT} | ${nic_name} +| | Set Numeric Frame Sizes +| | ${rate} = | Evaluate | ${bps_limit} / ((${avg_frame_size} + 20.0) * 8) +| | ${max_rate} = | Set Variable If | ${rate} > ${pps_limit} +| | ... | ${pps_limit} | ${rate} +| | ${pptad} = | Get Packets Per Transaction And Direction +| | ${max_rate} = | Evaluate | ${max_rate} / ${pptad} +| | Set Test Variable | \${max_rate} +| | Set Jumbo + +| Set Numeric Frame Sizes +| | [Documentation] +| | ... | Framesize can be either integer in case of a single packet +| | ... | in stream, or set of packets in case of IMIX type or simmilar. +| | ... | For jumbo decisions, we need a numeric size of the biggest packet. +| | ... | For max rate decisions, we need a numeric average packet size. +| | ... | This keyword computes both and sets them as test variables. +| | +| | ... | Each suite sets a value named \${overhead}, +| | ... | which describes by how many bytes the frames on DUT-DUT link +| | ... | are larger (due to encapsulation) than those +| | ... | on the primary TG-DUT link. But for some suites that value +| | ... | can be negaive (if TG-DUT is encapsulated more heavily). +| | ... | For calculations in this keyword, we need largest sizes +| | ... | across links, so zero is used if \${overhead} is negative. +| | +| | ... | *Test variables read:* +| | ... | - frame_size - Framesize. Type: integer or string +| | ... | - overhead - Overhead in bytes; default value: ${0}. Type: integer +| | +| | ... | *Test variables set* +| | ... | avg_frame_size - Average frame size including overhead. Type: float +| | ... | max_frame_size - Maximal frame size including overhead. Type: float +| | +| | ... | *Example:* +| | +| | ... | \| Set Numeric Frame Sizes \| +| | +| | ${max_overhead} = | Set Variable If | ${overhead} >= 0 | ${overhead} | ${0} +| | ${bare_avg_frame_size} = | Run Keyword If | '${frame_size}' == 'IMIX_v4_1' +| | ... | Set Variable | ${353.83333} +| | ... | ELSE +| | ... | Convert To Number | ${frame_size} +| | ${avg_frame_size} = | Evaluate | $bare_avg_frame_size + $max_overhead +| | Set Test Variable | \${avg_frame_size} +| | ${bare_max_frame_size} = | Run Keyword If | '${frame_size}' == 'IMIX_v4_1' +| | ... | Set Variable | ${1518} +| | ... | ELSE +| | ... | Convert To Number | ${frame_size} +| | ${max_frame_size} = | Evaluate | $bare_max_frame_size + $max_overhead +| | Set Test Variable | ${max_frame_size} + +| Set Rates For Policer +| | [Documentation] +| | ... | Policer tests need these values, +| | ... | currently computed from \${avg_frame_size}. +| | ... | TODO: Verify the units match and computation is correct. +| | +| | ... | *Test (or broader scope) variables read:* +| | ... | - avg_frame_size - Average L2 Frame Size [B]. Type: float +| | ... | Set by Set Max Rate And Jumbo keyword. +| | +| | ... | *Test variables set:* +| | ... | - eb - Excess burst rate for policer. Type: float +| | ... | - cb - Committed burst rate for policer. Type: float +| | +| | ... | *Example:* +| | +| | ... | \| Set Rates For Policer \| +| | +| | Set Test Variable | \${eb} | ${avg_frame_size} +| | Set Test Variable | \${cb} | ${avg_frame_size} diff --git a/resources/libraries/robot/shared/default.robot b/resources/libraries/robot/shared/default.robot index f060032b0c..dedbdcd2aa 100644 --- a/resources/libraries/robot/shared/default.robot +++ b/resources/libraries/robot/shared/default.robot @@ -58,7 +58,6 @@ | Resource | resources/libraries/robot/overlay/lisp.robot | Resource | resources/libraries/robot/overlay/lispgpe.robot | Resource | resources/libraries/robot/overlay/lisp_api.robot -| Resource | resources/libraries/robot/performance/performance_limits.robot | Resource | resources/libraries/robot/performance/performance_utils.robot | Resource | resources/libraries/robot/shared/interfaces.robot | Resource | resources/libraries/robot/shared/container.robot @@ -74,6 +73,26 @@ | ${cpu_alloc_str}= | ${0} *** Keywords *** +# TODO: Sort keywords alphabetically. + +| Call Resetter +| | [Documentation] +| | ... | Check for a presence of test variable \${resetter}. +| | ... | If it exists (and not None), call the resetter (as a Python callable). +| | ... | This is usually used to reset any state on DUT before next trial. +| | +| | ... | TODO: Move to a more specific library if needed. +| | +| | ... | *Example:* +| | +| | ... | \| Call Resetter \| +| | +| | ${resetter} = | Get Resetter +| | # See http://robotframework.org/robotframework/3.1.2/libraries/BuiltIn.html +| | # #Evaluating%20expressions for $variable (without braces) syntax. +| | # Parens are there to perform the call. +| | Run Keyword If | $resetter | Evaluate | $resetter() + | Configure crypto device on all DUTs | | [Documentation] | Verify if Crypto QAT device virtual functions are | | ... | initialized on all DUTs. If parameter force_init is set to True, then diff --git a/resources/libraries/robot/shared/test_teardown.robot b/resources/libraries/robot/shared/test_teardown.robot index 93c4574c83..09ebd9d01e 100644 --- a/resources/libraries/robot/shared/test_teardown.robot +++ b/resources/libraries/robot/shared/test_teardown.robot @@ -64,11 +64,21 @@ | Additional Test Tear Down Action For performance | | [Documentation] | | ... | Additional teardown for tests which uses performance measurement. -| | -| | Run Keyword If Test Failed -| | ... | Send traffic at specified rate | ${1.0} | 10000 -| | ... | ${frame_size} | ${traffic_profile} | trial_multiplicity=${1} -| | ... | extended_debug=${True} +| | ... | Optionally, call \${resetter} (if defined) to reset DUT state. +| | +| | ... | TODO: Document what test variables are required or optional. +| | +| | Run Keyword If Test Passed | Return From Keyword +| | ${use_latency} = | Get Use Latency +| | ${rate_for_teardown} = | Get Rate For Teardown +| | Call Resetter +| | Set Test Variable | \${extended_debug} | ${True} +| | Send traffic at specified rate +| | ... | trial_duration=${1.0} +| | ... | rate=${rate_for_teardown} +| | ... | trial_multiplicity=${1} +| | ... | use_latency=${use_latency} +| | ... | duration_limit=${1.0} | Additional Test Tear Down Action For packet_trace | | [Documentation] |