diff options
Diffstat (limited to 'scripts/automation/trex_control_plane')
5 files changed, 218 insertions, 24 deletions
diff --git a/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py index f044f623..c8827afe 100755 --- a/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py +++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py @@ -535,7 +535,7 @@ class CTRexClient(object): finally: self.prompt_verbose_data() - def sample_until_condition (self, condition_func, time_between_samples = 5): + def sample_until_condition (self, condition_func, time_between_samples = 1): """ Automatically sets ongoing sampling of TRex data, with sampling rate described by time_between_samples. @@ -549,7 +549,7 @@ class CTRexClient(object): time_between_samples : int determines the time between each sample of the server - default value : **5** + default value : **1** :return: the first result object (see :class:`CTRexResult` for further details) of the TRex run on which the condition has been met. @@ -579,7 +579,7 @@ class CTRexClient(object): # this could come from provided method 'condition_func' raise - def sample_to_run_finish (self, time_between_samples = 5): + def sample_to_run_finish (self, time_between_samples = 1): """ Automatically sets automatically sampling of TRex data with sampling rate described by time_between_samples until TRex run finished. @@ -587,7 +587,7 @@ class CTRexClient(object): time_between_samples : int determines the time between each sample of the server - default value : **5** + default value : **1** :return: the latest result object (see :class:`CTRexResult` for further details) with sampled data. @@ -609,7 +609,7 @@ class CTRexClient(object): results = self.get_result_obj() return results - def sample_x_seconds (self, sample_time, time_between_samples = 5): + def sample_x_seconds (self, sample_time, time_between_samples = 1): """ Automatically sets ongoing sampling of TRex data for sample_time seconds, with sampling rate described by time_between_samples. Does not stop the TRex afterwards! @@ -623,7 +623,7 @@ class CTRexClient(object): time_between_samples : int determines the time between each sample of the server - default value : **5** + default value : **1** :return: the first result object (see :class:`CTRexResult` for further details) of the TRex run after given sample_time. @@ -1271,7 +1271,7 @@ class CTRexResult(object): .. tip:: | Use '.' to enter one level deeper in dictionary hierarchy. | Use '[i]' to access the i'th indexed object of an array. - tree_path_to_key : regex + regex : regex apply a regex to filter results out from a multiple results set. Filter applies only on keys of dictionary type. @@ -1299,7 +1299,7 @@ class CTRexResult(object): .. tip:: | Use '.' to enter one level deeper in dictionary hierarchy. | Use '[i]' to access the i'th indexed object of an array. - tree_path_to_key : regex + regex : regex apply a regex to filter results out from a multiple results set. Filter applies only on keys of dictionary type. @@ -1352,7 +1352,7 @@ class CTRexResult(object): if not len(self._history): return -1 - return len(self.__get_value_by_path(self._history[-1], 'trex-global.data', 'opackets-\d+')) + return len(self.get_last_value('trex-global.data', 'opackets-\d+')) def update_result_data (self, latest_dump): @@ -1383,6 +1383,7 @@ class CTRexResult(object): # check for up to 2% change between expected and actual if (self._current_tx_rate['m_tx_bps'] > 0.98 * self._expected_tx_rate['m_tx_expected_bps']): self._done_warmup = True + latest_dump['warmup_barrier'] = True # handle latency data if self.latency_checked: @@ -1427,12 +1428,12 @@ class CTRexResult(object): for i, p in re.findall(r'(\d+)|([\w|-]+)', tree_path): dct = dct[p or int(i)] if regex is not None and isinstance(dct, dict): - res = {} - for key,val in dct.items(): - match = re.match(regex, key) - if match: - res[key]=val - return res + res = {} + for key,val in dct.items(): + match = re.match(regex, key) + if match: + res[key]=val + return res else: return dct except (KeyError, TypeError): @@ -1499,6 +1500,42 @@ class CTRexResult(object): return result + # history iterator after warmup period + def _get_steady_state_history_iterator(self): + if not self.is_done_warmup(): + raise Exception('Warm-up period not finished') + for index, res in enumerate(self._history): + if 'warmup_barrier' in res: + for steady_state_index in range(index, max(index, len(self._history) - 1)): + yield self._history[steady_state_index] + return + for index in range(len(self._history) - 1): + yield self._history[index] + + + def get_avg_steady_state_value(self, tree_path_to_key): + ''' + Gets average value after warmup period. + For example: <result object>.get_avg_steady_state_value('trex-global.data.m_tx_bps') + Usually more accurate than latest history value. + + :parameters: + tree_path_to_key : str + defines a path to desired data. + + :return: + average value at steady state + + :raises: + Exception in case steady state period was not reached or tree_path_to_key was not found in result. + ''' + values_arr = [self.__get_value_by_path(res, tree_path_to_key) for res in self._get_steady_state_history_iterator()] + values_arr = list(filter(lambda x: x is not None, values_arr)) + if not values_arr: + raise Exception('All the keys are None, probably wrong tree_path_to_key: %s' % tree_path_to_key) + return sum(values_arr) / float(len(values_arr)) + + if __name__ == "__main__": c = CTRexClient('127.0.0.1') print('restarting daemon') diff --git a/scripts/automation/trex_control_plane/stl/console/trex_tui.py b/scripts/automation/trex_control_plane/stl/console/trex_tui.py index d3be4435..a2ffcad6 100644 --- a/scripts/automation/trex_control_plane/stl/console/trex_tui.py +++ b/scripts/automation/trex_control_plane/stl/console/trex_tui.py @@ -210,7 +210,7 @@ class TrexTUILatencyStats(TrexTUIPanel): super(TrexTUILatencyStats, self).__init__(mng, "lstats") self.key_actions = OrderedDict() self.key_actions['c'] = {'action': self.action_clear, 'legend': 'clear', 'show': True} - self.key_actions['t'] = {'action': self.action_toggle_histogram, 'legend': 'toggle histogram', 'show': True} + self.key_actions['h'] = {'action': self.action_toggle_histogram, 'legend': 'histogram toggle', 'show': True} self.is_histogram = False @@ -238,6 +238,23 @@ class TrexTUILatencyStats(TrexTUIPanel): self.stateless_client.latency_stats.clear_stats() return "" + +# utilization stats +class TrexTUIUtilizationStats(TrexTUIPanel): + def __init__ (self, mng): + super(TrexTUIUtilizationStats, self).__init__(mng, "ustats") + self.key_actions = {} + + def show (self): + stats = self.stateless_client._get_formatted_stats(port_id_list = None, stats_mask = trex_stl_stats.UT_COMPAT) + # print stats to screen + for stat_type, stat_data in stats.items(): + text_tables.print_table_with_header(stat_data.text_table, stat_type) + + def get_key_actions (self): + return self.key_actions + + # log class TrexTUILog(): def __init__ (self): @@ -270,12 +287,14 @@ class TrexTUIPanelManager(): self.panels['dashboard'] = TrexTUIDashBoard(self) self.panels['sstats'] = TrexTUIStreamsStats(self) self.panels['lstats'] = TrexTUILatencyStats(self) + self.panels['ustats'] = TrexTUIUtilizationStats(self) self.key_actions = OrderedDict() self.key_actions['q'] = {'action': self.action_quit, 'legend': 'quit', 'show': True} self.key_actions['d'] = {'action': self.action_show_dash, 'legend': 'dashboard', 'show': True} self.key_actions['s'] = {'action': self.action_show_sstats, 'legend': 'streams', 'show': True} self.key_actions['l'] = {'action': self.action_show_lstats, 'legend': 'latency', 'show': True} + self.key_actions['u'] = {'action': self.action_show_ustats, 'legend': 'util', 'show': True} # start with dashboard self.main_panel = self.panels['dashboard'] @@ -392,6 +411,11 @@ class TrexTUIPanelManager(): self.init(self.show_log) return "" + def action_show_ustats(self): + self.main_panel = self.panels['ustats'] + self.init(self.show_log) + return "" + # shows a textual top style window class TrexTUI(): diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py index c1833754..dc0035b3 100755 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py @@ -555,14 +555,17 @@ class STLClient(object): self.latency_stats = trex_stl_stats.CLatencyStats(self.ports) + self.util_stats = trex_stl_stats.CUtilStats(self) + self.stats_generator = trex_stl_stats.CTRexInfoGenerator(self.global_stats, self.ports, self.flow_stats, self.latency_stats, + self.util_stats, self.async_client.monitor) - + ############# private functions - used by the class itself ########### @@ -1664,6 +1667,23 @@ class STLClient(object): if not rc: raise STLError(rc) + @__api_check(True) + def get_util_stats(self): + """ + Get utilization stats: + History of TRex CPU utilization per thread (list of lists) + MBUFs memory consumption per CPU socket. + + :parameters: + None + + :raises: + + :exc:`STLError` + + """ + self.logger.pre_cmd('Getting Utilization stats') + return self.util_stats.get_stats() + @__api_check(True) def reset(self, ports = None): @@ -1878,8 +1898,6 @@ class STLClient(object): raise STLError(rc) - - @__api_check(True) def stop (self, ports = None, rx_delay_ms = 10): """ diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py index 3effa1f0..678adb4e 100644 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py @@ -11,7 +11,6 @@ import datetime import time import re import math -import copy import threading import pprint @@ -22,13 +21,16 @@ PORT_STATUS = 'ps' STREAMS_STATS = 's' LATENCY_STATS = 'ls' LATENCY_HISTOGRAM = 'lh' +CPU_STATS = 'c' +MBUF_STATS = 'm' -ALL_STATS_OPTS = [GLOBAL_STATS, PORT_STATS, PORT_STATUS, STREAMS_STATS, LATENCY_STATS, PORT_GRAPH, LATENCY_HISTOGRAM] +ALL_STATS_OPTS = [GLOBAL_STATS, PORT_STATS, PORT_STATUS, STREAMS_STATS, LATENCY_STATS, PORT_GRAPH, LATENCY_HISTOGRAM, CPU_STATS, MBUF_STATS] COMPACT = [GLOBAL_STATS, PORT_STATS] GRAPH_PORT_COMPACT = [GLOBAL_STATS, PORT_GRAPH] SS_COMPAT = [GLOBAL_STATS, STREAMS_STATS] # stream stats LS_COMPAT = [GLOBAL_STATS, LATENCY_STATS] # latency stats LH_COMPAT = [GLOBAL_STATS, LATENCY_HISTOGRAM] # latency histogram +UT_COMPAT = [GLOBAL_STATS, CPU_STATS, MBUF_STATS] # utilization ExportableStats = namedtuple('ExportableStats', ['raw_data', 'text_table']) @@ -104,6 +106,13 @@ def calculate_diff_raw (samples): return total +get_number_of_bytes_cache = {} +# get number of bytes: '64b'->64, '9kb'->9000 etc. +def get_number_of_bytes(val): + if val not in get_number_of_bytes_cache: + get_number_of_bytes_cache[val] = int(val[:-1].replace('k', '000')) + return get_number_of_bytes_cache[val] + # a simple object to keep a watch over a field class WatchedField(object): @@ -138,11 +147,12 @@ class CTRexInfoGenerator(object): STLClient and the ports. """ - def __init__(self, global_stats_ref, ports_dict_ref, rx_stats_ref, latency_stats_ref, async_monitor): + def __init__(self, global_stats_ref, ports_dict_ref, rx_stats_ref, latency_stats_ref, util_stats_ref, async_monitor): self._global_stats = global_stats_ref self._ports_dict = ports_dict_ref self._rx_stats_ref = rx_stats_ref self._latency_stats_ref = latency_stats_ref + self._util_stats_ref = util_stats_ref self._async_monitor = async_monitor def generate_single_statistic(self, port_id_list, statistic_type): @@ -167,6 +177,12 @@ class CTRexInfoGenerator(object): elif statistic_type == LATENCY_HISTOGRAM: return self._generate_latency_histogram() + elif statistic_type == CPU_STATS: + return self._generate_cpu_util_stats() + + elif statistic_type == MBUF_STATS: + return self._generate_mbuf_util_stats() + else: # ignore by returning empty object return {} @@ -404,6 +420,73 @@ class CTRexInfoGenerator(object): stats_table.header(header) return {"latency_histogram": ExportableStats(None, stats_table)} + def _generate_cpu_util_stats(self): + util_stats = self._util_stats_ref.get_stats(use_1sec_cache = True) + stats_table = text_tables.TRexTextTable() + if util_stats: + if 'cpu' not in util_stats: + raise Exception("Excepting 'cpu' section in stats %s" % util_stats) + cpu_stats = util_stats['cpu'] + hist_len = len(cpu_stats[0]) + avg_len = min(5, hist_len) + show_len = min(15, hist_len) + stats_table.header(['Thread', 'Avg', 'Latest'] + list(range(-1, 0 - show_len, -1))) + stats_table.set_cols_align(['l'] + ['r'] * (show_len + 1)) + stats_table.set_cols_width([8, 3, 6] + [3] * (show_len - 1)) + stats_table.set_cols_dtype(['t'] * (show_len + 2)) + for i in range(min(14, len(cpu_stats))): + avg = int(round(sum(cpu_stats[i][:avg_len]) / avg_len)) + stats_table.add_row([i, avg] + cpu_stats[i][:show_len]) + else: + stats_table.add_row(['No Data.']) + return {'cpu_util(%)': ExportableStats(None, stats_table)} + + def _generate_mbuf_util_stats(self): + util_stats = self._util_stats_ref.get_stats(use_1sec_cache = True) + stats_table = text_tables.TRexTextTable() + if util_stats: + if 'mbuf_stats' not in util_stats: + raise Exception("Excepting 'mbuf_stats' section in stats %s" % util_stats) + mbuf_stats = util_stats['mbuf_stats'] + for mbufs_per_socket in mbuf_stats.values(): + first_socket_mbufs = mbufs_per_socket + break + if not self._util_stats_ref.mbuf_types_list: + mbuf_keys = list(first_socket_mbufs.keys()) + mbuf_keys.sort(key = get_number_of_bytes) + self._util_stats_ref.mbuf_types_list = mbuf_keys + types_len = len(self._util_stats_ref.mbuf_types_list) + stats_table.set_cols_align(['l'] + ['r'] * (types_len + 1)) + stats_table.set_cols_width([10] + [7] * (types_len + 1)) + stats_table.set_cols_dtype(['t'] * (types_len + 2)) + stats_table.header([''] + self._util_stats_ref.mbuf_types_list + ['RAM(MB)']) + total_list = [] + sum_totals = 0 + for mbuf_type in self._util_stats_ref.mbuf_types_list: + sum_totals += first_socket_mbufs[mbuf_type][1] * get_number_of_bytes(mbuf_type) + 64 + total_list.append(first_socket_mbufs[mbuf_type][1]) + sum_totals *= len(list(mbuf_stats.values())) + total_list.append(int(sum_totals/1e6)) + stats_table.add_row(['Total:'] + total_list) + stats_table.add_row(['Used:'] + [''] * (types_len + 1)) + for socket_name in sorted(list(mbuf_stats.keys())): + mbufs = mbuf_stats[socket_name] + socket_show_name = socket_name.replace('cpu-', '').replace('-', ' ').capitalize() + ':' + sum_used = 0 + used_list = [] + percentage_list = [] + for mbuf_type in self._util_stats_ref.mbuf_types_list: + used = mbufs[mbuf_type][1] - mbufs[mbuf_type][0] + sum_used += used * get_number_of_bytes(mbuf_type) + 64 + used_list.append(used) + percentage_list.append('%s%%' % int(100 * used / mbufs[mbuf_type][1])) + used_list.append(int(sum_used/1e6)) + stats_table.add_row([socket_show_name] + used_list) + stats_table.add_row(['Percent:'] + percentage_list + ['']) + else: + stats_table.add_row(['No Data.']) + return {'mbuf_util': ExportableStats(None, stats_table)} + @staticmethod def _get_rational_block_char(value, range_start, interval): # in Konsole, utf-8 is sometimes printed with artifacts, return ascii for now @@ -1276,7 +1359,27 @@ class CRxStats(CTRexStats): return stats - +class CUtilStats(CTRexStats): + + def __init__(self, client): + super(CUtilStats, self).__init__() + self.client = client + self.history = deque(maxlen = 1) + self.mbuf_types_list = None + self.last_update_ts = 0 + + def get_stats(self, use_1sec_cache = False): + time_now = time.time() + if self.last_update_ts + 1 < time_now or not self.history or not use_1sec_cache: + if self.client.is_connected(): + rc = self.client._transmit('get_utilization') + if not rc: + raise Exception(rc) + self.last_update_ts = time_now + self.history.append(rc.data()) + else: + self.history.append({}) + return self.history[-1] if __name__ == "__main__": pass diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py index bb28d3af..ceaf1af8 100755 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py @@ -39,6 +39,8 @@ PORT_STATS = 51 PORT_STATUS = 52 STREAMS_STATS = 53 STATS_MASK = 54 +CPU_STATS = 55 +MBUF_STATS = 56 STREAMS_MASK = 60 # ALL_STREAMS = 61 @@ -340,6 +342,14 @@ OPTIONS_DB = {MULTIPLIER: ArgumentPack(['-m', '--multiplier'], {'action': 'store_true', 'help': "Fetch only streams stats"}), + CPU_STATS: ArgumentPack(['-c'], + {'action': 'store_true', + 'help': "Fetch only CPU utilization stats"}), + + MBUF_STATS: ArgumentPack(['-m'], + {'action': 'store_true', + 'help': "Fetch only MBUF utilization stats"}), + STREAMS_MASK: ArgumentPack(['--streams'], {"nargs": '+', 'dest':'streams', @@ -365,7 +375,9 @@ OPTIONS_DB = {MULTIPLIER: ArgumentPack(['-m', '--multiplier'], STATS_MASK: ArgumentGroup(MUTEX, [GLOBAL_STATS, PORT_STATS, PORT_STATUS, - STREAMS_STATS], + STREAMS_STATS, + CPU_STATS, + MBUF_STATS], {}) } |