From cb13e66205717a8fcf69185ba350adab3438ffa0 Mon Sep 17 00:00:00 2001 From: Ido Barnea Date: Sun, 8 May 2016 14:24:37 +0300 Subject: Latency stat python API work --- .../regression/stateless_tests/stl_rx_test.py | 33 ++++++---- .../stl/trex_stl_lib/trex_stl_async_client.py | 3 + .../stl/trex_stl_lib/trex_stl_client.py | 5 ++ .../stl/trex_stl_lib/trex_stl_stats.py | 31 +++++++++ src/flow_stat.cpp | 12 +--- src/main_dpdk.cpp | 77 +--------------------- 6 files changed, 64 insertions(+), 97 deletions(-) diff --git a/scripts/automation/regression/stateless_tests/stl_rx_test.py b/scripts/automation/regression/stateless_tests/stl_rx_test.py index 3b979744..a5bc115f 100644 --- a/scripts/automation/regression/stateless_tests/stl_rx_test.py +++ b/scripts/automation/regression/stateless_tests/stl_rx_test.py @@ -7,7 +7,7 @@ class STLRX_Test(CStlGeneral_Test): """Tests for RX feature""" def setUp(self): - per_driver_params = {"rte_vmxnet3_pmd": [1, 50], "rte_ixgbe_pmd": [30, 5000], "rte_i40e_pmd": [80, 5000], + per_driver_params = {"rte_vmxnet3_pmd": [1, 50], "rte_ixgbe_pmd": [10, 5000], "rte_i40e_pmd": [80, 5000], "rte_igb_pmd": [80, 500], "rte_em_pmd": [1, 50], "rte_virtio_pmd": [1, 50]} CStlGeneral_Test.setUp(self) @@ -19,8 +19,9 @@ class STLRX_Test(CStlGeneral_Test): port_info = self.c.get_port_info(ports = self.rx_port)[0] cap = port_info['rx']['caps'] - if cap != 1: - self.skip('port {0} does not support RX'.format(self.rx_port)) +# print cap; ???? +# if cap != 1: +# self.skip('port {0} does not support RX'.format(self.rx_port)) self.rate_percent = per_driver_params[port_info['driver']][0] self.total_pkts = per_driver_params[port_info['driver']][1] @@ -36,14 +37,24 @@ class STLRX_Test(CStlGeneral_Test): def __verify_flow (self, pg_id, total_pkts, pkt_len): - flow_stats = self.c.get_stats()['flow_stats'].get(pg_id) + latency_stats = self.c.get_stats()['latency'].get(pg_id) + if not flow_stats: assert False, "no flow stats available" tx_pkts = flow_stats['tx_pkts'].get(self.tx_port, 0) tx_bytes = flow_stats['tx_bytes'].get(self.tx_port, 0) rx_pkts = flow_stats['rx_pkts'].get(self.rx_port, 0) + drops = latency_stats['err_cntrs']['dropped'] + ooo = latency_stats['err_cntrs']['out_of_order'] + jitter = latency_stats['jitter'] + latency = latency_stats['latency'] + + if drops != 0 or ooo != 0: + pprint.pprint(latency_stats) + tmp='Dropped or out of order packets - dropped: {0}, ooo: {1}'.format(drops, ooo) + assert False, tmp if tx_pkts != total_pkts: pprint.pprint(flow_stats) @@ -78,7 +89,7 @@ class STLRX_Test(CStlGeneral_Test): try: s1 = STLStream(name = 'rx', packet = self.pkt, - flow_stats = STLFlowStats(pg_id = 5), + flow_stats = STLFlowLatencyStats(pg_id = 5), mode = STLTXSingleBurst(total_pkts = total_pkts, percentage = self.rate_percent )) @@ -99,13 +110,11 @@ class STLRX_Test(CStlGeneral_Test): # one simple stream on TX --> RX def test_multiple_streams(self): - num_streams = 10 + num_streams = 128 total_pkts = int(self.total_pkts / num_streams) if total_pkts == 0: total_pkts = 1 - percent = int(self.rate_percent / num_streams) - if percent == 0: - percent = 1 + percent = float(self.rate_percent) / num_streams try: streams = [] @@ -115,7 +124,7 @@ class STLRX_Test(CStlGeneral_Test): streams.append(STLStream(name = 'rx {0}'.format(pg_id), packet = self.pkt, - flow_stats = STLFlowStats(pg_id = pg_id), + flow_stats = STLFlowLatencyStats(pg_id = pg_id), mode = STLTXSingleBurst(total_pkts = total_pkts+pg_id, percentage = percent))) exp.append({'pg_id': pg_id, 'total_pkts': total_pkts+pg_id, 'pkt_len': self.pkt.get_pkt_len()}) @@ -135,7 +144,7 @@ class STLRX_Test(CStlGeneral_Test): try: s1 = STLStream(name = 'rx', packet = self.pkt, - flow_stats = STLFlowStats(pg_id = 5), + flow_stats = STLFlowLatencyStats(pg_id = 5), mode = STLTXSingleBurst(total_pkts = total_pkts, percentage = self.rate_percent )) @@ -143,7 +152,7 @@ class STLRX_Test(CStlGeneral_Test): # add both streams to ports self.c.add_streams([s1], ports = [self.tx_port]) - #print "\ninjecting {0} packets on port {1}\n".format(total_pkts, self.tx_port) + print "\ninjecting {0} packets on port {1}\n".format(total_pkts, self.tx_port) exp = {'pg_id': 5, 'total_pkts': total_pkts, 'pkt_len': self.pkt.get_pkt_len()} diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_async_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_async_client.py index 5c9faf0f..731ddb10 100644 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_async_client.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_async_client.py @@ -293,6 +293,9 @@ class CTRexAsyncClient(): elif name == "flow_stats": self.event_handler.on_async_rx_stats_event(data, baseline) + elif name == "latency_stats": + self.event_handler.on_async_latency_stats_event(data, baseline) + else: pass diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py index e7b46aea..88c64e3c 100755 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py @@ -187,6 +187,8 @@ class EventsHandler(object): def on_async_rx_stats_event (self, data, baseline): self.client.flow_stats.update(data, baseline) + def on_async_latency_stats_event (self, data, baseline): + self.client.latency_stats.update(data, baseline) # handles an async stats update from the subscriber def on_async_stats_update(self, dump_data, baseline): @@ -551,6 +553,8 @@ class STLClient(object): self.flow_stats = trex_stl_stats.CRxStats(self.ports) + self.latency_stats = trex_stl_stats.CLatencyStats(self.ports) + self.stats_generator = trex_stl_stats.CTRexInfoGenerator(self.global_stats, self.ports, self.flow_stats) @@ -891,6 +895,7 @@ class STLClient(object): stats['total'] = total stats['flow_stats'] = self.flow_stats.get_stats() + stats['latency'] = self.latency_stats.get_stats() return stats diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py index e30da00e..a65ef9e0 100644 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py @@ -831,6 +831,37 @@ class CPortStats(CTRexStats): } +class CLatencyStats(CTRexStats): + def __init__(self, ports): + super(CLatencyStats, self).__init__() + + # for API + def get_stats (self): + return self.latest_stats + + def process_snapshot (self, current): + output = {} + + # we care only about the current active keys + pg_ids = list(filter(is_intable, current.keys())) + + for pg_id in pg_ids: + current_pg = current.get(pg_id) + int_pg_id = int(pg_id) + output[int_pg_id] = {} + for field in ['err_cntrs', 'jitter', 'latency']: + output[int_pg_id][field] = current_pg[field] + return output + + def update (self, snapshot, baseline): + # generate a new snapshot + if (snapshot is not None): + new_snapshot = self.process_snapshot(snapshot) + else: + return + + self.latest_stats = new_snapshot + return True # RX stats objects - COMPLEX :-( diff --git a/src/flow_stat.cpp b/src/flow_stat.cpp index d63dc778..b2c3f487 100644 --- a/src/flow_stat.cpp +++ b/src/flow_stat.cpp @@ -1048,20 +1048,10 @@ bool CFlowStatRuleMgr::dump_json(std::string & s_json, std::string & l_json, boo l_data_section[str_user_id]["jitter"] = user_id_info_p->get_jitter(); } ///????? add last 10 samples - l_data_section[str_user_id]["err_cntrs"]["lost"] + l_data_section[str_user_id]["err_cntrs"]["dropped"] = Json::Value::UInt64(user_id_info_p->get_seq_err_cnt()); l_data_section[str_user_id]["err_cntrs"]["out_of_order"] = Json::Value::UInt64(user_id_info_p->get_ooo_cnt()); - - //??? temp - remove -#if 0 - s_data_section[str_user_id]["tx_bytes"]["0"] - = Json::Value::UInt64(user_id_info_p->get_seq_err_cnt()); - s_data_section[str_user_id]["tx_bytes"]["1"] = 0; - s_data_section[str_user_id]["rx_bytes"]["0"] - = Json::Value::UInt64(user_id_info_p->get_ooo_cnt()); - s_data_section[str_user_id]["rx_bytes"]["1"] = 0; -#endif } } diff --git a/src/main_dpdk.cpp b/src/main_dpdk.cpp index ccd1aa8d..b0fd3180 100644 --- a/src/main_dpdk.cpp +++ b/src/main_dpdk.cpp @@ -2008,8 +2008,8 @@ void CCoreEthIF::update_mac_addr(CGenNode * node,uint8_t *p){ int CCoreEthIFStateless::send_node_flow_stat(rte_mbuf *m, CGenNodeStateless * node_sl, CCorePerPort * lp_port , CVirtualIFPerSideStats * lp_stats) { - //??? remove -# if 0 + // Defining this makes 10% percent packet loss. 1% packet reorder. +# ifdef ERR_CNTRS_TEST static int temp=1; temp++; #endif @@ -2038,20 +2038,15 @@ int CCoreEthIFStateless::send_node_flow_stat(rte_mbuf *m, CGenNodeStateless * no lp_stats->m_seq_num[hw_id_payload]++; - //??? remove -#if 0 +#ifdef ERR_CNTRS_TEST if (temp % 10 == 0) { fsp_head->seq = lp_stats->m_seq_num[hw_id_payload]++; } - if ((temp - 1) % 100 == 0) { fsp_head->seq = lp_stats->m_seq_num[hw_id_payload] - 4; - // lp_stats->m_seq_num[hw_id_payload]--; } #endif - - if (rte_pktmbuf_is_contiguous(m)) { // We have only the const mbuf mi = CGlobalInfo::pktmbuf_alloc_small(get_socket_id()); @@ -2081,72 +2076,6 @@ int CCoreEthIFStateless::send_node_flow_stat(rte_mbuf *m, CGenNodeStateless * no return 0; } -#if 0 -//??? remove -// Maybe make it part of send_node somehow -int CCoreEthIFStateless::send_node_flow_stat(CGenNodeStateless * node_sl) { - //??? remove - static int temp=1; - temp++; - - uint16_t hw_id = node_sl->get_stat_hw_id(); - tx_per_flow_t *lp_s; - /* check that we have mbuf */ - rte_mbuf_t *temp_m = node_sl->get_cache_mbuf(); - rte_mbuf_t *m; - - if (temp_m) { - /* cache case */ - m = node_sl->alloc_flow_stat_mbuf(temp_m); - }else{ - temp_m = node_sl->alloc_node_with_vm(); - assert(temp_m); - m = node_sl->alloc_flow_stat_mbuf(temp_m); - rte_pktmbuf_free(temp_m); - } - - pkt_dir_t dir=(pkt_dir_t)node_sl->get_mbuf_cache_dir(); - CCorePerPort * lp_port=&m_ports[dir]; - CVirtualIFPerSideStats * lp_stats = &m_stats[dir]; - - if (hw_id >= MAX_FLOW_STATS) { - // payload rule hw_ids are in the range right above ip id rules - uint16_t hw_id_payload = hw_id - MAX_FLOW_STATS; - if (hw_id_payload > max_stat_hw_id_seen_payload) { - max_stat_hw_id_seen_payload = hw_id_payload; - } - uint8_t *p = rte_pktmbuf_mtod(m, uint8_t*); - struct flow_stat_payload_header *fsp_head = (struct flow_stat_payload_header *) - (p + m->pkt_len - sizeof(struct flow_stat_payload_header)); - fsp_head->seq = lp_stats->m_seq_num[hw_id_payload]; - fsp_head->time_stamp = os_get_hr_tick_64(); - lp_stats->m_seq_num[hw_id_payload]++; - // remove ??? - - if (temp % 10 == 0) { - fsp_head->seq = lp_stats->m_seq_num[hw_id_payload]++; - } -#if 1 - if ((temp - 1) % 100 == 0) { - fsp_head->seq = lp_stats->m_seq_num[hw_id_payload] - 4; - // lp_stats->m_seq_num[hw_id_payload]--; - } -#endif - } else { - // ip id rule - if (hw_id > max_stat_hw_id_seen) { - max_stat_hw_id_seen = hw_id; - } - } - lp_s = &lp_stats->m_tx_per_flow[hw_id]; - lp_s->add_pkts(1); - lp_s->add_bytes(m->pkt_len); - - send_pkt(lp_port,m,lp_stats); - return 0; -} -#endif - int CCoreEthIFStateless::send_node(CGenNode * no) { /* if a node is marked as slow path - single IF to redirect it to slow path */ if (no->get_is_slow_path()) { -- cgit 1.2.3-korg