From dc66a3a82aab2ce41aa6ded38087e02b1eeb9493 Mon Sep 17 00:00:00 2001 From: Hanoh Haim Date: Sun, 25 Dec 2016 14:50:14 +0200 Subject: add a way to change tw configuration Signed-off-by: Hanoh Haim --- doc/trex_book.asciidoc | 37 ++++++++++++++++++++-- linux/ws_main.py | 1 + linux_dpdk/ws_main.py | 1 + scripts/cap2/dns_tw.yaml | 27 ++++++++++++++++ src/bp_sim.cpp | 32 +++++++++++++++++-- src/bp_sim.h | 45 +++++++++++++++++++++++--- src/main_dpdk.cpp | 19 +++++++++-- src/platform_cfg.cpp | 6 ++++ src/platform_cfg.h | 3 ++ src/tw_cfg.cpp | 82 ++++++++++++++++++++++++++++++++++++++++++++++++ src/tw_cfg.h | 64 +++++++++++++++++++++++++++++++++++++ 11 files changed, 306 insertions(+), 11 deletions(-) create mode 100644 scripts/cap2/dns_tw.yaml create mode 100644 src/tw_cfg.cpp create mode 100644 src/tw_cfg.h diff --git a/doc/trex_book.asciidoc b/doc/trex_book.asciidoc index a2cdde1e..b82c5765 100755 --- a/doc/trex_book.asciidoc +++ b/doc/trex_book.asciidoc @@ -1307,6 +1307,12 @@ Cpu Utilization : 0.1 % <7> Enable vlan feature. See xref:trex_vlan[trex_vlan section] for info. <8> Enable MAC address replacement by client IP. + +==== Timer Wheel section configuration + +(from v2.13) +see xref:timer_w[Timer Wheel section] + ==== Per template section // clarify "per template" @@ -1527,6 +1533,31 @@ We added configuration to the /etc/trex_cfg.yaml: This gave best results: with *\~98 Gb/s* TX BW and c=7, CPU utilization became *~21%*! (40% with c=4) + +==== Timer Wheeel section configuration + +anchor:timer_w[] + +The memory section is optional. It is used when there is a need to tune the amount of memory used by TRex packet manager. +Default values (from the TRex source code), are usually good for most users. Unless you have some unusual needs, you can +eliminate this section. + +==== Timer Wheel section configuration +The flow scheduler uses timer wheel to schedule flows. To tune it for a large number of flows it is possible to change the default values. +This is an advance configuration, don't use it if you don't know what you are doing. it can be configure in trex_cfg file and trex traffic profile. + +[source,python] +---- + tw : + buckets : 1024 <1> + levels : 3 <2> + bucket_time_usec : 20.0 <3> +---- +<1> the number of buckets in each level, higher number will improve performance, but will reduce the maximum levels. +<2> how many levels. +<3> bucket time in usec. higher number will create more bursts + + === Command line options anchor:cml-line[] @@ -2532,10 +2563,10 @@ image:images/xl710_vs_mlx5_var_size.png[title="Stateless variable size packet"] *Comments*:: -1. For Stateless 64B profiles, ConnectX-4 uses 50-90% more CPU cycles per packet (it is actually even more because there is the TRex scheduler overhead) - it means that in worst case scenario, you will need x2 CPU for the same total MPPS. +1. MLX5 can reach ~50MPPS while XL710 is limited to 35MPPS. (With potential future fix it will be ~65MPPS) 2. For Stateless/Stateful 256B profiles, ConnectX-4 uses half of the CPU cycles per packet. ConnectX-4 probably can handle in a better way chained mbufs (scatter gather). -3. In average stateful senario, ConnectX-4 is slightly better. -4. MLX5 can reach ~90MPPS while XL710 is limited to 35MPPS. +3. In the average stateful scenario, ConnectX-4 is the same as XL710. +4. For Stateless 64B/IMIX profiles, ConnectX-4 uses 50-90% more CPU cycles per packet (it is actually even more because there is the TRex scheduler overhead) - it means that in worst case scenario, you will need x2 CPU for the same total MPPS. [NOTE] diff --git a/linux/ws_main.py b/linux/ws_main.py index d885c597..711b4c89 100755 --- a/linux/ws_main.py +++ b/linux/ws_main.py @@ -112,6 +112,7 @@ main_src = SrcGroup(dir='src', 'tuple_gen.cpp', 'platform_cfg.cpp', 'utl_yaml.cpp', + 'tw_cfg.cpp', 'rx_check_header.cpp', 'nat_check.cpp', 'nat_check_flow_table.cpp', diff --git a/linux_dpdk/ws_main.py b/linux_dpdk/ws_main.py index 2327f28d..f3e25605 100755 --- a/linux_dpdk/ws_main.py +++ b/linux_dpdk/ws_main.py @@ -206,6 +206,7 @@ main_src = SrcGroup(dir='src', 'flow_stat_parser.cpp', 'inet_pton.cpp', 'pkt_gen.cpp', + 'tw_cfg.cpp', 'platform_cfg.cpp', 'pre_test.cpp', 'tuple_gen.cpp', diff --git a/scripts/cap2/dns_tw.yaml b/scripts/cap2/dns_tw.yaml new file mode 100644 index 00000000..9ddbc3c8 --- /dev/null +++ b/scripts/cap2/dns_tw.yaml @@ -0,0 +1,27 @@ +- duration : 10.0 + generator : + distribution : "seq" + clients_start : "16.0.0.1" + clients_end : "16.0.1.255" + servers_start : "48.0.0.1" + servers_end : "48.0.0.255" + clients_per_gb : 201 + min_clients : 101 + dual_port_mask : "1.0.0.0" + tcp_aging : 1 + udp_aging : 1 + tw : # set timer wheel configuration options + buckets : 32768 + levels : 2 + bucket_time_usec : 20.0 + mac : [0x00,0x00,0x00,0x01,0x00,0x00] + #vlan : { enable : 1 , vlan0 : 100 , vlan1 : 200 } + #mac_override_by_ip : true + cap_info : + - name: cap2/dns.pcap + cps : 1.0 + ipg : 10000 + rtt : 10000 + w : 1 + + diff --git a/src/bp_sim.cpp b/src/bp_sim.cpp index 68cc325d..baf891ad 100755 --- a/src/bp_sim.cpp +++ b/src/bp_sim.cpp @@ -2862,6 +2862,11 @@ void operator >> (const YAML::Node& node, CFlowsYamlInfo & flows_info) { flows_info.m_tuple_gen.get_server_pool_id(fi.m_server_pool_name); flows_info.m_vec.push_back(fi); } + + if ( node.FindValue("tw") ){ + node["tw"] >> flows_info.m_tw; + } + } void CVlanYamlInfo::Dump(FILE *fd){ @@ -2931,6 +2936,7 @@ void CFlowsYamlInfo::Dump(FILE *fd){ for (i=0; i<(int)m_vec.size(); i++) { m_vec[i].Dump(fd); } + m_tw.Dump(fd); } @@ -3015,6 +3021,7 @@ bool CFlowsYamlInfo::verify_correctness(uint32_t num_threads) { int CFlowsYamlInfo::load_from_yaml_file(std::string file_name){ m_vec.clear(); + m_tw.reset(); if ( !utl_is_file_exists (file_name) ){ printf(" ERROR file %s does not exist \n",file_name.c_str()); @@ -3064,6 +3071,21 @@ int CFlowsYamlInfo::load_from_yaml_file(std::string file_name){ m_is_plugin_configured=true; } } + + if ( m_tw.m_info_exist ){ + + CTimerWheelYamlInfo *lp=&m_tw; + std::string err; + if (!lp->Verify(err)){ + std::cout << err << "\n"; + exit(-1); + } + + CParserOption* po = &CGlobalInfo::m_options; + po->set_tw_bucket_time_in_usec(lp->m_bucket_time_usec); + po->set_tw_buckets(lp->m_buckets); + po->set_tw_levels(lp->m_levels); + } return 0; } @@ -3476,9 +3498,10 @@ bool CFlowGenListPerThread::Create(uint32_t thread_id, 0 , socket_id); - RC_HTW_t tw_res=m_tw.Create(TW_BUCKETS,3); + RC_HTW_t tw_res=m_tw.Create(TW_BUCKETS,TW_LEVELS); if (tw_res != RC_HTW_OK){ CHTimerWheelErrorStr err(tw_res); + printf("Timer wheel configuration error,please look into the manual for details \n"); printf("ERROR %-30s - %s \n",err.get_str(),err.get_help_str()); exit(1); } @@ -5194,7 +5217,12 @@ void CParserOption::dump(FILE *fd){ fprintf(fd," latency : %d pkt/sec \n",m_latency_rate); fprintf(fd," zmq_port : %d \n",m_zmq_port); fprintf(fd," telnet_port : %d \n",m_telnet_port); - fprintf(fd," expected_ports : %d \n",m_expected_portd); + fprintf(fd," expected_ports : %d \n",m_expected_portd); + fprintf(fd," tw_bucket_usec : %f usec \n",get_tw_bucket_time_in_sec()*1000000.0); + fprintf(fd," tw_buckets : %lu usec \n",(ulong)get_tw_buckets()); + fprintf(fd," tw_levels : %lu usec \n",(ulong)get_tw_levels()); + + if (preview.get_vlan_mode_enable() ) { fprintf(fd," vlans : [%d,%d] \n",m_vlan_port[0],m_vlan_port[1]); } diff --git a/src/bp_sim.h b/src/bp_sim.h index 328820cd..7f48601f 100755 --- a/src/bp_sim.h +++ b/src/bp_sim.h @@ -62,6 +62,7 @@ limitations under the License. #include "trex_watchdog.h" #include "trex_client_config.h" #include "h_timer.h" +#include "tw_cfg.h" #include @@ -723,10 +724,16 @@ public: m_l_pkt_mode = 0; m_rx_thread_enabled = false; m_arp_ref_per = 120; // in seconds + m_tw_buckets = 1024; + m_tw_levels = 3; + m_tw_bucket_time_sec = (20.0/1000000.0); + } CPreviewMode preview; + uint16_t m_tw_buckets; + uint16_t m_tw_levels; float m_factor; float m_mbuf_factor; float m_duration; @@ -765,6 +772,8 @@ public: CMacAddrCfg m_mac_addr[TREX_MAX_PORTS]; + double m_tw_bucket_time_sec; + uint8_t * get_src_mac_addr(int if_index){ return (m_mac_addr[if_index].u.m_mac.src); @@ -804,6 +813,32 @@ public: m_rx_thread_enabled = true; } + inline double get_tw_bucket_time_in_sec(void){ + return (m_tw_bucket_time_sec); + } + + void set_tw_bucket_time_in_usec(double usec){ + m_tw_bucket_time_sec=(usec/1000000.0); + } + + void set_tw_buckets(uint16_t buckets){ + m_tw_buckets=buckets; + } + + inline uint16_t get_tw_buckets(void){ + return (m_tw_buckets); + } + + void set_tw_levels(uint16_t levels){ + m_tw_levels=levels; + } + + inline uint16_t get_tw_levels(void){ + return (m_tw_levels); + } + + + inline void set_rxcheck_const_ts(){ m_run_flags |= RUN_FLAGS_RXCHECK_CONST_TS; } @@ -3584,6 +3619,9 @@ public: std::vector m_vec; bool m_is_plugin_configured; /* any plugin is configured */ + + CTimerWheelYamlInfo m_tw; + public: void Dump(FILE *fd); int load_from_yaml_file(std::string file_name); @@ -3755,11 +3793,10 @@ private: bool server_seq_init; /* TCP seq been init for server? */ }; -#define BUCKET_TIME_USEC (20) -#define TW_BUCKETS (1024) -#define BUCKET_TIME_SEC ((double)BUCKET_TIME_USEC/1000000.0) +#define TW_BUCKETS (CGlobalInfo::m_options.get_tw_buckets()) +#define TW_LEVELS (CGlobalInfo::m_options.get_tw_levels()) +#define BUCKET_TIME_SEC (CGlobalInfo::m_options.get_tw_bucket_time_in_sec()) -#define TW_BUCKETS_MAX_TIME (BUCKET_TIME_USEC *TW_BUCKETS) ///////////////////////////////////////////////////////////////////////////////// diff --git a/src/main_dpdk.cpp b/src/main_dpdk.cpp index e1ddc021..4c448063 100644 --- a/src/main_dpdk.cpp +++ b/src/main_dpdk.cpp @@ -1171,11 +1171,22 @@ static int parse_options(int argc, char *argv[], CParserOption* po, bool first_t parse_err("Single core is not supported with interactive (stateless) mode "); } - } - else { + } else { if ( !po->m_duration ) { po->m_duration = 3600.0; } + if ( global_platform_cfg_info.m_tw.m_info_exist ){ + + CTimerWheelYamlInfo *lp=&global_platform_cfg_info.m_tw; + std::string err; + if (!lp->Verify(err)){ + parse_err(err); + } + + po->set_tw_bucket_time_in_usec(lp->m_bucket_time_usec); + po->set_tw_buckets(lp->m_buckets); + po->set_tw_levels(lp->m_levels); + } } return 0; } @@ -3815,6 +3826,10 @@ bool CGlobalTRex::Create(){ m_stats_cnt =0; if (!get_is_stateless()) { pre_yaml_info.load_from_yaml_file(CGlobalInfo::m_options.cfg_file); + if ( CGlobalInfo::m_options.preview.getVMode() > 0){ + CGlobalInfo::m_options.dump(stdout); + CGlobalInfo::m_memory_cfg.Dump(stdout); + } } if ( !m_zmq_publisher.Create( CGlobalInfo::m_options.m_zmq_port, diff --git a/src/platform_cfg.cpp b/src/platform_cfg.cpp index a8f7997c..575c4c72 100755 --- a/src/platform_cfg.cpp +++ b/src/platform_cfg.cpp @@ -404,6 +404,11 @@ void operator >> (const YAML::Node& node, CPlatformYamlInfo & plat_info) { plat_info.m_platform.m_is_exists=true; } + if ( node.FindValue("tw") ){ + node["tw"] >> plat_info.m_tw; + } + + if ( node.FindValue("port_info") ) { const YAML::Node& mac_info = node["port_info"]; for(unsigned i=0;i #include #include +#include "tw_cfg.h" #define CONST_NB_MBUF_2_10G (16380/2) @@ -206,6 +207,7 @@ public: m_prefix=""; m_limit_memory="" ; m_thread_per_dual_if=1; + m_tw.reset(); } @@ -240,6 +242,7 @@ public: std::vector m_mac_info; CPlatformMemoryYamlInfo m_memory; CPlatformCoresYamlInfo m_platform; + CTimerWheelYamlInfo m_tw; public: std::string get_use_if_comma_seperated(); diff --git a/src/tw_cfg.cpp b/src/tw_cfg.cpp new file mode 100644 index 00000000..af079532 --- /dev/null +++ b/src/tw_cfg.cpp @@ -0,0 +1,82 @@ +/* + Hanoh Haim + Cisco Systems, Inc. +*/ + +/* +Copyright (c) 2015-2015 Cisco Systems, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#include "tw_cfg.h" + + +bool CTimerWheelYamlInfo::Verify(std::string & err){ + bool res=true; + if ( (m_levels>4) || (m_levels <1) ){ + err="tw number of levels should be betwean 1..4"; + res=false; + } + if ( (m_buckets >=(0xffff)) || ((m_buckets <256)) ){ + err="tw number of bucket should be betwean 256-2^16 and log2"; + res=false; + } + + if ( (m_bucket_time_usec <10.0) || (((m_bucket_time_usec> 200.0))) ){ + res=false; + err="bucket time should be betwean 10-200 usec "; + } + return (res); +} + +void CTimerWheelYamlInfo::Dump(FILE *fd){ + + if ( m_info_exist ==false ){ + fprintf(fd,"CTimerWheelYamlInfo does not exist \n"); + return; + } + + fprintf(fd," tw buckets : %lu \n",(ulong)m_buckets); + fprintf(fd," tw levels : %lu \n",(ulong)m_levels); + fprintf(fd," tw bucket time : %.02f usec\n",(double)m_bucket_time_usec); + +} + +void operator >> (const YAML::Node& node, CTimerWheelYamlInfo & info) { + + uint32_t val; + + if ( node.FindValue("buckets") ){ + node["buckets"] >> val; + info.m_buckets=(uint16_t)val; + info.m_info_exist = true; + } + + if ( node.FindValue("levels") ){ + node["levels"] >> val; + info.m_levels = (uint16_t)val; + info.m_info_exist = true; + } + + if ( node.FindValue("bucket_time_usec") ){ + node["bucket_time_usec"] >>info.m_bucket_time_usec; + info.m_info_exist = true; + } +} + + + + + + diff --git a/src/tw_cfg.h b/src/tw_cfg.h new file mode 100644 index 00000000..4e5c19f6 --- /dev/null +++ b/src/tw_cfg.h @@ -0,0 +1,64 @@ +#ifndef CTW_CFG_H +#define CTW_CFG_H + +/* + Hanoh Haim + Cisco Systems, Inc. +*/ + +/* +Copyright (c) 2015-2015 Cisco Systems, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#include +#include +#include +#include +#include + + + +struct CTimerWheelYamlInfo { +public: + CTimerWheelYamlInfo(){ + reset(); + } + + + void reset(){ + m_info_exist =false; + m_buckets=1024; + m_levels=3; + m_bucket_time_usec=20.0; + } + + bool m_info_exist; /* file exist ?*/ + uint32_t m_buckets; + uint32_t m_levels; + double m_bucket_time_usec; + + +public: + void Dump(FILE *fd); + bool Verify(std::string & err); + +}; + +void operator >> (const YAML::Node& node, CTimerWheelYamlInfo & mac_info); + + + + +#endif -- cgit 1.2.3-korg