diff options
Diffstat (limited to 'scripts/automation/regression')
7 files changed, 139 insertions, 50 deletions
diff --git a/scripts/automation/regression/functional_tests/stl_basic_tests.py b/scripts/automation/regression/functional_tests/stl_basic_tests.py index e03c0742..2843d619 100644 --- a/scripts/automation/regression/functional_tests/stl_basic_tests.py +++ b/scripts/automation/regression/functional_tests/stl_basic_tests.py @@ -23,6 +23,7 @@ import os import subprocess import shlex from threading import Thread +from collections import defaultdict @attr('run_on_trex') class CStlBasic_Test(functional_general_test.CGeneralFunctional_Test): @@ -82,8 +83,26 @@ class CStlBasic_Test(functional_general_test.CGeneralFunctional_Test): def compare_caps (self, output, golden, max_diff_sec = 0.01): - pkts1 = list(RawPcapReader(output)) - pkts2 = list(RawPcapReader(golden)) + pkts1 = [] + pkts2 = [] + pkts_ts_buckets = defaultdict(list) + + for pkt in RawPcapReader(output): + ts = pkt[1][0] * 1e6 + pkt[1][1] + pkts_ts_buckets[ts].append(pkt) + # don't take last ts bucket, it can be cut in middle and packets inside bucket might be different + #for ts in sorted(pkts_ts_buckets.keys())[:-1]: + for ts in sorted(pkts_ts_buckets.keys()): + pkts1.extend(sorted(pkts_ts_buckets[ts])) + pkts_ts_buckets.clear() + + for pkt in RawPcapReader(golden): + ts = pkt[1][0] * 1e6 + pkt[1][1] + pkts_ts_buckets[ts].append(pkt) + # don't take last ts bucket, it can be cut in middle and packets inside bucket might be different + #for ts in sorted(pkts_ts_buckets.keys())[:-1]: + for ts in sorted(pkts_ts_buckets.keys()): + pkts2.extend(sorted(pkts_ts_buckets[ts])) assert_equal(len(pkts1), len(pkts2), 'Lengths of generated pcap (%s) and golden (%s) are different' % (output, golden)) @@ -117,7 +136,7 @@ class CStlBasic_Test(functional_general_test.CGeneralFunctional_Test): raise AssertionError(errmsg) - def run_sim (self, yaml, output, options = "", silent = False, obj = None): + def run_sim (self, yaml, output, options = "", silent = False, obj = None, tunables = None): if output: user_cmd = "-f {0} -o {1} {2} -p {3}".format(yaml, output, options, self.scripts_path) else: @@ -126,6 +145,11 @@ class CStlBasic_Test(functional_general_test.CGeneralFunctional_Test): if silent: user_cmd += " --silent" + if tunables: + user_cmd += " -t" + for k, v in tunables.items(): + user_cmd += " {0}={1}".format(k, v) + rc = trex_stl_sim.main(args = shlex.split(user_cmd)) if obj: obj['rc'] = (rc == 0) @@ -134,7 +158,16 @@ class CStlBasic_Test(functional_general_test.CGeneralFunctional_Test): - def run_py_profile_path (self, profile, options,silent = False, do_no_remove=False,compare =True, test_generated=True, do_no_remove_generated = False): + def run_py_profile_path (self, + profile, + options, + silent = False, + do_no_remove = False, + compare = True, + test_generated = True, + do_no_remove_generated = False, + tunables = None): + print('Testing profile: %s' % profile) output_cap = "a.pcap" input_file = os.path.join('stl/', profile) @@ -142,7 +175,11 @@ class CStlBasic_Test(functional_general_test.CGeneralFunctional_Test): if os.path.exists(output_cap): os.unlink(output_cap) try: - rc = self.run_sim(input_file, output_cap, options, silent) + rc = self.run_sim(yaml = input_file, + output = output_cap, + options = options, + silent = silent, + tunables = tunables) assert_equal(rc, True, 'Simulation on profile %s failed.' % profile) #s='cp '+output_cap+' '+golden_file; #print s @@ -158,20 +195,21 @@ class CStlBasic_Test(functional_general_test.CGeneralFunctional_Test): try: generated_filename = input_file.replace('.py', '_GENERATED.py').replace('.yaml', '_GENERATED.py') if input_file.endswith('.py'): - profile = STLProfile.load_py(input_file) + profile = STLProfile.load_py(input_file, **(tunables if tunables else {})) elif input_file.endswith('.yaml'): profile = STLProfile.load_yaml(input_file) profile.dump_to_code(generated_filename) - rc = self.run_sim(generated_filename, output_cap, options, silent) + rc = self.run_sim(yaml = generated_filename, + output = output_cap, + options = options, + silent = silent) assert_equal(rc, True, 'Simulation on profile %s (generated) failed.' % profile) if compare: self.compare_caps(output_cap, golden_file) - except Exception as e: - print(e) finally: if not do_no_remove_generated: @@ -187,20 +225,20 @@ class CStlBasic_Test(functional_general_test.CGeneralFunctional_Test): p = [ ["udp_1pkt_1mac_override.py","-m 1 -l 50",True], - ["syn_attack.py","-m 1 -l 50",True], # can't compare random now + ["syn_attack.py","-m 1 -l 50",True], ["udp_1pkt_1mac.py","-m 1 -l 50",True], ["udp_1pkt_mac.py","-m 1 -l 50",True], ["udp_1pkt.py","-m 1 -l 50",True], ["udp_1pkt_tuple_gen.py","-m 1 -l 50",True], - ["udp_rand_len_9k.py","-m 1 -l 50",True], # can't do the compare + ["udp_rand_len_9k.py","-m 1 -l 50",True], ["udp_1pkt_mpls.py","-m 1 -l 50",True], ["udp_1pkt_mpls_vm.py","-m 1 ",True], ["imix.py","-m 1 -l 100",True], ["udp_inc_len_9k.py","-m 1 -l 100",True], ["udp_1pkt_range_clients.py","-m 1 -l 100",True], ["multi_burst_2st_1000pkt.py","-m 1 -l 100",True], - ["pcap.py", "-m 1", True], - ["pcap_with_vm.py", "-m 1", True], + ["pcap.py", "-m 1", True, False], + ["pcap_with_vm.py", "-m 1", True, False], ["flow_stats.py", "-m 1 -l 1", True], ["flow_stats_latency.py", "-m 1 -l 1", True], @@ -212,10 +250,10 @@ class CStlBasic_Test(functional_general_test.CGeneralFunctional_Test): ["yaml/imix_1pkt_2.yaml","-m 1 -l 100",True], ["yaml/imix_1pkt_tuple_gen.yaml","-m 1 -l 100",True], ["yaml/imix_1pkt_vm.yaml","-m 1 -l 100",True], - ["udp_1pkt_pcap.py","-m 1 -l 10",True], - ["udp_3pkt_pcap.py","-m 1 -l 10",True], + ["udp_1pkt_pcap.py","-m 1 -l 10",True, False], + ["udp_3pkt_pcap.py","-m 1 -l 10",True, False], #["udp_1pkt_simple.py","-m 1 -l 3",True], - ["udp_1pkt_pcap_relative_path.py","-m 1 -l 3",True], + ["udp_1pkt_pcap_relative_path.py","-m 1 -l 3",True, False], ["udp_1pkt_tuple_gen_split.py","-m 1 -c 2 -l 100",True], ["udp_1pkt_range_clients_split.py","-m 1 -c 2 -l 100",True], ["udp_1pkt_vxlan.py","-m 1 -c 1 -l 17",True, False], # can't generate: no VXLAN in Scapy, only in profile @@ -230,8 +268,8 @@ class CStlBasic_Test(functional_general_test.CGeneralFunctional_Test): ["udp_1pkt_mac_mask1.py","-m 1 -l 20 ",True] , ["udp_1pkt_mac_mask2.py","-m 1 -l 20 ",True], ["udp_1pkt_mac_mask3.py","-m 1 -l 20 ",True], - ["udp_1pkt_simple_test2.py","-m 1 -l 10 ",True], # test split of packet with ip option - ["udp_1pkt_simple_test.py","-m 1 -l 10 ",True], + ["udp_1pkt_simple_test2.py","-m 1 -l 10 ",True, False], # test split of packet with ip option + ["udp_1pkt_simple_test.py","-m 1 -l 10 ",True, False], ["udp_1pkt_mac_mask5.py","-m 1 -l 30 ",True], ["udp_1pkt_range_clients_split_garp.py","-m 1 -l 50",True], ["udp_1pkt_src_ip_split.py","-m 1 -l 50 --cores 2",True] @@ -254,31 +292,31 @@ class CStlBasic_Test(functional_general_test.CGeneralFunctional_Test): def test_hlt_profiles (self): p = ( - ['hlt/hlt_udp_inc_dec_len_9k.py', '-m 1 -l 20', True], - ['hlt/hlt_imix_default.py', '-m 1 -l 20', True], - ['hlt/hlt_imix_4rates.py', '-m 1 -l 20', True], - ['hlt/hlt_david1.py', '-m 1 -l 20', True], - ['hlt/hlt_david2.py', '-m 1 -l 20', True], - ['hlt/hlt_david3.py', '-m 1 -l 20', True], - ['hlt/hlt_david4.py', '-m 1 -l 20', True], - ['hlt/hlt_wentong1.py', '-m 1 -l 20', True], - ['hlt/hlt_wentong2.py', '-m 1 -l 20', True], - ['hlt/hlt_tcp_ranges.py', '-m 1 -l 20', True], - ['hlt/hlt_udp_ports.py', '-m 1 -l 20', True], - ['hlt/hlt_udp_random_ports.py', '-m 1 -l 20', True], - ['hlt/hlt_ip_ranges.py', '-m 1 -l 20', True], - ['hlt/hlt_framesize_vm.py', '-m 1 -l 20', True], - ['hlt/hlt_l3_length_vm.py', '-m 1 -l 20', True], - ['hlt/hlt_vlan_default.py', '-m 1 -l 20', True], - ['hlt/hlt_4vlans.py', '-m 1 -l 20', True], - ['hlt/hlt_vlans_vm.py', '-m 1 -l 20', True], - ['hlt/hlt_ipv6_default.py', '-m 1 -l 20', True], - ['hlt/hlt_ipv6_ranges.py', '-m 1 -l 20', True], - ['hlt/hlt_mac_ranges.py', '-m 1 -l 20', True], + ['hlt/hlt_udp_inc_dec_len_9k.py', '-m 1 -l 20', True, None], + ['hlt/hlt_imix_default.py', '-m 1 -l 20', True, None], + ['hlt/hlt_imix_4rates.py', '-m 1 -l 20', True, None], + ['hlt/hlt_david1.py', '-m 1 -l 20', True, None], + ['hlt/hlt_david2.py', '-m 1 -l 20', True, None], + ['hlt/hlt_david3.py', '-m 1 -l 20', True, None], + ['hlt/hlt_david4.py', '-m 1 -l 20', True, None], + ['hlt/hlt_wentong1.py', '-m 1 -l 20', True, None], + ['hlt/hlt_wentong2.py', '-m 1 -l 20', True, None], + ['hlt/hlt_tcp_ranges.py', '-m 1 -l 20', True, None], + ['hlt/hlt_udp_ports.py', '-m 1 -l 20', True, None], + ['hlt/hlt_udp_random_ports.py', '-m 1 -l 20', True, None], + ['hlt/hlt_ip_ranges.py', '-m 1 -l 20', True, None], + ['hlt/hlt_framesize_vm.py', '-m 1 -l 20', True, None], + ['hlt/hlt_l3_length_vm.py', '-m 1 -l 20', True, None], + ['hlt/hlt_vlan_default.py', '-m 1 -l 20', True, None], + ['hlt/hlt_4vlans.py', '-m 1 -l 20', True, None], + ['hlt/hlt_vlans_vm.py', '-m 1 -l 20', True, {'random_seed': 1}], + ['hlt/hlt_ipv6_default.py', '-m 1 -l 20', True, None], + ['hlt/hlt_ipv6_ranges.py', '-m 1 -l 20', True, None], + ['hlt/hlt_mac_ranges.py', '-m 1 -l 20', True, None], ) for obj in p: - self.run_py_profile_path (obj[0], obj[1], compare =obj[2], do_no_remove=True, do_no_remove_generated = False) + self.run_py_profile_path (obj[0], obj[1], compare =obj[2], do_no_remove=True, do_no_remove_generated = False, tunables = obj[3]) # valgrind tests - this runs in multi thread as it safe (no output) def test_valgrind_various_profiles (self): diff --git a/scripts/automation/regression/setups/trex08/benchmark.yaml b/scripts/automation/regression/setups/trex08/benchmark.yaml index aeeca004..8f83e8f9 100644 --- a/scripts/automation/regression/setups/trex08/benchmark.yaml +++ b/scripts/automation/regression/setups/trex08/benchmark.yaml @@ -34,19 +34,25 @@ test_ipv6_simple: bw_per_core : 25.948 -test_rx_check_http: +test_rx_check_http: &rx_http multiplier : 99000 cores : 3 rx_sample_rate : 128 bw_per_core : 49.464 +test_rx_check_http_ipv6: + << : *rx_http + bw_per_core : 49.237 -test_rx_check_sfr: +test_rx_check_sfr: &rx_sfr multiplier : 80 cores : 7 rx_sample_rate : 128 bw_per_core : 20.871 +test_rx_check_sfr_ipv6: + << : *rx_sfr + bw_per_core : 19.198 ### stateless ### diff --git a/scripts/automation/regression/setups/trex09/benchmark.yaml b/scripts/automation/regression/setups/trex09/benchmark.yaml index 1ddd6f03..1e3acdd8 100644 --- a/scripts/automation/regression/setups/trex09/benchmark.yaml +++ b/scripts/automation/regression/setups/trex09/benchmark.yaml @@ -34,19 +34,27 @@ test_ipv6_simple: bw_per_core : 22.808 -test_rx_check_http: +test_rx_check_http: &rx_http multiplier : 90000 cores : 2 rx_sample_rate : 32 bw_per_core : 46.075 -test_rx_check_sfr: +test_rx_check_http_ipv6: + << : *rx_http + bw_per_core : 49.237 + +test_rx_check_sfr: &rx_sfr multiplier : 50 cores : 3 rx_sample_rate : 32 bw_per_core : 20.469 +test_rx_check_sfr_ipv6: + << : *rx_sfr + bw_per_core : 19.198 + ### stateless ### diff --git a/scripts/automation/regression/setups/trex11/config.yaml b/scripts/automation/regression/setups/trex11/config.yaml index b57b302d..782b7542 100644 --- a/scripts/automation/regression/setups/trex11/config.yaml +++ b/scripts/automation/regression/setups/trex11/config.yaml @@ -35,4 +35,4 @@ trex: hostname : csi-trex-11 cores : 1 - modes : ['loopback', 'virtual'] + modes : ['loopback', 'VM', 'virt_nics'] diff --git a/scripts/automation/regression/stateless_tests/stl_client_test.py b/scripts/automation/regression/stateless_tests/stl_client_test.py index ed125cde..1984e49d 100644 --- a/scripts/automation/regression/stateless_tests/stl_client_test.py +++ b/scripts/automation/regression/stateless_tests/stl_client_test.py @@ -297,3 +297,29 @@ class STLClient_Test(CStlGeneral_Test): finally: self.c.set_port_attr(ports = [self.tx_port, self.rx_port], promiscuous = False) + + + # see https://trex-tgn.cisco.com/youtrack/issue/trex-226 + def test_latency_pause_resume (self): + + try: + + s1 = STLStream(name = 'latency', + packet = self.pkt, + mode = STLTXCont(percentage = self.percentage), + flow_stats = STLFlowLatencyStats(pg_id = 1)) + + self.c.add_streams([s1], ports = self.tx_port) + + self.c.clear_stats() + + self.c.start(ports = self.tx_port) + + for i in range(100): + self.c.pause() + self.c.resume() + + self.c.stop() + + except STLError as e: + assert False , '{0}'.format(e) diff --git a/scripts/automation/regression/stateless_tests/stl_general_test.py b/scripts/automation/regression/stateless_tests/stl_general_test.py index 5ae2b326..a5fb905e 100644 --- a/scripts/automation/regression/stateless_tests/stl_general_test.py +++ b/scripts/automation/regression/stateless_tests/stl_general_test.py @@ -67,10 +67,11 @@ class STLBasic_Test(CStlGeneral_Test): def test_connectivity(self): if not self.is_loopback: try: - if CTRexScenario.router_cfg['forceImageReload']: + if CTRexScenario.router_cfg['forceCleanConfig']: CTRexScenario.router.load_clean_config() CTRexScenario.router.configure_basic_interfaces() CTRexScenario.router.config_pbr(mode = "config") + CTRexScenario.router.config_ipv6_pbr(mode = "config") except Exception as e: CTRexScenario.stl_init_error = 'Could not configure device, err: %s' % e self.fail(CTRexScenario.stl_init_error) diff --git a/scripts/automation/regression/stateless_tests/stl_rx_test.py b/scripts/automation/regression/stateless_tests/stl_rx_test.py index 090261ff..d447ad4e 100644 --- a/scripts/automation/regression/stateless_tests/stl_rx_test.py +++ b/scripts/automation/regression/stateless_tests/stl_rx_test.py @@ -29,6 +29,10 @@ class STLRX_Test(CStlGeneral_Test): self.cap = cap drv_name = port_info['driver'] + if drv_name == "rte_ixgbe_pmd": + self.ipv6_support = False + else: + self.ipv6_support = True self.rate_percent = per_driver_params[drv_name][0] self.total_pkts = per_driver_params[drv_name][1] if len(per_driver_params[drv_name]) > 2: @@ -50,6 +54,8 @@ class STLRX_Test(CStlGeneral_Test): ); self.pkt = STLPktBuilder(pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)/('Your_paylaod_comes_here')) + self.ipv6pkt = STLPktBuilder(pkt = Ether()/IPv6(dst="2001:0:4137:9350:8000:f12a:b9c8:2815",src="2001:4860:0:2001::68") + /UDP(dport=12,sport=1025)/('Your_paylaod_comes_here')) self.large_pkt = STLPktBuilder(pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)/('a'*1000)) self.pkt_9k = STLPktBuilder(pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)/('a'*9000)) self.vm_pkt = STLPktBuilder(pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1") @@ -249,6 +255,10 @@ class STLRX_Test(CStlGeneral_Test): if self.latency_9k_enable: streams_data.append({'name': 'Latency, 9k packet with field engine', 'pkt': self.vm_9k_pkt, 'lat': True}) + if self.ipv6_support: + streams_data.append({'name': 'IPv6 flow stat. No latency', 'pkt': self.ipv6pkt, 'lat': False}) + streams_data.append({'name': 'IPv6 latency, no field engine', 'pkt': self.ipv6pkt, 'lat': True}) + streams = [] for data in streams_data: if data['lat']: @@ -333,7 +343,7 @@ class STLRX_Test(CStlGeneral_Test): - # check low latency when you have stream of 9K stream + # check low latency when you have stream of 9K stream def test_9k_stream(self): if self.is_virt_nics: self.skip('Skip this for virtual NICs') @@ -353,7 +363,7 @@ class STLRX_Test(CStlGeneral_Test): s_port=random.sample(all_ports, random.randint(1, len(all_ports)) ) s_port=sorted(s_port) if self.speed == 40 : - # the NIC does not support all full rate in case both port works let's filter odd ports + # the NIC does not support all full rate in case both port works let's filter odd ports s_port=list(filter(lambda x: x % 2==0, s_port)) if len(s_port)==0: s_port=[0]; @@ -446,7 +456,7 @@ class STLRX_Test(CStlGeneral_Test): ls=stats['flow_stats'][5+ pid] self.check_stats (stats,ls['rx_pkts']['total'], pkts,"ls['rx_pkts']['total']") self.check_stats (stats,ls['rx_pkts'][dpid], pkts,"ls['rx_pkts'][dpid]") - + self.check_stats (stats,ls['tx_pkts']['total'], pkts,"ls['tx_pkts']['total']") self.check_stats (stats,ls['tx_pkts'][pid], pkts,"ls['tx_pkts'][pid]") @@ -473,7 +483,7 @@ class STLRX_Test(CStlGeneral_Test): print(" test port {0} latency : {1} ".format(port,l)) self.send_1_burst(port,l,100) - + # this test adds more and more latency streams and re-test with incremental def test_incremental_latency_streams (self): if self.is_virt_nics: |