diff options
Diffstat (limited to 'scripts/automation/regression')
29 files changed, 1276 insertions, 237 deletions
diff --git a/scripts/automation/regression/CPlatform.py b/scripts/automation/regression/CPlatform.py index 0017e7db..606235a6 100755 --- a/scripts/automation/regression/CPlatform.py +++ b/scripts/automation/regression/CPlatform.py @@ -499,6 +499,8 @@ class CPlatform(object): client_net_next_hop = misc_methods.get_single_net_client_addr(dual_if.server_if.get_ipv6_addr(), {'7':1}, ip_type = 'ipv6' ) server_net_next_hop = misc_methods.get_single_net_client_addr(dual_if.client_if.get_ipv6_addr(), {'7':1}, ip_type = 'ipv6' ) + client_net_next_hop_v4 = misc_methods.get_single_net_client_addr(dual_if.server_if.get_ipv4_addr() ) + server_net_next_hop_v4 = misc_methods.get_single_net_client_addr(dual_if.client_if.get_ipv4_addr() ) client_if_command_set.append ('{mode}ipv6 enable'.format(mode = unconfig_str)) @@ -526,12 +528,23 @@ class CPlatform(object): next_hop = server_net_next_hop, intf = dual_if.client_if.get_name(), dest_mac = dual_if.client_if.get_ipv6_dest_mac())) + # For latency packets (which are IPv4), we need to configure also static ARP + conf_t_command_set.append('{mode}arp {next_hop} {dest_mac} arpa'.format( + mode = unconfig_str, + next_hop = server_net_next_hop_v4, + dest_mac = dual_if.client_if.get_ipv6_dest_mac())) + if dual_if.server_if.get_ipv6_dest_mac(): conf_t_command_set.append('{mode}ipv6 neighbor {next_hop} {intf} {dest_mac}'.format( mode = unconfig_str, - next_hop = client_net_next_hop, + next_hop = client_net_next_hop, intf = dual_if.server_if.get_name(), dest_mac = dual_if.server_if.get_ipv6_dest_mac())) + # For latency packets (which are IPv4), we need to configure also static ARP + conf_t_command_set.append('{mode}arp {next_hop} {dest_mac} arpa'.format( + mode = unconfig_str, + next_hop = client_net_next_hop_v4, + dest_mac = dual_if.server_if.get_ipv6_dest_mac())) conf_t_command_set.append('{mode}route-map {pre}_{p1}_to_{p2} permit 10'.format( mode = unconfig_str, diff --git a/scripts/automation/regression/aggregate_results.py b/scripts/automation/regression/aggregate_results.py index c7c61ea6..130e0545 100755 --- a/scripts/automation/regression/aggregate_results.py +++ b/scripts/automation/regression/aggregate_results.py @@ -8,10 +8,8 @@ import sys, os from collections import OrderedDict import copy import datetime, time -try: - import cPickle as pickle -except: - import pickle +import traceback +import yaml import subprocess, shlex from ansi2html import Ansi2HTMLConverter @@ -25,6 +23,15 @@ FUNCTIONAL_CATEGORY = 'Functional' # how to display those categories ERROR_CATEGORY = 'Error' +def try_write(file, text): + try: + file.write(text) + except: + try: + file.write(text.encode('utf-8')) + except: + file.write(text.decode('utf-8')) + def pad_tag(text, tag): return '<%s>%s</%s>' % (tag, text, tag) @@ -256,6 +263,7 @@ if __name__ == '__main__': build_url = os.environ.get('BUILD_URL') build_id = os.environ.get('BUILD_ID') trex_repo = os.environ.get('TREX_CORE_REPO') + last_commit_info_file = os.environ.get('LAST_COMMIT_INFO') python_ver = os.environ.get('PYTHON_VER') if not scenario: print('Warning: no environment variable SCENARIO, using default') @@ -283,19 +291,25 @@ if __name__ == '__main__': trex_last_commit_info = '' trex_last_commit_hash = trex_info_dict.get('Git SHA') - if trex_last_commit_hash and trex_repo: + if last_commit_info_file and os.path.exists(last_commit_info_file): + with open(last_commit_info_file) as f: + trex_last_commit_info = f.read().strip().replace('\n', '<br>\n') + elif trex_last_commit_hash and trex_repo: try: - print('Getting TRex commit with hash %s' % trex_last_commit_hash) - command = 'git --git-dir %s show %s --quiet' % (trex_repo, trex_last_commit_hash) + command = 'git show %s -s' % trex_last_commit_hash print('Executing: %s' % command) - proc = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (trex_last_commit_info, stderr) = proc.communicate() - print('Stdout:\n\t' + trex_last_commit_info.replace('\n', '\n\t')) - print('Stderr:', stderr) - print('Return code:', proc.returncode) - trex_last_commit_info = trex_last_commit_info.replace('\n', '<br>') + proc = subprocess.Popen(shlex.split(command), stdout = subprocess.PIPE, stderr = subprocess.STDOUT, cwd = trex_repo) + (stdout, stderr) = proc.communicate() + stdout = stdout.decode('utf-8', errors = 'replace') + print('Stdout:\n\t' + stdout.replace('\n', '\n\t')) + if stderr or proc.returncode: + print('Return code: %s' % proc.returncode) + trex_last_commit_info = stdout.replace('\n', '<br>\n') except Exception as e: + traceback.print_exc() print('Error getting last commit: %s' % e) + else: + print('Could not find info about commit!') ##### get xmls: report_<setup name>.xml @@ -520,7 +534,7 @@ if __name__ == '__main__': # save html with open(args.output_htmlfile, 'w') as f: print('Writing output file: %s' % args.output_htmlfile) - f.write(html_output) + try_write(f, html_output) html_output = None # mail report (only error tests, expanded) @@ -596,7 +610,7 @@ if __name__ == '__main__': else: mail_output += add_category_of_tests(ERROR_CATEGORY, error_tests, expanded=True) else: - mail_output += '<table><tr style="font-size:120;color:green;font-family:arial"><td>☺</td><td style="font-size:20">All passed.</td></tr></table>\n' + mail_output += u'<table><tr style="font-size:120;color:green;font-family:arial"><td>☺</td><td style="font-size:20">All passed.</td></tr></table>\n' mail_output += '\n</body>\n</html>' ##### save outputs @@ -605,17 +619,17 @@ if __name__ == '__main__': # mail content with open(args.output_mailfile, 'w') as f: print('Writing output file: %s' % args.output_mailfile) - f.write(mail_output) + try_write(f, mail_output) # build status category_dict_status = {} if os.path.exists(args.build_status_file): print('Reading: %s' % args.build_status_file) - with open(args.build_status_file, 'rb') as f: + with open(args.build_status_file, 'r') as f: try: - category_dict_status = pickle.load(f) + category_dict_status = yaml.safe_load(f.read()) except Exception as e: - print('Error during pickle load: %s' % e) + print('Error during YAML load: %s' % e) if type(category_dict_status) is not dict: print('%s is corrupt, truncating' % args.build_status_file) category_dict_status = {} @@ -635,15 +649,15 @@ if __name__ == '__main__': current_status = 'Fixed' category_dict_status[scenario] = current_status - with open(args.build_status_file, 'wb') as f: + with open(args.build_status_file, 'w') as f: print('Writing output file: %s' % args.build_status_file) - pickle.dump(category_dict_status, f) + yaml.dump(category_dict_status, f) # last successful commit - if (current_status in ('Successful', 'Fixed')) and trex_last_commit_hash and jobs_list > 0 and scenario == 'nightly': + if (current_status in ('Successful', 'Fixed')) and trex_last_commit_hash and len(jobs_list) > 0 and scenario == 'nightly': with open(args.last_passed_commit, 'w') as f: print('Writing output file: %s' % args.last_passed_commit) - f.write(trex_last_commit_hash) + try_write(f, trex_last_commit_hash) # mail title mailtitle_output = scenario.capitalize() @@ -653,7 +667,8 @@ if __name__ == '__main__': with open(args.output_titlefile, 'w') as f: print('Writing output file: %s' % args.output_titlefile) - f.write(mailtitle_output) + try_write(f, mailtitle_output) # exit + print('Status: %s' % current_status) sys.exit(exit_status) diff --git a/scripts/automation/regression/cfg/client_cfg.yaml b/scripts/automation/regression/cfg/client_cfg.yaml new file mode 100644 index 00000000..5c3d19ef --- /dev/null +++ b/scripts/automation/regression/cfg/client_cfg.yaml @@ -0,0 +1,48 @@ +#vlan: true +vlan: false + +groups: + +- ip_start : 16.0.0.1 + ip_end : 16.0.1.255 + initiator : + next_hop: 1.1.1.1 + src_ip : 1.1.1.2 + responder : + next_hop: 1.1.2.1 + src_ip : 1.1.2.2 + + count : 1 + +- ip_start : 17.0.0.1 + ip_end : 17.0.1.255 + initiator : + next_hop: 1.1.3.1 + src_ip : 1.1.3.2 + responder : + next_hop: 1.1.4.1 + src_ip : 1.1.4.2 + + count : 1 + +- ip_start : 18.0.0.1 + ip_end : 18.0.1.255 + initiator : + next_hop: 1.1.5.1 + src_ip : 1.1.5.2 + responder : + next_hop: 1.1.6.1 + src_ip : 1.1.6.2 + + count : 1 + +- ip_start : 19.0.0.1 + ip_end : 19.0.1.255 + initiator : + next_hop: 1.1.7.1 + src_ip : 1.1.7.2 + responder : + next_hop: 1.1.8.1 + src_ip : 1.1.8.2 + + count : 1 diff --git a/scripts/automation/regression/functional_tests/trex_cfg_creator_test.py b/scripts/automation/regression/functional_tests/trex_cfg_creator_test.py index 5ff6b318..66cb666c 100755 --- a/scripts/automation/regression/functional_tests/trex_cfg_creator_test.py +++ b/scripts/automation/regression/functional_tests/trex_cfg_creator_test.py @@ -25,7 +25,7 @@ def compare_lines(golden, output): raise CompareLinesNumDiff('Number of lines on golden is: %s, in output: %s\nGolden:\n%s\nGenerated:\n%s\n' % (len(golden_lines), len(output_lines), golden, output)) for line_num, (golden_line, output_line) in enumerate(zip(golden_lines, output_lines)): if golden_line != output_line: - raise CompareLinesDiff('Produced YAML differs from golden at line %s.Golden: %s <-> Output: %s' % (line_num + 1, golden_line, output_line)) + raise CompareLinesDiff('Produced YAML differs from golden at line %s.\nGolden: %s <-> Output: %s' % (line_num + 1, golden_line, output_line)) def create_config(cpu_topology, interfaces, *args, **kwargs): config = ConfigCreator(cpu_topology, interfaces, *args, **kwargs) @@ -112,7 +112,7 @@ class TRexCfgCreator_Test: latency_thread_id: 1 dual_if: - socket: 0 - threads: [2] + threads: [2,3,4] ''' output = create_config(cpu_topology, interfaces) verify_master_core0(output) @@ -308,16 +308,16 @@ class TRexCfgCreator_Test: platform: master_thread_id: 0 - latency_thread_id: 16 + latency_thread_id: 12 dual_if: - socket: 0 - threads: [1,17,2,18,3,19,4] + threads: [1,2,3,16,17,18,19] - socket: 1 - threads: [8,24,9,25,10,26,11] + threads: [8,9,10,11,24,25,26] - socket: 0 - threads: [20,5,21,6,22,7,23] + threads: [4,5,6,7,20,21,22] ''' output = create_config(cpu_topology, interfaces) verify_master_core0(output) @@ -446,10 +446,10 @@ class TRexCfgCreator_Test: latency_thread_id: 31 dual_if: - socket: 0 - threads: [1,17,2,18,3,19,4,20,5,21,6,22,7,23,16] + threads: [1,2,3,4,5,6,7,16,17,18,19,20,21,22,23] - socket: 1 - threads: [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15] + threads: [8,9,10,11,12,13,14,15,24,25,26,27,28,29,30] ''' output = create_config(cpu_topology, interfaces) verify_master_core0(output) @@ -575,13 +575,13 @@ class TRexCfgCreator_Test: platform: master_thread_id: 0 - latency_thread_id: 16 + latency_thread_id: 8 dual_if: - socket: 0 - threads: [1,17,2,18,3,19,4] + threads: [1,2,3,16,17,18,19] - socket: 0 - threads: [20,5,21,6,22,7,23] + threads: [4,5,6,7,20,21,22] ''' output = create_config(cpu_topology, interfaces) verify_master_core0(output) @@ -694,5 +694,6 @@ class TRexCfgCreator_Test: @classmethod def tearDownClass(cls): - sys.path.remove(CTRexScenario.scripts_path) + if CTRexScenario.scripts_path in sys.path: + sys.path.remove(CTRexScenario.scripts_path) del sys.modules['dpdk_setup_ports'] diff --git a/scripts/automation/regression/setups/trex15/benchmark.yaml b/scripts/automation/regression/setups/trex03/benchmark.yaml index b366b3fb..b366b3fb 100644 --- a/scripts/automation/regression/setups/trex15/benchmark.yaml +++ b/scripts/automation/regression/setups/trex03/benchmark.yaml diff --git a/scripts/automation/regression/setups/trex15/config.yaml b/scripts/automation/regression/setups/trex03/config.yaml index c5fc3b22..17c4c91e 100644 --- a/scripts/automation/regression/setups/trex15/config.yaml +++ b/scripts/automation/regression/setups/trex03/config.yaml @@ -34,6 +34,6 @@ # expected_bw - the "golden" bandwidth (in Gbps) results planned on receiving from the test trex: - hostname : csi-trex-15 + hostname : csi-trex-03 cores : 1 modes : [loopback, virt_nics, VM] diff --git a/scripts/automation/regression/setups/trex17/benchmark.yaml b/scripts/automation/regression/setups/trex06/benchmark.yaml index 8bc9d29c..2f51a3fd 100644 --- a/scripts/automation/regression/setups/trex17/benchmark.yaml +++ b/scripts/automation/regression/setups/trex06/benchmark.yaml @@ -140,7 +140,7 @@ test_CPU_benchmark: bw_per_core : 1 - name : stl/pcap.py - kwargs : {ipg_usec: 2, loop_count: 0} + kwargs : {ipg_usec: 3, loop_count: 0} cpu_util : 1 bw_per_core : 1 diff --git a/scripts/automation/regression/setups/trex17/config.yaml b/scripts/automation/regression/setups/trex06/config.yaml index 7ad6a20a..da0eb2dd 100644 --- a/scripts/automation/regression/setups/trex17/config.yaml +++ b/scripts/automation/regression/setups/trex06/config.yaml @@ -34,6 +34,6 @@ # expected_bw - the "golden" bandwidth (in Gbps) results planned on receiving from the test trex: - hostname : csi-trex-17 + hostname : csi-trex-06 cores : 1 modes : [loopback, virt_nics, VM] diff --git a/scripts/automation/regression/setups/trex07/backup/benchmark.yaml b/scripts/automation/regression/setups/trex07/backup/benchmark.yaml new file mode 100644 index 00000000..0dc340b0 --- /dev/null +++ b/scripts/automation/regression/setups/trex07/backup/benchmark.yaml @@ -0,0 +1,244 @@ +############################################################### +#### TRex benchmark configuration file #### +############################################################### + +#### common templates ### + +stat_route_dict: &stat_route_dict + clients_start : 16.0.0.1 + servers_start : 48.0.0.1 + dual_port_mask : 1.0.0.0 + client_destination_mask : 255.0.0.0 + server_destination_mask : 255.0.0.0 + +nat_dict: &nat_dict + clients_net_start : 16.0.0.0 + client_acl_wildcard_mask : 0.0.0.255 + dual_port_mask : 1.0.0.0 + pool_start : 200.0.0.0 + pool_netmask : 255.255.255.0 + + +### stateful ### + +test_jumbo: + multiplier : 17 + cores : 1 + bw_per_core : 543.232 + + +test_routing_imix: + multiplier : 10 + cores : 1 + bw_per_core : 34.128 + + +test_routing_imix_64: + multiplier : 430 + cores : 1 + bw_per_core : 5.893 + + +test_static_routing_imix: &test_static_routing_imix + stat_route_dict : *stat_route_dict + multiplier : 8 + cores : 1 + bw_per_core : 34.339 + +test_static_routing_imix_asymmetric: *test_static_routing_imix + + +test_ipv6_simple: + multiplier : 9 + cores : 2 + bw_per_core : 19.064 + + +test_nat_simple_mode1: &test_nat_simple + stat_route_dict : *stat_route_dict + nat_dict : *nat_dict + multiplier : 6000 + cores : 1 + nat_opened : 500000 + allow_timeout_dev : True + bw_per_core : 44.445 + +test_nat_simple_mode2: *test_nat_simple + +test_nat_simple_mode3: *test_nat_simple + +test_nat_learning: *test_nat_simple + + +test_nbar_simple: + multiplier : 7.5 + cores : 2 + bw_per_core : 17.174 + nbar_classification: + rtp : 32.57 + http : 30.25 + oracle_sqlnet : 11.23 + exchange : 10.80 + citrix : 5.62 + rtsp : 2.84 + dns : 1.95 + smtp : 0.57 + pop3 : 0.36 + ssl : 0.17 + sctp : 0.13 + sip : 0.09 + unknown : 3.41 + + +test_rx_check_http: &rx_http + multiplier : 15000 + cores : 1 + rx_sample_rate : 16 + bw_per_core : 39.560 + +test_rx_check_http_ipv6: + << : *rx_http + bw_per_core : 49.237 + +test_rx_check_http_negative_disabled: + << : *rx_http + stat_route_dict : *stat_route_dict + nat_dict : *nat_dict + + +test_rx_check_sfr: &rx_sfr + multiplier : 10 + cores : 3 + rx_sample_rate : 16 + bw_per_core : 16.082 + +test_rx_check_sfr_ipv6: + << : *rx_sfr + bw_per_core : 19.198 + + + +### stateless ### + +test_CPU_benchmark: + profiles: + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 64, stream_count: 10} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 64, stream_count: 100} + cpu_util : 1 + bw_per_core : 1 + +# causes queue full +# - name : stl/udp_for_benchmarks.py +# kwargs : {packet_len: 64, stream_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 9000, stream_count: 10} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 9000, stream_count: 100} + cpu_util : 1 + bw_per_core : 1 + +# not enough memory + queue full if memory increase +# - name : stl/udp_for_benchmarks.py +# kwargs : {packet_len: 9000, stream_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/imix.py + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/pcap.py + kwargs : {ipg_usec: 2, loop_count: 0} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 + + - name : stl/hlt/hlt_udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 + + diff --git a/scripts/automation/regression/setups/trex07/backup/config.yaml b/scripts/automation/regression/setups/trex07/backup/config.yaml new file mode 100644 index 00000000..db6e9bf8 --- /dev/null +++ b/scripts/automation/regression/setups/trex07/backup/config.yaml @@ -0,0 +1,66 @@ +################################################################ +#### TRex nightly test configuration file #### +################################################################ + + +### TRex configuration: +# hostname - can be DNS name or IP for the TRex machine for ssh to the box +# password - root password for TRex machine +# is_dual - should the TRex inject with -p ? +# version_path - path to the TRex version and executable +# cores - how many cores should be used +# latency - rate of latency packets injected by the TRex +# modes - list of modes (tagging) of this setup (loopback etc.) +# * loopback - Trex works via loopback. Router and TFTP configurations may be skipped. +# * VM - Virtual OS (accept low CPU utilization in tests, latency can get spikes) +# * virt_nics - NICs are virtual (VMXNET3 etc.) + +### Router configuration: +# hostname - the router hostname as apears in ______# cli prefix +# ip_address - the router's ip that can be used to communicate with +# image - the desired imaged wished to be loaded as the router's running config +# line_password - router password when access via Telent +# en_password - router password when changing to "enable" mode +# interfaces - an array of client-server pairs, representing the interfaces configurations of the router +# configurations - an array of configurations that could possibly loaded into the router during the test. +# The "clean" configuration is a mandatory configuration the router will load with to run the basic test bench + +### TFTP configuration: +# hostname - the tftp hostname +# ip_address - the tftp's ip address +# images_path - the tftp's relative path in which the router's images are located + +### Test_misc configuration: +# expected_bw - the "golden" bandwidth (in Gbps) results planned on receiving from the test + +trex: + hostname : csi-trex-07 + cores : 4 + +router: + model : ASR1001x + hostname : csi-asr-01 + ip_address : 10.56.216.120 + image : asr1001x-universalk9.03.13.02.S.154-3.S2-ext.SPA.bin + line_password : cisco + en_password : cisco + mgmt_interface : GigabitEthernet0 + clean_config : clean_config.cfg + intf_masking : 255.255.255.0 + ipv6_mask : 64 + interfaces : + - client : + name : Te0/0/0 + src_mac_addr : 0000.0001.0002 + dest_mac_addr : 0000.0001.0001 + server : + name : Te0/0/1 + src_mac_addr : 0000.0002.0002 + dest_mac_addr : 0000.0002.0001 + vrf_name : null + +tftp: + hostname : ats-asr-srv-1 + ip_address : 10.56.217.7 + root_dir : /scratch/tftp/ + images_path : /asr1001x/ diff --git a/scripts/automation/regression/setups/trex07/benchmark.yaml b/scripts/automation/regression/setups/trex07/benchmark.yaml index 0dc340b0..6e861836 100644 --- a/scripts/automation/regression/setups/trex07/benchmark.yaml +++ b/scripts/automation/regression/setups/trex07/benchmark.yaml @@ -4,120 +4,57 @@ #### common templates ### -stat_route_dict: &stat_route_dict - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - -nat_dict: &nat_dict - clients_net_start : 16.0.0.0 - client_acl_wildcard_mask : 0.0.0.255 - dual_port_mask : 1.0.0.0 - pool_start : 200.0.0.0 - pool_netmask : 255.255.255.0 - - -### stateful ### - test_jumbo: - multiplier : 17 - cores : 1 - bw_per_core : 543.232 + multiplier : 120 + cores : 2 + bw_per_core : 962.464 test_routing_imix: - multiplier : 10 - cores : 1 - bw_per_core : 34.128 + multiplier : 60 + cores : 4 + bw_per_core : 48.130 test_routing_imix_64: - multiplier : 430 - cores : 1 - bw_per_core : 5.893 - + multiplier : 4000 + cores : 7 + bw_per_core : 12.699 -test_static_routing_imix: &test_static_routing_imix - stat_route_dict : *stat_route_dict - multiplier : 8 - cores : 1 - bw_per_core : 34.339 -test_static_routing_imix_asymmetric: *test_static_routing_imix +test_static_routing_imix_asymmetric: + multiplier : 50 + cores : 3 + bw_per_core : 50.561 test_ipv6_simple: - multiplier : 9 - cores : 2 - bw_per_core : 19.064 - - -test_nat_simple_mode1: &test_nat_simple - stat_route_dict : *stat_route_dict - nat_dict : *nat_dict - multiplier : 6000 - cores : 1 - nat_opened : 500000 - allow_timeout_dev : True - bw_per_core : 44.445 - -test_nat_simple_mode2: *test_nat_simple - -test_nat_simple_mode3: *test_nat_simple - -test_nat_learning: *test_nat_simple - - -test_nbar_simple: - multiplier : 7.5 - cores : 2 - bw_per_core : 17.174 - nbar_classification: - rtp : 32.57 - http : 30.25 - oracle_sqlnet : 11.23 - exchange : 10.80 - citrix : 5.62 - rtsp : 2.84 - dns : 1.95 - smtp : 0.57 - pop3 : 0.36 - ssl : 0.17 - sctp : 0.13 - sip : 0.09 - unknown : 3.41 + multiplier : 50 + cores : 7 + bw_per_core : 19.5 test_rx_check_http: &rx_http - multiplier : 15000 - cores : 1 - rx_sample_rate : 16 - bw_per_core : 39.560 + multiplier : 99000 + cores : 7 + rx_sample_rate : 128 + bw_per_core : 49.464 test_rx_check_http_ipv6: << : *rx_http bw_per_core : 49.237 -test_rx_check_http_negative_disabled: - << : *rx_http - stat_route_dict : *stat_route_dict - nat_dict : *nat_dict - - test_rx_check_sfr: &rx_sfr - multiplier : 10 - cores : 3 - rx_sample_rate : 16 - bw_per_core : 16.082 + multiplier : 35 + cores : 7 + rx_sample_rate : 128 + bw_per_core : 20.871 test_rx_check_sfr_ipv6: << : *rx_sfr bw_per_core : 19.198 - ### stateless ### test_CPU_benchmark: @@ -178,10 +115,10 @@ test_CPU_benchmark: cpu_util : 1 bw_per_core : 1 - - name : stl/udp_for_benchmarks.py - kwargs : {packet_len: 9000, stream_count: 100} - cpu_util : 1 - bw_per_core : 1 + #- name : stl/udp_for_benchmarks.py + # kwargs : {packet_len: 9000, stream_count: 100} + # cpu_util : 1 + # bw_per_core : 1 # not enough memory + queue full if memory increase # - name : stl/udp_for_benchmarks.py @@ -241,4 +178,56 @@ test_CPU_benchmark: cpu_util : 1 bw_per_core : 1 - +test_performance_vm_single_cpu: + cfg: + mult : "90%" + mpps_per_core_golden : + min: 9.6 + max: 13.3 + + +test_performance_vm_single_cpu_cached: + cfg: + mult : "10%" + mpps_per_core_golden : + min: 16.0 + max: 25.0 + + + +test_performance_syn_attack_single_cpu: + cfg: + mult : "90%" + mpps_per_core_golden : + min: 9.0 + max: 14.0 + +test_performance_vm_multi_cpus: + cfg: + core_count : 7 + mult : "90%" + mpps_per_core_golden : + min: 8.5 + max: 12.0 + + +test_performance_vm_multi_cpus_cached: + cfg: + core_count : 7 + mult : "35%" + mpps_per_core_golden : + min: 9.0 + max: 15.0 + +test_performance_syn_attack_multi_cpus: + cfg: + core_count : 7 + mult : "90%" + mpps_per_core_golden : + min: 8.0 + max: 16.0 + + +test_all_profiles : + mult : "5%" + diff --git a/scripts/automation/regression/setups/trex07/config.yaml b/scripts/automation/regression/setups/trex07/config.yaml index db6e9bf8..10472c4f 100644 --- a/scripts/automation/regression/setups/trex07/config.yaml +++ b/scripts/automation/regression/setups/trex07/config.yaml @@ -35,32 +35,7 @@ trex: hostname : csi-trex-07 - cores : 4 + cores : 8 + modes : ['loopback'] -router: - model : ASR1001x - hostname : csi-asr-01 - ip_address : 10.56.216.120 - image : asr1001x-universalk9.03.13.02.S.154-3.S2-ext.SPA.bin - line_password : cisco - en_password : cisco - mgmt_interface : GigabitEthernet0 - clean_config : clean_config.cfg - intf_masking : 255.255.255.0 - ipv6_mask : 64 - interfaces : - - client : - name : Te0/0/0 - src_mac_addr : 0000.0001.0002 - dest_mac_addr : 0000.0001.0001 - server : - name : Te0/0/1 - src_mac_addr : 0000.0002.0002 - dest_mac_addr : 0000.0002.0001 - vrf_name : null -tftp: - hostname : ats-asr-srv-1 - ip_address : 10.56.217.7 - root_dir : /scratch/tftp/ - images_path : /asr1001x/ diff --git a/scripts/automation/regression/setups/trex08/benchmark.yaml b/scripts/automation/regression/setups/trex08/benchmark.yaml index 8f83e8f9..935b3e55 100644 --- a/scripts/automation/regression/setups/trex08/benchmark.yaml +++ b/scripts/automation/regression/setups/trex08/benchmark.yaml @@ -179,3 +179,53 @@ test_CPU_benchmark: bw_per_core : 1 +test_performance_vm_single_cpu: + cfg: + mult : "90%" + mpps_per_core_golden : + min: 15.1 + max: 20.3 + + +test_performance_vm_single_cpu_cached: + cfg: + mult : "10%" + mpps_per_core_golden : + min: 29.1 + max: 32.0 + + + +test_performance_syn_attack_single_cpu: + cfg: + mult : "90%" + mpps_per_core_golden : + min: 13.2 + max: 15.0 + +test_performance_vm_multi_cpus: + cfg: + core_count : 7 + mult : "40%" + mpps_per_core_golden : + min: 15.0 + max: 20.0 + + +test_performance_vm_multi_cpus_cached: + cfg: + core_count : 7 + mult : "40%" + mpps_per_core_golden : + min: 29.0 + max: 34.0 + +test_performance_syn_attack_multi_cpus: + cfg: + core_count : 7 + mult : "40%" + mpps_per_core_golden : + min: 13.0 + max: 17.0 + + diff --git a/scripts/automation/regression/setups/trex09/benchmark.yaml b/scripts/automation/regression/setups/trex09/benchmark.yaml index d1f5f56c..50f08351 100644 --- a/scripts/automation/regression/setups/trex09/benchmark.yaml +++ b/scripts/automation/regression/setups/trex09/benchmark.yaml @@ -187,7 +187,7 @@ test_performance_vm_single_cpu: cfg: mult : "90%" mpps_per_core_golden : - min: 16.2 + min: 13.0 max: 17.3 @@ -195,7 +195,7 @@ test_performance_vm_single_cpu_cached: cfg: mult : "90%" mpps_per_core_golden : - min: 29.5 + min: 28.5 max: 31.2 @@ -221,7 +221,7 @@ test_performance_vm_multi_cpus_cached: core_count : 2 mult : "90%" mpps_per_core_golden : - min: 28.8 + min: 26.8 max: 29.5 test_performance_syn_attack_multi_cpus: diff --git a/scripts/automation/regression/setups/trex11/backup/benchmark.yaml b/scripts/automation/regression/setups/trex11/backup/benchmark.yaml new file mode 100644 index 00000000..b366b3fb --- /dev/null +++ b/scripts/automation/regression/setups/trex11/backup/benchmark.yaml @@ -0,0 +1,155 @@ +################################################################ +#### TRex benchmark configuration file #### +################################################################ + +### stateful ### + +test_jumbo: + multiplier : 2.8 + cores : 1 + bw_per_core : 106.652 + + +test_routing_imix: + multiplier : 0.5 + cores : 1 + bw_per_core : 11.577 + + +test_routing_imix_64: + multiplier : 28 + cores : 1 + bw_per_core : 2.030 + + +test_static_routing_imix_asymmetric: + multiplier : 0.8 + cores : 1 + bw_per_core : 13.742 + + + +### stateless ### + +test_CPU_benchmark: + profiles: + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 64, stream_count: 10} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 64, stream_count: 100} + cpu_util : 1 + bw_per_core : 1 + +# causes queue full +# - name : stl/udp_for_benchmarks.py +# kwargs : {packet_len: 64, stream_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 9000, stream_count: 10} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 9000, stream_count: 100} + cpu_util : 1 + bw_per_core : 1 + +# not enough memory + queue full if memory increase +# - name : stl/udp_for_benchmarks.py +# kwargs : {packet_len: 9000, stream_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/imix.py + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/pcap.py + kwargs : {ipg_usec: 4, loop_count: 0} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 + + - name : stl/hlt/hlt_udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 + + diff --git a/scripts/automation/regression/setups/trex11/backup/config.yaml b/scripts/automation/regression/setups/trex11/backup/config.yaml new file mode 100644 index 00000000..782b7542 --- /dev/null +++ b/scripts/automation/regression/setups/trex11/backup/config.yaml @@ -0,0 +1,38 @@ +################################################################ +#### TRex nightly test configuration file #### +################################################################ + + +### TRex configuration: +# hostname - can be DNS name or IP for the TRex machine for ssh to the box +# password - root password for TRex machine +# is_dual - should the TRex inject with -p ? +# version_path - path to the TRex version and executable +# cores - how many cores should be used +# latency - rate of latency packets injected by the TRex +# modes - list of modes (tagging) of this setup (loopback, virtual etc.) +# * loopback - Trex works via loopback. Router and TFTP configurations may be skipped. +# * virtual - virtual OS (accept low CPU utilization in tests) + +### Router configuration: +# hostname - the router hostname as apears in ______# cli prefix +# ip_address - the router's ip that can be used to communicate with +# image - the desired imaged wished to be loaded as the router's running config +# line_password - router password when access via Telent +# en_password - router password when changing to "enable" mode +# interfaces - an array of client-server pairs, representing the interfaces configurations of the router +# configurations - an array of configurations that could possibly loaded into the router during the test. +# The "clean" configuration is a mandatory configuration the router will load with to run the basic test bench + +### TFTP configuration: +# hostname - the tftp hostname +# ip_address - the tftp's ip address +# images_path - the tftp's relative path in which the router's images are located + +### Test_misc configuration: +# expected_bw - the "golden" bandwidth (in Gbps) results planned on receiving from the test + +trex: + hostname : csi-trex-11 + cores : 1 + modes : ['loopback', 'VM', 'virt_nics'] diff --git a/scripts/automation/regression/setups/trex11/benchmark.yaml b/scripts/automation/regression/setups/trex11/benchmark.yaml index b366b3fb..5ebcdd55 100644 --- a/scripts/automation/regression/setups/trex11/benchmark.yaml +++ b/scripts/automation/regression/setups/trex11/benchmark.yaml @@ -1,32 +1,58 @@ -################################################################ +############################################################### #### TRex benchmark configuration file #### -################################################################ +############################################################### -### stateful ### +#### common templates ### -test_jumbo: - multiplier : 2.8 - cores : 1 - bw_per_core : 106.652 +#test_jumbo: +# multiplier : 2.8 +# cores : 1 +# bw_per_core : 962.464 test_routing_imix: multiplier : 0.5 cores : 1 - bw_per_core : 11.577 + bw_per_core : 48.130 test_routing_imix_64: multiplier : 28 cores : 1 - bw_per_core : 2.030 + bw_per_core : 12.699 test_static_routing_imix_asymmetric: - multiplier : 0.8 + multiplier : 0.5 + cores : 1 + bw_per_core : 50.561 + + +test_ipv6_simple: + multiplier : 0.5 + cores : 1 + bw_per_core : 19.5 + + +test_rx_check_http: &rx_http + multiplier : 1000 + cores : 1 + rx_sample_rate : 128 + bw_per_core : 49.464 + +test_rx_check_http_ipv6: + << : *rx_http + bw_per_core : 49.237 + +test_rx_check_sfr: &rx_sfr + multiplier : 8 cores : 1 - bw_per_core : 13.742 + rx_sample_rate : 128 + bw_per_core : 20.9 +test_rx_check_sfr_ipv6: + << : *rx_sfr + bw_per_core : 23.9 ### stateless ### @@ -140,16 +166,21 @@ test_CPU_benchmark: bw_per_core : 1 - name : stl/pcap.py - kwargs : {ipg_usec: 4, loop_count: 0} + kwargs : {ipg_usec: 2, loop_count: 0} cpu_util : 1 bw_per_core : 1 - - name : stl/udp_rand_len_9k.py - cpu_util : 1 - bw_per_core : 1 + #- name : stl/udp_rand_len_9k.py + # cpu_util : 1 + # bw_per_core : 1 - - name : stl/hlt/hlt_udp_rand_len_9k.py - cpu_util : 1 - bw_per_core : 1 + #- name : stl/hlt/hlt_udp_rand_len_9k.py + # cpu_util : 1 + # bw_per_core : 1 + +test_all_profiles : + mult : "5%" + skip : ['udp_rand_len_9k.py','udp_inc_len_9k.py'] # due to VIC 9K defect trex-282 + diff --git a/scripts/automation/regression/setups/trex11/config.yaml b/scripts/automation/regression/setups/trex11/config.yaml index 782b7542..393c8749 100644 --- a/scripts/automation/regression/setups/trex11/config.yaml +++ b/scripts/automation/regression/setups/trex11/config.yaml @@ -4,15 +4,16 @@ ### TRex configuration: -# hostname - can be DNS name or IP for the TRex machine for ssh to the box -# password - root password for TRex machine -# is_dual - should the TRex inject with -p ? -# version_path - path to the TRex version and executable -# cores - how many cores should be used -# latency - rate of latency packets injected by the TRex -# modes - list of modes (tagging) of this setup (loopback, virtual etc.) -# * loopback - Trex works via loopback. Router and TFTP configurations may be skipped. -# * virtual - virtual OS (accept low CPU utilization in tests) +# hostname - can be DNS name or IP for the TRex machine for ssh to the box +# password - root password for TRex machine +# is_dual - should the TRex inject with -p ? +# version_path - path to the TRex version and executable +# cores - how many cores should be used +# latency - rate of latency packets injected by the TRex +# modes - list of modes (tagging) of this setup (loopback etc.) +# * loopback - Trex works via loopback. Router and TFTP configurations may be skipped. +# * VM - Virtual OS (accept low CPU utilization in tests, latency can get spikes) +# * virt_nics - NICs are virtual (VMXNET3 etc.) ### Router configuration: # hostname - the router hostname as apears in ______# cli prefix @@ -35,4 +36,6 @@ trex: hostname : csi-trex-11 cores : 1 - modes : ['loopback', 'VM', 'virt_nics'] + modes : ['loopback'] + + diff --git a/scripts/automation/regression/setups/trex14/BU/benchmark.yaml b/scripts/automation/regression/setups/trex14/BU/benchmark.yaml new file mode 100644 index 00000000..04f13e79 --- /dev/null +++ b/scripts/automation/regression/setups/trex14/BU/benchmark.yaml @@ -0,0 +1,245 @@ +############################################################### +#### TRex benchmark configuration file #### +############################################################### + +#### common templates ### + +stat_route_dict: &stat_route_dict + clients_start : 16.0.0.1 + servers_start : 48.0.0.1 + dual_port_mask : 1.0.0.0 + client_destination_mask : 255.0.0.0 + server_destination_mask : 255.0.0.0 + +nat_dict: &nat_dict + clients_net_start : 16.0.0.0 + client_acl_wildcard_mask : 0.0.0.255 + dual_port_mask : 1.0.0.0 + pool_start : 200.0.0.0 + pool_netmask : 255.255.255.0 + + +### stateful ### + +test_jumbo: + multiplier : 17 + cores : 1 + bw_per_core : 543.232 + + +test_routing_imix: + multiplier : 10 + cores : 1 + bw_per_core : 34.128 + + +test_routing_imix_64: + multiplier : 430 + cores : 1 + bw_per_core : 5.893 + + +test_static_routing_imix: &test_static_routing_imix + stat_route_dict : *stat_route_dict + multiplier : 8 + cores : 1 + bw_per_core : 34.339 + +test_static_routing_imix_asymmetric: *test_static_routing_imix + + +test_ipv6_simple: + multiplier : 9 + cores : 2 + bw_per_core : 19.064 + + +test_nat_simple_mode1: &test_nat_simple + stat_route_dict : *stat_route_dict + nat_dict : *nat_dict + multiplier : 6000 + cores : 1 + nat_opened : 500000 + allow_timeout_dev : True + bw_per_core : 44.445 + +test_nat_simple_mode2: *test_nat_simple + +test_nat_simple_mode3: *test_nat_simple + +test_nat_learning: *test_nat_simple + + +test_nbar_simple: + multiplier : 7.5 + cores : 2 + bw_per_core : 17.174 + nbar_classification: + http : 32.58 + rtp-audio : 21.21 + oracle_sqlnet : 11.41 + exchange : 11.22 + rtp : 11.2 + citrix : 5.65 + rtsp : 2.87 + dns : 1.96 + smtp : 0.57 + pop3 : 0.37 + ssl : 0.28 + sctp : 0.13 + sip : 0.09 + unknown : 0.45 + + +test_rx_check_http: &rx_http + multiplier : 15000 + cores : 1 + rx_sample_rate : 16 + bw_per_core : 39.560 + +test_rx_check_http_ipv6: + << : *rx_http + bw_per_core : 49.237 + +test_rx_check_http_negative_disabled: + << : *rx_http + stat_route_dict : *stat_route_dict + nat_dict : *nat_dict + + +test_rx_check_sfr: &rx_sfr + multiplier : 10 + cores : 3 + rx_sample_rate : 16 + bw_per_core : 16.082 + +test_rx_check_sfr_ipv6: + << : *rx_sfr + bw_per_core : 19.198 + + + +### stateless ### + +test_CPU_benchmark: + profiles: + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 64, stream_count: 10} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 64, stream_count: 100} + cpu_util : 1 + bw_per_core : 1 + +# causes queue full +# - name : stl/udp_for_benchmarks.py +# kwargs : {packet_len: 64, stream_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 9000, stream_count: 10} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_for_benchmarks.py + kwargs : {packet_len: 9000, stream_count: 100} + cpu_util : 1 + bw_per_core : 1 + +# not enough memory + queue full if memory increase +# - name : stl/udp_for_benchmarks.py +# kwargs : {packet_len: 9000, stream_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/imix.py + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/pcap.py + kwargs : {ipg_usec: 2, loop_count: 0} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 + + - name : stl/hlt/hlt_udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 + + diff --git a/scripts/automation/regression/setups/trex14/BU/config.yaml b/scripts/automation/regression/setups/trex14/BU/config.yaml new file mode 100644 index 00000000..0fd6b70e --- /dev/null +++ b/scripts/automation/regression/setups/trex14/BU/config.yaml @@ -0,0 +1,67 @@ +################################################################ +#### TRex nightly test configuration file #### +################################################################ + + +### TRex configuration: +# hostname - can be DNS name or IP for the TRex machine for ssh to the box +# password - root password for TRex machine +# is_dual - should the TRex inject with -p ? +# version_path - path to the TRex version and executable +# cores - how many cores should be used +# latency - rate of latency packets injected by the TRex +# modes - list of modes (tagging) of this setup (loopback etc.) +# * loopback - Trex works via loopback. Router and TFTP configurations may be skipped. +# * VM - Virtual OS (accept low CPU utilization in tests, latency can get spikes) +# * virt_nics - NICs are virtual (VMXNET3 etc.) + +### Router configuration: +# hostname - the router hostname as apears in ______# cli prefix +# ip_address - the router's ip that can be used to communicate with +# image - the desired imaged wished to be loaded as the router's running config +# line_password - router password when access via Telent +# en_password - router password when changing to "enable" mode +# interfaces - an array of client-server pairs, representing the interfaces configurations of the router +# configurations - an array of configurations that could possibly loaded into the router during the test. +# The "clean" configuration is a mandatory configuration the router will load with to run the basic test bench + +### TFTP configuration: +# hostname - the tftp hostname +# ip_address - the tftp's ip address +# images_path - the tftp's relative path in which the router's images are located + +### Test_misc configuration: +# expected_bw - the "golden" bandwidth (in Gbps) results planned on receiving from the test + +trex: + hostname : csi-trex-14 + cores : 4 + modes : [] + +router: + model : ASR1001x + hostname : csi-asr-01 + ip_address : 10.56.216.103 + image : asr1001x-universalk9.03.17.00.S.156-1.S-std.SPA.bin + line_password : cisco + en_password : cisco + mgmt_interface : GigabitEthernet0 + clean_config : /Configurations/danklei/asr1001_TRex_clean_config.cfg + intf_masking : 255.255.255.0 + ipv6_mask : 64 + interfaces : + - client : + name : Te0/0/0 + src_mac_addr : 0000.0001.0000 + dest_mac_addr : 0000.0001.0000 + server : + name : Te0/0/1 + src_mac_addr : 0000.0001.0000 + dest_mac_addr : 0000.0001.0000 + vrf_name : null + +tftp: + hostname : ats-asr-srv-1 + ip_address : 10.56.217.7 + root_dir : /scratch/tftp/ + images_path : /asr1001x/ diff --git a/scripts/automation/regression/setups/trex14/benchmark.yaml b/scripts/automation/regression/setups/trex14/benchmark.yaml index 04f13e79..0dc340b0 100644 --- a/scripts/automation/regression/setups/trex14/benchmark.yaml +++ b/scripts/automation/regression/setups/trex14/benchmark.yaml @@ -75,20 +75,19 @@ test_nbar_simple: cores : 2 bw_per_core : 17.174 nbar_classification: - http : 32.58 - rtp-audio : 21.21 - oracle_sqlnet : 11.41 - exchange : 11.22 - rtp : 11.2 - citrix : 5.65 - rtsp : 2.87 - dns : 1.96 + rtp : 32.57 + http : 30.25 + oracle_sqlnet : 11.23 + exchange : 10.80 + citrix : 5.62 + rtsp : 2.84 + dns : 1.95 smtp : 0.57 - pop3 : 0.37 - ssl : 0.28 + pop3 : 0.36 + ssl : 0.17 sctp : 0.13 sip : 0.09 - unknown : 0.45 + unknown : 3.41 test_rx_check_http: &rx_http diff --git a/scripts/automation/regression/setups/trex14/config.yaml b/scripts/automation/regression/setups/trex14/config.yaml index 0fd6b70e..ffb61763 100644 --- a/scripts/automation/regression/setups/trex14/config.yaml +++ b/scripts/automation/regression/setups/trex14/config.yaml @@ -36,28 +36,27 @@ trex: hostname : csi-trex-14 cores : 4 - modes : [] router: model : ASR1001x hostname : csi-asr-01 - ip_address : 10.56.216.103 - image : asr1001x-universalk9.03.17.00.S.156-1.S-std.SPA.bin + ip_address : 10.56.216.120 + image : asr1001x-universalk9.03.13.02.S.154-3.S2-ext.SPA.bin line_password : cisco en_password : cisco mgmt_interface : GigabitEthernet0 - clean_config : /Configurations/danklei/asr1001_TRex_clean_config.cfg + clean_config : clean_config.cfg intf_masking : 255.255.255.0 ipv6_mask : 64 interfaces : - client : name : Te0/0/0 - src_mac_addr : 0000.0001.0000 - dest_mac_addr : 0000.0001.0000 + src_mac_addr : 0000.0001.0002 + dest_mac_addr : 0000.0001.0001 server : name : Te0/0/1 - src_mac_addr : 0000.0001.0000 - dest_mac_addr : 0000.0001.0000 + src_mac_addr : 0000.0002.0002 + dest_mac_addr : 0000.0002.0001 vrf_name : null tftp: diff --git a/scripts/automation/regression/stateful_tests/trex_client_cfg_test.py b/scripts/automation/regression/stateful_tests/trex_client_cfg_test.py new file mode 100644 index 00000000..852e745d --- /dev/null +++ b/scripts/automation/regression/stateful_tests/trex_client_cfg_test.py @@ -0,0 +1,52 @@ +#!/router/bin/python +from .trex_general_test import CTRexGeneral_Test, CTRexScenario +from CPlatform import CStaticRouteConfig +from .tests_exceptions import * +#import sys +import time +from nose.tools import nottest + +# Testing client cfg ARP resolve. Actually, just need to check that TRex run finished with no errors. +# If resolve will fail, TRex will exit with exit code != 0 +class CTRexClientCfg_Test(CTRexGeneral_Test): + """This class defines the IMIX testcase of the TRex traffic generator""" + def __init__(self, *args, **kwargs): + # super(CTRexClientCfg_Test, self).__init__() + CTRexGeneral_Test.__init__(self, *args, **kwargs) + + def setUp(self): + if CTRexScenario.setup_name == 'kiwi02': + self.skip("Can't run currently on kiwi02") + super(CTRexClientCfg_Test, self).setUp() # launch super test class setUp process + pass + + def test_client_cfg(self): + # test initializtion + if self.is_loopback: + return + else: + self.router.configure_basic_interfaces() + self.router.config_pbr(mode = "config") + + ret = self.trex.start_trex( + c = 1, + m = 1, + d = 10, + f = 'cap2/dns.yaml', + v = 3, + client_cfg = 'automation/regression/cfg/client_cfg.yaml', + l = 1000) + + trex_res = self.trex.sample_to_run_finish() + + print("\nLATEST RESULT OBJECT:") + print(trex_res) + + self.check_general_scenario_results(trex_res) + + def tearDown(self): + CTRexGeneral_Test.tearDown(self) + pass + +if __name__ == "__main__": + pass diff --git a/scripts/automation/regression/stateful_tests/trex_general_test.py b/scripts/automation/regression/stateful_tests/trex_general_test.py index e968d380..fe38ed34 100755 --- a/scripts/automation/regression/stateful_tests/trex_general_test.py +++ b/scripts/automation/regression/stateful_tests/trex_general_test.py @@ -198,11 +198,14 @@ class CTRexGeneral_Test(unittest.TestCase): def check_for_trex_crash(self): pass - def get_benchmark_param (self, param, sub_param = None, test_name = None): + def get_benchmark_param (self, param, sub_param = None, test_name = None,default=None): if not test_name: test_name = self.get_name() if test_name not in self.benchmark: - self.skip('No data in benchmark.yaml for test: %s, param: %s. Skipping.' % (test_name, param)) + if default ==None: + self.skip('No data in benchmark.yaml for test: %s, param: %s. Skipping.' % (test_name, param)) + else: + return default if sub_param: return self.benchmark[test_name][param].get(sub_param) else: @@ -254,7 +257,7 @@ class CTRexGeneral_Test(unittest.TestCase): allowed_latency = 1000 if max(trex_res.get_max_latency().values()) > allowed_latency: self.fail('LatencyError: Maximal latency exceeds %s (usec)' % allowed_latency) - + # check that avg latency does not exceed 1 msec if self.is_VM: allowed_latency = 9999999 @@ -263,6 +266,15 @@ class CTRexGeneral_Test(unittest.TestCase): if max(trex_res.get_avg_latency().values()) > allowed_latency: self.fail('LatencyError: Average latency exceeds %s (usec)' % allowed_latency) + ports_names = trex_res.get_last_value('trex-latecny-v2.data', 'port\-\d+') + if not ports_names: + raise AbnormalResultError('Could not find ports info in TRex results, path: trex-latecny-v2.data.port-*') + for port_name in ports_names: + path = 'trex-latecny-v2.data.%s.hist.cnt' % port_name + lat_count = trex_res.get_last_value(path) + if lat_count == 0: + self.fail('LatencyError: Number of latency packets received on %s is 0' % port_name) + if not self.is_loopback: # check router number of drops --> deliberately masked- need to be figured out!!!!! pkt_drop_stats = self.router.get_drop_stats() @@ -356,7 +368,7 @@ class CTRexGeneral_Test(unittest.TestCase): print("Can't get TRex log:", e) if len(self.fail_reasons): sys.stdout.flush() - raise Exception('The test is failed, reasons:\n%s' % '\n'.join(self.fail_reasons)) + raise Exception('Test failed. Reasons:\n%s' % '\n'.join(self.fail_reasons)) sys.stdout.flush() def check_for_trex_crash(self): diff --git a/scripts/automation/regression/stateless_tests/stl_client_test.py b/scripts/automation/regression/stateless_tests/stl_client_test.py index 36ac0ee1..73dac734 100644 --- a/scripts/automation/regression/stateless_tests/stl_client_test.py +++ b/scripts/automation/regression/stateless_tests/stl_client_test.py @@ -240,10 +240,26 @@ class STLClient_Test(CStlGeneral_Test): self.skip('skipping profile tests for virtual / non loopback') return + default_mult = self.get_benchmark_param('mult',default="30%") + skip_tests = self.get_benchmark_param('skip',default=[]) + try: - + print("\n"); + + for profile in self.profiles: + skip=False + if skip_tests: + for skip_test in skip_tests: + if skip_test in profile: + skip=True; + break; + if skip: + print("skipping testing profile due to config file {0}...\n".format(profile)) + continue; + + print("now testing profile {0}...\n".format(profile)) p1 = STLProfile.load(profile, port_id = self.tx_port) @@ -269,7 +285,7 @@ class STLClient_Test(CStlGeneral_Test): self.c.clear_stats() - self.c.start(ports = [self.tx_port, self.rx_port], mult = "30%") + self.c.start(ports = [self.tx_port, self.rx_port], mult = default_mult) time.sleep(100 / 1000.0) if p1.is_pauseable() and p2.is_pauseable(): diff --git a/scripts/automation/regression/stateless_tests/stl_performance_test.py b/scripts/automation/regression/stateless_tests/stl_performance_test.py index a556daf3..641f0a33 100644 --- a/scripts/automation/regression/stateless_tests/stl_performance_test.py +++ b/scripts/automation/regression/stateless_tests/stl_performance_test.py @@ -296,6 +296,10 @@ class STLPerformance_Test(CStlGeneral_Test): # sample bps/pps for _ in range(0, 20): stats = self.c.get_stats(ports = 0) + if stats['global'][ 'queue_full']>10000: + assert 0, "Queue is full need to tune the multiplier" + + # CPU results are not valid cannot use them samples['bps'].append(stats[0]['tx_bps']) samples['pps'].append(stats[0]['tx_pps']) time.sleep(1) diff --git a/scripts/automation/regression/stateless_tests/stl_rx_test.py b/scripts/automation/regression/stateless_tests/stl_rx_test.py index 524ad4bf..4dad712f 100644 --- a/scripts/automation/regression/stateless_tests/stl_rx_test.py +++ b/scripts/automation/regression/stateless_tests/stl_rx_test.py @@ -51,6 +51,24 @@ class STLRX_Test(CStlGeneral_Test): 'latency_9k_enable': False, 'allow_packets_drop_num': 1, # allow 1 pkt drop }, + + 'librte_pmd_mlx5': { + 'rate_percent': 80, + 'total_pkts': 1000, + 'rate_latency': 1, + 'latency_9k_enable': True, + 'latency_9k_max_average': 100, + 'latency_9k_max_latency': 250, + }, + + 'rte_enic_pmd': { + 'rate_percent': 1, + 'total_pkts': 50, + 'rate_latency': 1, + 'latency_9k_enable': False, + }, + + } CStlGeneral_Test.setUp(self) @@ -63,7 +81,6 @@ class STLRX_Test(CStlGeneral_Test): port_info = self.c.get_port_info(ports = self.rx_port)[0] self.speed = port_info['speed'] - cap = port_info['rx']['caps'] if "flow_stats" not in cap or "latency" not in cap: self.skip('port {0} does not support RX'.format(self.rx_port)) @@ -400,12 +417,14 @@ class STLRX_Test(CStlGeneral_Test): s_port=random.sample(all_ports, random.randint(1, len(all_ports)) ) s_port=sorted(s_port) - if self.speed == 40 : + + if ((self.speed == 40) or (self.speed == 100)): # the NIC does not support all full rate in case both port works let's filter odd ports s_port=list(filter(lambda x: x % 2==0, s_port)) if len(s_port)==0: s_port=[0]; + error=1; for j in range(0,5): print(" {4} - duration {0} pgid {1} pkt_size {2} s_port {3} ".format(duration,pgid,pkt_size,s_port,j)); diff --git a/scripts/automation/regression/trex.py b/scripts/automation/regression/trex.py index 7b96f2f8..416a6e3b 100644 --- a/scripts/automation/regression/trex.py +++ b/scripts/automation/regression/trex.py @@ -35,7 +35,7 @@ class CTRexScenario: report_dir = 'reports' # logger = None test_types = {'functional_tests': [], 'stateful_tests': [], 'stateless_tests': []} - is_copied = False + pkg_updated = False GAManager = None no_daemon = False debug_image = False diff --git a/scripts/automation/regression/trex_unit_test.py b/scripts/automation/regression/trex_unit_test.py index daa1abaf..09614770 100755 --- a/scripts/automation/regression/trex_unit_test.py +++ b/scripts/automation/regression/trex_unit_test.py @@ -137,35 +137,29 @@ class CTRexTestConfiguringPlugin(Plugin): parser.add_option('--stf', '--stateful', action="store_true", default = False, dest="stateful", help="Run stateful tests.") - parser.add_option('--pkg', action="store", - dest="pkg", - help="Run with given TRex package. Make sure the path available at server machine.") + parser.add_option('--pkg', type = str, + help="Run with given TRex package. Make sure the path available at server machine. Implies --restart-daemon.") + parser.add_option('--restart-daemon', action="store_true", default = False, + help="Flag that specifies to restart daemon. Implied by --pkg.") parser.add_option('--collect', action="store_true", default = False, - dest="collect", help="Alias to --collect-only.") parser.add_option('--warmup', action="store_true", default = False, - dest="warmup", help="Warm up the system for stateful: run 30 seconds 9k imix test without check of results.") parser.add_option('--test-client-package', '--test_client_package', action="store_true", default = False, - dest="test_client_package", help="Includes tests of client package.") parser.add_option('--long', action="store_true", default = False, - dest="long", help="Flag of long tests (stability).") parser.add_option('--ga', action="store_true", default = False, - dest="ga", help="Flag to send benchmarks to GA.") parser.add_option('--no-daemon', action="store_true", default = False, dest="no_daemon", help="Flag that specifies to use running stl server, no need daemons.") parser.add_option('--debug-image', action="store_true", default = False, - dest="debug_image", help="Flag that specifies to use t-rex-64-debug as TRex executable.") - parser.add_option('--trex-args', action='store', default = '', - dest="trex_args", + parser.add_option('--trex-args', default = '', help="Additional TRex arguments (--no-watchdog etc.).") - parser.add_option('-t', '--test', action='store', default = '', dest='test', - help='Test name to run (without file, class etc.)') + parser.add_option('-t', '--test', type = str, + help = 'Test name to run (without file, class etc.)') def configure(self, options, conf): @@ -174,10 +168,13 @@ class CTRexTestConfiguringPlugin(Plugin): self.stateless = options.stateless self.stateful = options.stateful self.pkg = options.pkg + self.restart_daemon = options.restart_daemon self.json_verbose = options.json_verbose self.telnet_verbose = options.telnet_verbose self.no_daemon = options.no_daemon CTRexScenario.test = options.test + if self.no_daemon and (self.pkg or self.restart_daemon): + raise Exception('You have specified both --no-daemon and either --pkg or --restart-daemon at same time.') if self.collect_only or self.functional: return if CTRexScenario.setup_dir and options.config_path: @@ -212,6 +209,7 @@ class CTRexTestConfiguringPlugin(Plugin): verbose = self.json_verbose, debug_image = options.debug_image, trex_args = options.trex_args) + if self.pkg or self.restart_daemon: if not CTRexScenario.trex.check_master_connectivity(): print('Could not connect to master daemon') sys.exit(-1) @@ -228,20 +226,20 @@ class CTRexTestConfiguringPlugin(Plugin): def begin (self): client = CTRexScenario.trex - if self.pkg and not CTRexScenario.is_copied: + if self.pkg and not CTRexScenario.pkg_updated: if client.master_daemon.is_trex_daemon_running() and client.get_trex_cmds() and not self.kill_running: - print("Can't update TRex, it's running") + print("Can't update TRex, it's running. Consider adding --kill-running flag.") sys.exit(-1) print('Updating TRex to %s' % self.pkg) if not client.master_daemon.update_trex(self.pkg): - print('Failed updating TRex') + print('Failed updating TRex.') sys.exit(-1) else: - print('Updated') - CTRexScenario.is_copied = True + print('Updated.') + CTRexScenario.pkg_updated = True if self.functional or self.collect_only: return - if not self.no_daemon: + if self.pkg or self.restart_daemon: print('Restarting TRex daemon server') res = client.restart_trex_daemon() if not res: |