diff options
10 files changed, 526 insertions, 33 deletions
diff --git a/scripts/automation/regression/CPlatform.py b/scripts/automation/regression/CPlatform.py index 314aae63..36e87821 100755 --- a/scripts/automation/regression/CPlatform.py +++ b/scripts/automation/regression/CPlatform.py @@ -618,8 +618,8 @@ class CPlatform(object): """ pre_commit_cache = CCommandCache() - pre_commit_cache.add('EXEC', ['clear counters','\r'] ) - self.cmd_link.run_single_command( pre_commit_cache ) + pre_commit_cache.add('EXEC', ['clear counters', '\r'] ) + self.cmd_link.run_single_command( pre_commit_cache , read_until = ['#', '\[confirm\]']) def clear_nbar_stats(self): """ clear_nbar_stats(self) -> None @@ -725,7 +725,7 @@ class CPlatform(object): progress_thread = CProgressDisp.ProgressThread(notifyMessage = "Copying image via tftp, this may take a while...\n") progress_thread.start() - response = self.cmd_link.run_single_command(cache, timeout = 900, read_until = ['\?', '\#']) + response = self.cmd_link.run_single_command(cache, timeout = 900, read_until = ['\?', '#']) print("RESPONSE:") print(response) progress_thread.join() diff --git a/scripts/automation/regression/platform_cmd_link.py b/scripts/automation/regression/platform_cmd_link.py index d2143a5d..d034fac3 100755 --- a/scripts/automation/regression/platform_cmd_link.py +++ b/scripts/automation/regression/platform_cmd_link.py @@ -95,11 +95,9 @@ class CCommandLink(object): def __transmit (self, cmd_list, **kwargs): self.history.extend(cmd_list) - if not self.silent_mode: - print('\n'.join(cmd_list)) # prompting the pushed platform commands if not self.virtual_mode: # transmit the command to platform. - return self.telnet_con.write_ios_cmd(cmd_list, **kwargs) + return self.telnet_con.write_ios_cmd(cmd_list, verbose = not self.silent_mode, **kwargs) def run_command (self, cmd_list, **kwargs): response = '' @@ -420,7 +418,7 @@ class CIosTelnet(telnetlib.Telnet): except Exception as inst: raise - def write_ios_cmd (self, cmd_list, result_from = 0, timeout = 3, **kwargs): + def write_ios_cmd (self, cmd_list, result_from = 0, timeout = 60, **kwargs): assert (isinstance (cmd_list, list) == True) self.read_until(self.pr, timeout = 1) @@ -431,19 +429,22 @@ class CIosTelnet(telnetlib.Telnet): wf = self.pr for idx, cmd in enumerate(cmd_list): + start_time = time.time() self.write(cmd+'\r\n') - if idx < result_from: - # don't care for return string - if type(wf) is list: - self.expect(wf, timeout)[2] - else: - self.read_until(wf, timeout) + if kwargs.get('verbose'): + print('-->\n%s' % cmd) + if type(wf) is list: + output = self.expect(wf, timeout)[2] else: - # care for return string - if type(wf) is list: - res += self.expect(wf, timeout)[2] - else: - res += self.read_until(wf, timeout) + output = self.read_until(wf, timeout) + if idx >= result_from: + res += output + if kwargs.get('verbose'): + print('<-- (%ss)\n%s' % (round(time.time() - start_time, 2), output)) + if time.time() - start_time > timeout - 1: + raise Exception('Timeout while performing telnet command: %s' % cmd) + if 'Invalid' in res: + print('Warning: telnet command probably failed.\nCommand: %s\nResponse: %s' % (cmd_list, res)) # return res.split('\r\n') return res # return the received response as a string, each line is seperated by '\r\n'. diff --git a/scripts/automation/regression/setups/kiwi02/benchmark.yaml b/scripts/automation/regression/setups/kiwi02/benchmark.yaml index 60febc8f..f136a835 100644 --- a/scripts/automation/regression/setups/kiwi02/benchmark.yaml +++ b/scripts/automation/regression/setups/kiwi02/benchmark.yaml @@ -142,11 +142,12 @@ test_rx_check_http_ipv6: rx_sample_rate : 32 error_tolerance : 0.01 -test_rx_check_http_negative: - multiplier : 40000 - cores : 2 - rx_sample_rate : 32 - error_tolerance : 0.01 +#works badly on kiwi02, comment out for now +#test_rx_check_http_negative: +# multiplier : 40000 +# cores : 2 +# rx_sample_rate : 32 +# error_tolerance : 0.01 test_jumbo: multiplier : 55 diff --git a/scripts/automation/regression/setups/trex07/benchmark.yaml b/scripts/automation/regression/setups/trex07/benchmark.yaml new file mode 100644 index 00000000..4778de91 --- /dev/null +++ b/scripts/automation/regression/setups/trex07/benchmark.yaml @@ -0,0 +1,170 @@ +############################################################### +#### TRex benchmark configuration file #### +############################################################### + +test_nbar_simple : + multiplier : 7.5 + cores : 2 + exp_gbps : 3.5 + cpu_to_core_ratio : 20800000 + cpu2core_custom_dev: YES + cpu2core_dev : 0.07 + exp_max_latency : 1000 + + nbar_classification: + rtp : 32.57 + http : 30.25 + oracle-sqlnet : 11.23 + exchange : 10.80 + citrix : 5.62 + rtsp : 2.84 + dns : 1.95 + smtp : 0.57 + pop3 : 0.36 + ssl : 0.17 + sctp : 0.13 + sip : 0.09 + unknown : 3.41 + +test_rx_check : + multiplier : 13 + cores : 3 + rx_sample_rate : 128 + exp_gbps : 6 + cpu_to_core_ratio : 37270000 + exp_bw : 13 + exp_latency : 1 + +test_nat_simple : &test_nat_simple + stat_route_dict : + clients_start : 16.0.0.1 + servers_start : 48.0.0.1 + dual_port_mask : 1.0.0.0 + client_destination_mask : 255.0.0.0 + server_destination_mask : 255.0.0.0 + nat_dict : + clients_net_start : 16.0.0.0 + client_acl_wildcard_mask : 0.0.0.255 + dual_port_mask : 1.0.0.0 + pool_start : 200.0.0.0 + pool_netmask : 255.255.255.0 + multiplier : 12000 + cores : 1 + cpu_to_core_ratio : 37270000 + exp_bw : 1 + exp_latency : 1 + allow_timeout_dev : YES + +test_nat_simple_mode1 : *test_nat_simple +test_nat_simple_mode2 : *test_nat_simple + +test_nat_learning : + stat_route_dict : + clients_start : 16.0.0.1 + servers_start : 48.0.0.1 + dual_port_mask : 1.0.0.0 + client_destination_mask : 255.0.0.0 + server_destination_mask : 255.0.0.0 + multiplier : 12000 + cores : 1 + nat_opened : 40000 + cpu_to_core_ratio : 270 + exp_bw : 8 + exp_latency : 1 + allow_timeout_dev : YES + +test_routing_imix_64 : + multiplier : 430 + cores : 1 + cpu_to_core_ratio : 280 + exp_latency : 1 + +test_routing_imix : + multiplier : 10 + cores : 1 + cpu_to_core_ratio : 1800 + exp_latency : 1 + +test_static_routing_imix : + stat_route_dict : + clients_start : 16.0.0.1 + servers_start : 48.0.0.1 + dual_port_mask : 1.0.0.0 + client_destination_mask : 255.0.0.0 + server_destination_mask : 255.0.0.0 + multiplier : 8 + cores : 1 + cpu_to_core_ratio : 1800 + exp_latency : 1 + +test_static_routing_imix_asymmetric: + stat_route_dict : + clients_start : 16.0.0.1 + servers_start : 48.0.0.1 + dual_port_mask : 1.0.0.0 + client_destination_mask : 255.0.0.0 + server_destination_mask : 255.0.0.0 + multiplier : 8 + cores : 1 + cpu_to_core_ratio : 1800 + exp_latency : 1 + +test_ipv6_simple : + multiplier : 9 + cores : 2 + cpu_to_core_ratio : 30070000 + cpu2core_custom_dev: YES + cpu2core_dev : 0.07 + + +test_rx_check_sfr: + multiplier : 10 + cores : 2 + rx_sample_rate : 16 + # allow 0.03% errors, bad router + error_tolerance : 0.03 + +test_rx_check_http: + multiplier : 15000 + cores : 1 + rx_sample_rate : 16 + # allow 0.03% errors, bad routerifconfig + error_tolerance : 0.03 + +test_rx_check_sfr_ipv6: + multiplier : 10 + cores : 2 + rx_sample_rate : 16 + # allow 0.03% errors, bad router + error_tolerance : 0.03 + +test_rx_check_http_ipv6: + multiplier : 15000 + cores : 1 + rx_sample_rate : 16 + # allow 0.03% errors, bad router + error_tolerance : 0.03 + +test_rx_check_http_negative: + multiplier : 13000 + cores : 1 + rx_sample_rate : 16 + # allow 0.03% errors, bad router + error_tolerance : 0.03 + stat_route_dict : + clients_start : 16.0.0.1 + servers_start : 48.0.0.1 + dual_port_mask : 1.0.0.0 + client_destination_mask : 255.0.0.0 + server_destination_mask : 255.0.0.0 + nat_dict : + clients_net_start : 16.0.0.0 + client_acl_wildcard_mask : 0.0.0.255 + dual_port_mask : 1.0.0.0 + pool_start : 200.0.0.0 + pool_netmask : 255.255.255.0 + + +test_jumbo: + multiplier : 17 + cores : 1 diff --git a/scripts/automation/regression/setups/trex07/config.yaml b/scripts/automation/regression/setups/trex07/config.yaml new file mode 100644 index 00000000..beb73435 --- /dev/null +++ b/scripts/automation/regression/setups/trex07/config.yaml @@ -0,0 +1,66 @@ +################################################################ +#### T-Rex nightly test configuration file #### +################################################################ + + +### T-Rex configuration: +# hostname - can be DNS name or IP for the TRex machine for ssh to the box +# password - root password for TRex machine +# is_dual - should the TRex inject with -p ? +# version_path - path to the t-rex version and executable +# cores - how many cores should be used +# latency - rate of latency packets injected by the TRex +# modes - list of modes (tagging) of this setup (loopback etc.) +# * loopback - Trex works via loopback. Router and TFTP configurations may be skipped. +# * VM - Virtual OS (accept low CPU utilization in tests, latency can get spikes) +# * virt_nics - NICs are virtual (VMXNET3 etc.) + +### Router configuration: +# hostname - the router hostname as apears in ______# cli prefix +# ip_address - the router's ip that can be used to communicate with +# image - the desired imaged wished to be loaded as the router's running config +# line_password - router password when access via Telent +# en_password - router password when changing to "enable" mode +# interfaces - an array of client-server pairs, representing the interfaces configurations of the router +# configurations - an array of configurations that could possibly loaded into the router during the test. +# The "clean" configuration is a mandatory configuration the router will load with to run the basic test bench + +### TFTP configuration: +# hostname - the tftp hostname +# ip_address - the tftp's ip address +# images_path - the tftp's relative path in which the router's images are located + +### Test_misc configuration: +# expected_bw - the "golden" bandwidth (in Gbps) results planned on receiving from the test + +trex: + hostname : csi-trex-07 + cores : 4 + +router: + model : ASR1001x + hostname : csi-asr-01 + ip_address : 10.56.216.120 + image : asr1001x-universalk9.03.13.02.S.154-3.S2-ext.SPA.bin + line_password : cisco + en_password : cisco + mgmt_interface : GigabitEthernet0 + clean_config : clean_config.cfg + intf_masking : 255.255.255.0 + ipv6_mask : 64 + interfaces : + - client : + name : Te0/0/0 + src_mac_addr : 0000.0001.0002 + dest_mac_addr : 0000.0001.0001 + server : + name : Te0/0/1 + src_mac_addr : 0000.0002.0002 + dest_mac_addr : 0000.0002.0001 + vrf_name : null + +tftp: + hostname : ats-asr-srv-1 + ip_address : 10.56.217.7 + root_dir : /scratch/tftp/ + images_path : /asr1001x/ diff --git a/scripts/automation/regression/setups/trex14/config.yaml b/scripts/automation/regression/setups/trex14/config.yaml index 10938ff3..1a528a9b 100644 --- a/scripts/automation/regression/setups/trex14/config.yaml +++ b/scripts/automation/regression/setups/trex14/config.yaml @@ -35,7 +35,8 @@ trex: hostname : csi-trex-14 - cores : 1 + cores : 4 + modes : [] router: model : ASR1001x diff --git a/scripts/automation/regression/setups/trex25/benchmark.yaml b/scripts/automation/regression/setups/trex25/benchmark.yaml new file mode 100644 index 00000000..bd0a7e03 --- /dev/null +++ b/scripts/automation/regression/setups/trex25/benchmark.yaml @@ -0,0 +1,162 @@ +############################################################### +#### TRex benchmark configuration file #### +############################################################### + +test_nbar_simple : + multiplier : 6 + cores : 1 + exp_gbps : 0.5 + cpu_to_core_ratio : 20800000 + cpu2core_custom_dev: YES + cpu2core_dev : 0.07 + exp_max_latency : 1000 + + nbar_classification: + http : 24.55 + rtp : 19.15 + sqlnet : 10.38 + secure-http : 5.11 + citrix : 4.68 + mapi : 4.04 + dns : 1.56 + sctp : 0.66 + smtp : 0.48 + pop3 : 0.30 + novadigm : 0.09 + sip : 0.08 + h323 : 0.05 + rtsp : 0.04 + unknown : 28.52 + +test_rx_check : + multiplier : 3.2 + cores : 1 + rx_sample_rate : 128 + exp_gbps : 0.5 + cpu_to_core_ratio : 37270000 + exp_bw : 1 + exp_latency : 1 + +test_nat_simple : &test_nat_simple + stat_route_dict : + clients_start : 16.0.0.1 + servers_start : 48.0.0.1 + dual_port_mask : 1.0.0.0 + client_destination_mask : 255.0.0.0 + server_destination_mask : 255.0.0.0 + nat_dict : + clients_net_start : 16.0.0.0 + client_acl_wildcard_mask : 0.0.0.255 + dual_port_mask : 1.0.0.0 + pool_start : 200.0.0.0 + pool_netmask : 255.255.255.0 + multiplier : 2200 + cores : 1 + cpu_to_core_ratio : 37270000 + exp_bw : 1 + exp_latency : 1 + allow_timeout_dev : YES + +test_nat_simple_mode1 : *test_nat_simple +test_nat_simple_mode2 : *test_nat_simple + +test_nat_learning : + stat_route_dict : + clients_start : 16.0.0.1 + servers_start : 48.0.0.1 + dual_port_mask : 1.0.0.0 + client_destination_mask : 255.0.0.0 + server_destination_mask : 255.0.0.0 + multiplier : 2200 + cores : 1 + nat_opened : 40000 + cpu_to_core_ratio : 270 + exp_bw : 1 + exp_latency : 1 + allow_timeout_dev : YES + +test_routing_imix_64 : + multiplier : 600 + cores : 1 + cpu_to_core_ratio : 280 + exp_latency : 1 + +test_routing_imix : + multiplier : 4 + cores : 1 + cpu_to_core_ratio : 1800 + exp_latency : 1 + +test_static_routing_imix : + stat_route_dict : + clients_start : 16.0.0.1 + servers_start : 48.0.0.1 + dual_port_mask : 1.0.0.0 + client_destination_mask : 255.0.0.0 + server_destination_mask : 255.0.0.0 + multiplier : 2.8 + cores : 1 + cpu_to_core_ratio : 1800 + exp_latency : 1 + +test_static_routing_imix_asymmetric: + stat_route_dict : + clients_start : 16.0.0.1 + servers_start : 48.0.0.1 + dual_port_mask : 1.0.0.0 + client_destination_mask : 255.0.0.0 + server_destination_mask : 255.0.0.0 + multiplier : 3.2 + cores : 1 + cpu_to_core_ratio : 1800 + exp_latency : 1 + +test_ipv6_simple: + multiplier : 6 + cores : 1 + cpu_to_core_ratio : 30070000 + cpu2core_custom_dev: YES + cpu2core_dev : 0.07 + + +test_rx_check_sfr: + multiplier : 6.8 + cores : 1 + rx_sample_rate : 16 + +test_rx_check_http: + multiplier : 8800 + cores : 1 + rx_sample_rate : 16 + +test_rx_check_sfr_ipv6: + multiplier : 6.8 + cores : 1 + rx_sample_rate : 16 + +test_rx_check_http_ipv6: + multiplier : 8800 + cores : 1 + rx_sample_rate : 16 + +test_rx_check_http_negative: + multiplier : 8800 + cores : 1 + rx_sample_rate : 16 + stat_route_dict : + clients_start : 16.0.0.1 + servers_start : 48.0.0.1 + dual_port_mask : 1.0.0.0 + client_destination_mask : 255.0.0.0 + server_destination_mask : 255.0.0.0 + nat_dict : + clients_net_start : 16.0.0.0 + client_acl_wildcard_mask : 0.0.0.255 + dual_port_mask : 1.0.0.0 + pool_start : 200.0.0.0 + pool_netmask : 255.255.255.0 + + +test_jumbo: + multiplier : 6 + cores : 1 diff --git a/scripts/automation/regression/setups/trex25/config.yaml b/scripts/automation/regression/setups/trex25/config.yaml new file mode 100644 index 00000000..821208a5 --- /dev/null +++ b/scripts/automation/regression/setups/trex25/config.yaml @@ -0,0 +1,93 @@ +################################################################ +#### T-Rex nightly test configuration file #### +################################################################ + + +### T-Rex configuration: +# hostname - can be DNS name or IP for the TRex machine for ssh to the box +# is_dual - should the TRex inject with -p ? +# version_path - path to the t-rex version and executable +# cores - how many cores should be used +# latency - rate of latency packets injected by the TRex +# modes - list of modes (tagging) of this setup (loopback, virtual etc.) +# * loopback - Trex works via loopback. Router and TFTP configurations may be skipped. +# * VM - Virtual OS (accept low CPU utilization in tests, latency can get spikes) +# * virt_nics - NICs are virtual (VMXNET3 etc.) + +### Router configuration: +# hostname - the router hostname as apears in ______# cli prefix +# ip_address - the router's ip that can be used to communicate with +# image - the desired imaged wished to be loaded as the router's running config +# line_password - router password when access via Telent +# en_password - router password when changing to "enable" mode +# interfaces - an array of client-server pairs, representing the interfaces configurations of the router +# configurations - an array of configurations that could possibly loaded into the router during the test. +# The "clean" configuration is a mandatory configuration the router will load with to run the basic test bench + +### TFTP configuration: +# hostname - the tftp hostname +# ip_address - the tftp's ip address +# images_path - the tftp's relative path in which the router's images are located + +### Test_misc configuration: +# expected_bw - the "golden" bandwidth (in Gbps) results planned on receiving from the test + +trex: + hostname : csi-trex-25 + cores : 2 + modes : ['1G'] + +router: + model : ASR1004(RP2) + hostname : csi-mcp-asr1k-4ru-12 + ip_address : 10.56.217.181 + image : asr1000rp2-adventerprisek9.BLD_V151_1_S_XE32_THROTTLE_LATEST_20100926_034325_2.bin + line_password : cisco + en_password : cisco + mgmt_interface : GigabitEthernet0/0/0 + clean_config : clean_config.cfg + intf_masking : 255.255.255.0 + ipv6_mask : 64 + interfaces : + - client : + name : GigabitEthernet0/1/0 + src_mac_addr : 0000.0001.0000 + dest_mac_addr : 0000.0001.0000 + server : + name : GigabitEthernet0/1/1 + src_mac_addr : 0000.0001.0000 + dest_mac_addr : 0000.0001.0000 + vrf_name : + - client : + name : GigabitEthernet0/1/2 + src_mac_addr : 0000.0001.0000 + dest_mac_addr : 0000.0001.0000 + server : + name : GigabitEthernet0/1/4 + src_mac_addr : 0000.0001.0000 + dest_mac_addr : 0000.0001.0000 + vrf_name : + - client : + name : GigabitEthernet0/1/5 + src_mac_addr : 0000.0001.0000 + dest_mac_addr : 0000.0001.0000 + server : + name : GigabitEthernet0/1/3 + src_mac_addr : 0000.0001.0000 + dest_mac_addr : 0000.0001.0000 + vrf_name : + - client : + name : GigabitEthernet0/1/6 + src_mac_addr : 0000.0001.0000 + dest_mac_addr : 0000.0001.0000 + server : + name : GigabitEthernet0/1/7 + src_mac_addr : 0000.0001.0000 + dest_mac_addr : 0000.0001.0000 + vrf_name : + +tftp: + hostname : ats-asr-srv-1 + ip_address : 10.56.128.23 + root_dir : /auto/avc-devtest/ + images_path : /images/1RU/ diff --git a/scripts/automation/regression/stateful_tests/trex_general_test.py b/scripts/automation/regression/stateful_tests/trex_general_test.py index 1a44970a..2e5adc40 100755 --- a/scripts/automation/regression/stateful_tests/trex_general_test.py +++ b/scripts/automation/regression/stateful_tests/trex_general_test.py @@ -135,15 +135,12 @@ class CTRexGeneral_Test(unittest.TestCase): if res[name] != float(val): self.fail('TRex results[%s]==%f and not as expected %f ' % (name, res[name], val)) - def check_CPU_benchmark (self, trex_res, err = 10, minimal_cpu = None, maximal_cpu = 85): + def check_CPU_benchmark (self, trex_res, err = 10, minimal_cpu = 30, maximal_cpu = 85): #cpu_util = float(trex_res.get_last_value("trex-global.data.m_cpu_util")) cpu_util = sum([float(x) for x in trex_res.get_value_list("trex-global.data.m_cpu_util")[-4:-1]]) / 3 # mean of 3 values before last - if minimal_cpu is None: - if '1G' in self.modes: - minimal_cpu = 1 - else: - minimal_cpu = 30 + if '1G' in self.modes: + minimal_cpu /= 10 if not self.is_virt_nics: if cpu_util > maximal_cpu: diff --git a/scripts/automation/regression/stateful_tests/trex_rx_test.py b/scripts/automation/regression/stateful_tests/trex_rx_test.py index 2f0a24f4..a6cc4bc6 100755 --- a/scripts/automation/regression/stateful_tests/trex_rx_test.py +++ b/scripts/automation/regression/stateful_tests/trex_rx_test.py @@ -52,9 +52,10 @@ class CTRexRx_Test(CTRexGeneral_Test): path = 'rx-check.data.stats.m_total_rx' total_rx = trex_res.get_last_value(path) - if not total_rx: + if total_rx is None: raise AbnormalResultError('No TRex results by path: %s' % path) - + elif not total_rx: + raise AbnormalResultError('Total rx_check (%s) packets is zero.' % path) print('Total packets checked: %s' % total_rx) print('Latency counters: %s' % latency_counters_display) @@ -255,7 +256,8 @@ class CTRexRx_Test(CTRexGeneral_Test): self.router.config_zbf() trex_res = self.trex.sample_to_run_finish() self.router.config_no_zbf() - self.router.clear_nat_translations() + self.router.config_no_nat(nat_obj) + #self.router.clear_nat_translations() print("\nLATEST RESULT OBJECT:") print(trex_res) self.check_rx_errors(trex_res, allow_error_tolerance = False) |