diff options
| author |   Hanoh Haim <hhaim@cisco.com> | 2016-05-03 16:09:00 +0300 |
|---|---|---|
| committer | Hanoh Haim <hhaim@cisco.com> | 2016-05-03 16:09:00 +0300 |
| commit | f27bfbab72f9221b221a7f36f4063e9ff8e9d307 (patch) | |
| tree | a8775aa1c5063b5ed02fd13fbb94d541b696e91b /scripts/automation/regression | |
| parent | dbe44f64ec7cff8d55400327e5f6994ac1fc87bb (diff) | |
| parent | 737d92948a1bfd9f94fd31a4e207d0f9e6f028c0 (diff) | |
Merge trex-204
Diffstat (limited to 'scripts/automation/regression')
28 files changed, 2313 insertions, 1030 deletions
diff --git a/scripts/automation/regression/CPlatform.py b/scripts/automation/regression/CPlatform.py index 314aae63..de1c22ce 100755 --- a/scripts/automation/regression/CPlatform.py +++ b/scripts/automation/regression/CPlatform.py @@ -73,7 +73,8 @@ class CPlatform(object): if i < 4: continue raise Exception('Could not load clean config, response: %s' % res) - return + if i > 0: # were errors, better to wait + time.sleep(2) def config_pbr (self, mode = 'config'): idx = 1 @@ -174,7 +175,8 @@ class CPlatform(object): # finish handling pre-config cache pre_commit_set = list(pre_commit_set) -# pre_commit_set.append('exit') + if len(pre_commit_set): + pre_commit_set.append('exit') pre_commit_cache.add('CONF', pre_commit_set ) # deploy the configs (order is important!) self.cmd_link.run_command( [pre_commit_cache, cache] ) @@ -227,7 +229,7 @@ class CPlatform(object): # define the relevant VRF name pre_commit_set.add('{mode}ip vrf {dup}'.format( mode = unconfig_str, dup = dual_if.get_vrf_name()) ) - + # assign VRF to interfaces, config interfaces with relevant route-map client_if_command_set.append ('{mode}ip vrf forwarding {dup}'.format( mode = unconfig_str, dup = dual_if.get_vrf_name()) ) server_if_command_set.append ('{mode}ip vrf forwarding {dup}'.format( mode = unconfig_str, dup = dual_if.get_vrf_name()) ) @@ -290,7 +292,8 @@ class CPlatform(object): # finish handling pre-config cache pre_commit_set = list(pre_commit_set) -# pre_commit_set.append('exit') + if len(pre_commit_set): + pre_commit_set.append('exit') pre_commit_cache.add('CONF', pre_commit_set ) # assign generated config list to cache cache.add('CONF', conf_t_command_set) @@ -618,8 +621,8 @@ class CPlatform(object): """ pre_commit_cache = CCommandCache() - pre_commit_cache.add('EXEC', ['clear counters','\r'] ) - self.cmd_link.run_single_command( pre_commit_cache ) + pre_commit_cache.add('EXEC', ['clear counters', '\r'] ) + self.cmd_link.run_single_command( pre_commit_cache , read_until = ['#', '\[confirm\]']) def clear_nbar_stats(self): """ clear_nbar_stats(self) -> None @@ -725,7 +728,7 @@ class CPlatform(object): progress_thread = CProgressDisp.ProgressThread(notifyMessage = "Copying image via tftp, this may take a while...\n") progress_thread.start() - response = self.cmd_link.run_single_command(cache, timeout = 900, read_until = ['\?', '\#']) + response = self.cmd_link.run_single_command(cache, timeout = 900, read_until = ['\?', '#']) print("RESPONSE:") print(response) progress_thread.join() diff --git a/scripts/automation/regression/aggregate_results.py b/scripts/automation/regression/aggregate_results.py index 35ec80d0..eb0632ec 100755 --- a/scripts/automation/regression/aggregate_results.py +++ b/scripts/automation/regression/aggregate_results.py @@ -457,7 +457,7 @@ if __name__ == '__main__': if len(error_tests): html_output += '\n<button onclick=tgl_cat("cat_tglr_{error}")>{error}</button>'.format(error = ERROR_CATEGORY) # Setups buttons - for category in setups.keys(): + for category in sorted(setups.keys()): category_arr.append(category) html_output += '\n<button onclick=tgl_cat("cat_tglr_%s")>%s</button>' % (category_arr[-1], category) # Functional buttons diff --git a/scripts/automation/regression/functional_tests/filters_test.py b/scripts/automation/regression/functional_tests/filters_test.py new file mode 100644 index 00000000..fbb8a126 --- /dev/null +++ b/scripts/automation/regression/functional_tests/filters_test.py @@ -0,0 +1,100 @@ +#!/router/bin/python + +import functional_general_test +from trex_stl_lib.utils import filters +from nose.tools import assert_equal +from nose.tools import assert_not_equal +from nose.tools import assert_raises +from nose.tools import assert_true, assert_false +from nose.tools import raises + + +class ToggleFilter_Test(functional_general_test.CGeneralFunctional_Test): + + def setUp(self): + self.list_db = [1, 2, 3, 4, 5] + self.set_db = {1, 2, 3, 4, 5} + self.tuple_db = (1, 2, 3, 4, 5) + self.dict_db = {str(x): x**2 + for x in range(5)} + + def test_init_with_dict(self): + toggle_filter = filters.ToggleFilter(self.dict_db) + assert_equal(toggle_filter._toggle_db, set(self.dict_db.keys())) + assert_equal(toggle_filter.filter_items(), self.dict_db) + + + def test_init_with_list(self): + toggle_filter = filters.ToggleFilter(self.list_db) + assert_equal(toggle_filter._toggle_db, set(self.list_db)) + assert_equal(toggle_filter.filter_items(), self.list_db) + + def test_init_with_set(self): + toggle_filter = filters.ToggleFilter(self.set_db) + assert_equal(toggle_filter._toggle_db, self.set_db) + assert_equal(toggle_filter.filter_items(), self.set_db) + + def test_init_with_tuple(self): + toggle_filter = filters.ToggleFilter(self.tuple_db) + assert_equal(toggle_filter._toggle_db, set(self.tuple_db)) + assert_equal(toggle_filter.filter_items(), self.tuple_db) + + @raises(TypeError) + def test_init_with_non_iterable(self): + toggle_filter = filters.ToggleFilter(15) + + def test_dict_toggeling(self): + toggle_filter = filters.ToggleFilter(self.dict_db) + assert_false(toggle_filter.toggle_item("3")) + assert_equal(toggle_filter._toggle_db, {'0', '1', '2', '4'}) + assert_true(toggle_filter.toggle_item("3")) + assert_equal(toggle_filter._toggle_db, {'0', '1', '2', '3', '4'}) + assert_false(toggle_filter.toggle_item("2")) + assert_false(toggle_filter.toggle_item("4")) + self.dict_db.update({'5': 25, '6': 36}) + assert_true(toggle_filter.toggle_item("6")) + + assert_equal(toggle_filter.filter_items(), {'0': 0, '1': 1, '3': 9, '6': 36}) + + del self.dict_db['1'] + assert_equal(toggle_filter.filter_items(), {'0': 0, '3': 9, '6': 36}) + + def test_dict_toggeling_negative(self): + toggle_filter = filters.ToggleFilter(self.dict_db) + assert_raises(KeyError, toggle_filter.toggle_item, "100") + + def test_list_toggeling(self): + toggle_filter = filters.ToggleFilter(self.list_db) + assert_false(toggle_filter.toggle_item(3)) + assert_equal(toggle_filter._toggle_db, {1, 2, 4, 5}) + assert_true(toggle_filter.toggle_item(3)) + assert_equal(toggle_filter._toggle_db, {1, 2, 3, 4, 5}) + assert_false(toggle_filter.toggle_item(2)) + assert_false(toggle_filter.toggle_item(4)) + self.list_db.extend([6 ,7]) + assert_true(toggle_filter.toggle_item(6)) + + assert_equal(toggle_filter.filter_items(), [1, 3 , 5, 6]) + + self.list_db.remove(1) + assert_equal(toggle_filter.filter_items(), [3, 5, 6]) + + def test_list_toggling_negative(self): + toggle_filter = filters.ToggleFilter(self.list_db) + assert_raises(KeyError, toggle_filter.toggle_item, 10) + + def test_toggle_multiple_items(self): + toggle_filter = filters.ToggleFilter(self.list_db) + assert_false(toggle_filter.toggle_items(1, 3, 5)) + assert_equal(toggle_filter._toggle_db, {2, 4}) + assert_true(toggle_filter.toggle_items(1, 5)) + assert_equal(toggle_filter._toggle_db, {1, 2, 4, 5}) + + def test_dont_show_after_init(self): + toggle_filter = filters.ToggleFilter(self.list_db, show_by_default = False) + assert_equal(toggle_filter._toggle_db, set()) + assert_equal(toggle_filter.filter_items(), []) + + + def tearDown(self): + pass diff --git a/scripts/automation/regression/platform_cmd_link.py b/scripts/automation/regression/platform_cmd_link.py index d2143a5d..d034fac3 100755 --- a/scripts/automation/regression/platform_cmd_link.py +++ b/scripts/automation/regression/platform_cmd_link.py @@ -95,11 +95,9 @@ class CCommandLink(object): def __transmit (self, cmd_list, **kwargs): self.history.extend(cmd_list) - if not self.silent_mode: - print('\n'.join(cmd_list)) # prompting the pushed platform commands if not self.virtual_mode: # transmit the command to platform. - return self.telnet_con.write_ios_cmd(cmd_list, **kwargs) + return self.telnet_con.write_ios_cmd(cmd_list, verbose = not self.silent_mode, **kwargs) def run_command (self, cmd_list, **kwargs): response = '' @@ -420,7 +418,7 @@ class CIosTelnet(telnetlib.Telnet): except Exception as inst: raise - def write_ios_cmd (self, cmd_list, result_from = 0, timeout = 3, **kwargs): + def write_ios_cmd (self, cmd_list, result_from = 0, timeout = 60, **kwargs): assert (isinstance (cmd_list, list) == True) self.read_until(self.pr, timeout = 1) @@ -431,19 +429,22 @@ class CIosTelnet(telnetlib.Telnet): wf = self.pr for idx, cmd in enumerate(cmd_list): + start_time = time.time() self.write(cmd+'\r\n') - if idx < result_from: - # don't care for return string - if type(wf) is list: - self.expect(wf, timeout)[2] - else: - self.read_until(wf, timeout) + if kwargs.get('verbose'): + print('-->\n%s' % cmd) + if type(wf) is list: + output = self.expect(wf, timeout)[2] else: - # care for return string - if type(wf) is list: - res += self.expect(wf, timeout)[2] - else: - res += self.read_until(wf, timeout) + output = self.read_until(wf, timeout) + if idx >= result_from: + res += output + if kwargs.get('verbose'): + print('<-- (%ss)\n%s' % (round(time.time() - start_time, 2), output)) + if time.time() - start_time > timeout - 1: + raise Exception('Timeout while performing telnet command: %s' % cmd) + if 'Invalid' in res: + print('Warning: telnet command probably failed.\nCommand: %s\nResponse: %s' % (cmd_list, res)) # return res.split('\r\n') return res # return the received response as a string, each line is seperated by '\r\n'. diff --git a/scripts/automation/regression/setups/kiwi02/benchmark.yaml b/scripts/automation/regression/setups/kiwi02/benchmark.yaml index 60febc8f..343d4120 100644 --- a/scripts/automation/regression/setups/kiwi02/benchmark.yaml +++ b/scripts/automation/regression/setups/kiwi02/benchmark.yaml @@ -1,153 +1,246 @@ -################################################################ -#### T-Rex benchmark configuration file #### -################################################################ - -test_nbar_simple : - multiplier : 20 - cores : 2 - exp_gbps : 4.5 - cpu_to_core_ratio : 37270000 - cpu2core_custom_dev: YES - cpu2core_dev : 0.07 - exp_max_latency : 1000 - - nbar_classification: - http : 30.41 - rtp_audio : 21.22 - rtp : 11.4 - oracle_sqlnet : 11.3 - exchange : 10.95 - citrix : 5.65 - rtsp : 2.67 - dns : 1.95 - smtp : 0.57 - pop3 : 0.36 - sctp : 0.09 - sip : 0.09 - ssl : 0.06 - unknown : 3.2 - -test_rx_check : - multiplier : 25 - cores : 4 - rx_sample_rate : 128 - exp_gbps : 0.5 - cpu_to_core_ratio : 37270000 - exp_bw : 1 - exp_latency : 1 - -test_nat_simple : &test_nat_simple - stat_route_dict : +############################################################### +#### TRex benchmark configuration file #### +############################################################### + +#### common templates ### + +stat_route_dict: &stat_route_dict clients_start : 16.0.0.1 servers_start : 48.0.0.1 dual_port_mask : 1.0.0.0 client_destination_mask : 255.0.0.0 server_destination_mask : 255.0.0.0 - nat_dict : + +nat_dict: &nat_dict clients_net_start : 16.0.0.0 client_acl_wildcard_mask : 0.0.0.255 dual_port_mask : 1.0.0.0 pool_start : 200.0.0.0 pool_netmask : 255.255.255.0 - multiplier : 10000 - cpu_to_core_ratio : 37270000 - cores : 1 - exp_bw : 1 - exp_latency : 1 - allow_timeout_dev : YES - -test_nat_simple_mode1 : *test_nat_simple -test_nat_simple_mode2 : *test_nat_simple - -test_nat_learning : - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - multiplier : 10000 - cores : 1 - nat_opened : 100000 - cpu_to_core_ratio : 37270000 - exp_bw : 1 - exp_latency : 1 - allow_timeout_dev : YES - -test_routing_imix_64 : - multiplier : 2500 - cores : 4 - cpu_to_core_ratio : 8900 - exp_latency : 1 - -test_routing_imix : - multiplier : 32 - cores : 2 - cpu_to_core_ratio : 8900 - exp_latency : 1 - -test_static_routing_imix : - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - multiplier : 32 - cores : 2 - cpu_to_core_ratio : 3766666 - exp_latency : 1 -test_static_routing_imix_asymmetric: - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - multiplier : 16 - cores : 1 - cpu_to_core_ratio : 3766666 - exp_latency : 1 - -test_ipv6_simple : - multiplier : 32 - cores : 4 - cpu_to_core_ratio : 30070000 - cpu2core_custom_dev: YES - cpu2core_dev : 0.07 - - -test_rx_check_sfr: - multiplier : 25 - cores : 4 - rx_sample_rate : 32 - error_tolerance : 0.01 - -test_rx_check_http: - multiplier : 40000 - cores : 2 - rx_sample_rate : 32 - error_tolerance : 0.01 -test_rx_check_sfr_ipv6: - multiplier : 25 - cores : 4 - rx_sample_rate : 32 - error_tolerance : 0.01 +### stateful ### + +test_jumbo: + multiplier : 55 + cores : 1 + bw_per_core : 647.305 + + +test_routing_imix: + multiplier : 32 + cores : 2 + bw_per_core : 39.131 + + +test_routing_imix_64: + multiplier : 2500 + cores : 4 + bw_per_core : 7.427 + + +test_static_routing_imix: + stat_route_dict : *stat_route_dict + multiplier : 32 + cores : 2 + bw_per_core : 39.039 + + +test_static_routing_imix_asymmetric: + stat_route_dict : *stat_route_dict + multiplier : 16 + cores : 1 + bw_per_core : 38.796 + + +test_ipv6_simple: + multiplier : 32 + cores : 4 + bw_per_core : 19.283 + + +test_nat_simple_mode1: &test_nat_simple + stat_route_dict : *stat_route_dict + nat_dict : *nat_dict + multiplier : 10000 + cores : 1 + allow_timeout_dev : True + bw_per_core : 45.304 + +test_nat_simple_mode2: *test_nat_simple + +test_nat_learning: + << : *test_nat_simple + nat_opened : 100000 + + +test_nbar_simple: + multiplier : 20 + cores : 2 + bw_per_core : 18.243 + nbar_classification: + http : 30.41 + rtp_audio : 21.22 + rtp : 11.4 + oracle_sqlnet : 11.3 + exchange : 10.95 + citrix : 5.65 + rtsp : 2.67 + dns : 1.95 + smtp : 0.57 + pop3 : 0.36 + sctp : 0.09 + sip : 0.09 + ssl : 0.06 + unknown : 3.2 + + +test_rx_check_http: &rx_http + multiplier : 40000 + cores : 2 + rx_sample_rate : 32 + error_tolerance : 0.01 + bw_per_core : 38.071 test_rx_check_http_ipv6: - multiplier : 40000 - cores : 2 - rx_sample_rate : 32 - error_tolerance : 0.01 + << : *rx_http + bw_per_core : 46.733 + + +test_rx_check_sfr: &rx_sfr + multiplier : 25 + cores : 4 + rx_sample_rate : 32 + error_tolerance : 0.01 + bw_per_core : 16.915 + +test_rx_check_sfr_ipv6: + << : *rx_sfr + bw_per_core : 20.323 + + + +### stateless ### + +test_CPU_benchmark: + profiles: + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64, packet_count: 10} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64, packet_count: 100} + cpu_util : 1 + bw_per_core : 1 + +# causes queue full +# - name : stl/udp_1pkt_simple.py +# kwargs : {packet_len: 64, packet_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000, packet_count: 10} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000, packet_count: 100} + cpu_util : 1 + bw_per_core : 1 + +# not enough memory + queue full if memory increase +# - name : stl/udp_1pkt_simple.py +# kwargs : {packet_len: 9000, packet_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/imix.py + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/pcap.py + kwargs : {ipg_usec: 2, loop_count: 0} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 + + - name : stl/hlt/hlt_udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 -test_rx_check_http_negative: - multiplier : 40000 - cores : 2 - rx_sample_rate : 32 - error_tolerance : 0.01 -test_jumbo: - multiplier : 55 - cores : 1 diff --git a/scripts/automation/regression/setups/trex-dan/benchmark.yaml b/scripts/automation/regression/setups/trex-dan/benchmark.yaml index 33e5a771..c8b046d4 100644 --- a/scripts/automation/regression/setups/trex-dan/benchmark.yaml +++ b/scripts/automation/regression/setups/trex-dan/benchmark.yaml @@ -2,160 +2,250 @@ #### TRex benchmark configuration file #### ############################################################### -test_nbar_simple : - multiplier : 1.5 - cores : 2 - exp_gbps : 0.5 - cpu_to_core_ratio : 20800000 - cpu2core_custom_dev: YES - cpu2core_dev : 0.07 - exp_max_latency : 1000 - - nbar_classification: - http : 30.3 - rtp_audio : 21.06 - oracle_sqlnet : 11.25 - rtp : 11.1 - exchange : 10.16 - citrix : 5.6 - rtsp : 2.84 - sctp : 0.65 - ssl : 0.8 - sip : 0.09 - dns : 1.95 - smtp : 0.57 - pop3 : 0.36 - unknown : 3.19 - -test_rx_check : - multiplier : 0.8 - cores : 1 - rx_sample_rate : 128 - exp_gbps : 0.5 - cpu_to_core_ratio : 37270000 - exp_bw : 1 - exp_latency : 1 - -test_nat_simple : &test_nat_simple - stat_route_dict : +#### common templates ### + +stat_route_dict: &stat_route_dict clients_start : 16.0.0.1 servers_start : 48.0.0.1 dual_port_mask : 1.0.0.0 client_destination_mask : 255.0.0.0 server_destination_mask : 255.0.0.0 - nat_dict : + +nat_dict: &nat_dict clients_net_start : 16.0.0.0 client_acl_wildcard_mask : 0.0.0.255 dual_port_mask : 1.0.0.0 pool_start : 200.0.0.0 pool_netmask : 255.255.255.0 - multiplier : 550 - cores : 1 - cpu_to_core_ratio : 37270000 - exp_bw : 1 - exp_latency : 1 - allow_timeout_dev : YES - -test_nat_simple_mode1 : *test_nat_simple -test_nat_simple_mode2 : *test_nat_simple - -test_nat_learning : - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - multiplier : 550 - cores : 1 - nat_opened : 40000 - cpu_to_core_ratio : 270 - exp_bw : 1 - exp_latency : 1 - allow_timeout_dev : YES - -test_routing_imix_64 : - multiplier : 150 - cores : 4 - cpu_to_core_ratio : 280 - exp_latency : 1 - -test_routing_imix : - multiplier : 1 - cores : 1 - cpu_to_core_ratio : 1800 - exp_latency : 1 - -test_static_routing_imix : - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - multiplier : 0.7 - cores : 1 - cpu_to_core_ratio : 1800 - exp_latency : 1 -test_static_routing_imix_asymmetric: - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - multiplier : 0.8 - cores : 1 - cpu_to_core_ratio : 1800 - exp_latency : 1 -test_ipv6_simple : - multiplier : 1.5 - cores : 2 - cpu_to_core_ratio : 30070000 - cpu2core_custom_dev: YES - cpu2core_dev : 0.07 +### stateful ### +test_jumbo: + multiplier : 2.8 + cores : 1 + bw_per_core : 67.030 -test_rx_check_sfr: - multiplier : 1.7 - cores : 2 - rx_sample_rate : 16 -test_rx_check_http: - multiplier : 2200 - cores : 1 - rx_sample_rate : 16 +test_routing_imix: + multiplier : 1 + cores : 1 + bw_per_core : 3.979 + + +test_routing_imix_64: + multiplier : 150 + cores : 4 + bw_per_core : .681 + + +test_static_routing_imix: + stat_route_dict : *stat_route_dict + multiplier : 0.7 + cores : 1 + bw_per_core : 3.837 -test_rx_check_sfr_ipv6: - multiplier : 1.7 - cores : 2 - rx_sample_rate : 16 + +test_static_routing_imix_asymmetric: + stat_route_dict : *stat_route_dict + multiplier : 0.8 + cores : 1 + bw_per_core : 3.939 + + +test_ipv6_simple: + multiplier : 1.5 + cores : 2 + bw_per_core : 4.719 + + +test_nat_simple_mode1: &test_nat_simple + stat_route_dict : *stat_route_dict + nat_dict : *nat_dict + multiplier : 550 + cores : 1 + allow_timeout_dev : True + bw_per_core : 7.465 + +test_nat_simple_mode2: *test_nat_simple + +test_nat_learning: + << : *test_nat_simple + bw_per_core : 7.377 + nat_opened : 40000 + + +test_nbar_simple: + multiplier : 1.5 + cores : 2 + bw_per_core : 4.465 + nbar_classification: + http : 30.3 + rtp_audio : 21.06 + oracle_sqlnet : 11.25 + rtp : 11.1 + exchange : 10.16 + citrix : 5.6 + rtsp : 2.84 + sctp : 0.65 + ssl : 0.8 + sip : 0.09 + dns : 1.95 + smtp : 0.57 + pop3 : 0.36 + unknown : 3.19 + + +test_rx_check_http: &rx_http + multiplier : 2200 + cores : 1 + rx_sample_rate : 16 + bw_per_core : 8.142 test_rx_check_http_ipv6: - multiplier : 2200 - cores : 1 - rx_sample_rate : 16 + << : *rx_http + bw_per_core : 8.591 test_rx_check_http_negative: - multiplier : 2200 - cores : 1 - rx_sample_rate : 16 - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - nat_dict : - clients_net_start : 16.0.0.0 - client_acl_wildcard_mask : 0.0.0.255 - dual_port_mask : 1.0.0.0 - pool_start : 200.0.0.0 - pool_netmask : 255.255.255.0 + << : *rx_http + stat_route_dict : *stat_route_dict + nat_dict : *nat_dict + bw_per_core : 8.037 + + +test_rx_check_sfr: &rx_sfr + multiplier : 1.7 + cores : 2 + rx_sample_rate : 16 + bw_per_core : 4.473 + +test_rx_check_sfr_ipv6: + << : *rx_sfr + bw_per_core : 4.773 + + + +### stateless ### + +test_CPU_benchmark: + profiles: + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64, packet_count: 10} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64, packet_count: 100} + cpu_util : 1 + bw_per_core : 1 + +# causes queue full +# - name : stl/udp_1pkt_simple.py +# kwargs : {packet_len: 64, packet_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000, packet_count: 10} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000, packet_count: 100} + cpu_util : 1 + bw_per_core : 1 + +# not enough memory + queue full if memory increase +# - name : stl/udp_1pkt_simple.py +# kwargs : {packet_len: 9000, packet_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/imix.py + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/pcap.py + kwargs : {ipg_usec: 2, loop_count: 0} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 + + - name : stl/hlt/hlt_udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 -test_jumbo: - multiplier : 2.8 - cores : 1 diff --git a/scripts/automation/regression/setups/trex04/benchmark.yaml b/scripts/automation/regression/setups/trex04/benchmark.yaml index e5459dce..d2b1c4f2 100644 --- a/scripts/automation/regression/setups/trex04/benchmark.yaml +++ b/scripts/automation/regression/setups/trex04/benchmark.yaml @@ -2,61 +2,154 @@ #### T-Rex benchmark configuration file #### ################################################################ +### stateful ### + +test_jumbo: + multiplier : 2.8 + cores : 1 + bw_per_core : 106.652 + + +test_routing_imix: + multiplier : 0.5 + cores : 1 + bw_per_core : 11.577 + + +test_routing_imix_64: + multiplier : 28 + cores : 1 + bw_per_core : 2.030 -test_rx_check : - multiplier : 0.8 - cores : 1 - rx_sample_rate : 128 - exp_gbps : 0.5 - cpu_to_core_ratio : 37270000 - exp_bw : 1 - exp_latency : 1 - - -test_routing_imix_64 : - multiplier : 28 - cores : 1 - cpu_to_core_ratio : 280 - exp_latency : 1 - -test_routing_imix : - multiplier : 0.5 - cores : 1 - cpu_to_core_ratio : 1800 - exp_latency : 1 - -test_static_routing_imix : - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - multiplier : 0.8 - cores : 1 - cpu_to_core_ratio : 1800 - exp_latency : 1 test_static_routing_imix_asymmetric: - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - multiplier : 0.8 - cores : 1 - cpu_to_core_ratio : 1800 - exp_latency : 1 - -test_ipv6_simple : - multiplier : 0.5 - cores : 1 - cpu_to_core_ratio : 30070000 - cpu2core_custom_dev: YES - cpu2core_dev : 0.07 + multiplier : 0.8 + cores : 1 + bw_per_core : 13.742 + + + +### stateless ### + +test_CPU_benchmark: + profiles: + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64, packet_count: 10} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64, packet_count: 100} + cpu_util : 1 + bw_per_core : 1 + +# causes queue full +# - name : stl/udp_1pkt_simple.py +# kwargs : {packet_len: 64, packet_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000, packet_count: 10} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000, packet_count: 100} + cpu_util : 1 + bw_per_core : 1 + +# not enough memory + queue full if memory increase +# - name : stl/udp_1pkt_simple.py +# kwargs : {packet_len: 9000, packet_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/imix.py + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/pcap.py + kwargs : {ipg_usec: 4, loop_count: 0} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 + + - name : stl/hlt/hlt_udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 -test_jumbo: - multiplier : 2.8 - cores : 1
\ No newline at end of file diff --git a/scripts/automation/regression/setups/trex07/benchmark.yaml b/scripts/automation/regression/setups/trex07/benchmark.yaml new file mode 100644 index 00000000..4778de91 --- /dev/null +++ b/scripts/automation/regression/setups/trex07/benchmark.yaml @@ -0,0 +1,170 @@ +############################################################### +#### TRex benchmark configuration file #### +############################################################### + +test_nbar_simple : + multiplier : 7.5 + cores : 2 + exp_gbps : 3.5 + cpu_to_core_ratio : 20800000 + cpu2core_custom_dev: YES + cpu2core_dev : 0.07 + exp_max_latency : 1000 + + nbar_classification: + rtp : 32.57 + http : 30.25 + oracle-sqlnet : 11.23 + exchange : 10.80 + citrix : 5.62 + rtsp : 2.84 + dns : 1.95 + smtp : 0.57 + pop3 : 0.36 + ssl : 0.17 + sctp : 0.13 + sip : 0.09 + unknown : 3.41 + +test_rx_check : + multiplier : 13 + cores : 3 + rx_sample_rate : 128 + exp_gbps : 6 + cpu_to_core_ratio : 37270000 + exp_bw : 13 + exp_latency : 1 + +test_nat_simple : &test_nat_simple + stat_route_dict : + clients_start : 16.0.0.1 + servers_start : 48.0.0.1 + dual_port_mask : 1.0.0.0 + client_destination_mask : 255.0.0.0 + server_destination_mask : 255.0.0.0 + nat_dict : + clients_net_start : 16.0.0.0 + client_acl_wildcard_mask : 0.0.0.255 + dual_port_mask : 1.0.0.0 + pool_start : 200.0.0.0 + pool_netmask : 255.255.255.0 + multiplier : 12000 + cores : 1 + cpu_to_core_ratio : 37270000 + exp_bw : 1 + exp_latency : 1 + allow_timeout_dev : YES + +test_nat_simple_mode1 : *test_nat_simple +test_nat_simple_mode2 : *test_nat_simple + +test_nat_learning : + stat_route_dict : + clients_start : 16.0.0.1 + servers_start : 48.0.0.1 + dual_port_mask : 1.0.0.0 + client_destination_mask : 255.0.0.0 + server_destination_mask : 255.0.0.0 + multiplier : 12000 + cores : 1 + nat_opened : 40000 + cpu_to_core_ratio : 270 + exp_bw : 8 + exp_latency : 1 + allow_timeout_dev : YES + +test_routing_imix_64 : + multiplier : 430 + cores : 1 + cpu_to_core_ratio : 280 + exp_latency : 1 + +test_routing_imix : + multiplier : 10 + cores : 1 + cpu_to_core_ratio : 1800 + exp_latency : 1 + +test_static_routing_imix : + stat_route_dict : + clients_start : 16.0.0.1 + servers_start : 48.0.0.1 + dual_port_mask : 1.0.0.0 + client_destination_mask : 255.0.0.0 + server_destination_mask : 255.0.0.0 + multiplier : 8 + cores : 1 + cpu_to_core_ratio : 1800 + exp_latency : 1 + +test_static_routing_imix_asymmetric: + stat_route_dict : + clients_start : 16.0.0.1 + servers_start : 48.0.0.1 + dual_port_mask : 1.0.0.0 + client_destination_mask : 255.0.0.0 + server_destination_mask : 255.0.0.0 + multiplier : 8 + cores : 1 + cpu_to_core_ratio : 1800 + exp_latency : 1 + +test_ipv6_simple : + multiplier : 9 + cores : 2 + cpu_to_core_ratio : 30070000 + cpu2core_custom_dev: YES + cpu2core_dev : 0.07 + + +test_rx_check_sfr: + multiplier : 10 + cores : 2 + rx_sample_rate : 16 + # allow 0.03% errors, bad router + error_tolerance : 0.03 + +test_rx_check_http: + multiplier : 15000 + cores : 1 + rx_sample_rate : 16 + # allow 0.03% errors, bad routerifconfig + error_tolerance : 0.03 + +test_rx_check_sfr_ipv6: + multiplier : 10 + cores : 2 + rx_sample_rate : 16 + # allow 0.03% errors, bad router + error_tolerance : 0.03 + +test_rx_check_http_ipv6: + multiplier : 15000 + cores : 1 + rx_sample_rate : 16 + # allow 0.03% errors, bad router + error_tolerance : 0.03 + +test_rx_check_http_negative: + multiplier : 13000 + cores : 1 + rx_sample_rate : 16 + # allow 0.03% errors, bad router + error_tolerance : 0.03 + stat_route_dict : + clients_start : 16.0.0.1 + servers_start : 48.0.0.1 + dual_port_mask : 1.0.0.0 + client_destination_mask : 255.0.0.0 + server_destination_mask : 255.0.0.0 + nat_dict : + clients_net_start : 16.0.0.0 + client_acl_wildcard_mask : 0.0.0.255 + dual_port_mask : 1.0.0.0 + pool_start : 200.0.0.0 + pool_netmask : 255.255.255.0 + + +test_jumbo: + multiplier : 17 + cores : 1 diff --git a/scripts/automation/regression/setups/trex07/config.yaml b/scripts/automation/regression/setups/trex07/config.yaml new file mode 100644 index 00000000..beb73435 --- /dev/null +++ b/scripts/automation/regression/setups/trex07/config.yaml @@ -0,0 +1,66 @@ +################################################################ +#### T-Rex nightly test configuration file #### +################################################################ + + +### T-Rex configuration: +# hostname - can be DNS name or IP for the TRex machine for ssh to the box +# password - root password for TRex machine +# is_dual - should the TRex inject with -p ? +# version_path - path to the t-rex version and executable +# cores - how many cores should be used +# latency - rate of latency packets injected by the TRex +# modes - list of modes (tagging) of this setup (loopback etc.) +# * loopback - Trex works via loopback. Router and TFTP configurations may be skipped. +# * VM - Virtual OS (accept low CPU utilization in tests, latency can get spikes) +# * virt_nics - NICs are virtual (VMXNET3 etc.) + +### Router configuration: +# hostname - the router hostname as apears in ______# cli prefix +# ip_address - the router's ip that can be used to communicate with +# image - the desired imaged wished to be loaded as the router's running config +# line_password - router password when access via Telent +# en_password - router password when changing to "enable" mode +# interfaces - an array of client-server pairs, representing the interfaces configurations of the router +# configurations - an array of configurations that could possibly loaded into the router during the test. +# The "clean" configuration is a mandatory configuration the router will load with to run the basic test bench + +### TFTP configuration: +# hostname - the tftp hostname +# ip_address - the tftp's ip address +# images_path - the tftp's relative path in which the router's images are located + +### Test_misc configuration: +# expected_bw - the "golden" bandwidth (in Gbps) results planned on receiving from the test + +trex: + hostname : csi-trex-07 + cores : 4 + +router: + model : ASR1001x + hostname : csi-asr-01 + ip_address : 10.56.216.120 + image : asr1001x-universalk9.03.13.02.S.154-3.S2-ext.SPA.bin + line_password : cisco + en_password : cisco + mgmt_interface : GigabitEthernet0 + clean_config : clean_config.cfg + intf_masking : 255.255.255.0 + ipv6_mask : 64 + interfaces : + - client : + name : Te0/0/0 + src_mac_addr : 0000.0001.0002 + dest_mac_addr : 0000.0001.0001 + server : + name : Te0/0/1 + src_mac_addr : 0000.0002.0002 + dest_mac_addr : 0000.0002.0001 + vrf_name : null + +tftp: + hostname : ats-asr-srv-1 + ip_address : 10.56.217.7 + root_dir : /scratch/tftp/ + images_path : /asr1001x/ diff --git a/scripts/automation/regression/setups/trex08/benchmark.yaml b/scripts/automation/regression/setups/trex08/benchmark.yaml index d4bb8283..c5c2fde7 100644 --- a/scripts/automation/regression/setups/trex08/benchmark.yaml +++ b/scripts/automation/regression/setups/trex08/benchmark.yaml @@ -2,58 +2,174 @@ #### TRex benchmark configuration file #### ############################################################### -test_routing_imix_64 : - multiplier : 8000 - cores : 7 - cpu_to_core_ratio : 280 - exp_latency : 1 - -test_routing_imix : - multiplier : 80 - cores : 4 - cpu_to_core_ratio : 1800 - exp_latency : 1 +### stateful ### + +test_jumbo: + multiplier : 150 + cores : 2 + bw_per_core : 962.464 + + +test_routing_imix: + multiplier : 80 + cores : 4 + bw_per_core : 55.130 + + +test_routing_imix_64: + multiplier : 8000 + cores : 7 + bw_per_core : 11.699 + test_static_routing_imix_asymmetric: - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - multiplier : 70 - cores : 3 - cpu_to_core_ratio : 1800 - exp_latency : 1 - -test_ipv6_simple : - multiplier : 80 - cores : 7 - cpu_to_core_ratio : 30070000 - cpu2core_custom_dev: YES - cpu2core_dev : 0.07 + multiplier : 70 + cores : 3 + bw_per_core : 50.561 -test_rx_check_sfr: - multiplier : 80 - cores : 7 - rx_sample_rate : 128 +test_ipv6_simple: + multiplier : 80 + cores : 7 + bw_per_core : 25.948 -test_rx_check_sfr_ipv6_disabled: - multiplier : 80 - cores : 7 - rx_sample_rate : 128 test_rx_check_http: - multiplier : 99000 - cores : 3 - rx_sample_rate : 128 + multiplier : 99000 + cores : 3 + rx_sample_rate : 128 + bw_per_core : 49.464 + + +test_rx_check_sfr: + multiplier : 80 + cores : 7 + rx_sample_rate : 128 + bw_per_core : 20.871 + + + +### stateless ### + +test_CPU_benchmark: + profiles: + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64, packet_count: 10} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64, packet_count: 100} + cpu_util : 1 + bw_per_core : 1 + +# causes queue full +# - name : stl/udp_1pkt_simple.py +# kwargs : {packet_len: 64, packet_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000, packet_count: 10} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000, packet_count: 100} + cpu_util : 1 + bw_per_core : 1 + +# not enough memory + queue full if memory increase +# - name : stl/udp_1pkt_simple.py +# kwargs : {packet_len: 9000, packet_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/imix.py + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/pcap.py + kwargs : {ipg_usec: 2, loop_count: 0} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 + + - name : stl/hlt/hlt_udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 -test_rx_check_http_ipv6_disabled: - multiplier : 99000 - cores : 3 - rx_sample_rate : 128 -test_jumbo: - multiplier : 150 - cores : 2
\ No newline at end of file diff --git a/scripts/automation/regression/setups/trex09/benchmark.yaml b/scripts/automation/regression/setups/trex09/benchmark.yaml index 3f7b9a95..1f8fe47a 100644 --- a/scripts/automation/regression/setups/trex09/benchmark.yaml +++ b/scripts/automation/regression/setups/trex09/benchmark.yaml @@ -1,118 +1,175 @@ -############################################################### -#### TRex benchmark configuration file #### -############################################################### - -test_nbar_simple : - multiplier : 1.5 - cores : 1 - exp_gbps : 0.5 - cpu_to_core_ratio : 20800000 - cpu2core_custom_dev: YES - cpu2core_dev : 0.07 - exp_max_latency : 1000 - - nbar_classification: - http : 30.3 - rtp_audio : 21.06 - oracle_sqlnet : 11.25 - rtp : 11.1 - exchange : 10.16 - citrix : 5.6 - rtsp : 2.84 - sctp : 0.65 - ssl : 0.8 - sip : 0.09 - dns : 1.95 - smtp : 0.57 - pop3 : 0.36 - unknown : 3.19 - -test_rx_check : - multiplier : 0.8 - cores : 1 - rx_sample_rate : 128 - exp_gbps : 0.5 - cpu_to_core_ratio : 37270000 - exp_bw : 1 - exp_latency : 1 - -test_nat_simple : - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - nat_dict : - clients_net_start : 16.0.0.0 - client_acl_wildcard_mask : 0.0.0.255 - dual_port_mask : 1.0.0.0 - pool_start : 200.0.0.0 - pool_netmask : 255.255.255.0 - multiplier : 150 - cores : 1 - cpu_to_core_ratio : 37270000 - exp_bw : 1 - exp_latency : 1 - allow_timeout_dev : YES - -test_nat_learning : - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - multiplier : 150 - cores : 1 - nat_opened : 40000 - cpu_to_core_ratio : 270 - exp_bw : 1 - exp_latency : 1 - allow_timeout_dev : YES - -test_routing_imix_64 : - multiplier : 28 - cores : 1 - cpu_to_core_ratio : 280 - exp_latency : 1 - -test_routing_imix : - multiplier : 1 - cores : 1 - cpu_to_core_ratio : 1800 - exp_latency : 1 - -test_static_routing_imix : - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - multiplier : 0.7 - cores : 1 - cpu_to_core_ratio : 1800 - exp_latency : 1 +################################################################ +#### T-Rex benchmark configuration file #### +################################################################ + +### stateful ### + +test_jumbo: + multiplier : 110 + cores : 1 + bw_per_core : 767.198 + + +test_routing_imix: + multiplier : 64 + cores : 2 + bw_per_core : 35.889 + + +test_routing_imix_64: + multiplier : 5000 + cores : 2 + bw_per_core : 10.672 + test_static_routing_imix_asymmetric: - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - multiplier : 0.8 - cores : 1 - cpu_to_core_ratio : 1800 - exp_latency : 1 - -test_ipv6_simple : - multiplier : 1.5 - cores : 1 - cpu_to_core_ratio : 30070000 - cpu2core_custom_dev: YES - cpu2core_dev : 0.07 + multiplier : 32 + cores : 1 + bw_per_core : 52.738 + + +test_ipv6_simple: + multiplier : 64 + cores : 4 + bw_per_core : 22.808 + + +test_rx_check_http: + multiplier : 90000 + cores : 2 + rx_sample_rate : 32 + bw_per_core : 46.075 + + +test_rx_check_sfr: + multiplier : 50 + cores : 3 + rx_sample_rate : 32 + bw_per_core : 20.469 + + + +### stateless ### + +test_CPU_benchmark: + profiles: + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64, packet_count: 10} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64, packet_count: 100} + cpu_util : 1 + bw_per_core : 1 + +# causes queue full +# - name : stl/udp_1pkt_simple.py +# kwargs : {packet_len: 64, packet_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000, packet_count: 10} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000, packet_count: 100} + cpu_util : 1 + bw_per_core : 1 + +# not enough memory + queue full if memory increase +# - name : stl/udp_1pkt_simple.py +# kwargs : {packet_len: 9000, packet_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/imix.py + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/pcap.py + kwargs : {ipg_usec: 2, loop_count: 0} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 + - name : stl/hlt/hlt_udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 diff --git a/scripts/automation/regression/setups/trex11/config.yaml b/scripts/automation/regression/setups/trex11/config.yaml index 876a1afd..7bb5b6d4 100644 --- a/scripts/automation/regression/setups/trex11/config.yaml +++ b/scripts/automation/regression/setups/trex11/config.yaml @@ -34,36 +34,5 @@ trex: hostname : csi-trex-11 - cores : 2 + cores : 1 modes : ['loopback', 'virtual'] - -router: - model : 1RU - hostname : ASR1001_T-Rex -# ip_address : 10.56.199.247 - ip_address : 10.56.199.247123123123 - image : asr1001-universalk9.BLD_V155_1_S_XE314_THROTTLE_LATEST_20141112_090734-std.bin - #image : asr1001-universalk9.BLD_V155_2_S_XE315_THROTTLE_LATEST_20150121_110036-std.bin - #image : asr1001-universalk9.BLD_V155_2_S_XE315_THROTTLE_LATEST_20150324_100047-std.bin - line_password : lab - en_password : lab - mgmt_interface : GigabitEthernet0/0/0 - clean_config : /Configurations/danklei/asr1001_TRex_clean_config.cfg - intf_masking : 255.255.255.0 - ipv6_mask : 64 - interfaces : - - client : - name : GigabitEthernet0/0/1 - src_mac_addr : 0000.0001.0000 - dest_mac_addr : 0000.0001.0000 - server : - name : GigabitEthernet0/0/2 - src_mac_addr : 0000.0001.0000 - dest_mac_addr : 0000.0001.0000 - vrf_name : null - -tftp: - hostname : ats-asr-srv-1 - ip_address : 10.56.128.23 - root_dir : /auto/avc-devtest/ - images_path : /images/1RU/ diff --git a/scripts/automation/regression/setups/trex12/benchmark.yaml b/scripts/automation/regression/setups/trex12/benchmark.yaml index 7985f15e..0ebc2bcb 100644 --- a/scripts/automation/regression/setups/trex12/benchmark.yaml +++ b/scripts/automation/regression/setups/trex12/benchmark.yaml @@ -2,165 +2,181 @@ #### TRex benchmark configuration file #### ############################################################### -test_nbar_simple : - multiplier : 7.5 - cores : 2 - exp_gbps : 3.5 - cpu_to_core_ratio : 20800000 - cpu2core_custom_dev: YES - cpu2core_dev : 0.07 - exp_max_latency : 1000 - - nbar_classification: - http : 30.18 - rtp-audio : 21.27 - rtp : 11.26 - oracle_sqlnet : 11.2 - exchange : 10.78 - citrix : 5.61 - rtsp : 2.82 - dns : 1.94 - smtp : 0.57 - pop3 : 0.36 - ssl : 0.16 - sctp : 0.13 - sip : 0.09 - unknown : 3.54 - -test_rx_check : - multiplier : 13 - cores : 4 - rx_sample_rate : 128 - exp_gbps : 6 - cpu_to_core_ratio : 37270000 - exp_bw : 13 - exp_latency : 1 - -test_nat_simple : - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - nat_dict : - clients_net_start : 16.0.0.0 - client_acl_wildcard_mask : 0.0.0.255 - dual_port_mask : 1.0.0.0 - pool_start : 200.0.0.0 - pool_netmask : 255.255.255.0 - multiplier : 12000 - cores : 1 - cpu_to_core_ratio : 37270000 - exp_bw : 1 - exp_latency : 1 - allow_timeout_dev : YES - -test_nat_learning : - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - multiplier : 12000 - cores : 1 - nat_opened : 40000 - cpu_to_core_ratio : 270 - exp_bw : 8 - exp_latency : 1 - allow_timeout_dev : YES - -test_routing_imix_64 : - multiplier : 430 - cores : 1 - cpu_to_core_ratio : 280 - exp_latency : 1 - -test_routing_imix : - multiplier : 10 - cores : 1 - cpu_to_core_ratio : 1800 - exp_latency : 1 - -test_static_routing_imix : - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - multiplier : 10 - cores : 1 - cpu_to_core_ratio : 1800 - exp_latency : 1 +### stateful ### + +test_jumbo: + multiplier : 14 + cores : 1 + bw_per_core : 689.664 + + +test_routing_imix: + multiplier : 8 + cores : 1 + bw_per_core : 45.422 + + +test_routing_imix_64: + multiplier : 2200 + cores : 1 + bw_per_core : 11.655 + test_static_routing_imix_asymmetric: - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - multiplier : 8 - cores : 1 - cpu_to_core_ratio : 1800 - exp_latency : 1 - -test_ipv6_simple : - multiplier : 18 - cores : 4 - cpu_to_core_ratio : 30070000 - cpu2core_custom_dev: YES - cpu2core_dev : 0.07 - - -test_rx_check_sfr: - multiplier : 15 - cores : 3 - rx_sample_rate : 16 - # allow 0.03% errors, bad router - error_tolerance : 0.03 - -test_rx_check_http: - multiplier : 15000 - cores : 1 - rx_sample_rate : 16 - # allow 0.03% errors, bad router - error_tolerance : 0.03 + multiplier : 4 + cores : 1 + bw_per_core : 45.294 -test_rx_check_sfr_ipv6: - multiplier : 15 - cores : 3 - rx_sample_rate : 16 - # allow 0.03% errors, bad router - error_tolerance : 0.03 + +test_ipv6_simple: + multiplier : 8 + cores : 1 + bw_per_core : 29.332 + + +test_rx_check_http: &rx_http + multiplier : 11000 + cores : 1 + rx_sample_rate : 16 + bw_per_core : 47.813 test_rx_check_http_ipv6: - multiplier : 15000 - cores : 1 - rx_sample_rate : 16 - # allow 0.03% errors, bad router - error_tolerance : 0.03 - -test_rx_check_http_negative: - multiplier : 13000 - cores : 1 - rx_sample_rate : 16 - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - nat_dict : - clients_net_start : 16.0.0.0 - client_acl_wildcard_mask : 0.0.0.255 - dual_port_mask : 1.0.0.0 - pool_start : 200.0.0.0 - pool_netmask : 255.255.255.0 + << : *rx_http + bw_per_core : 55.607 + + +test_rx_check_sfr: &rx_sfr + multiplier : 8 + cores : 1 + rx_sample_rate : 16 + bw_per_core : 24.203 + +test_rx_check_sfr_ipv6: + << : *rx_sfr + bw_per_core : 28.867 + + +### stateless ### + +test_CPU_benchmark: + profiles: + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64, packet_count: 10} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64, packet_count: 100} + cpu_util : 1 + bw_per_core : 1 + +# causes queue full +# - name : stl/udp_1pkt_simple.py +# kwargs : {packet_len: 64, packet_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000, packet_count: 10} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000, packet_count: 100} + cpu_util : 1 + bw_per_core : 1 + +# not enough memory + queue full if memory increase +# - name : stl/udp_1pkt_simple.py +# kwargs : {packet_len: 9000, packet_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/imix.py + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/pcap.py + kwargs : {ipg_usec: 2, loop_count: 0} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 + + - name : stl/hlt/hlt_udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 -test_jumbo: - multiplier : 28 - cores : 1 diff --git a/scripts/automation/regression/setups/trex12/config.yaml b/scripts/automation/regression/setups/trex12/config.yaml index af17db45..56471ac7 100644 --- a/scripts/automation/regression/setups/trex12/config.yaml +++ b/scripts/automation/regression/setups/trex12/config.yaml @@ -35,34 +35,6 @@ trex: hostname : csi-trex-12 -# version_path : /auto/proj-pcube-b/apps/PL-b/tools/bp_sim2/v1.57/ #/auto/srg-sce-swinfra-usr/emb/users/danklei/Work/asr1k/emb/private/bpsim/main/scripts cores : 1 - modes : [VM] + modes : ['loopback', '1G'] -router: - model : ASR1001x - hostname : csi-asr-01 - ip_address : 10.56.216.103 - image : asr1001x-universalk9_npe.BLD_V155_2_S_XE315_THROTTLE_LATEST_20151121_110441-std_2.SSA.bin - line_password : cisco - en_password : cisco - mgmt_interface : GigabitEthernet0 - clean_config : /Configurations/danklei/asr1001_TRex_clean_config.cfg - intf_masking : 255.255.255.0 - ipv6_mask : 64 - interfaces : - - client : - name : Te0/0/0 - src_mac_addr : 0000.0001.0000 - dest_mac_addr : 0000.0001.0000 - server : - name : Te0/0/1 - src_mac_addr : 0000.0001.0000 - dest_mac_addr : 0000.0001.0000 - vrf_name : null - -tftp: - hostname : ats-asr-srv-1 - ip_address : 10.56.128.23 - root_dir : /auto/avc-devtest/ - images_path : /images/RP2/ diff --git a/scripts/automation/regression/setups/trex14/benchmark.yaml b/scripts/automation/regression/setups/trex14/benchmark.yaml index e602ad1a..afa27a82 100644 --- a/scripts/automation/regression/setups/trex14/benchmark.yaml +++ b/scripts/automation/regression/setups/trex14/benchmark.yaml @@ -2,170 +2,244 @@ #### TRex benchmark configuration file #### ############################################################### -test_nbar_simple : - multiplier : 7.5 - cores : 2 - exp_gbps : 3.5 - cpu_to_core_ratio : 20800000 - cpu2core_custom_dev: YES - cpu2core_dev : 0.07 - exp_max_latency : 1000 - - nbar_classification: - http : 32.58 - rtp-audio : 21.21 - oracle_sqlnet : 11.41 - exchange : 11.22 - rtp : 11.2 - citrix : 5.65 - rtsp : 2.87 - dns : 1.96 - smtp : 0.57 - pop3 : 0.37 - ssl : 0.28 - sctp : 0.13 - sip : 0.09 - unknown : 0.45 - -test_rx_check : - multiplier : 13 - cores : 4 - rx_sample_rate : 128 - exp_gbps : 6 - cpu_to_core_ratio : 37270000 - exp_bw : 13 - exp_latency : 1 - -test_nat_simple : &test_nat_simple - stat_route_dict : +#### common templates ### + +stat_route_dict: &stat_route_dict clients_start : 16.0.0.1 servers_start : 48.0.0.1 dual_port_mask : 1.0.0.0 client_destination_mask : 255.0.0.0 server_destination_mask : 255.0.0.0 - nat_dict : + +nat_dict: &nat_dict clients_net_start : 16.0.0.0 client_acl_wildcard_mask : 0.0.0.255 dual_port_mask : 1.0.0.0 pool_start : 200.0.0.0 pool_netmask : 255.255.255.0 - multiplier : 12000 - cores : 1 - cpu_to_core_ratio : 37270000 - exp_bw : 1 - exp_latency : 1 - allow_timeout_dev : YES - -test_nat_simple_mode1 : *test_nat_simple -test_nat_simple_mode2 : *test_nat_simple - -test_nat_learning : - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - multiplier : 12000 - cores : 1 - nat_opened : 40000 - cpu_to_core_ratio : 270 - exp_bw : 8 - exp_latency : 1 - allow_timeout_dev : YES - -test_routing_imix_64 : - multiplier : 430 - cores : 1 - cpu_to_core_ratio : 280 - exp_latency : 1 - -test_routing_imix : - multiplier : 10 - cores : 1 - cpu_to_core_ratio : 1800 - exp_latency : 1 - -test_static_routing_imix : - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - multiplier : 8 - cores : 1 - cpu_to_core_ratio : 1800 - exp_latency : 1 -test_static_routing_imix_asymmetric: - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - multiplier : 8 - cores : 1 - cpu_to_core_ratio : 1800 - exp_latency : 1 - -test_ipv6_simple : - multiplier : 9 - cores : 2 - cpu_to_core_ratio : 30070000 - cpu2core_custom_dev: YES - cpu2core_dev : 0.07 - - -test_rx_check_sfr: - multiplier : 10 - cores : 3 - rx_sample_rate : 16 - # allow 0.03% errors, bad router - error_tolerance : 0.03 - -test_rx_check_http: - multiplier : 15000 - cores : 1 - rx_sample_rate : 16 - # allow 0.03% errors, bad routerifconfig - error_tolerance : 0.03 -test_rx_check_sfr_ipv6: - multiplier : 10 - cores : 3 - rx_sample_rate : 16 - # allow 0.03% errors, bad router - error_tolerance : 0.03 +### stateful ### + +test_jumbo: + multiplier : 17 + cores : 1 + bw_per_core : 543.232 + + +test_routing_imix: + multiplier : 10 + cores : 1 + bw_per_core : 34.128 + + +test_routing_imix_64: + multiplier : 430 + cores : 1 + bw_per_core : 5.893 + + +test_static_routing_imix: &test_static_routing_imix + stat_route_dict : *stat_route_dict + multiplier : 8 + cores : 1 + bw_per_core : 34.339 + +test_static_routing_imix_asymmetric: *test_static_routing_imix + + +test_ipv6_simple: + multiplier : 9 + cores : 2 + bw_per_core : 19.064 + + +test_nat_simple_mode1: &test_nat_simple + stat_route_dict : *stat_route_dict + nat_dict : *nat_dict + multiplier : 12000 + cores : 1 + nat_opened : 40000 + allow_timeout_dev : True + bw_per_core : 44.445 + +test_nat_simple_mode2: *test_nat_simple + +test_nat_learning: + << : *test_nat_simple + nat_opened : 40000 + + +test_nbar_simple: + multiplier : 7.5 + cores : 2 + bw_per_core : 17.174 + nbar_classification: + http : 32.58 + rtp-audio : 21.21 + oracle_sqlnet : 11.41 + exchange : 11.22 + rtp : 11.2 + citrix : 5.65 + rtsp : 2.87 + dns : 1.96 + smtp : 0.57 + pop3 : 0.37 + ssl : 0.28 + sctp : 0.13 + sip : 0.09 + unknown : 0.45 + + +test_rx_check_http: &rx_http + multiplier : 15000 + cores : 1 + rx_sample_rate : 16 + bw_per_core : 39.560 test_rx_check_http_ipv6: - multiplier : 15000 - cores : 1 - rx_sample_rate : 16 - # allow 0.03% errors, bad router - error_tolerance : 0.03 + << : *rx_http + bw_per_core : 49.237 test_rx_check_http_negative: - multiplier : 13000 - cores : 1 - rx_sample_rate : 16 - # allow 0.03% errors, bad router - error_tolerance : 0.03 - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - nat_dict : - clients_net_start : 16.0.0.0 - client_acl_wildcard_mask : 0.0.0.255 - dual_port_mask : 1.0.0.0 - pool_start : 200.0.0.0 - pool_netmask : 255.255.255.0 + << : *rx_http + stat_route_dict : *stat_route_dict + nat_dict : *nat_dict + + +test_rx_check_sfr: &rx_sfr + multiplier : 10 + cores : 3 + rx_sample_rate : 16 + bw_per_core : 16.082 + +test_rx_check_sfr_ipv6: + << : *rx_sfr + bw_per_core : 19.198 + + + +### stateless ### + +test_CPU_benchmark: + profiles: + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64, packet_count: 10} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64, packet_count: 100} + cpu_util : 1 + bw_per_core : 1 + +# causes queue full +# - name : stl/udp_1pkt_simple.py +# kwargs : {packet_len: 64, packet_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000, packet_count: 10} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000, packet_count: 100} + cpu_util : 1 + bw_per_core : 1 + +# not enough memory + queue full if memory increase +# - name : stl/udp_1pkt_simple.py +# kwargs : {packet_len: 9000, packet_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/imix.py + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/pcap.py + kwargs : {ipg_usec: 2, loop_count: 0} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 + + - name : stl/hlt/hlt_udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 -test_jumbo: - multiplier : 17 - cores : 1 diff --git a/scripts/automation/regression/setups/trex14/config.yaml b/scripts/automation/regression/setups/trex14/config.yaml index 10938ff3..1a528a9b 100644 --- a/scripts/automation/regression/setups/trex14/config.yaml +++ b/scripts/automation/regression/setups/trex14/config.yaml @@ -35,7 +35,8 @@ trex: hostname : csi-trex-14 - cores : 1 + cores : 4 + modes : [] router: model : ASR1001x diff --git a/scripts/automation/regression/setups/trex17/benchmark.yaml b/scripts/automation/regression/setups/trex17/benchmark.yaml index e5459dce..d9191a42 100644 --- a/scripts/automation/regression/setups/trex17/benchmark.yaml +++ b/scripts/automation/regression/setups/trex17/benchmark.yaml @@ -2,61 +2,154 @@ #### T-Rex benchmark configuration file #### ################################################################ +### stateful ### + +test_jumbo: + multiplier : 2.8 + cores : 1 + bw_per_core : 66.489 + + +test_routing_imix: + multiplier : 0.5 + cores : 1 + bw_per_core : 5.530 + + +test_routing_imix_64: + multiplier : 28 + cores : 1 + bw_per_core : 0.859 -test_rx_check : - multiplier : 0.8 - cores : 1 - rx_sample_rate : 128 - exp_gbps : 0.5 - cpu_to_core_ratio : 37270000 - exp_bw : 1 - exp_latency : 1 - - -test_routing_imix_64 : - multiplier : 28 - cores : 1 - cpu_to_core_ratio : 280 - exp_latency : 1 - -test_routing_imix : - multiplier : 0.5 - cores : 1 - cpu_to_core_ratio : 1800 - exp_latency : 1 - -test_static_routing_imix : - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - multiplier : 0.8 - cores : 1 - cpu_to_core_ratio : 1800 - exp_latency : 1 test_static_routing_imix_asymmetric: - stat_route_dict : - clients_start : 16.0.0.1 - servers_start : 48.0.0.1 - dual_port_mask : 1.0.0.0 - client_destination_mask : 255.0.0.0 - server_destination_mask : 255.0.0.0 - multiplier : 0.8 - cores : 1 - cpu_to_core_ratio : 1800 - exp_latency : 1 - -test_ipv6_simple : - multiplier : 0.5 - cores : 1 - cpu_to_core_ratio : 30070000 - cpu2core_custom_dev: YES - cpu2core_dev : 0.07 + multiplier : 0.8 + cores : 1 + bw_per_core : 9.635 + + + +### stateless ### + +test_CPU_benchmark: + profiles: + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64, packet_count: 10} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64, packet_count: 100} + cpu_util : 1 + bw_per_core : 1 + +# causes queue full +# - name : stl/udp_1pkt_simple.py +# kwargs : {packet_len: 64, packet_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000, packet_count: 10} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000, packet_count: 100} + cpu_util : 1 + bw_per_core : 1 + +# not enough memory + queue full if memory increase +# - name : stl/udp_1pkt_simple.py +# kwargs : {packet_len: 9000, packet_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/imix.py + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/pcap.py + kwargs : {ipg_usec: 2, loop_count: 0} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 + + - name : stl/hlt/hlt_udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 -test_jumbo: - multiplier : 2.8 - cores : 1
\ No newline at end of file diff --git a/scripts/automation/regression/setups/trex25/benchmark.yaml b/scripts/automation/regression/setups/trex25/benchmark.yaml new file mode 100644 index 00000000..f87759f9 --- /dev/null +++ b/scripts/automation/regression/setups/trex25/benchmark.yaml @@ -0,0 +1,252 @@ +############################################################### +#### TRex benchmark configuration file #### +############################################################### + +#### common templates ### + +stat_route_dict: &stat_route_dict + clients_start : 16.0.0.1 + servers_start : 48.0.0.1 + dual_port_mask : 1.0.0.0 + client_destination_mask : 255.0.0.0 + server_destination_mask : 255.0.0.0 + +nat_dict: &nat_dict + clients_net_start : 16.0.0.0 + client_acl_wildcard_mask : 0.0.0.255 + dual_port_mask : 1.0.0.0 + pool_start : 200.0.0.0 + pool_netmask : 255.255.255.0 + + +### stateful ### + +test_jumbo: + multiplier : 6 + cores : 1 + bw_per_core : 443.970 + + +test_routing_imix: + multiplier : 4 + cores : 1 + bw_per_core : 26.509 + + +test_routing_imix_64: + multiplier : 600 + cores : 1 + bw_per_core : 6.391 + + +test_static_routing_imix: + stat_route_dict : *stat_route_dict + multiplier : 2.8 + cores : 1 + bw_per_core : 24.510 + + + +test_static_routing_imix_asymmetric: + stat_route_dict : *stat_route_dict + multiplier : 3.2 + cores : 1 + bw_per_core : 28.229 + + +test_ipv6_simple: + multiplier : 6 + cores : 1 + bw_per_core : 19.185 + + +test_nat_simple_mode1: &test_nat_simple + stat_route_dict : *stat_route_dict + nat_dict : *nat_dict + multiplier : 2200 + cores : 1 + allow_timeout_dev : True + bw_per_core : 32.171 + +test_nat_simple_mode2: *test_nat_simple + +test_nat_learning: + << : *test_nat_simple + nat_opened : 40000 + + +test_nbar_simple: + multiplier : 6 + cores : 1 + bw_per_core : 16.645 + nbar_classification: + http : 24.55 + rtp : 19.15 + sqlnet : 10.38 + secure-http : 5.11 + citrix : 4.68 + mapi : 4.04 + dns : 1.56 + sctp : 0.66 + smtp : 0.48 + pop3 : 0.30 + novadigm : 0.09 + sip : 0.08 + h323 : 0.05 + rtsp : 0.04 + unknown : 28.52 + + +test_rx_check_http: &rx_http + multiplier : 8800 + cores : 1 + rx_sample_rate : 16 + bw_per_core : 31.389 + +test_rx_check_http_ipv6: + << : *rx_http + bw_per_core : 37.114 + +test_rx_check_http_negative: + << : *rx_http + stat_route_dict : *stat_route_dict + nat_dict : *nat_dict + + +test_rx_check_sfr: &rx_sfr + multiplier : 6.8 + cores : 1 + rx_sample_rate : 16 + bw_per_core : 16.063 + +test_rx_check_sfr_ipv6: + << : *rx_sfr + bw_per_core : 19.663 + + +### stateless ### + +test_CPU_benchmark: + profiles: + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64, packet_count: 10} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 64, packet_count: 100} + cpu_util : 1 + bw_per_core : 1 + +# causes queue full +# - name : stl/udp_1pkt_simple.py +# kwargs : {packet_len: 64, packet_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_simple.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + +# problem stabilizing CPU utilization at this setup +# - name : stl/udp_1pkt_simple.py +# kwargs : {packet_len: 9000, packet_count: 10} +# cpu_util : 1 +# bw_per_core : 1 + +# problem stabilizing CPU utilization at this setup +# - name : stl/udp_1pkt_simple.py +# kwargs : {packet_len: 9000, packet_count: 100} +# cpu_util : 1 +# bw_per_core : 1 + +# not enough memory + queue full if memory increase +# - name : stl/udp_1pkt_simple.py +# kwargs : {packet_len: 9000, packet_count: 1000} +# cpu_util : 1 +# bw_per_core : 1 + + - name : stl/imix.py + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 64} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 128} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 256} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 512} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 1500} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 4000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_1pkt_tuple_gen.py + kwargs : {packet_len: 9000} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/pcap.py + kwargs : {ipg_usec: 2, loop_count: 0} + cpu_util : 1 + bw_per_core : 1 + + - name : stl/udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 + + - name : stl/hlt/hlt_udp_rand_len_9k.py + cpu_util : 1 + bw_per_core : 1 + + diff --git a/scripts/automation/regression/setups/trex25/config.yaml b/scripts/automation/regression/setups/trex25/config.yaml new file mode 100644 index 00000000..821208a5 --- /dev/null +++ b/scripts/automation/regression/setups/trex25/config.yaml @@ -0,0 +1,93 @@ +################################################################ +#### T-Rex nightly test configuration file #### +################################################################ + + +### T-Rex configuration: +# hostname - can be DNS name or IP for the TRex machine for ssh to the box +# is_dual - should the TRex inject with -p ? +# version_path - path to the t-rex version and executable +# cores - how many cores should be used +# latency - rate of latency packets injected by the TRex +# modes - list of modes (tagging) of this setup (loopback, virtual etc.) +# * loopback - Trex works via loopback. Router and TFTP configurations may be skipped. +# * VM - Virtual OS (accept low CPU utilization in tests, latency can get spikes) +# * virt_nics - NICs are virtual (VMXNET3 etc.) + +### Router configuration: +# hostname - the router hostname as apears in ______# cli prefix +# ip_address - the router's ip that can be used to communicate with +# image - the desired imaged wished to be loaded as the router's running config +# line_password - router password when access via Telent +# en_password - router password when changing to "enable" mode +# interfaces - an array of client-server pairs, representing the interfaces configurations of the router +# configurations - an array of configurations that could possibly loaded into the router during the test. +# The "clean" configuration is a mandatory configuration the router will load with to run the basic test bench + +### TFTP configuration: +# hostname - the tftp hostname +# ip_address - the tftp's ip address +# images_path - the tftp's relative path in which the router's images are located + +### Test_misc configuration: +# expected_bw - the "golden" bandwidth (in Gbps) results planned on receiving from the test + +trex: + hostname : csi-trex-25 + cores : 2 + modes : ['1G'] + +router: + model : ASR1004(RP2) + hostname : csi-mcp-asr1k-4ru-12 + ip_address : 10.56.217.181 + image : asr1000rp2-adventerprisek9.BLD_V151_1_S_XE32_THROTTLE_LATEST_20100926_034325_2.bin + line_password : cisco + en_password : cisco + mgmt_interface : GigabitEthernet0/0/0 + clean_config : clean_config.cfg + intf_masking : 255.255.255.0 + ipv6_mask : 64 + interfaces : + - client : + name : GigabitEthernet0/1/0 + src_mac_addr : 0000.0001.0000 + dest_mac_addr : 0000.0001.0000 + server : + name : GigabitEthernet0/1/1 + src_mac_addr : 0000.0001.0000 + dest_mac_addr : 0000.0001.0000 + vrf_name : + - client : + name : GigabitEthernet0/1/2 + src_mac_addr : 0000.0001.0000 + dest_mac_addr : 0000.0001.0000 + server : + name : GigabitEthernet0/1/4 + src_mac_addr : 0000.0001.0000 + dest_mac_addr : 0000.0001.0000 + vrf_name : + - client : + name : GigabitEthernet0/1/5 + src_mac_addr : 0000.0001.0000 + dest_mac_addr : 0000.0001.0000 + server : + name : GigabitEthernet0/1/3 + src_mac_addr : 0000.0001.0000 + dest_mac_addr : 0000.0001.0000 + vrf_name : + - client : + name : GigabitEthernet0/1/6 + src_mac_addr : 0000.0001.0000 + dest_mac_addr : 0000.0001.0000 + server : + name : GigabitEthernet0/1/7 + src_mac_addr : 0000.0001.0000 + dest_mac_addr : 0000.0001.0000 + vrf_name : + +tftp: + hostname : ats-asr-srv-1 + ip_address : 10.56.128.23 + root_dir : /auto/avc-devtest/ + images_path : /images/1RU/ diff --git a/scripts/automation/regression/stateful_tests/trex_general_test.py b/scripts/automation/regression/stateful_tests/trex_general_test.py index 1a44970a..42720f70 100755 --- a/scripts/automation/regression/stateful_tests/trex_general_test.py +++ b/scripts/automation/regression/stateful_tests/trex_general_test.py @@ -48,6 +48,7 @@ def tearDownModule(module): class CTRexGeneral_Test(unittest.TestCase): """This class defines the general stateful testcase of the T-Rex traffic generator""" def __init__ (self, *args, **kwargs): + sys.stdout.flush() unittest.TestCase.__init__(self, *args, **kwargs) if CTRexScenario.is_test_list: return @@ -57,6 +58,7 @@ class CTRexGeneral_Test(unittest.TestCase): self.trex = CTRexScenario.trex self.trex_crashed = CTRexScenario.trex_crashed self.modes = CTRexScenario.modes + self.GAManager = CTRexScenario.GAManager self.skipping = False self.fail_reasons = [] if not hasattr(self, 'unsupported_modes'): @@ -135,15 +137,12 @@ class CTRexGeneral_Test(unittest.TestCase): if res[name] != float(val): self.fail('TRex results[%s]==%f and not as expected %f ' % (name, res[name], val)) - def check_CPU_benchmark (self, trex_res, err = 10, minimal_cpu = None, maximal_cpu = 85): + def check_CPU_benchmark (self, trex_res, err = 25, minimal_cpu = 30, maximal_cpu = 85): #cpu_util = float(trex_res.get_last_value("trex-global.data.m_cpu_util")) - cpu_util = sum([float(x) for x in trex_res.get_value_list("trex-global.data.m_cpu_util")[-4:-1]]) / 3 # mean of 3 values before last + cpu_util = sum(trex_res.get_value_list("trex-global.data.m_cpu_util")[-4:-1]) / 3.0 # mean of 3 values before last - if minimal_cpu is None: - if '1G' in self.modes: - minimal_cpu = 1 - else: - minimal_cpu = 30 + if '1G' in self.modes: + minimal_cpu /= 10.0 if not self.is_virt_nics: if cpu_util > maximal_cpu: @@ -151,23 +150,25 @@ class CTRexGeneral_Test(unittest.TestCase): if cpu_util < minimal_cpu: self.fail("CPU is too low (%s%%), can't verify performance in such low CPU%%." % cpu_util ) - cores = self.get_benchmark_param('cores') - trex_tx_bps = trex_res.get_last_value("trex-global.data.m_total_tx_bytes") - test_norm_cpu = 100.0*(trex_tx_bps/(cores*cpu_util))/1e6 + test_norm_cpu = sum(trex_res.get_value_list("trex-global.data.m_bw_per_core")[-4:-1]) / 3.0 - print("TRex CPU utilization: %g%%, norm_cpu is : %d Mb/core" % (round(cpu_util), int(test_norm_cpu))) + print("TRex CPU utilization: %g%%, norm_cpu is : %g Gb/core" % (round(cpu_util, 2), round(test_norm_cpu))) - #expected_norm_cpu = self.get_benchmark_param('cpu_to_core_ratio') + expected_norm_cpu = self.get_benchmark_param('bw_per_core') + if not expected_norm_cpu: + expected_norm_cpu = 1 - #calc_error_precent = abs(100.0*(test_norm_cpu/expected_norm_cpu)-100.0) - -# if calc_error_precent > err: -# msg ='Normalized bandwidth to CPU utilization ratio is %2.0f Mb/core expected %2.0f Mb/core more than %2.0f %% - ERROR' % (test_norm_cpu, expected_norm_cpu, err) -# raise AbnormalResultError(msg) -# else: -# msg ='Normalized bandwidth to CPU utilization ratio is %2.0f Mb/core expected %2.0f Mb/core less than %2.0f %% - OK' % (test_norm_cpu, expected_norm_cpu, err) -# print msg + calc_error_precent = abs(100.0 * test_norm_cpu / expected_norm_cpu - 100) + print('Err percent: %s' % calc_error_precent) + if calc_error_precent > err and cpu_util > 10: + self.fail('Excepted bw_per_core ratio: %s, got: %g' % (expected_norm_cpu, round(test_norm_cpu))) + # report benchmarks + if self.GAManager: + setup_test = '%s.%s' % (CTRexScenario.setup_name, self.get_name()) + self.GAManager.gaAddAction(Event = 'stateful_test', action = setup_test, label = 'bw_per_core', value = int(test_norm_cpu)) + self.GAManager.gaAddAction(Event = 'stateful_test', action = setup_test, label = 'bw_per_core_exp', value = int(expected_norm_cpu)) + self.GAManager.emptyAndReportQ() def check_results_gt (self, res, name, val): if res is None: @@ -341,7 +342,9 @@ class CTRexGeneral_Test(unittest.TestCase): except Exception as e: print("Can't get TRex log:", e) if len(self.fail_reasons): + sys.stdout.flush() raise Exception('The test is failed, reasons:\n%s' % '\n'.join(self.fail_reasons)) + sys.stdout.flush() def check_for_trex_crash(self): pass diff --git a/scripts/automation/regression/stateful_tests/trex_imix_test.py b/scripts/automation/regression/stateful_tests/trex_imix_test.py index 95a5471d..cafa1c55 100755 --- a/scripts/automation/regression/stateful_tests/trex_imix_test.py +++ b/scripts/automation/regression/stateful_tests/trex_imix_test.py @@ -161,7 +161,7 @@ class CTRexIMIX_Test(CTRexGeneral_Test): self.check_general_scenario_results(trex_res) - self.check_CPU_benchmark(trex_res) + self.check_CPU_benchmark(trex_res, minimal_cpu = 25) def test_jumbo(self, duration = 100, **kwargs): diff --git a/scripts/automation/regression/stateful_tests/trex_nat_test.py b/scripts/automation/regression/stateful_tests/trex_nat_test.py index 512ad4e4..b2df684d 100755 --- a/scripts/automation/regression/stateful_tests/trex_nat_test.py +++ b/scripts/automation/regression/stateful_tests/trex_nat_test.py @@ -54,9 +54,9 @@ class CTRexNoNat_Test(CTRexGeneral_Test):#(unittest.TestCase): learning_stats = trex_res.get_last_value("trex-global.data", ".*nat.*") # extract all nat data if self.get_benchmark_param('allow_timeout_dev'): - nat_timeout_ratio = learning_stats['m_total_nat_time_out']/learning_stats['m_total_nat_open'] + nat_timeout_ratio = float(learning_stats['m_total_nat_time_out']) / learning_stats['m_total_nat_open'] if nat_timeout_ratio > 0.005: - self.fail('TRex nat_timeout ratio %f > 0.005 (0.5%) and not as expected to be less than 0.5%' %(nat_timeout_ratio)) + self.fail('TRex nat_timeout ratio %f > 0.5%%' % nat_timeout_ratio) else: self.check_results_eq (learning_stats, 'm_total_nat_time_out', 0.0) self.check_results_eq (learning_stats, 'm_total_nat_no_fid', 0.0) @@ -128,7 +128,7 @@ class CTRexNat_Test(CTRexGeneral_Test):#(unittest.TestCase): trex_nat_stats = trex_res.get_last_value("trex-global.data", ".*nat.*") # extract all nat data if self.get_benchmark_param('allow_timeout_dev'): - nat_timeout_ratio = trex_nat_stats['m_total_nat_time_out']/trex_nat_stats['m_total_nat_open'] + nat_timeout_ratio = float(trex_nat_stats['m_total_nat_time_out']) / trex_nat_stats['m_total_nat_open'] if nat_timeout_ratio > 0.005: self.fail('TRex nat_timeout ratio %f > 0.5%%' % nat_timeout_ratio) else: diff --git a/scripts/automation/regression/stateful_tests/trex_nbar_test.py b/scripts/automation/regression/stateful_tests/trex_nbar_test.py index 69c3f605..fa3f3485 100755 --- a/scripts/automation/regression/stateful_tests/trex_nbar_test.py +++ b/scripts/automation/regression/stateful_tests/trex_nbar_test.py @@ -82,70 +82,10 @@ class CTRexNbar_Test(CTRexGeneral_Test): print("\nLATEST DUMP:") print(trex_res.get_latest_dump()) - self.check_general_scenario_results(trex_res, check_latency = False) # NBAR can cause latency - # test_norm_cpu = 2*(trex_res.result['total-tx']/(core*trex_res.result['cpu_utilization'])) - trex_tx_pckt = trex_res.get_last_value("trex-global.data.m_total_tx_pkts") - cpu_util = trex_res.get_last_value("trex-global.data.m_cpu_util") - cpu_util_hist = trex_res.get_value_list("trex-global.data.m_cpu_util") - print("cpu util is:", cpu_util) - print(cpu_util_hist) - test_norm_cpu = 2 * trex_tx_pckt / (core * cpu_util) - print("test_norm_cpu is:", test_norm_cpu) - - - if self.get_benchmark_param('cpu2core_custom_dev'): - # check this test by custom deviation - deviation_compare_value = self.get_benchmark_param('cpu2core_dev') - print("Comparing test with custom deviation value- {dev_val}%".format( dev_val = int(deviation_compare_value*100) )) - - # need to be fixed ! - #if ( abs((test_norm_cpu/self.get_benchmark_param('cpu_to_core_ratio')) - 1) > deviation_compare_value): - # raise AbnormalResultError('Normalized bandwidth to CPU utilization ratio exceeds benchmark boundaries') - + self.check_CPU_benchmark(trex_res) self.match_classification() - assert True - - @nottest - def test_rx_check (self): - # test initializtion - self.router.configure_basic_interfaces() - - self.router.config_pbr(mode = "config") - self.router.config_nbar_pd() - - mult = self.get_benchmark_param('multiplier') - core = self.get_benchmark_param('cores') - sample_rate = self.get_benchmark_param('rx_sample_rate') - - ret = self.trex.start_trex( - c = core, - m = mult, - p = True, - nc = True, - rx_check = sample_rate, - d = 100, - f = 'cap2/sfr.yaml', - l = 1000) - - trex_res = self.trex.sample_to_run_finish() - - # trex_res is a CTRexResult instance- and contains the summary of the test results - # you may see all the results keys by simply calling here for 'print trex_res.result' - print("\nLATEST RESULT OBJECT:") - print(trex_res) - print("\nLATEST DUMP:") - print(trex_res.get_latest_dump()) - - self.check_general_scenario_results(trex_res) - - self.check_CPU_benchmark(trex_res, 10) - -# if trex_res.result['rx_check_tx']==trex_res.result['rx_check_rx']: # rx_check verification shoud pass -# assert trex_res.result['rx_check_verification'] == "OK" -# else: -# assert trex_res.result['rx_check_verification'] == "FAIL" # the name intentionally not matches nose default pattern, including the test should be specified explicitly def NBarLong(self): diff --git a/scripts/automation/regression/stateful_tests/trex_rx_test.py b/scripts/automation/regression/stateful_tests/trex_rx_test.py index 2f0a24f4..c097b180 100755 --- a/scripts/automation/regression/stateful_tests/trex_rx_test.py +++ b/scripts/automation/regression/stateful_tests/trex_rx_test.py @@ -52,9 +52,10 @@ class CTRexRx_Test(CTRexGeneral_Test): path = 'rx-check.data.stats.m_total_rx' total_rx = trex_res.get_last_value(path) - if not total_rx: + if total_rx is None: raise AbnormalResultError('No TRex results by path: %s' % path) - + elif not total_rx: + raise AbnormalResultError('Total rx_check (%s) packets is zero.' % path) print('Total packets checked: %s' % total_rx) print('Latency counters: %s' % latency_counters_display) @@ -69,7 +70,7 @@ class CTRexRx_Test(CTRexGeneral_Test): error_tolerance = self.get_benchmark_param('error_tolerance') if not error_tolerance or not allow_error_tolerance: error_tolerance = 0 - error_percentage = float(total_errors) * 100 / total_rx + error_percentage = total_errors * 100.0 / total_rx if total_errors > 0: if self.is_loopback or error_percentage > error_tolerance: @@ -255,7 +256,8 @@ class CTRexRx_Test(CTRexGeneral_Test): self.router.config_zbf() trex_res = self.trex.sample_to_run_finish() self.router.config_no_zbf() - self.router.clear_nat_translations() + self.router.config_no_nat(nat_obj) + #self.router.clear_nat_translations() print("\nLATEST RESULT OBJECT:") print(trex_res) self.check_rx_errors(trex_res, allow_error_tolerance = False) diff --git a/scripts/automation/regression/stateless_tests/stl_benchmark_test.py b/scripts/automation/regression/stateless_tests/stl_benchmark_test.py new file mode 100755 index 00000000..ef4c435f --- /dev/null +++ b/scripts/automation/regression/stateless_tests/stl_benchmark_test.py @@ -0,0 +1,69 @@ +#!/router/bin/python +from .stl_general_test import CStlGeneral_Test, CTRexScenario +from trex_stl_lib.api import * +import os, sys +from collections import deque +from time import time, sleep + +class STLBenchmark_Test(CStlGeneral_Test): + """Benchark stateless performance""" + + def test_CPU_benchmark(self): + timeout = 60 # max time to wait for stabilization + stabilize = 5 # ensure stabilization over this period + print('') + + for profile_bench in self.get_benchmark_param('profiles'): + cpu_utils = deque([0] * stabilize, maxlen = stabilize) + bws_per_core = deque([0] * stabilize, maxlen = stabilize) + kwargs = profile_bench.get('kwargs', {}) + print('Testing profile %s, kwargs: %s' % (profile_bench['name'], kwargs)) + profile = STLProfile.load(os.path.join(CTRexScenario.scripts_path, profile_bench['name']), **kwargs) + + self.stl_trex.reset() + self.stl_trex.clear_stats() + sleep(1) + self.stl_trex.add_streams(profile) + self.stl_trex.start(mult = '10%') + start_time = time() + + for i in range(timeout + 1): + stats = self.stl_trex.get_stats() + cpu_utils.append(stats['global']['cpu_util']) + bws_per_core.append(stats['global']['bw_per_core']) + if i > stabilize and min(cpu_utils) > max(cpu_utils) * 0.95: + break + sleep(0.5) + + agv_cpu_util = sum(cpu_utils) / stabilize + agv_bw_per_core = sum(bws_per_core) / stabilize + + if i == timeout and agv_cpu_util > 10: + raise Exception('Timeout on waiting for stabilization, last CPU util values: %s' % list(cpu_utils)) + if stats[0]['opackets'] < 1000 or stats[1]['opackets'] < 1000: + raise Exception('Too few opackets, port0: %s, port1: %s' % (stats[0]['opackets'], stats[1]['opackets'])) + if stats['global']['queue_full'] > 100000: + raise Exception('Too much queue_full: %s' % stats['global']['queue_full']) + if not cpu_utils[-1]: + raise Exception('CPU util is zero, last values: %s' % list(cpu_utils)) + print('Done (%ss), CPU util: %4g, bw_per_core: %6sGb/core' % (int(time() - start_time), agv_cpu_util, round(agv_bw_per_core, 2))) + # TODO: add check of benchmark based on results from regression + + # report benchmarks + if self.GAManager: + profile_repr = '%s.%s %s' % (CTRexScenario.setup_name, + os.path.basename(profile_bench['name']), + repr(kwargs).replace("'", '')) + self.GAManager.gaAddAction(Event = 'stateless_test', action = profile_repr, + label = 'bw_per_core', value = int(agv_bw_per_core)) + # TODO: report expected once acquired + #self.GAManager.gaAddAction(Event = 'stateless_test', action = profile_repr, + # label = 'bw_per_core_exp', value = int(expected_norm_cpu)) + self.GAManager.emptyAndReportQ() + + def tearDown(self): + self.stl_trex.reset() + self.stl_trex.clear_stats() + sleep(1) + CStlGeneral_Test.tearDown(self) + diff --git a/scripts/automation/regression/stateless_tests/trex_client_pkg_test.py b/scripts/automation/regression/stateless_tests/trex_client_pkg_test.py index 64d5000e..fe666ac3 100755 --- a/scripts/automation/regression/stateless_tests/trex_client_pkg_test.py +++ b/scripts/automation/regression/stateless_tests/trex_client_pkg_test.py @@ -12,7 +12,7 @@ class CTRexClientPKG_Test(CStlGeneral_Test): CStlGeneral_Test.setUp(self) self.unzip_client_package() - def run_client_package_stf_example(self, python_version): + def run_client_package_stl_example(self, python_version): commands = [ 'cd %s' % CTRexScenario.scripts_path, 'source find_python.sh --%s' % python_version, @@ -25,7 +25,7 @@ class CTRexClientPKG_Test(CStlGeneral_Test): self.fail('Error in running stf_example using %s: %s' % (python_version, stderr)) def test_client_python2(self): - self.run_client_package_stf_example(python_version = 'python2') + self.run_client_package_stl_example(python_version = 'python2') def test_client_python3(self): - self.run_client_package_stf_example(python_version = 'python3') + self.run_client_package_stl_example(python_version = 'python3') diff --git a/scripts/automation/regression/trex.py b/scripts/automation/regression/trex.py index 9459e7c6..7440d76d 100644 --- a/scripts/automation/regression/trex.py +++ b/scripts/automation/regression/trex.py @@ -36,6 +36,7 @@ class CTRexScenario: # logger = None test_types = {'functional_tests': [], 'stateful_tests': [], 'stateless_tests': []} is_copied = False + GAManager = None class CTRexRunner: """This is an instance for generating a CTRexRunner""" diff --git a/scripts/automation/regression/trex_unit_test.py b/scripts/automation/regression/trex_unit_test.py index 5d29ff31..b9f0c05c 100755 --- a/scripts/automation/regression/trex_unit_test.py +++ b/scripts/automation/regression/trex_unit_test.py @@ -39,6 +39,7 @@ from trex import CTRexScenario from trex_stf_lib.trex_client import * from trex_stf_lib.trex_exceptions import * from trex_stl_lib.api import * +from trex_stl_lib.utils.GAObjClass import GAmanager import trex import socket from pprint import pprint @@ -117,9 +118,12 @@ class CTRexTestConfiguringPlugin(Plugin): parser.add_option('--log-path', '--log_path', action='store', dest='log_path', help='Specify path for the tests` log to be saved at. Once applied, logs capturing by nose will be disabled.') # Default is CURRENT/WORKING/PATH/trex_log/trex_log.log') - parser.add_option('--verbose-mode', '--verbose_mode', action="store_true", default = False, - dest="verbose_mode", - help="Print RPC command and router commands.") + parser.add_option('--json-verbose', '--json_verbose', action="store_true", default = False, + dest="json_verbose", + help="Print JSON-RPC commands.") + parser.add_option('--telnet-verbose', '--telnet_verbose', action="store_true", default = False, + dest="telnet_verbose", + help="Print telnet commands and responces.") parser.add_option('--server-logs', '--server_logs', action="store_true", default = False, dest="server_logs", help="Print server side (TRex and trex_daemon) logs per test.") @@ -150,17 +154,21 @@ class CTRexTestConfiguringPlugin(Plugin): parser.add_option('--test-client-package', '--test_client_package', action="store_true", default = False, dest="test_client_package", help="Includes tests of client package.") + parser.add_option('--long', action="store_true", default = False, + dest="long", + help="Flag of long tests (stability).") def configure(self, options, conf): self.collect_only = options.collect_only if self.collect_only: return - self.functional = options.functional - self.stateless = options.stateless - self.stateful = options.stateful - self.pkg = options.pkg - self.no_ssh = options.no_ssh - self.verbose_mode = options.verbose_mode + self.functional = options.functional + self.stateless = options.stateless + self.stateful = options.stateful + self.pkg = options.pkg + self.no_ssh = options.no_ssh + self.json_verbose = options.json_verbose + self.telnet_verbose = options.telnet_verbose if self.functional and (not self.pkg or self.no_ssh): return if CTRexScenario.setup_dir and options.config_path: @@ -215,7 +223,7 @@ class CTRexTestConfiguringPlugin(Plugin): if self.stateful: if not self.no_ssh: trex_remote_command(self.configuration.trex, STATEFUL_RUN_COMMAND) - CTRexScenario.trex = CTRexClient(trex_host = self.configuration.trex['trex_name'], verbose = self.verbose_mode) + CTRexScenario.trex = CTRexClient(trex_host = self.configuration.trex['trex_name'], verbose = self.json_verbose) elif self.stateless: if not self.no_ssh: cores = self.configuration.trex.get('trex_cores', 1) @@ -224,11 +232,11 @@ class CTRexTestConfiguringPlugin(Plugin): trex_remote_command(self.configuration.trex, './t-rex-64 -i -c %s' % cores, background = True) CTRexScenario.stl_trex = STLClient(username = 'TRexRegression', server = self.configuration.trex['trex_name'], - verbose_level = self.verbose_mode) + verbose_level = self.json_verbose) if 'loopback' not in self.modes: CTRexScenario.router_cfg = dict(config_dict = self.configuration.router, forceImageReload = self.load_image, - silent_mode = not self.verbose_mode, + silent_mode = not self.telnet_verbose, forceCleanConfig = self.clean_config, tftp_config_dict = self.configuration.tftp) try: @@ -297,6 +305,7 @@ if __name__ == "__main__": xml_name = 'unit_test.xml' if CTRexScenario.setup_dir: CTRexScenario.setup_name = os.path.basename(CTRexScenario.setup_dir) + CTRexScenario.GAManager = GAmanager(GoogleID='UA-75220362-4', UserID=CTRexScenario.setup_name, QueueSize=100, Timeout=5, UserPermission=1, BlockingMode=1, appName='TRex', appVer='1.11.232') #timeout in seconds xml_name = 'report_%s.xml' % CTRexScenario.setup_name xml_arg= '--xunit-file=%s/%s' % (CTRexScenario.report_dir, xml_name) set_report_dir(CTRexScenario.report_dir) |