diff options
| -rw-r--r-- | scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py | 56 | ||||
| -rw-r--r-- | scripts/stl/tests/multi_core_test.py | 79 | ||||
| -rw-r--r-- | src/stateless/cp/trex_stream_vm.h | 238 | ||||
| -rw-r--r-- | src/stateless/cp/trex_vm_splitter.cpp | 127 | ||||
| -rw-r--r-- | src/stateless/cp/trex_vm_splitter.h | 4 |
5 files changed, 290 insertions, 214 deletions
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py index 3e63c4e2..0394cf43 100644 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py @@ -24,6 +24,9 @@ from .trex_stl_client import STLClient from .utils import pcap from trex_stl_lib.trex_stl_packet_builder_scapy import RawPcapReader, RawPcapWriter, hexdump +from random import randint +from random import choice as rand_choice + from yaml import YAMLError import re @@ -473,7 +476,31 @@ def compare_caps (cap1, cap2, max_diff_sec = (5 * 1e-6)): return True - +def hexdiff (d1, d2): + rc = [] + + if len(d1) != len(d2): + return rc + + for i in range(len(d1)): + if d1[i] != d2[i]: + rc.append(i) + return rc + +def prettyhex (h, diff_list): + if type(h[0]) == str: + h = [ord(x) for x in h] + + for i in range(len(h)): + + if i in diff_list: + sys.stdout.write("->'0x%02x'<-" % h[i]) + else: + sys.stdout.write(" '0x%02x' " % h[i]) + if ((i % 9) == 8): + print("") + + print("") # a more strict comparsion 1 <--> 1 def compare_caps_strict (cap1, cap2, max_diff_sec = (5 * 1e-6)): @@ -495,18 +522,31 @@ def compare_caps_strict (cap1, cap2, max_diff_sec = (5 * 1e-6)): if pkt1[0] != pkt2[0]: print(format_text("RAW error: cap files '{0}', '{1}' differ in cap #{2}\n".format(cap1, cap2, i), 'bold')) - print(hexdump(pkt1[0])) - print("") - print(hexdump(pkt2[0])) + + #d1 = hexdump(pkt1[0]) + #d2 = hexdump(pkt2[0]) + + diff_list = hexdiff(pkt1[0], pkt2[0]) + + print("{0} - packet #{1}:\n".format(cap1, i)) + prettyhex(pkt1[0], diff_list) + + print("\n{0} - packet #{1}:\n".format(cap2, i)) + prettyhex(pkt2[0], diff_list) + + #print(hexdump(pkt1[0])) + #print("") + #print(hexdump(pkt2[0])) + #print("") print("") return False return True -# + def test_multi_core (r, options): - for core_count in [1, 2, 4, 6, 8]: + for core_count in range(1, 9): r.run(input_list = options.input_file, outfile = '{0}.cap'.format(core_count), dp_core_count = core_count, @@ -516,12 +556,12 @@ def test_multi_core (r, options): duration = options.duration, mode = 'none', silent = True, - tunables = options.tunables) + tunables = [{'seed': 5}]) print("") print(format_text("comparing 2 cores to 1 core:\n", 'underline')) - rc = compare_caps('1.cap', '2.cap') + rc = compare_caps_strict('1.cap', '2.cap') if rc: print("[Passed]\n") diff --git a/scripts/stl/tests/multi_core_test.py b/scripts/stl/tests/multi_core_test.py new file mode 100644 index 00000000..a0ec3b2d --- /dev/null +++ b/scripts/stl/tests/multi_core_test.py @@ -0,0 +1,79 @@ +from trex_stl_lib.api import * +import random + +class STLMultiCore(object): + + def __init__ (self): + # default IMIX properties + self.streams_def = [ {'size': 300, 'pps': 2884, 'isg':0 }, + #{'size': 590, 'pps': 20, 'isg':0.1 }, + #{'size': 1514, 'pps': 4, 'isg':0.2 } + ] + + + def create_stream (self, size, pps, isg, vm ): + # Create base packet and pad it to size + base_pkt = Ether()/IP()/UDP() + pad = max(0, size - len(base_pkt)) * 'x' + + pkt = STLPktBuilder(pkt = base_pkt/pad, + vm = vm) + + return STLStream(isg = isg, + packet = pkt, + mode = STLTXCont(pps = pps)) + + + def generate_var (self, rng, i): + + d = {'name': str(i)} + + d['size'] = rng.choice([1, 2, 4]) + max_val = (1 << d['size'] * 8) + + d['start'] = rng.randint(0, max_val - 1) + d['end'] = rng.randint(d['start'], max_val) + d['step'] = rng.randint(1, 1000) + d['op'] = rng.choice(['inc', 'dec']) + + return d + + def dump_var (self, var): + return 'name: {:}, start: {:}, end: {:}, size: {:}, op: {:}, step {:}'.format(var['name'], var['start'], var['end'], var['size'], var['op'], var['step']) + + + def get_streams (self, direction = 0, **kwargs): + + rng = random.Random(kwargs.get('seed', 1)) + + # base offset + pkt_offset = 42 + vm = [] + print("\nusing the following vars:\n") + for i in range(10): + var = self.generate_var(rng, i) + print("at offset {:} - var: {:}".format(pkt_offset, self.dump_var(var))) + vm += [STLVmFlowVar(name = var['name'], + min_value = var['start'], + max_value = var['end'], + size = var['size'], + op = var['op']), + STLVmWrFlowVar(fv_name = var['name'], pkt_offset = pkt_offset), + ] + pkt_offset += var['size'] + + + + + print("\n") + # create imix streams + return [self.create_stream(x['size'], x['pps'],x['isg'] , vm) for x in self.streams_def] + + + +# dynamic load - used for trex console or simulator +def register(): + return STLMultiCore() + + + diff --git a/src/stateless/cp/trex_stream_vm.h b/src/stateless/cp/trex_stream_vm.h index 5e3665c6..ab2a4fa8 100644 --- a/src/stateless/cp/trex_stream_vm.h +++ b/src/stateless/cp/trex_stream_vm.h @@ -30,7 +30,27 @@ limitations under the License. #include "pal_utl.h" #include "mbuf.h" +static inline +uint64_t inc_mod(uint64_t a, uint64_t b, uint64_t c, uint64_t step) { + /* check if we have enough left for simple inc */ + uint64_t left = b - c; + if (step <= left) { + return (c + step); + } else { + return (a + (step - left - 1)); // restart consumes also 1 + } +} +static inline +uint64_t dec_mod(uint64_t a, uint64_t b, uint64_t c, uint64_t step) { + /* check if we have enough left for simple dec */ + uint64_t left = c - a; + if (step <= left) { + return (c - step); + } else { + return (b - (step - left - 1)); // restart consumes also 1 + } +} //https://software.intel.com/en-us/articles/fast-random-number-generator-on-the-intel-pentiumr-4-processor/ @@ -211,20 +231,12 @@ public: inline void run_inc(uint8_t * flow_var) { uint8_t *p = (flow_var + m_flow_offset); - if (*p == m_max_val) { - *p = m_min_val; - } else { - *p = *p + 1; - } + *p = inc_mod(m_min_val, m_max_val, *p, 1); } inline void run_dec(uint8_t * flow_var) { uint8_t *p = (flow_var + m_flow_offset); - if (*p == m_min_val) { - *p = m_max_val; - } else { - *p = *p - 1; - } + *p = dec_mod(m_min_val, m_max_val, *p, 1); } inline void run_rand(uint8_t * flow_var,uint32_t *per_thread_random) { @@ -245,20 +257,12 @@ public: inline void run_inc(uint8_t * flow_var) { uint16_t *p = (uint16_t *)(flow_var + m_flow_offset); - if (*p == m_max_val) { - *p = m_min_val; - } else { - *p = *p + 1; - } + *p = inc_mod(m_min_val, m_max_val, *p, 1); } inline void run_dec(uint8_t * flow_var) { uint16_t *p = (uint16_t *)(flow_var + m_flow_offset); - if (*p == m_min_val) { - *p = m_max_val; - } else { - *p = *p - 1; - } + *p = dec_mod(m_min_val, m_max_val, *p, 1); } inline void run_rand(uint8_t * flow_var,uint32_t *per_thread_random) { @@ -280,20 +284,12 @@ public: inline void run_inc(uint8_t * flow_var) { uint32_t *p = (uint32_t *)(flow_var + m_flow_offset); - if (*p == m_max_val) { - *p = m_min_val; - } else { - *p = *p + 1; - } + *p = inc_mod(m_min_val, m_max_val, *p, 1); } inline void run_dec(uint8_t * flow_var) { uint32_t *p = (uint32_t *)(flow_var + m_flow_offset); - if (*p == m_min_val) { - *p = m_max_val; - } else { - *p = *p - 1; - } + *p = dec_mod(m_min_val, m_max_val, *p, 1); } inline void run_rand(uint8_t * flow_var,uint32_t *per_thread_random) { @@ -313,20 +309,12 @@ public: inline void run_inc(uint8_t * flow_var) { uint64_t *p = (uint64_t *)(flow_var + m_flow_offset); - if (*p == m_max_val) { - *p = m_min_val; - } else { - *p = *p + 1; - } + *p = inc_mod(m_min_val, m_max_val, *p, 1); } inline void run_dec(uint8_t * flow_var) { uint64_t *p = (uint64_t *)(flow_var + m_flow_offset); - if (*p == m_min_val) { - *p = m_max_val; - } else { - *p = *p - 1; - } + *p = dec_mod(m_min_val, m_max_val, *p, 1); } inline void run_rand(uint8_t * flow_var,uint32_t *per_thread_random) { @@ -355,20 +343,12 @@ public: inline void run_inc(uint8_t * flow_var) { uint8_t *p = (flow_var + m_flow_offset); - if (*p > (m_max_val-m_step)) { - *p = m_min_val; - } else { - *p = *p + m_step; - } + *p = inc_mod(m_min_val, m_max_val, *p, m_step); } inline void run_dec(uint8_t * flow_var) { uint8_t *p = (flow_var + m_flow_offset); - if (*p < (m_min_val+m_step)) { - *p = m_max_val; - } else { - *p = *p - m_step; - } + *p = dec_mod(m_min_val, m_max_val, *p, m_step); } } __attribute__((packed)) ; @@ -385,20 +365,12 @@ public: inline void run_inc(uint8_t * flow_var) { uint16_t *p = (uint16_t *)(flow_var + m_flow_offset); - if (*p > (m_max_val-m_step)) { - *p = m_min_val; - } else { - *p = *p + m_step; - } + *p = inc_mod(m_min_val, m_max_val, *p, m_step); } inline void run_dec(uint8_t * flow_var) { uint16_t *p = (uint16_t *)(flow_var + m_flow_offset); - if (*p < (m_min_val+m_step)) { - *p = m_max_val; - } else { - *p = *p - m_step; - } + *p = dec_mod(m_min_val, m_max_val, *p, m_step); } } __attribute__((packed)) ; @@ -414,20 +386,12 @@ public: inline void run_inc(uint8_t * flow_var) { uint32_t *p = (uint32_t *)(flow_var + m_flow_offset); - if (*p > (m_max_val-m_step)) { - *p = m_min_val; - } else { - *p = *p + m_step; - } + *p = inc_mod(m_min_val, m_max_val, *p, m_step); } inline void run_dec(uint8_t * flow_var) { uint32_t *p = (uint32_t *)(flow_var + m_flow_offset); - if (*p < (m_min_val+m_step)) { - *p = m_max_val; - } else { - *p = *p - m_step; - } + *p = dec_mod(m_min_val, m_max_val, *p, m_step); } } __attribute__((packed)) ; @@ -443,20 +407,12 @@ public: inline void run_inc(uint8_t * flow_var) { uint64_t *p = (uint64_t *)(flow_var + m_flow_offset); - if (*p > (m_max_val-m_step) ) { - *p = m_min_val; - } else { - *p = *p + m_step; - } + *p = inc_mod(m_min_val, m_max_val, *p, m_step); } inline void run_dec(uint8_t * flow_var) { uint64_t *p = (uint64_t *)(flow_var + m_flow_offset); - if (*p < m_min_val+m_step) { - *p = m_max_val; - } else { - *p = *p - m_step; - } + *p = dec_mod(m_min_val, m_max_val, *p, m_step); } @@ -981,10 +937,11 @@ public: virtual StreamVmInstruction * clone() = 0; - /* by default an instruction is not splitable */ - virtual bool is_splitable() const { - return false; + bool is_var_instruction() const { + instruction_type_t type = get_instruction_type(); + return ( (type == itFLOW_MAN) || (type == itFLOW_CLIENT) ); } + /* nothing to init */ virtual uint8_t bss_init_value(uint8_t *p){ return (0); @@ -1007,19 +964,41 @@ public: } - const std::string & get_var_name() { + const std::string & get_var_name() const { return m_var_name; } - virtual bool is_splitable() const { - return true; - } + virtual bool need_split() const = 0; /** * what is the split range for this var * */ - virtual uint64_t get_splitable_range() const = 0; + virtual uint64_t get_range() const = 0; + + /** + * allows a var instruction to be updated + * for multicore (split) + * + */ + virtual void update(uint64_t phase, uint64_t step_multiplier) = 0; + + uint64_t peek_next(uint64_t skip = 1) const { + return peek(skip, true); + } + + uint64_t peek_prev(uint64_t skip = 1) const { + return peek(skip, false); + } + + +protected: + /** + * a var instruction should be able to peek back/forward with + * any number of steps in the series + * + */ + virtual uint64_t peek(int skip = 1, bool forward = true) const = 0; public: @@ -1064,12 +1043,12 @@ public: return ( StreamVmInstruction::itFLOW_MAN); } - virtual bool is_valid_for_split() const { - return (m_step==1?true:false); + virtual bool need_split() const { + /* random does not need split */ + return (m_op != FLOW_VAR_OP_RANDOM); } - - virtual uint64_t get_splitable_range() const { + virtual uint64_t get_range() const { return (m_max_value - m_min_value + 1); } @@ -1094,19 +1073,7 @@ public: * */ uint64_t get_bss_init_value() const { - uint64_t init = m_init_value; - - switch (m_op) { - case FLOW_VAR_OP_INC: - return (init == m_min_value ? m_max_value : (init - 1)); - - case FLOW_VAR_OP_DEC: - return (init == m_max_value ? m_min_value : (init + 1)); - - default: - return init; - } - + return peek_prev(); } StreamVmInstructionFlowMan(const std::string &var_name, @@ -1117,15 +1084,27 @@ public: uint64_t max_value, uint64_t step=1) : StreamVmInstructionVar(var_name) { - m_op = op; - m_size_bytes = size; - m_init_value = init_value; - m_min_value = min_value; - m_max_value = max_value; - m_step = step; + m_op = op; + m_size_bytes = size; + m_init_value = init_value; + m_min_value = min_value; + m_max_value = max_value; + m_step = step % get_range(); // support step overflow by modulu + + assert(m_init_value >= m_min_value); + assert(m_init_value <= m_max_value); + } + virtual void update(uint64_t phase, uint64_t step_multiplier) { + /* update the init value to be with a phase */ + m_init_value = peek_next(phase); + m_step = (m_step * step_multiplier) % get_range(); + + assert(m_init_value >= m_min_value); + assert(m_init_value <= m_max_value); } + virtual void Dump(FILE *fd); void sanity_check(uint32_t ins_id,StreamVm *lp); @@ -1140,6 +1119,29 @@ public: m_step); } + +protected: + + /* fetch the next value in the variable (used for core phase and etc.) */ + virtual uint64_t peek(int skip = 1, bool forward = true) const { + + if (m_op == FLOW_VAR_OP_RANDOM) { + return m_init_value; + } + + assert( (m_op == FLOW_VAR_OP_INC) || (m_op == FLOW_VAR_OP_DEC) ); + bool add = ( (m_op == FLOW_VAR_OP_INC) ? forward : !forward ); + + uint64_t next_step = (m_step * skip) % get_range(); + + if (add) { + return inc_mod(m_min_value, m_max_value, m_init_value, next_step); + } else { + return dec_mod(m_min_value, m_max_value, m_init_value, next_step); + } + } + + private: void sanity_check_valid_size(uint32_t ins_id,StreamVm *lp); void sanity_check_valid_opt(uint32_t ins_id,StreamVm *lp); @@ -1313,6 +1315,9 @@ public: return ( StreamVmInstruction::itFLOW_CLIENT); } + virtual bool need_split() const { + return true; + } StreamVmInstructionFlowClient(const std::string &var_name, uint32_t client_min_value, @@ -1347,7 +1352,7 @@ public: return (m_port_max - m_port_min + 1); } - virtual uint64_t get_splitable_range() const { + virtual uint64_t get_range() const { return get_ip_range(); } @@ -1370,6 +1375,15 @@ public: m_flags); } + virtual void update(uint64_t phase, uint64_t step_multiplier) { + } + + +protected: + virtual uint64_t peek(int skip = 1, bool forward = true) const { + return (0); + } + public: uint32_t m_client_min; // min ip diff --git a/src/stateless/cp/trex_vm_splitter.cpp b/src/stateless/cp/trex_vm_splitter.cpp index 5069c535..1ff43829 100644 --- a/src/stateless/cp/trex_vm_splitter.cpp +++ b/src/stateless/cp/trex_vm_splitter.cpp @@ -89,27 +89,28 @@ TrexVmSplitter::split(TrexStream *stream, std::vector<TrexStream *> core_streams bool TrexVmSplitter::split_internal() { - const StreamVmInstructionVar *split_instr = m_stream->m_vm.get_split_instruction(); + duplicate_vm(); + + /* search for splitable instructions */ + for (StreamVmInstruction *instr : m_stream->m_vm.get_instruction_list()) { + if (!instr->is_var_instruction()) { + continue; + } + + split_flow_var( (const StreamVmInstructionVar *)instr ); - /* if no split instruction was specified - fall back*/ - if (split_instr == NULL) { - return false; } - if (split_instr->get_instruction_type() == StreamVmInstruction::itFLOW_MAN) { - return split_by_flow_var( (const StreamVmInstructionFlowMan *)split_instr ); - } else if (split_instr->get_instruction_type() == StreamVmInstruction::itFLOW_CLIENT) { - return split_by_flow_client_var( (const StreamVmInstructionFlowClient *)split_instr ); + /* done - now compile for all cores */ + compile_vm(); - } else { - throw TrexException("VM splitter : cannot split by instruction which is not flow var or flow client var"); - } + return true; } /** - * split VM by flow var + * split a flow var instruction * * @author imarom (20-Dec-15) * @@ -117,98 +118,29 @@ TrexVmSplitter::split_internal() { * * @return bool */ -bool -TrexVmSplitter::split_by_flow_var(const StreamVmInstructionFlowMan *instr) { - /* no point in splitting random */ - if (instr->m_op == StreamVmInstructionFlowMan::FLOW_VAR_OP_RANDOM) { - return false; - } - - /* if the range is too small - it is unsplitable */ - if (instr->get_splitable_range() < m_dp_core_count) { - return false; - } - - /* split only step of 1 */ - if (!instr->is_valid_for_split() ){ - return false; +void +TrexVmSplitter::split_flow_var(const StreamVmInstructionVar *src) { + /* a var might not need split (random) */ + if (!src->need_split()) { + return; } - /* we need to split - duplicate VM now */ - duplicate_vm(); - - /* calculate range splitting */ - uint64_t range = instr->get_splitable_range(); - - uint64_t range_part = range / m_dp_core_count; - uint64_t leftover = range % m_dp_core_count; - - /* first core handles a bit more */ - uint64_t start = instr->m_min_value; - uint64_t end = start + range_part + leftover - 1; - - /* do work */ + int core_id = 0; for (TrexStream *core_stream : *m_core_streams) { - /* get the per-core instruction to split */ - StreamVmInstructionFlowMan *per_core_instr = (StreamVmInstructionFlowMan *)core_stream->m_vm.get_split_instruction(); - - per_core_instr->m_min_value = start; - per_core_instr->m_max_value = end; + StreamVmInstructionVar *dst = core_stream->m_vm.lookup_var_by_name(src->get_var_name()); + assert(dst); - /* after split this has no meaning - choose it as we see fit */ - per_core_instr->m_init_value = (per_core_instr->m_op == StreamVmInstructionFlowMan::FLOW_VAR_OP_DEC ? end : start); + /* for each core we need to give a phase and multiply the step frequency */ + dst->update(core_id, m_dp_core_count); - core_stream->vm_compile(); - - start = end + 1; - end = start + range_part - 1; + core_id++; } - return true; } -bool -TrexVmSplitter::split_by_flow_client_var(const StreamVmInstructionFlowClient *instr) { - - /* if the range is too small - it is unsplitable */ - if (instr->get_ip_range() < m_dp_core_count) { - return false; - } - - /* we need to split - duplicate VM now */ - duplicate_vm(); - - /* calculate range splitting */ - uint64_t range = instr->get_ip_range(); - - uint64_t range_part = range / m_dp_core_count; - uint64_t leftover = range % m_dp_core_count; - - /* first core handles a bit more */ - uint64_t start = instr->m_client_min; - uint64_t end = start + range_part + leftover - 1; - - - /* do work */ - for (TrexStream *core_stream : *m_core_streams) { - - /* get the per-core instruction to split */ - StreamVmInstructionFlowClient *per_core_instr = (StreamVmInstructionFlowClient *)core_stream->m_vm.get_split_instruction(); - - per_core_instr->m_client_min = start; - per_core_instr->m_client_max = end; - - core_stream->vm_compile(); - - start = end + 1; - end = start + range_part - 1; - } - - return true; -} /** * duplicate the VM instructions @@ -222,3 +154,14 @@ TrexVmSplitter::duplicate_vm() { } } +/** + * now compile the updated VM + */ +void +TrexVmSplitter::compile_vm() { + /* for each core - duplicate the instructions */ + for (TrexStream *core_stream : *m_core_streams) { + core_stream->vm_compile(); + } +} + diff --git a/src/stateless/cp/trex_vm_splitter.h b/src/stateless/cp/trex_vm_splitter.h index dac71c21..d24cd2cc 100644 --- a/src/stateless/cp/trex_vm_splitter.h +++ b/src/stateless/cp/trex_vm_splitter.h @@ -46,10 +46,10 @@ public: private: bool split_internal(); - bool split_by_flow_var(const StreamVmInstructionFlowMan *instr); - bool split_by_flow_client_var(const StreamVmInstructionFlowClient *instr); + void split_flow_var(const StreamVmInstructionVar *instr); void duplicate_vm(); + void compile_vm(); TrexStream *m_stream; std::vector<TrexStream *> *m_core_streams; |