ignore python3 cache, generated code files

add option to export Profile (and thus stream too) to Python code
add for each profile in functional tests generating it to Python code and comparing to same pcap result
add loading Prifile via add_streams()
fix IMIX rates in HLTAPI used with line % and bps
Scapy packet builder: fixes to load of pcap/yaml - use MAC from pcap (with flag), remove FCS (with flag), support for VM with offsets by name

6 files changed, 253 insertions, 60 deletions

diff --git a/scripts/automation/regression/unit_tests/functional_tests/stl_basic_tests.py b/scripts/automation/regression/unit_tests/functional_tests/stl_basic_tests.py
index 2099c953..44735435 100644
--- a/scripts/automation/regression/unit_tests/functional_tests/stl_basic_tests.py
+++ b/scripts/automation/regression/unit_tests/functional_tests/stl_basic_tests.py
@@ -13,7 +13,6 @@ import sys
 import os
 import subprocess
 
-# should be set to run explicitly, not as part of all regression tests
 @attr('run_on_trex')
 class CStlBasic_Test(functional_general_test.CGeneralFunctional_Test):
     def setUp (self):
@@ -117,7 +116,7 @@ class CStlBasic_Test(functional_general_test.CGeneralFunctional_Test):
             os.unlink(output_cap)
 
 
-    def run_py_profile_path (self, profile, options,silent = False, do_no_remove=False,compare =True):
+    def run_py_profile_path (self, profile, options,silent = False, do_no_remove=False,compare =True, test_generated=True):
         output_cap = "a.pcap"
         input_file =  os.path.join('stl/', profile)
         golden_file = os.path.join('exp',os.path.basename(profile).split('.')[0]+'.pcap');
@@ -133,8 +132,26 @@ class CStlBasic_Test(functional_general_test.CGeneralFunctional_Test):
             if compare:
                 self.compare_caps(output_cap, golden_file)
         finally:
-            if  not do_no_remove: 
+            if not do_no_remove: 
                 os.unlink(output_cap)
+        if test_generated:
+            try:
+                from trex_stl_lib.api import STLProfile # if test is skipped, don't load it
+                generated_filename = input_file.replace('.py', '_GENERATED.py').replace('.yaml', '_GENERATED.py')
+                if input_file.endswith('.py'):
+                    profile = STLProfile.load_py(input_file)
+                elif input_file.endswith('.yaml'):
+                    profile = STLProfile.load_yaml(input_file)
+                profile.dump_to_code(generated_filename)
+                rc = self.run_sim(generated_filename, output_cap, options, silent)
+                assert_equal(rc, True)
+    
+                if compare:
+                    self.compare_caps(output_cap, golden_file)
+            finally:
+                if not do_no_remove: 
+                    os.unlink(generated_filename)
+                    os.unlink(output_cap)
 
 
     def test_stl_profiles (self):
@@ -170,7 +187,7 @@ class CStlBasic_Test(functional_general_test.CGeneralFunctional_Test):
             ["udp_1pkt_pcap_relative_path.py","-m 1 -l 3",True],
             ["udp_1pkt_tuple_gen_split.py","-m 1 -c 2 -l 100",True],
             ["udp_1pkt_range_clients_split.py","-m 1 -c 2 -l 100",True],
-            ["udp_1pkt_vxlan.py","-m 1 -c 1 -l 17",True],
+            ["udp_1pkt_vxlan.py","-m 1 -c 1 -l 17",True, False], # can't generate: no VXLAN in Scapy, only in profile
             ["udp_1pkt_ipv6_in_ipv4.py","-m 1 -c 1 -l 17",True],
             ["yaml/imix_3pkt.yaml","-m 50kpps --limit 20 --cores 2",True],
             ["yaml/imix_3pkt_vm.yaml","-m 50kpps --limit 20 --cores 2",True],
@@ -195,7 +212,11 @@ class CStlBasic_Test(functional_general_test.CGeneralFunctional_Test):
         
 
         for obj in p:
-            self.run_py_profile_path (obj[0],obj[1],compare =obj[2], do_no_remove=True)
+            try:
+                test_generated = obj[3]
+            except: # check generated if not said otherwise
+                test_generated = True
+            self.run_py_profile_path (obj[0],obj[1],compare =obj[2], test_generated = test_generated, do_no_remove=True)
 
 
     def test_hlt_profiles (self):
@@ -212,7 +233,7 @@ class CStlBasic_Test(functional_general_test.CGeneralFunctional_Test):
             ['hlt/hlt_tcp_ranges.py', '-m 1 -l 20', True],
             ['hlt/hlt_udp_ports.py', '-m 1 -l 20', True],
             ['hlt/hlt_udp_random_ports.py', '-m 1 -l 20', True],
-            #['hlt/hlt_ip_ranges.py', '-m 1 -l 20', True], # can't run now, random on full range issue
+            ['hlt/hlt_ip_ranges.py', '-m 1 -l 20', True],
             ['hlt/hlt_framesize_vm.py', '-m 1 -l 20', True],
             ['hlt/hlt_l3_length_vm.py', '-m 1 -l 20', True],
             ['hlt/hlt_vlan_default.py', '-m 1 -l 20', True],
diff --git a/scripts/automation/trex_control_plane/stl/examples/hlt_udp_simple.py b/scripts/automation/trex_control_plane/stl/examples/hlt_udp_simple.py
index 19752f4f..6cd6debc 100644
--- a/scripts/automation/trex_control_plane/stl/examples/hlt_udp_simple.py
+++ b/scripts/automation/trex_control_plane/stl/examples/hlt_udp_simple.py
@@ -16,17 +16,13 @@ from trex_stl_lib.trex_stl_hltapi import *
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(usage=""" 
-    Connect to TRex and send burst of packets
+    Connect to TRex and send bidirectional continuous traffic
 
-    examples
+    examples:
 
-     hlt_udp_simple.py -s 9000 -d 30
+     hlt_udp_simple.py --server <hostname/ip>
 
-     hlt_udp_simple.py -s 9000 -d 30 -rate_percent 10
-
-     hlt_udp_simple.py -s 300 -d 30 -rate_pps 5000000
-
-     hlt_udp_simple.py -s 800 -d 30 -rate_bps 500000000 --debug
+     hlt_udp_simple.py -s 300 -d 30 -rate_pps 5000000 --src <MAC> --dst <MAC>
 
      then run the simulator on the output 
        ./stl-sim -f example.yaml -o a.pcap  ==> a.pcap include the packet
@@ -35,9 +31,9 @@ if __name__ == "__main__":
     description="Example for TRex HLTAPI",
     epilog=" based on hhaim's stl_run_udp_simple example");
 
-    parser.add_argument("--ip", 
-                        dest="ip",
-                        help='Remote trex ip',
+    parser.add_argument("--server", 
+                        dest="server",
+                        help='Remote trex address',
                         default="127.0.0.1",
                         type = str)
 
@@ -72,7 +68,7 @@ if __name__ == "__main__":
 
     hltapi = CTRexHltApi()
     print 'Connecting to TRex'
-    res = hltapi.connect(device = args.ip, port_list = [0, 1], reset = True, break_locks = True)
+    res = hltapi.connect(device = args.server, port_list = [0, 1], reset = True, break_locks = True)
     check_res(res)
     ports = res['port_handle']
     if len(ports) < 2:
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
index 39d69178..6ff007b1 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
@@ -1183,7 +1183,7 @@ class STLClient(object):
             ports : list
                 ports to execute the command
             streams: list
-                streams to attach
+                streams to attach (or profile)
 
         :returns:
             list of stream IDs in order of the stream list
@@ -1198,6 +1198,9 @@ class STLClient(object):
         ports = ports if ports is not None else self.get_acquired_ports()
         ports = self._validate_port_list(ports)
 
+        if isinstance(streams, STLProfile):
+            streams = streams.get_streams()
+
         # transform single stream
         if not isinstance(streams, list):
             streams = [streams]
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py
index 5e8fddfa..2f08cc48 100755
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py
@@ -21,9 +21,12 @@ cleanup_session_kwargs = {
 traffic_config_kwargs = {
     'mode': None,                           # ( create | modify | remove | reset )
     'split_by_cores': 'split',              # ( split | duplicate | single ) TRex extention: split = split traffic by cores, duplicate = duplicate traffic for all cores, single = run only with sinle core (not implemented yet)
-    'consistent_random': False,             # TRex extention: False (default): random sequence will be different every run, True: random sequence will be same every run
+    'load_profile': None,                   # TRex extention: path to filename with stream profile (stream builder parameters will be ignored, limitation: modify)
+    'consistent_random': False,             # TRex extention: False (default) = random sequence will be different every run, True = random sequence will be same every run
+    'ignore_macs': False,                   # TRex extention: True = use MACs from server configuration , no MAC VM (workaround on lack of ARP)
     'port_handle': None,
     'port_handle2': None,
+    'bidirectional': False,
     # stream builder parameters
     'transmit_mode': 'continuous',          # ( continuous | multi_burst | single_burst )
     'rate_pps': None,
@@ -31,19 +34,18 @@ traffic_config_kwargs = {
     'rate_percent': 10,
     'stream_id': None,
     'name': None,
-    'bidirectional': 0,
-    'direction': 0,                         # ( 0 | 1 ) TRex extention: 1 = exchange sources and destinations
+    'direction': 0,                         # TRex extention: 1 = exchange sources and destinations, 0 = do nothing
     'pkts_per_burst': 1,
     'burst_loop_count': 1,
     'inter_burst_gap': 12,
     'length_mode': 'fixed',                 # ( auto | fixed | increment | decrement | random | imix )
-    'l3_imix1_size': 60,
-    'l3_imix1_ratio': 28,
-    'l3_imix2_size': 590,
-    'l3_imix2_ratio': 20,
-    'l3_imix3_size': 1514,
-    'l3_imix3_ratio': 4,
-    'l3_imix4_size': 9226,
+    'l3_imix1_size': 64,
+    'l3_imix1_ratio': 7,
+    'l3_imix2_size': 570,
+    'l3_imix2_ratio': 4,
+    'l3_imix3_size': 1518,
+    'l3_imix3_ratio': 1,
+    'l3_imix4_size': 9230,
     'l3_imix4_ratio': 0,
     #L2
     'frame_size': 64,
@@ -301,6 +303,7 @@ class CStreamsPerPort(defaultdict):
 
     # save HLT args to modify streams later
     def save_stream_args(self, ports_list, stream_id, stream_hlt_args):
+        if stream_hlt_args.get('load_profile'): return # can't modify profiles, don't save
         if not self.hlt_history: raise STLError('CStreamsPerPort: this object works only with HLT history, try init with hlt_history = True')
         if type(stream_id) not in (int, long): raise STLError('CStreamsPerPort: stream_id should be number')
         if not isinstance(stream_hlt_args, dict): raise STLError('CStreamsPerPort: stream_hlt_args should be dict')
@@ -351,8 +354,7 @@ class CTRexHltApi(object):
                 return HLT_ERR('Could not translate hostname "%s" to IP: %s' % (device, e))
 
         try:
-            # sync = RPC, async = ZMQ
-            self.trex_client = STLClient(kwargs['username'], device, sync_port = 4501, async_port = 4500, verbose_level = self.verbose)
+            self.trex_client = STLClient(kwargs['username'], device, verbose_level = self.verbose)
         except Exception as e:
             return HLT_ERR('Could not init stateless client %s: %s' % (device, e if isinstance(e, STLError) else traceback.format_exc()))
 
@@ -552,20 +554,26 @@ class CTRexHltApi(object):
                     return HLT_OK()
 
             try:
-                stream_obj = STLHltStream(**user_kwargs)
+                if kwargs['load_profile']:
+                    stream_obj = STLProfile.load_py(kwargs['load_profile'], direction = kwargs['direction'])
+                else:
+                    stream_obj = STLHltStream(**user_kwargs)
             except Exception as e:
                 return HLT_ERR('Could not create stream: %s' % e if isinstance(e, STLError) else traceback.format_exc())
 
             # try adding the stream per ports
             try:
-                stream_id_arr = self.trex_client.add_streams(streams=stream_obj,
-                                                             ports=port_handle)
+                stream_id_arr = self.trex_client.add_streams(streams = stream_obj,
+                                                             ports = port_handle)
                 for port in port_handle:
                     self._streams_history.save_stream_args(port_handle, stream_id_arr[0], user_kwargs)
             except Exception as e:
                 return HLT_ERR('Could not add stream to ports: %s' % e if isinstance(e, STLError) else traceback.format_exc())
             if mode == 'create':
-                return HLT_OK(stream_id = dict((port, stream_id_arr[0]) for port in port_handle))
+                if len(stream_id_arr) == 1:
+                    return HLT_OK(stream_id = dict((port, stream_id_arr[0]) for port in port_handle))
+                else:
+                    return HLT_OK(stream_id = dict((port, stream_id_arr) for port in port_handle))
             else:
                 return HLT_OK()
 
@@ -662,11 +670,11 @@ class CTRexHltApi(object):
                 self._remove_stream(each_stream_id, port_handle)                        # recurse
             return
         if type(stream_id) is str: # range or list in string
-            if stream_id.find(',') != -1:
+            if ',' in stream_id:
                 for each_stream_id_element in stream_id.split(','):
                     self._remove_stream(each_stream_id_element, port_handle)            # recurse
                 return
-            if stream_id.find('-') != -1:
+            if '-' in stream_id:
                 stream_id_min, stream_id_max = stream_id.split('-', 1)
                 stream_id_min = get_number(stream_id_min)
                 stream_id_max = get_number(stream_id_max)
@@ -715,34 +723,49 @@ def STLHltStream(**user_kwargs):
         user_kwargs['length_mode'] = 'fixed'
         if kwargs['l3_imix1_size'] < 32 or kwargs['l3_imix2_size'] < 32 or kwargs['l3_imix3_size'] < 32 or kwargs['l3_imix4_size'] < 32:
             raise STLError('l3_imix*_size should be at least 32')
-        total_ratio = kwargs['l3_imix1_ratio'] + kwargs['l3_imix2_ratio'] + kwargs['l3_imix3_ratio'] + kwargs['l3_imix4_ratio']
-        if total_ratio == 0:
-            raise STLError('Used length_mode imix, but all the ratios are 0')
         save_to_yaml = kwargs.get('save_to_yaml')
         total_rate = float(kwargs[rate_key])
+        if rate_key == 'rate_pps': # ratio in packets as is
+            imix1_weight = kwargs['l3_imix1_ratio']
+            imix2_weight = kwargs['l3_imix2_ratio']
+            imix3_weight = kwargs['l3_imix3_ratio']
+            imix4_weight = kwargs['l3_imix4_ratio']
+        if rate_key == 'rate_bps': # ratio dependent on L2 size too
+            imix1_weight = kwargs['l3_imix1_ratio'] * kwargs['l3_imix1_size']
+            imix2_weight = kwargs['l3_imix2_ratio'] * kwargs['l3_imix2_size']
+            imix3_weight = kwargs['l3_imix3_ratio'] * kwargs['l3_imix3_size']
+            imix4_weight = kwargs['l3_imix4_ratio'] * kwargs['l3_imix4_size']
+        elif rate_key == 'rate_percent': # ratio dependent on L1 size too
+            imix1_weight = kwargs['l3_imix1_ratio'] * (kwargs['l3_imix1_size'] + 20)
+            imix2_weight = kwargs['l3_imix2_ratio'] * (kwargs['l3_imix2_size'] + 20)
+            imix3_weight = kwargs['l3_imix3_ratio'] * (kwargs['l3_imix3_size'] + 20)
+            imix4_weight = kwargs['l3_imix4_ratio'] * (kwargs['l3_imix4_size'] + 20)
+        total_weight = float(imix1_weight + imix2_weight + imix3_weight + imix4_weight)
+        if total_weight == 0:
+            raise STLError('Used length_mode imix, but all the ratios are 0')
         if kwargs['l3_imix1_ratio'] > 0:
             if save_to_yaml and type(save_to_yaml) is str:
                 user_kwargs['save_to_yaml'] = save_to_yaml.replace('.yaml', '_imix1.yaml')
             user_kwargs['frame_size'] = kwargs['l3_imix1_size']
-            user_kwargs[rate_key] = total_rate * kwargs['l3_imix1_ratio'] / total_ratio
+            user_kwargs[rate_key] = total_rate * imix1_weight / total_weight
             streams_arr.append(STLHltStream(**user_kwargs))
         if kwargs['l3_imix2_ratio'] > 0:
             if save_to_yaml and type(save_to_yaml) is str:
                 user_kwargs['save_to_yaml'] = save_to_yaml.replace('.yaml', '_imix2.yaml')
             user_kwargs['frame_size'] = kwargs['l3_imix2_size']
-            user_kwargs[rate_key] = total_rate * kwargs['l3_imix2_ratio'] / total_ratio
+            user_kwargs[rate_key] = total_rate * imix2_weight / total_weight
             streams_arr.append(STLHltStream(**user_kwargs))
         if kwargs['l3_imix3_ratio'] > 0:
             if save_to_yaml and type(save_to_yaml) is str:
                 user_kwargs['save_to_yaml'] = save_to_yaml.replace('.yaml', '_imix3.yaml')
             user_kwargs['frame_size'] = kwargs['l3_imix3_size']
-            user_kwargs[rate_key] = total_rate * kwargs['l3_imix3_ratio'] / total_ratio
+            user_kwargs[rate_key] = total_rate * imix3_weight / total_weight
             streams_arr.append(STLHltStream(**user_kwargs))
         if kwargs['l3_imix4_ratio'] > 0:
             if save_to_yaml and type(save_to_yaml) is str:
                 user_kwargs['save_to_yaml'] = save_to_yaml.replace('.yaml', '_imix4.yaml')
             user_kwargs['frame_size'] = kwargs['l3_imix4_size']
-            user_kwargs[rate_key] = total_rate * kwargs['l3_imix4_ratio'] / total_ratio
+            user_kwargs[rate_key] = total_rate * imix4_weight / total_weight
             streams_arr.append(STLHltStream(**user_kwargs))
         return streams_arr
 
@@ -781,13 +804,14 @@ def STLHltStream(**user_kwargs):
 
     debug_filename = kwargs.get('save_to_yaml')
     if type(debug_filename) is str:
-        print 'saving to %s' % debug_filename
+        print('saving to %s' % debug_filename)
         stream.dump_to_yaml(debug_filename)
     return stream
 
 def generate_packet(**user_kwargs):
     correct_macs(user_kwargs)
     kwargs = merge_kwargs(traffic_config_kwargs, user_kwargs)
+    correct_sizes(kwargs) # we are producing the packet - 4 bytes fcs
     correct_direction(kwargs, kwargs)
 
     vm_cmds = []
@@ -797,12 +821,17 @@ def generate_packet(**user_kwargs):
     ### L2 ###
     if kwargs['l2_encap'] in ('ethernet_ii', 'ethernet_ii_vlan'):
                 #fields_desc = [ MACField("dst","00:00:00:01:00:00"),
-                #                MACField("src","00:00:00:02:00:00"), 
+                #                MACField("src","00:00:00:02:00:00"),
                 #                XShortEnumField("type", 0x9000, ETHER_TYPES) ]
+        if kwargs['ignore_macs']: # workaround for lack of ARP
+            kwargs['mac_src'] = None
+            kwargs['mac_dst'] = None
+            kwargs['mac_src_mode'] = 'fixed'
+            kwargs['mac_dst_mode'] = 'fixed'
+
         l2_layer = Ether(src = kwargs['mac_src'], dst = kwargs['mac_dst'])
 
-        # Eth VM
-        # !!!! Need 8 bytes mask and vars !!!! for now will change only 32 lsb
+        # Eth VM, change only 32 lsb
         if kwargs['mac_src_mode'] != 'fixed':
             count = int(kwargs['mac_src_count']) - 1
             if count < 0:
@@ -1055,7 +1084,7 @@ def generate_packet(**user_kwargs):
             ipv6_kwargs['nh'] = kwargs['ipv6_next_header']
         l3_layer = IPv6(**ipv6_kwargs)
 
-        # IPv6 VM
+        # IPv6 VM, change only 32 lsb
         if kwargs['ipv6_src_mode'] != 'fixed':
             count = int(kwargs['ipv6_src_count']) - 1
             if count < 0:
@@ -1452,3 +1481,9 @@ def correct_direction(user_kwargs, kwargs):
             if 'ipv6_src_' in arg:
                 dst_arg = 'ipv6_dst_' + arg[9:]
                 user_kwargs[arg], user_kwargs[dst_arg] = kwargs[dst_arg], kwargs[arg]
+
+def correct_sizes(kwargs):
+    for arg in kwargs.keys():
+        if type(arg) in (int, long):
+            if arg.endswith(('_length', '_size', '_size_min', '_size_max', '_length_min', '_length_max')):
+                kwargs[arg] -= 4
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py
index dc787263..a3430c79 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py
@@ -674,7 +674,7 @@ class CScapyTRexPktBuilder(CTrexPktBuilderInterface):
 
     When path_relative_to_profile is True load pcap file from path relative to the profile
     """
-    def __init__(self, pkt = None, pkt_buffer = None, vm = None, path_relative_to_profile = False):
+    def __init__(self, pkt = None, pkt_buffer = None, vm = None, path_relative_to_profile = False, build_raw = True, remove_fcs = True):
         """
         Instantiate a CTRexPktBuilder object
 
@@ -688,8 +688,10 @@ class CScapyTRexPktBuilder(CTrexPktBuilderInterface):
         self.pkt_raw = None # from raw pcap file
         self.vm_scripts = [] # list of high level instructions
         self.vm_low_level = None
+        self.is_pkt_built = False
         self.metadata=""
         self.path_relative_to_profile = path_relative_to_profile
+        self.remove_fcs = remove_fcs
         
 
         if pkt != None and pkt_buffer != None:
@@ -709,13 +711,17 @@ class CScapyTRexPktBuilder(CTrexPktBuilderInterface):
 
             self.add_command(vm if isinstance(vm, CTRexScRaw) else CTRexScRaw(vm))
 
+        # raw source build to see MAC presence/ fields offset by name in VM
+        if build_raw and self.pkt_raw and not self.pkt:
+            self.__lazy_build_packet()
+
         # if we have packet and VM - compile now
         if (self.pkt or self.pkt_raw) and (self.vm_scripts):
             self.compile()
 
 
     def dump_vm_data_as_yaml(self):
-       print yaml.dump(self.get_vm_data(), default_flow_style=False)
+        print yaml.dump(self.get_vm_data(), default_flow_style=False)
 
     def get_vm_data(self):
         """
@@ -861,9 +867,6 @@ class CScapyTRexPktBuilder(CTrexPktBuilderInterface):
 
         
         self.vm_low_level = CTRexVmEngine()
-    
-        # before VM compile set this to false
-        self.is_pkt_built = False
 
         # compile the VM
         for sc in self.vm_scripts:
@@ -947,6 +950,8 @@ class CScapyTRexPktBuilder(CTrexPktBuilderInterface):
         # for buffer, promote to a scapy packet
         if self.pkt_raw:
             self.pkt = Ether(self.pkt_raw)
+            if self.remove_fcs and self.pkt.lastlayer().name == 'Padding':
+                self.pkt.lastlayer().underlayer.remove_payload()
             self.pkt.build()
             self.pkt_raw = None
 
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py
index 2dfc5f29..c41fa0c2 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py
@@ -6,6 +6,7 @@ from trex_stl_packet_builder_interface import CTrexPktBuilderInterface
 from trex_stl_packet_builder_scapy import CScapyTRexPktBuilder, Ether, IP, UDP, TCP, RawPcapReader
 from collections import OrderedDict, namedtuple
 
+from scapy.utils import ltoa
 import random
 import yaml
 import base64
@@ -14,7 +15,6 @@ import traceback
 from types import NoneType
 import copy
 
-
 # base class for TX mode
 class STLTXMode(object):
     def __init__ (self, pps = None, bps_L1 = None, bps_L2 = None, percentage = None):
@@ -69,7 +69,8 @@ class STLTXCont(STLTXMode):
 
         self.fields['type'] = 'continuous'
 
-    def __str__ (self):
+    @staticmethod
+    def __str__ ():
         return "Continuous"
 
 # single burst mode
@@ -85,7 +86,8 @@ class STLTXSingleBurst(STLTXMode):
         self.fields['type'] = 'single_burst'
         self.fields['total_pkts'] = total_pkts
 
-    def __str__ (self):
+    @staticmethod
+    def __str__ ():
         return "Single Burst"
 
 # multi burst mode
@@ -113,7 +115,8 @@ class STLTXMultiBurst(STLTXMode):
         self.fields['ibg'] = ibg
         self.fields['count'] = count
 
-    def __str__ (self):
+    @staticmethod
+    def __str__ ():
         return "Multi Burst"
 
 STLStreamDstMAC_CFG_FILE=0
@@ -172,6 +175,8 @@ class STLStream(object):
         self.name = name
         self.next = next
 
+        self.mac_src_override_by_pkt = mac_src_override_by_pkt # save for easy construct code from stream object
+        self.mac_dst_override_mode = mac_dst_override_mode
         self.id = stream_id
 
 
@@ -329,7 +334,109 @@ class STLStream(object):
         del y['stream']['mode']['rate']
 
         return y
-  
+
+    # returns the Python code (text) to build this stream, inside the code it will be in variable "stream"
+    def to_code (self):
+        packet = Ether(self.pkt)
+        packet.hide_defaults()
+        payload = packet.getlayer('Raw')
+        packet_command = packet.command()
+        imports_arr = []
+        if 'MPLS(' in packet_command:
+            imports_arr.append('from scapy.contrib.mpls import MPLS')
+        if 'VXLAN(' in packet_command:
+            imports_arr.append('from scapy.contrib.mpls import MPLS')
+            
+        imports = '\n'.join(imports_arr)
+        if payload:
+            payload.remove_payload() # fcs etc.
+            data = payload.fields.get('load', '')
+            replchars = re.compile('(\s|/|\'|\\\|[^' + re.escape(string.printable) + '])') # convert bad chars to hex
+            new_data =  replchars.sub(self.__replchars_to_hex, data)
+            payload_start = packet_command.find("Raw(load='")
+            if payload_start != -1:
+                packet_command = packet_command[:payload_start-1]
+        layers = packet_command.split('/')
+        if payload:
+            if len(new_data) and new_data == new_data[0] * len(new_data):
+                layers.append("Raw(load='%s' * %s)" % (new_data[0], len(new_data)))
+            else:
+                layers.append("Raw(load='%s')" % new_data)
+        packet_code = 'packet = (' + (' / \n          ').join(layers) + ')'
+        vm_list = []
+        for inst in self.fields['vm']['instructions']:
+            if inst['type'] == 'flow_var':
+                vm_list.append("CTRexVmDescFlowVar(name='{name}', size={size}, op='{op}', init_value={init_value}, min_value={min_value}, max_value={max_value}, step={step})".format(**inst))
+            elif inst['type'] == 'write_flow_var':
+                vm_list.append("CTRexVmDescWrFlowVar(fv_name='{name}', pkt_offset={pkt_offset}, add_val={add_value}, is_big={is_big_endian})".format(**inst))
+            elif inst['type'] == 'write_mask_flow_var':
+                inst = copy.copy(inst)
+                inst['mask'] = hex(inst['mask'])
+                vm_list.append("CTRexVmDescWrMaskFlowVar(fv_name='{name}', pkt_offset={pkt_offset}, pkt_cast_size={pkt_cast_size}, mask={mask}, shift={shift}, add_value={add_value}, is_big={is_big_endian})".format(**inst))
+            elif inst['type'] == 'fix_checksum_ipv4':
+                vm_list.append("CTRexVmDescFixIpv4(offset={pkt_offset})".format(**inst))
+            elif inst['type'] == 'trim_pkt_size':
+                vm_list.append("CTRexVmDescTrimPktSize(fv_name='{name}')".format(**inst))
+            elif inst['type'] == 'tuple_flow_var':
+                inst = copy.copy(inst)
+                inst['ip_min'] = ltoa(inst['ip_min'])
+                inst['ip_max'] = ltoa(inst['ip_max'])
+                vm_list.append("CTRexVmDescTupleGen(name='{name}', ip_min='{ip_min}', ip_max='{ip_max}', port_min={port_min}, port_max={port_max}, limit_flows={limit_flows}, flags={flags})".format(**inst))
+        vm_code = 'vm = CTRexScRaw([' + ',\n                 '.join(vm_list) + '], split_by_field = %s)' % STLStream.__add_quotes(self.fields['vm'].get('split_by_var'))
+        stream_params_list = []
+        stream_params_list.append('packet = CScapyTRexPktBuilder(pkt = packet, vm = vm)')
+        if default_STLStream.name != self.name:
+            stream_params_list.append('name = %s' % STLStream.__add_quotes(self.name))
+        if default_STLStream.fields['enabled'] != self.fields['enabled']:
+            stream_params_list.append('enabled = %s' % self.fields['enabled'])
+        if default_STLStream.fields['self_start'] != self.fields['self_start']:
+            stream_params_list.append('self_start = %s' % self.fields['self_start'])
+        if default_STLStream.fields['isg'] != self.fields['isg']:
+            stream_params_list.append('isg = %s' % self.fields['isg'])
+        if default_STLStream.fields['rx_stats'] != self.fields['rx_stats']:
+            stream_params_list.append('rx_stats = STLRxStats(%s)' % self.fields['rx_stats']['stream_id'])
+        if default_STLStream.next != self.next:
+            stream_params_list.append('next = %s' % STLStream.__add_quotes(self.next))
+        if default_STLStream.id != self.id:
+            stream_params_list.append('stream_id = %s' % self.id)
+        if default_STLStream.fields['action_count'] != self.fields['action_count']:
+            stream_params_list.append('action_count = %s' % self.fields['action_count'])
+        if 'random_seed' in self.fields:
+            stream_params_list.append('random_seed = %s' % self.fields.get('random_seed', 0))
+        if default_STLStream.mac_src_override_by_pkt != self.mac_src_override_by_pkt:
+            stream_params_list.append('mac_src_override_by_pkt = %s' % self.mac_src_override_by_pkt)
+        if default_STLStream.mac_dst_override_mode != self.mac_dst_override_mode:
+            stream_params_list.append('mac_dst_override_mode = %s' % self.mac_dst_override_mode)
+
+        mode_args = ''
+        for key, value in self.fields['mode'].items():
+            if key not in ('rate', 'type'):
+                mode_args += '%s = %s, ' % (key, value)
+        mode_args += '%s = %s' % (self.fields['mode']['rate']['type'], self.fields['mode']['rate']['value'])
+        if self.mode_desc == STLTXCont.__str__():
+            stream_params_list.append('mode = STLTXCont(%s)' % mode_args)
+        elif self.mode_desc == STLTXSingleBurst().__str__():
+            stream_params_list.append('mode = STLTXSingleBurst(%s)' % mode_args)
+        elif self.mode_desc == STLTXMultiBurst().__str__():
+            stream_params_list.append('mode = STLTXMultiBurst(%s)' % mode_args)
+        else:
+            raise STLError('Could not determine mode: %s' % self.mode_desc)
+
+        stream = "stream = STLStream(" + ',\n                   '.join(stream_params_list) + ')'
+        return '\n'.join([imports, packet_code, vm_code, stream])
+
+    # add quoted for string, or leave as is if other type
+    @staticmethod
+    def __add_quotes(arg):
+        if type(arg) is str:
+            return "'%s'" % arg
+        return arg
+
+    # used to replace non-printable characters with hex
+    @staticmethod
+    def __replchars_to_hex(match):
+        return r'\x{0:02x}'.format(ord(match.group()))
+
     def dump_to_yaml (self, yaml_file = None):
         yaml_dump = yaml.dump([self.to_yaml()], default_flow_style = False)
 
@@ -439,7 +546,7 @@ class YAMLLoader(object):
         rx_stats = self.__parse_rx_stats(s_obj.get('rx_stats'))
         
 
-        defaults = STLStream()
+        defaults = default_STLStream
         # create the stream
         stream = STLStream(name       = yaml_object.get('name'),
                            packet     = builder,
@@ -628,7 +735,33 @@ class STLProfile(object):
 
         return yaml_str
 
+    def dump_to_code (self, profile_file = None):
+        profile_dump = '''# !!! Auto-generated code !!!
+from trex_stl_lib.api import *
+
+class STLS1(object):
+    def get_streams(self):
+        streams = []
+'''
+        for stream in self.streams:
+            profile_dump += ' '*8 + stream.to_code().replace('\n', '\n' + ' '*8) + '\n'
+            profile_dump += ' '*8 + 'streams.append(stream)\n'
+        profile_dump += '''
+        return streams
+
+def register():
+    return STLS1()
+'''
+        # write to file if provided
+        if profile_file:
+            with open(profile_file, 'w') as f:
+                f.write(profile_dump)
+
+        return profile_dump
+
+
 
     def __len__ (self):
         return len(self.streams)
 
+default_STLStream = STLStream()
\ No newline at end of file