HLTAPI update + HLT packet builder functional tests (no simulation)

5 files changed, 790 insertions, 236 deletions

diff --git a/scripts/automation/regression/hltapi_example.py b/scripts/automation/regression/hltapi_example.py
index 2a30d08c..0040219a 100755
--- a/scripts/automation/regression/hltapi_example.py
+++ b/scripts/automation/regression/hltapi_example.py
@@ -56,45 +56,66 @@ if __name__ == "__main__":
         print('Connected.')
 
         print('Create single_burst 100 packets rate_pps=100 on port 0')
-        res = check_res(hlt_client.traffic_config('create', port_handle = port_handle[0], transmit_mode = 'single_burst', pkts_per_burst = 100, rate_pps = 100))
-        save_streams_id(res, streams_id_arr)
+        res = check_res(hlt_client.traffic_config(mode = 'create', port_handle = port_handle[0], transmit_mode = 'single_burst', pkts_per_burst = 100, rate_pps = 100))
+        #save_streams_id(res, streams_id_arr)
+
+        # playground - creating various streams on port 1
+        res = check_res(hlt_client.traffic_config(mode = 'create', port_handle = port_handle[1], save_to_yaml = '/tmp/hlt2.yaml',
+                        tcp_src_port_mode = 'decrement',
+                        tcp_src_port_count = 10, tcp_dst_port_count = 10, tcp_dst_port_mode = 'random'))
+
+        res = check_res(hlt_client.traffic_config(mode = 'create', port_handle = port_handle[1], save_to_yaml = '/tmp/hlt3.yaml',
+                        l4_protocol = 'udp',
+                        udp_src_port_mode = 'decrement',
+                        udp_src_port_count = 10, udp_dst_port_count = 10, udp_dst_port_mode = 'random'))
+
+        res = check_res(hlt_client.traffic_config(mode = 'create', port_handle = port_handle[1], save_to_yaml = '/tmp/hlt4.yaml',
+                        length_mode = 'increment',
+                        #ip_src_addr = '192.168.1.1', ip_src_mode = 'increment', ip_src_count = 5,
+                        ip_dst_addr = '5.5.5.5', ip_dst_mode = 'random', ip_dst_count = 2))
+
+        res = check_res(hlt_client.traffic_config(mode = 'create', port_handle = port_handle[1], save_to_yaml = '/tmp/hlt5.yaml',
+                        length_mode = 'decrement', frame_size_min = 100, frame_size_max = 3000,
+                        #ip_src_addr = '192.168.1.1', ip_src_mode = 'increment', ip_src_count = 5,
+                        #ip_dst_addr = '5.5.5.5', ip_dst_mode = 'random', ip_dst_count = 2
+                        ))
+        # remove the playground
+        check_res(hlt_client.traffic_config(mode = 'reset', port_handle = port_handle[1]))
 
         print('Create continuous stream for port 1, rate_pps = 1')
-        res = check_res(hlt_client.traffic_config('create', port_handle = port_handle[1]))
-        save_streams_id(res, streams_id_arr)
+        res = check_res(hlt_client.traffic_config(mode = 'create', port_handle = port_handle[1], save_to_yaml = '/tmp/hlt1.yaml',
+                        #length_mode = 'increment', l3_length_min = 200,
+                        ip_src_addr = '192.168.1.1', ip_src_mode = 'increment', ip_src_count = 5,
+                        ip_dst_addr = '5.5.5.5', ip_dst_mode = 'random', ip_dst_count = 2))
         
-        print('Run traffic on both ports for 20s')
-        check_res(hlt_client.traffic_control(action = 'run', port_handle = port_handle, mul = {'type': 'raw', 'op': 'abs', 'value': 1}, duration = 20))
-        wait_with_progress(5)
-
-        print('All available HLT stats after 5s')
+        check_res(hlt_client.traffic_control(action = 'run', port_handle = port_handle))
+        wait_with_progress(1)
+        print('Sample after 1 seconds (only packets count)')
         res = check_res(hlt_client.traffic_stats(mode = 'all', port_handle = port_handle))
-        pprint(res)
+        print_brief_stats(res)
+        print ''
 
-        # delete the single_burst
-        print('Port 0 has finished the burst, delete it and put continuous with rate 1000. No stopping of other ports.')
-        check_res(hlt_client.traffic_config('reset', port_handle = port_handle[0]))
-        res = check_res(hlt_client.traffic_config('create', port_handle = port_handle[0], rate_pps = 1000))
+        print('Port 0 has finished the burst, put continuous instead with rate 1000. No stopping of other ports.')
+        check_res(hlt_client.traffic_control(action = 'stop', port_handle = port_handle[0]))
+        check_res(hlt_client.traffic_config(mode = 'reset', port_handle = port_handle[0]))
+        res = check_res(hlt_client.traffic_config(mode = 'create', port_handle = port_handle[0], rate_pps = 1000))
         save_streams_id(res, streams_id_arr)
-        check_res(hlt_client.traffic_control(action = 'run', port_handle = port_handle[0], mul = {'type': 'raw', 'op': 'abs', 'value': 1}, duration = 15))
+        check_res(hlt_client.traffic_control(action = 'run', port_handle = port_handle[0]))
         wait_with_progress(5)
-        print('Sample after another %s seconds (only packets count)' % 5)
+        print('Sample after another 5 seconds (only packets count)')
         res = check_res(hlt_client.traffic_stats(mode = 'aggregate', port_handle = port_handle))
         print_brief_stats(res)
-
-        #print('Stop the traffic on port 1')
-        #res = check_res(hlt_client.traffic_control('stop', port_handle = port_handle[1]))
-        #wait_with_progress(sample_duration)
-        #print('Sample after another %s seconds (only packets count)' % sample_duration)
-        #res = check_res(hlt_client.traffic_stats(mode = 'all', port_handle = port_handle))
-        #print_brief_stats(res)
+        print ''
 
         print('Stop traffic at port 1')
-        res = check_res(hlt_client.traffic_control('stop', port_handle = port_handle[1]))
+        res = check_res(hlt_client.traffic_control(action = 'stop', port_handle = port_handle[1]))
         wait_with_progress(5)
         print('Sample after another %s seconds (only packets count)' % 5)
         res = check_res(hlt_client.traffic_stats(mode = 'aggregate', port_handle = port_handle))
         print_brief_stats(res)
+        print ''
+        print('Full HLT stats:')
+        pprint(res)
 
         check_res(hlt_client.cleanup_session())
     except Exception as e:
diff --git a/scripts/automation/regression/unit_tests/functional_tests/hltapi_stream_builder_test.py b/scripts/automation/regression/unit_tests/functional_tests/hltapi_stream_builder_test.py
new file mode 100755
index 00000000..eefa1947
--- /dev/null
+++ b/scripts/automation/regression/unit_tests/functional_tests/hltapi_stream_builder_test.py
@@ -0,0 +1,343 @@
+#!/router/bin/python
+
+from client.trex_hltapi import CTRexHltApiPktBuilder
+import os
+import unittest
+
+gen_stream = CTRexHltApiPktBuilder.generate_stream
+
+def compare_yamls(yaml1, yaml2):
+    if type(yaml1) is not str:
+        raise Exception("yaml1 is '%s', expected str" % type(yaml1))
+    if type(yaml2) is not str:
+        raise Exception("yaml2 is '%s', expected str" % type(yaml2))
+    i = 0
+    for line1, line2 in zip(yaml1.strip().split('\n'), yaml2.strip().split('\n')):
+        i += 1
+        if line1 != line2:
+            raise Exception('yamls are not equal starting from line %s:\n%s\n\t<->\n%s' % (i, line1.strip(), line2.strip()))
+
+class CTRexHltApi_Test(unittest.TestCase):
+    ''' Checks correct HLTAPI creation of packet/VM '''
+
+    def setUp(self):
+        self.golden_yaml = None
+        self.test_yaml = None
+
+    def tearDown(self):
+        compare_yamls(self.golden_yaml, self.test_yaml)
+
+    # Eth/IP/TCP, all values default, no VM instructions
+    def test_default(self):
+        test_stream = gen_stream()
+        self.test_yaml = test_stream.dump_to_yaml(self.yaml_save_location(), test_stream)
+        self.golden_yaml = '''
+- name: stream-0
+  stream:
+    enabled: true
+    isg: 0.0
+    mode:
+      pps: 1
+      type: continuous
+    packet:
+      binary: AAAAAAAAAAABAAABCABFAAAyAAAAAEAGusUAAAAAwAAAAQQAAFAAAAABAAAAAVAAD+U1/QAAISEhISEhISEhIQ==
+      meta: ''
+    rx_stats:
+      enabled: false
+    self_start: true
+    vm:
+      instructions: []
+      split_by_var: ''
+'''
+
+    # Eth/IP/TCP, ip src and dest is changed by VM
+    def test_ip_ranges(self):
+        test_stream = gen_stream(ip_src_addr = '192.168.1.1',
+                                 ip_src_mode = 'increment',
+                                 ip_src_count = 5,
+                                 ip_dst_addr = '5.5.5.5',
+                                 ip_dst_count = 2,
+                                 ip_dst_mode = 'random')
+        self.test_yaml = test_stream.dump_to_yaml(self.yaml_save_location(), test_stream)
+        self.golden_yaml = '''
+- name: stream-0
+  stream:
+    enabled: true
+    isg: 0.0
+    mode:
+      pps: 1
+      type: continuous
+    packet:
+      binary: AAAAAAAAAAABAAABCABFAAAyAAAAAEAGrxPAqAEBBQUFBQQAAFAAAAABAAAAAVAAD+UqSwAAISEhISEhISEhIQ==
+      meta: ''
+    rx_stats:
+      enabled: false
+    self_start: true
+    vm:
+      instructions:
+      - init_value: 3232235777
+        max_value: 3232235781
+        min_value: 3232235777
+        name: ip_src
+        op: inc
+        size: 4
+        type: flow_var
+      - add_value: 0
+        is_big_endian: true
+        name: ip_src
+        pkt_offset: 26
+        type: write_flow_var
+      - init_value: 84215045
+        max_value: 84215046
+        min_value: 84215045
+        name: ip_dst
+        op: random
+        size: 4
+        type: flow_var
+      - add_value: 0
+        is_big_endian: true
+        name: ip_dst
+        pkt_offset: 30
+        type: write_flow_var
+      - pkt_offset: 14
+        type: fix_checksum_ipv4
+      split_by_var: ''
+
+'''
+
+    # Eth / IP / TCP, tcp ports are changed by VM
+    def test_tcp_ranges(self):
+        test_stream = gen_stream(tcp_src_port_mode = 'decrement',
+                                 tcp_src_port_count = 10,
+                                 tcp_dst_port_mode = 'random',
+                                 tcp_dst_port_count = 10,
+                                 tcp_dst_port = 1234)
+        self.test_yaml = test_stream.dump_to_yaml(self.yaml_save_location(), test_stream)
+        self.golden_yaml = '''
+- name: stream-0
+  stream:
+    enabled: true
+    isg: 0.0
+    mode:
+      pps: 1
+      type: continuous
+    packet:
+      binary: AAAAAAAAAAABAAABCABFAAAyAAAAAEAGusUAAAAAwAAAAQQABNIAAAABAAAAAVAAD+UxewAAISEhISEhISEhIQ==
+      meta: ''
+    rx_stats:
+      enabled: false
+    self_start: true
+    vm:
+      instructions:
+      - init_value: 1024
+        max_value: 1024
+        min_value: 1015
+        name: tcp_src
+        op: dec
+        size: 2
+        type: flow_var
+      - add_value: 0
+        is_big_endian: true
+        name: tcp_src
+        pkt_offset: 34
+        type: write_flow_var
+      - init_value: 1234
+        max_value: 1243
+        min_value: 1234
+        name: tcp_dst
+        op: random
+        size: 2
+        type: flow_var
+      - add_value: 0
+        is_big_endian: true
+        name: tcp_dst
+        pkt_offset: 36
+        type: write_flow_var
+      - pkt_offset: 14
+        type: fix_checksum_ipv4
+      split_by_var: ''
+'''
+
+    # Eth / IP / UDP, udp ports are changed by VM
+    def test_udp_ranges(self):
+        # UDP is not set, expecting ignore of wrong UDP arguments
+        gen_stream(udp_src_port_mode = 'qwerqwer',
+                   udp_src_port_count = 'weqwer',
+                   udp_src_port = 'qwerqwer',
+                   udp_dst_port_mode = 'qwerqwe',
+                   udp_dst_port_count = 'sfgsdfg',
+                   udp_dst_port = 'sdfgsdfg')
+        # UDP is set, expecting fail due to wrong UDP arguments
+        with self.assertRaises(Exception):
+            gen_stream(l4_protocol = 'udp',
+                       udp_src_port_mode = 'qwerqwer',
+                       udp_src_port_count = 'weqwer',
+                       udp_src_port = 'qwerqwer',
+                       udp_dst_port_mode = 'qwerqwe',
+                       udp_dst_port_count = 'sfgsdfg',
+                       udp_dst_port = 'sdfgsdfg')
+        # generate it already with correct arguments
+        test_stream = gen_stream(l4_protocol = 'udp',
+                                 udp_src_port_mode = 'decrement',
+                                 udp_src_port_count = 10,
+                                 udp_src_port = 1234,
+                                 udp_dst_port_mode = 'increment',
+                                 udp_dst_port_count = 10,
+                                 udp_dst_port = 1234)
+        self.test_yaml = test_stream.dump_to_yaml(self.yaml_save_location(), test_stream)
+        self.golden_yaml = '''
+- name: stream-0
+  stream:
+    enabled: true
+    isg: 0.0
+    mode:
+      pps: 1
+      type: continuous
+    packet:
+      binary: AAAAAAAAAAABAAABCABFAAAyAAAAAEARuroAAAAAwAAAAQTSBNIAHsmgISEhISEhISEhISEhISEhISEhISEhIQ==
+      meta: ''
+    rx_stats:
+      enabled: false
+    self_start: true
+    vm:
+      instructions:
+      - init_value: 1234
+        max_value: 1234
+        min_value: 1225
+        name: udp_src
+        op: dec
+        size: 2
+        type: flow_var
+      - add_value: 0
+        is_big_endian: true
+        name: udp_src
+        pkt_offset: 34
+        type: write_flow_var
+      - init_value: 1234
+        max_value: 1243
+        min_value: 1234
+        name: udp_dst
+        op: inc
+        size: 2
+        type: flow_var
+      - add_value: 0
+        is_big_endian: true
+        name: udp_dst
+        pkt_offset: 36
+        type: write_flow_var
+      - pkt_offset: 14
+        type: fix_checksum_ipv4
+      split_by_var: ''
+'''
+
+    # Eth/IP/TCP, packet length is changed in VM by frame_size
+    def test_pkt_len_by_framesize(self):
+        # frame_size_step should be 1 (as default)
+        with self.assertRaises(Exception):
+            test_stream = gen_stream(length_mode = 'decrement',
+                                     frame_size_min = 100,
+                                     frame_size_max = 3000,
+                                     frame_size_step = 20)
+        # just check errors, no compare to golden
+        gen_stream(length_mode = 'increment',
+                   frame_size_min = 100,
+                   frame_size_max = 3000)
+        test_stream = gen_stream(length_mode = 'decrement',
+                                 frame_size_min = 100,
+                                 frame_size_max = 3000)
+        self.test_yaml = test_stream.dump_to_yaml(self.yaml_save_location(), test_stream)
+        self.golden_yaml = '''
+- name: stream-0
+  stream:
+    enabled: true
+    isg: 0.0
+    mode:
+      pps: 1
+      type: continuous
+    packet:
+      binary: AAAAAAAAAAABAAABCABFAAAyAAAAAEAGusUAAAAAwAAAAQQAAFAAAAABAAAAAVAAD+U8AwAAISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEh
+      meta: ''
+    rx_stats:
+      enabled: false
+    self_start: true
+    vm:
+      instructions:
+      - init_value: 3000
+        max_value: 3000
+        min_value: 100
+        name: fv_rand
+        op: dec
+        size: 2
+        type: flow_var
+      - name: fv_rand
+        type: trim_pkt_size
+      - add_value: -14
+        is_big_endian: true
+        name: fv_rand
+        pkt_offset: 16
+        type: write_flow_var
+      - pkt_offset: 14
+        type: fix_checksum_ipv4
+      split_by_var: ''
+'''
+
+    # Eth/IP/UDP, packet length is changed in VM by l3_length
+    def test_pkt_len_by_l3length(self):
+        # l3_length_step should be 1
+        with self.assertRaises(Exception):
+            gen_stream(l4_protocol = 'udp',
+                       length_mode = 'random',
+                       l3_length_min = 100,
+                       l3_length_max = 400,
+                       l3_length_step = 20)
+        test_stream = gen_stream(l4_protocol = 'udp',
+                                 length_mode = 'random',
+                                 l3_length_min = 100,
+                                 l3_length_max = 400)
+        self.test_yaml = test_stream.dump_to_yaml(self.yaml_save_location(), test_stream)
+        self.golden_yaml = '''
+- name: stream-0
+  stream:
+    enabled: true
+    isg: 0.0
+    mode:
+      pps: 1
+      type: continuous
+    packet:
+      binary: AAAAAAAAAAABAAABCABFAAAyAAAAAEARuroAAAAAwAAAAQQAAFABfCfxISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEh
+      meta: ''
+    rx_stats:
+      enabled: false
+    self_start: true
+    vm:
+      instructions:
+      - init_value: 114
+        max_value: 414
+        min_value: 114
+        name: fv_rand
+        op: random
+        size: 2
+        type: flow_var
+      - name: fv_rand
+        type: trim_pkt_size
+      - add_value: -14
+        is_big_endian: true
+        name: fv_rand
+        pkt_offset: 16
+        type: write_flow_var
+      - add_value: -34
+        is_big_endian: true
+        name: fv_rand
+        pkt_offset: 38
+        type: write_flow_var
+      - pkt_offset: 14
+        type: fix_checksum_ipv4
+      split_by_var: ''
+'''
+
+    def yaml_save_location(self):
+        return os.devnull
+        # debug/deveopment, comment line above
+        return '/tmp/%s.yaml' % self._testMethodName
+
+
diff --git a/scripts/automation/trex_control_plane/client/trex_hltapi.py b/scripts/automation/trex_control_plane/client/trex_hltapi.py
index 3bbf62e0..5d3f506c 100755
--- a/scripts/automation/trex_control_plane/client/trex_hltapi.py
+++ b/scripts/automation/trex_control_plane/client/trex_hltapi.py
@@ -1,78 +1,123 @@
 #!/router/bin/python
 
 '''
-Supported function/arguments:
-connect()
-    device
-    port_list
-    username
-    reset
-    break_locks
-
-cleanup_session()
-    maintain_lock
-    port_list
-    port_handle
-
-traffic_config()
-    mode                ( create | modify | remove | reset )
-    port_handle
-    transmit_mode       ( continuous | multi_burst | single_burst )
-    rate_pps
-    stream_id
-    bidirectional
-    l2_encap
-    mac_src
-    mac_src2
-    mac_dst
-    mac_dst2
-    l3_protocol
-    ip_tos_field
-    l3_length
-    ip_id
-    ip_fragment_offset
-    ip_ttl
-    ip_checksum
-    ip_src_addr
-    ip_dst_addr
-    l4_protocol
-    tcp_src_port
-    tcp_dst_port
-    tcp_seq_num
-    tcp_ack_num
-    tcp_data_offset
-    tcp_fin_flag
-    tcp_syn_flag
-    tcp_rst_flag
-    tcp_psh_flag
-    tcp_ack_flag
-    tcp_urg_flag
-    tcp_window
-    tcp_checksum
-    tcp_urgent_ptr
-
-traffic_control()
-    action              ( run | stop )
-    port_handle
-
-traffic_stats()
-    mode                ( aggregate )
-    port_handle
-
+Supported function/arguments/defaults:
 '''
-
-
-
-import trex_root_path
+# connect()
+connect_kwargs = {
+    'device': 'localhost',                  # ip or hostname of TRex
+    'port_list': None,                      # list of ports
+    'username': 'TRexUser',
+    'reset': True,
+    'break_locks': False,
+}
+
+# cleanup_session()
+cleanup_session_kwargs = {
+    'maintain_lock': False,                 # release ports at the end or not
+    'port_list': None,
+    'port_handle': None,
+}
+
+# traffic_config()
+traffic_config_kwargs = {
+    'mode': None,                           # ( create | modify | remove | reset )
+    'port_handle': None,
+    'transmit_mode': 'continuous',          # ( continuous | multi_burst | single_burst )
+    'rate_pps': 1,
+    'stream_id': None,
+    'bidirectional': 0,
+    # stream builder parameters
+    'pkts_per_burst': 1,
+    'burst_loop_count': 1,
+    'inter_burst_gap': 12,
+    'length_mode': 'fixed',                 #  ( auto | fixed | increment | decrement | random )
+    #L2
+    'frame_size': 64,
+    'frame_size_min': 64,
+    'frame_size_max': 64,
+    'frame_size_step': 1,                   # has to be 1
+    'l2_encap': 'ethernet_ii',              # ( ethernet_ii )
+    'mac_src': '00:00:01:00:00:01',
+    'mac_dst': '00:00:00:00:00:00',
+    #L3, IP
+    'l3_protocol': 'ipv4',                  # ( ipv4 )
+    'ip_tos_field': 0,
+    'l3_length': 50,
+    'ip_id': 0,
+    'ip_fragment_offset': 0,
+    'ip_ttl': 64,
+    'ip_checksum': None,
+    'ip_src_addr': '0.0.0.0',
+    'ip_dst_addr': '192.0.0.1',
+    'ip_src_mode': 'fixed',                 # ( fixed | increment | decrement | random )
+    'ip_src_step': 1,                       # has to be 1
+    'ip_src_count': 1,
+    'ip_dst_mode': 'fixed',                 # ( fixed | increment | decrement | random )
+    'ip_dst_step': 1,                       # has to be 1
+    'ip_dst_count': 1,
+    'l3_length_min': 110,
+    'l3_length_max': 238,
+    'l3_length_step': 1,                    # has to be 1
+    #L3, IPv6 (TODO: add)
+    #L4, TCP
+    'l4_protocol': 'tcp',                   # ( tcp | udp )
+    'tcp_src_port': 1024,
+    'tcp_dst_port': 80,
+    'tcp_seq_num': 1,
+    'tcp_ack_num': 1,
+    'tcp_data_offset': 5,
+    'tcp_fin_flag': 0,
+    'tcp_syn_flag': 0,
+    'tcp_rst_flag': 0,
+    'tcp_psh_flag': 0,
+    'tcp_ack_flag': 0,
+    'tcp_urg_flag': 0,
+    'tcp_window': 4069,
+    'tcp_checksum': None,
+    'tcp_urgent_ptr': 0,
+    'tcp_src_port_mode': 'increment',       # ( increment | decrement | random )
+    'tcp_src_port_step': 1,                 # has to be 1
+    'tcp_src_port_count': 1,
+    'tcp_dst_port_mode': 'increment',       # ( increment | decrement | random )
+    'tcp_dst_port_step': 1,                 # has to be 1
+    'tcp_dst_port_count': 1,
+    # L4, UDP
+    'udp_src_port': 1024,
+    'udp_dst_port': 80,
+    'udp_length': None,
+    'udp_checksum': None,
+    'udp_dst_port_mode': 'increment',       # ( increment | decrement | random )
+    'udp_src_port_step': 1,                 # has to be 1
+    'udp_src_port_count': 1,
+    'udp_src_port_mode': 'increment',       # ( increment | decrement | random )
+    'udp_dst_port_step': 1,                 # has to be 1
+    'udp_dst_port_count': 1,
+}
+
+# traffic_control()
+traffic_control_kwargs = {
+    'action': None,                         # ( run | stop )
+    'port_handle': None
+}
+
+# traffic_stats()
+traffic_stats_kwargs = {
+    'mode': 'aggregate',                    # ( aggregate )
+    'port_handle': None
+}
+
+
+#import trex_root_path
 import client_utils.scapy_packet_builder as pkt_bld
+from client_utils.scapy_packet_builder import CTRexVmDescFlowVar, CTRexVmDescWrFlowVar
 from trex_stateless_client import STLClient
 from common.trex_streams import *
 from client_utils.general_utils import get_integer
-import dpkt
 import socket
+import copy
 from misc_methods import print_r
-import traceback
-import time
+
 
 class HLT_ERR(dict):
     def __init__(self, log = 'Unknown error', **kwargs):
@@ -91,6 +136,16 @@ class HLT_OK(dict):
         dict.update(self, init_dict)
         dict.update(self, kwargs)
 
+def merge_kwargs(default_kwargs, user_kwargs):
+    extra_args = []
+    kwargs = copy.deepcopy(default_kwargs)
+    for key, value in user_kwargs.items():
+        if key in kwargs:
+            kwargs[key] = value
+        elif key not in ('save_to_yaml', 'save_to_pcap'): # internal debug arguments
+            print("Warning: provided parameter '%s' not supported" % key) 
+    return kwargs
+
 
 class CTRexHltApi(object):
 
@@ -105,9 +160,9 @@ class CTRexHltApi(object):
 #    Session functions    #
 ###########################
 
-    # device: ip or hostname
-    def connect(self, device, port_list, username = '', reset = False, break_locks = False):
-
+    def connect(self, **user_kwargs):
+        kwargs = merge_kwargs(connect_kwargs, user_kwargs)
+        device = kwargs['device']
         try:
             device = socket.gethostbyname(device) # work with ip
         except: # give it another try
@@ -118,7 +173,7 @@ class CTRexHltApi(object):
 
         try:
             # sync = RPC, async = ZMQ
-            self.trex_client = STLClient(username, device, sync_port = 4501, async_port = 4500, verbose_level = self.verbose)
+            self.trex_client = STLClient(kwargs['username'], device, sync_port = 4501, async_port = 4500, verbose_level = self.verbose)
         except Exception as e:
             self.trex_client = None
             return HLT_ERR('Could not init stateless client %s: %s' % (device, e))
@@ -130,8 +185,9 @@ class CTRexHltApi(object):
 
         # connection successfully created with server, try acquiring ports of TRex
         try:
-            port_list = self.parse_port_list(port_list)
-            self.trex_client.acquire(ports = port_list, force = break_locks)
+            port_list = self.parse_port_list(kwargs['port_list'])
+            print kwargs['break_locks']
+            self.trex_client.acquire(ports = port_list, force = kwargs['break_locks'])
         except Exception as e:
             self.trex_client = None
             return HLT_ERR('Could not acquire ports %s: %s' % (port_list, e))
@@ -140,9 +196,11 @@ class CTRexHltApi(object):
         port_handle = self.trex_client.get_acquired_ports()
 
         # arrived here, all desired ports were successfully acquired
-        if reset:
+        if kwargs['reset']:
             # remove all port traffic configuration from TRex
             try:
+                print 'reseting'
+                self.trex_client.stop(ports = port_list)
                 self.trex_client.reset(ports = port_list)
             except Exception as e:
                 self.trex_client = None
@@ -151,10 +209,11 @@ class CTRexHltApi(object):
         self.connected = True
         return HLT_OK(port_handle = port_handle)
 
-    def cleanup_session(self, maintain_lock = False, **kwargs):
-        if not maintain_lock:
+    def cleanup_session(self, **user_kwargs):
+        kwargs = merge_kwargs(cleanup_session_kwargs, user_kwargs)
+        if not kwargs['maintain_lock']:
             # release taken ports
-            port_list = kwargs.get('port_list', kwargs.get('port_handle', 'all'))
+            port_list = kwargs['port_list'] or kwargs['port_handle'] or 'all'
             try:
                 if port_list == 'all':
                     port_list = self.trex_client.get_acquired_ports()
@@ -186,16 +245,19 @@ class CTRexHltApi(object):
 #    Traffic functions    #
 ###########################
 
-    def traffic_config(self, mode, port_handle, **kwargs):
-        stream_id = kwargs.get('stream_id')
+    def traffic_config(self, **user_kwargs):
+        kwargs = merge_kwargs(traffic_config_kwargs, user_kwargs)
+        stream_id = kwargs['stream_id']
         if type(stream_id) is list:
             del kwargs['stream_id']
             for each_stream_id in stream_id:
-                res = self.traffic_config(mode, port_handle, stream_id = each_stream_id, **kwargs)
+                res = self.traffic_config(stream_id = each_stream_id, **kwargs)
                 if type(res) is HLT_ERR:
                     return res
             return HLT_OK()
 
+        mode = kwargs['mode']
+        port_handle = kwargs['port_handle']
         if type(port_handle) is not list:
             port_handle = [port_handle]
         ALLOWED_MODES = ['create', 'modify', 'remove', 'enable', 'disable', 'reset']
@@ -240,7 +302,6 @@ class CTRexHltApi(object):
             if stream_id not in self._hlt_streams_history:
                 return HLT_ERR('This stream_id (%s) was not used before, please create new.' % stream_id)
             self._hlt_streams_history[stream_id].update(kwargs) # <- the modification
-            kwargs = self._hlt_streams_history[stream_id]
             #for port_id in port_handle:
             #    if stream_id not in self.trex_client.get_stream_id_list(port_id):
             #        return HLT_ERR('Port %s does not have stream_id %s.' % (port_id, stream_id))
@@ -252,27 +313,27 @@ class CTRexHltApi(object):
         if mode == 'create' or mode == 'modify':
             # create a new stream with desired attributes, starting by creating packet
 
-            if 'bidirectional' in kwargs: # two streams with opposite src and dst MAC
+            if kwargs['bidirectional']: # two streams with opposite directions
                 del kwargs['bidirectional']
+                del kwargs['port_handle']
                 bidirect_err = 'When using bidirectional flag, '
                 if len(port_handle) != 2:
                     return HLT_ERR(bidirect_err + 'number of ports should be exactly 2')
                 try:
-                    res1 = self.traffic_config(mode, port_handle[0], **kwargs)
-                    res2 = self.traffic_config(mode, port_handle[1],
-                            mac_src = kwargs.get(mac_src2, '00-00-01-00-00-01'),
-                            mac_dst = kwargs.get(mac_dst2, '00-00-00-00-00-00'),
-                            ip_src_addr = kwargs.get(ip_dst_addr, '192.0.0.1'),
-                            ip_dst_addr = kwargs.get(ip_src_addr, '0.0.0.0'),
-                            ipv6_src_addr = kwargs.get(ipv6_dst_addr, 'fe80:0:0:0:0:0:0:22'),
-                            ipv6_dst_addr = kwargs.get(ipv6_src_addr, 'fe80:0:0:0:0:0:0:12'),
-                            **kwargs)
+                    res1 = self.traffic_config(port_handle = port_handle[0], **kwargs)
+                    kwargs['mac_src'] = kwargs['mac_src2']
+                    kwargs['mac_dst'] = kwargs['mac_dst2']
+                    kwargs['ip_src_addr'] = kwargs['ip_dst_addr']
+                    kwargs['ip_dst_addr'] = kwargs['ip_src_addr']
+                    kwargs['ipv6_src_addr'] = kwargs['ipv6_dst_addr']
+                    kwargs['ipv6_dst_addr'] = kwargs['ipv6_src_addr']
+                    res2 = self.traffic_config(port_handle = port_handle[1], **kwargs)
                 except Exception as e:
                     return HLT_ERR('Could not generate bidirectional traffic: %s' % e)
                 return HLT_OK(stream_id = {port_handle[0]: res1['stream_id'], port_handle[1]: res2['stream_id']})
-
+            stream_obj = CTRexHltApiPktBuilder.generate_stream(**kwargs)
             try:
-                stream_obj = CTRexHltApi._generate_stream(**kwargs)
+                stream_obj = CTRexHltApiPktBuilder.generate_stream(**kwargs)
             except Exception as e:
                 return HLT_ERR('Could not create stream: %s' % e)
             stream_id = stream_obj.get_id()
@@ -288,7 +349,10 @@ class CTRexHltApi(object):
 
         return HLT_ERR('Got to the end of traffic_config, mode not implemented or forgot "return" function somewhere.')
 
-    def traffic_control(self, action, port_handle, **kwargs):
+    def traffic_control(self, **user_kwargs):
+        kwargs = merge_kwargs(traffic_control_kwargs, user_kwargs)
+        action = kwargs['action']
+        port_handle = kwargs['port_handle']
         ALLOWED_ACTIONS = ['clear_stats', 'run', 'stop', 'sync_run']
         if action not in ALLOWED_ACTIONS:
             return HLT_ERR('Action must be one of the following values: {actions}'.format(actions=ALLOWED_ACTIONS))
@@ -298,7 +362,7 @@ class CTRexHltApi(object):
         if action == 'run':
 
             try:
-                self.trex_client.start(duration = kwargs.get('duration', -1), ports = port_handle)
+                self.trex_client.start(ports = port_handle)
             except Exception as e:
                 return HLT_ERR('Could not start traffic: %s' % e)
             return HLT_OK(stopped = 0)
@@ -316,7 +380,10 @@ class CTRexHltApi(object):
         # if we arrived here, this means that operation FAILED!
         return HLT_ERR("Probably action '%s' is not implemented" % action)
 
-    def traffic_stats(self, port_handle, mode):
+    def traffic_stats(self, **user_kwargs):
+        kwargs = merge_kwargs(traffic_stats_kwargs, user_kwargs)
+        mode = kwargs['mode']
+        port_handle = kwargs['port_handle']
         ALLOWED_MODES = ['aggregate', 'streams', 'all']
         if mode not in ALLOWED_MODES:
             return HLT_ERR("'mode' must be one of the following values: %s" % ALLOWED_MODES)
@@ -417,19 +484,21 @@ class CTRexHltApi(object):
             return '\n'. join([str(response) for response in responses])
         return responses
 
+class CTRexHltApiPktBuilder:
     @staticmethod
-    def _generate_stream(**kwargs):
+    def generate_stream(**user_kwargs):
+        kwargs = merge_kwargs(traffic_config_kwargs, user_kwargs)
         try:
-            packet = CTRexHltApi._generate_packet(**kwargs)
+            packet = CTRexHltApiPktBuilder.generate_packet(**kwargs)
         except Exception as e:
             raise Exception('Could not generate packet: %s' % e)
 
         try:
-            transmit_mode = kwargs.get('transmit_mode', 'continuous')
-            rate_pps = kwargs.get('rate_pps', 1)
-            pkts_per_burst = kwargs.get('pkts_per_burst', 1)
-            burst_loop_count = kwargs.get('burst_loop_count', 1)
-            inter_burst_gap = kwargs.get('inter_burst_gap', 12)
+            transmit_mode = kwargs['transmit_mode']
+            rate_pps = kwargs['rate_pps']
+            pkts_per_burst = kwargs['pkts_per_burst']
+            burst_loop_count = kwargs['burst_loop_count']
+            inter_burst_gap = kwargs['inter_burst_gap']
             if transmit_mode == 'continuous':
                 transmit_mode_class = STLTXCont(pps = rate_pps)
             elif transmit_mode == 'single_burst':
@@ -458,59 +527,25 @@ class CTRexHltApi(object):
         return stream_obj
 
     @staticmethod
-    def _generate_packet(
-                # L2
-                frame_size = 64,
-                l2_encap = 'ethernet_ii',
-                mac_src = '00:00:01:00:00:01',
-                mac_dst = '00:00:00:00:00:00',
-
-                # L3 IPv4
-                l3_protocol = 'ipv4',
-                ip_tos_field = 0,
-                l3_length = 50,
-                ip_id = 0,
-                ip_fragment_offset = 0,
-                ip_ttl = 64,
-                ip_checksum = None,
-                ip_src_addr = '0.0.0.0',
-                ip_dst_addr = '192.0.0.1',
-
-                # L3 IPv4 FE
-                ip_src_mode = 'fixed',
-                ip_src_step = 1,
-                ip_src_count = 1,
-                
-                l4_protocol = 'tcp',
-                tcp_src_port = 1024,
-                tcp_dst_port = 80,
-                tcp_seq_num = 1,
-                tcp_ack_num = 1,
-                tcp_data_offset = 1,
-                tcp_fin_flag = 0,
-                tcp_syn_flag = 0,
-                tcp_rst_flag = 0,
-                tcp_psh_flag = 0,
-                tcp_ack_flag = 0,
-                tcp_urg_flag = 0,
-                tcp_window = 4069,
-                tcp_checksum = None,
-                tcp_urgent_ptr = 0,
-                **kwargs):
-
-        pkt_plus_vm = pkt_bld.CScapyTRexPktBuilder()
+    def generate_packet(**user_kwargs):
+        kwargs = merge_kwargs(traffic_config_kwargs, user_kwargs)
+        pkt = pkt_bld.CScapyTRexPktBuilder()
+
+        vm_cmds = []
+        fix_ipv4_checksum = False
 
         ### L2 ###
-        if l2_encap == 'ethernet_ii':
+        if kwargs['l2_encap'] == 'ethernet_ii':
                     #fields_desc = [ MACField("dst","00:00:00:01:00:00"),
                     #                MACField("src","00:00:00:02:00:00"), 
                     #                XShortEnumField("type", 0x9000, ETHER_TYPES) ]
-            pkt = pkt_bld.Ether(src = mac_src, dst = mac_dst)
+            l2_layer = pkt_bld.Ether(src = kwargs['mac_src'], dst = kwargs['mac_dst'])
         else:
-            raise NotImplementedError("l2_encap does not support the desired encapsulation '%s'" % l2_encap)
+            raise NotImplementedError("l2_encap does not support the desired encapsulation '%s'" % kwargs['l2_encap'])
+        base_pkt = l2_layer
 
         ### L3 ###
-        if l3_protocol == 'ipv4':
+        if kwargs['l3_protocol'] == 'ipv4':
                     #fields_desc = [ BitField("version" , 4 , 4),
                     #                BitField("ihl", None, 4),
                     #                XByteField("tos", 0),
@@ -526,39 +561,67 @@ class CTRexHltApi(object):
                     #                Emph(IPField("src", "16.0.0.1")),
                     #                Emph(IPField("dst", "48.0.0.1")),
                     #                PacketListField("options", [], IPOption, length_from=lambda p:p.ihl*4-20) ]
-            pkt /= pkt_bld.IP(tos = ip_tos_field,
-                              len = l3_length,
-                              id = ip_id,
-                              frag = ip_fragment_offset,
-                              ttl = ip_ttl,
-                              chksum = ip_checksum,
-                              src = ip_src_addr,
-                              dst = ip_dst_addr
-                              )
-            # IP FE (Field Engine)
-            print dir(pkt_bld.IP.src.fld)
-            print_r(pkt_bld.IP.src.fld)
-            if ip_src_mode == 'increment':
-                ip_src_vm = pkt_bld.CTRexScRaw([ pkt_bld.CTRexVmDescFlowVar(name='a', min_value='16.0.0.1', max_value='16.0.0.10', init_value='16.0.0.1', size=4, op='inc'),
-                          pkt_bld.CTRexVmDescWrFlowVar (fv_name='a', pkt_offset = 'IP.src'),
-                          pkt_bld.CTRexVmDescFixIpv4(offset = 'IP')])
-                pkt_plus_vm.add_command(ip_src_vm)
-
-                #print '>> is inc'
-                #if ip_src_step != 1:
-                #    return HLT_ERR('ip_src_step has to be 1 (TRex limitation)')
-                #FE_var_ip_src_addr = pkt_bld.vm.CTRexVMFlowVariable('ip_src_addr')
-                #FE_var_ip_src_addr.set_field('size', 4)
-                #FE_var_ip_src_addr.set_field('operation', 'inc')
-                #FE_var_ip_src_addr.set_field('init_value', ip_src_addr)
-                #FE_var_ip_src_addr.set_field('init_value', 1)
-                #FE_var_ip_src_addr.set_field('max_value', ip_src_count)
-                #pkt_bld.vm.vm_var.set_field('split_by_core', False)
+            l3_layer = pkt_bld.IP(tos    = kwargs['ip_tos_field'],
+                                  len    = None if kwargs['length_mode'] == 'auto' else kwargs['l3_length'],
+                                  id     = kwargs['ip_id'],
+                                  frag   = kwargs['ip_fragment_offset'],
+                                  ttl    = kwargs['ip_ttl'],
+                                  chksum = kwargs['ip_checksum'],
+                                  src    = kwargs['ip_src_addr'],
+                                  dst    = kwargs['ip_dst_addr']
+                                  )
+            # IPv4 VM
+            if kwargs['ip_src_mode'] != 'fixed':
+                fix_ipv4_checksum = True
+                if kwargs['ip_src_step'] != 1:
+                    raise Exception('ip_src_step has to be 1 (TRex limitation)')
+                if kwargs['ip_src_count'] < 1:
+                    raise Exception('ip_src_count has to be at least 1')
+                ip_src_addr_num = pkt_bld.ipv4_str_to_num(pkt_bld.is_valid_ipv4(kwargs['ip_src_addr']))
+                if kwargs['ip_src_mode'] == 'increment':
+                    vm_cmds.append(CTRexVmDescFlowVar(name = 'ip_src', size = 4, op = 'inc',
+                                                      min_value = ip_src_addr_num,
+                                                      max_value = ip_src_addr_num + kwargs['ip_src_count'] - 1))
+                elif kwargs['ip_src_mode'] == 'decrement':
+                    vm_cmds.append(CTRexVmDescFlowVar(name = 'ip_src', size = 4, op = 'dec',
+                                                       min_value = ip_src_addr_num - kwargs['ip_src_count'] + 1,
+                                                       max_value = ip_src_addr_num))
+                elif kwargs['ip_src_mode'] == 'random':
+                    vm_cmds.append(CTRexVmDescFlowVar(name = 'ip_src', size = 4, op = 'random',
+                                                       min_value = ip_src_addr_num,
+                                                       max_value = ip_src_addr_num + kwargs['ip_src_count'] - 1))
+                else:
+                    raise Exception('ip_src_mode %s is not supported' % kwargs['ip_src_mode'])
+                vm_cmds.append(CTRexVmDescWrFlowVar(fv_name='ip_src', pkt_offset = 'IP.src'))
+
+            if kwargs['ip_dst_mode'] != 'fixed':
+                fix_ipv4_checksum = True
+                if kwargs['ip_dst_step'] != 1:
+                    raise Exception('ip_dst_step has to be 1 (TRex limitation)')
+                if kwargs['ip_dst_count'] < 1:
+                    raise Exception('ip_dst_count has to be at least 1')
+                ip_dst_addr_num = pkt_bld.ipv4_str_to_num(pkt_bld.is_valid_ipv4(kwargs['ip_dst_addr']))
+                if kwargs['ip_dst_mode'] == 'increment':
+                    vm_cmds.append(CTRexVmDescFlowVar(name = 'ip_dst', size = 4, op = 'inc',
+                                                       min_value = ip_dst_addr_num,
+                                                       max_value = ip_dst_addr_num + kwargs['ip_dst_count'] - 1))
+                elif kwargs['ip_dst_mode'] == 'decrement':
+                    vm_cmds.append(CTRexVmDescFlowVar(name = 'ip_dst', size = 4, op = 'dec',
+                                                       min_value = ip_dst_addr_num - kwargs['ip_dst_count'] + 1,
+                                                       max_value = ip_dst_addr_num))
+                elif kwargs['ip_dst_mode'] == 'random':
+                    vm_cmds.append(CTRexVmDescFlowVar(name = 'ip_dst', size = 4, op = 'random',
+                                                       min_value = ip_dst_addr_num,
+                                                       max_value = ip_dst_addr_num + kwargs['ip_dst_count'] - 1))
+                else:
+                    raise Exception('ip_dst_mode %s is not supported' % kwargs['ip_dst_mode'])
+                vm_cmds.append(CTRexVmDescWrFlowVar(fv_name='ip_dst', pkt_offset = 'IP.dst'))
         else:
-            raise NotImplementedError("l3_protocol '%s' is not supported by TRex yet." % l3_protocol)
+            raise NotImplementedError("l3_protocol '%s' is not supported by TRex yet." % kwargs['l3_protocol'])
+        base_pkt /= l3_layer
 
         ### L4 ###
-        if l4_protocol == 'tcp':
+        if kwargs['l4_protocol'] == 'tcp':
                     #fields_desc = [ ShortEnumField("sport", 20, TCP_SERVICES),
                     #                ShortEnumField("dport", 80, TCP_SERVICES),
                     #                IntField("seq", 0),
@@ -570,41 +633,168 @@ class CTRexHltApi(object):
                     #                XShortField("chksum", None),
                     #                ShortField("urgptr", 0),
                     #                TCPOptionsField("options", {}) ]
-            tcp_flags = ('F' if tcp_fin_flag else '' +
-                         'S' if tcp_syn_flag else '' +
-                         'R' if tcp_rst_flag else '' +
-                         'P' if tcp_psh_flag else '' +
-                         'A' if tcp_ack_flag else '' +
-                         'U' if tcp_urg_flag else '')
-
-            pkt /= pkt_bld.TCP(sport = tcp_src_port,
-                               dport = tcp_dst_port,
-                               seq = tcp_seq_num,
-                               ack = tcp_ack_num,
-                               dataofs = tcp_data_offset,
-                               flags = tcp_flags,
-                               window = tcp_window,
-                               chksum = tcp_checksum,
-                               urgptr = tcp_urgent_ptr,
-                               )
-        else:
-            raise NotImplementedError("l4_protocol '%s' is not supported by TRex yet." % l3_protocol)
-        pkt /= 'payload'
-        pkt_plus_vm.set_packet(pkt)
-        #payload = frame_size - pkt
-        #payload = pkt_bld.payload_gen.gen_repeat_ptrn('Hello, World!')
-        #pkt_bld.set_pkt_payload(payload)
-            
-        #print pkt_bld.vm
-        # debug (only base packet, without FE)
-        debug_filename = kwargs.get('save_to_pcap')
-        if type(debug_filename) is str:
-            pkt_plus_vm.dump_pkt_to_pcap(debug_filename)
-        pkt_plus_vm.compile()
-        return pkt_plus_vm
+            tcp_flags = ('F' if kwargs['tcp_fin_flag'] else '' +
+                         'S' if kwargs['tcp_syn_flag'] else '' +
+                         'R' if kwargs['tcp_rst_flag'] else '' +
+                         'P' if kwargs['tcp_psh_flag'] else '' +
+                         'A' if kwargs['tcp_ack_flag'] else '' +
+                         'U' if kwargs['tcp_urg_flag'] else '')
+
+            l4_layer = pkt_bld.TCP(sport   = kwargs['tcp_src_port'],
+                                   dport   = kwargs['tcp_dst_port'],
+                                   seq     = kwargs['tcp_seq_num'],
+                                   ack     = kwargs['tcp_ack_num'],
+                                   dataofs = kwargs['tcp_data_offset'],
+                                   flags   = tcp_flags,
+                                   window  = kwargs['tcp_window'],
+                                   chksum  = kwargs['tcp_checksum'],
+                                   urgptr  = kwargs['tcp_urgent_ptr'],
+                                   )
+            # TCP VM
+            if kwargs['tcp_src_port_count'] != 1:
+                fix_ipv4_checksum = True
+                if kwargs['tcp_src_port_step'] != 1:
+                    raise Exception('tcp_src_port_step has to be 1 (TRex limitation)')
+                if kwargs['tcp_src_port_count'] < 1:
+                    raise Exception('tcp_src_port_count has to be at least 1')
+                if kwargs['tcp_src_port_mode'] == 'increment':
+                    vm_cmds.append(CTRexVmDescFlowVar(name = 'tcp_src', size = 2, op = 'inc',
+                                                      min_value = kwargs['tcp_src_port'],
+                                                      max_value = kwargs['tcp_src_port'] + kwargs['tcp_src_port_count'] - 1))
+                elif kwargs['tcp_src_port_mode'] == 'decrement':
+                    vm_cmds.append(CTRexVmDescFlowVar(name = 'tcp_src', size = 2, op = 'dec',
+                                                      min_value = kwargs['tcp_src_port'] - kwargs['tcp_src_port_count'] +1,
+                                                      max_value = kwargs['tcp_src_port']))
+                elif kwargs['tcp_src_port_mode'] == 'random':
+                    vm_cmds.append(CTRexVmDescFlowVar(name = 'tcp_src', size = 2, op = 'random',
+                                                      min_value = kwargs['tcp_src_port'],
+                                                      max_value = kwargs['tcp_src_port'] + kwargs['tcp_src_port_count'] - 1))
+                else:
+                    raise Exception('tcp_src_port_mode %s is not supported' % kwargs['tcp_src_port_mode'])
+                vm_cmds.append(CTRexVmDescWrFlowVar(fv_name='tcp_src', pkt_offset = 'TCP.sport'))
+
+            if kwargs['tcp_dst_port_count'] != 1:
+                fix_ipv4_checksum = True
+                if kwargs['tcp_dst_port_step'] != 1:
+                    raise Exception('tcp_dst_port_step has to be 1 (TRex limitation)')
+                if kwargs['tcp_dst_port_count'] < 1:
+                    raise Exception('tcp_dst_port_count has to be at least 1')
+                if kwargs['tcp_dst_port_mode'] == 'increment':
+                    vm_cmds.append(CTRexVmDescFlowVar(name = 'tcp_dst', size = 2, op = 'inc',
+                                                      min_value = kwargs['tcp_dst_port'],
+                                                      max_value = kwargs['tcp_dst_port'] + kwargs['tcp_dst_port_count'] - 1))
+                elif kwargs['tcp_dst_port_mode'] == 'decrement':
+                    vm_cmds.append(CTRexVmDescFlowVar(name = 'tcp_dst', size = 2, op = 'dec',
+                                                      min_value = kwargs['tcp_dst_port'] - kwargs['tcp_dst_port_count'] +1,
+                                                      max_value = kwargs['tcp_dst_port']))
+                elif kwargs['tcp_dst_port_mode'] == 'random':
+                    vm_cmds.append(CTRexVmDescFlowVar(name = 'tcp_dst', size = 2, op = 'random',
+                                                      min_value = kwargs['tcp_dst_port'],
+                                                      max_value = kwargs['tcp_dst_port'] + kwargs['tcp_dst_port_count'] - 1))
+                else:
+                    raise Exception('tcp_dst_port_mode %s is not supported' % kwargs['tcp_dst_port_mode'])
+                vm_cmds.append(CTRexVmDescWrFlowVar(fv_name='tcp_dst', pkt_offset = 'TCP.dport'))
 
 
+        elif kwargs['l4_protocol'] == 'udp':
+                    #fields_desc = [ ShortEnumField("sport", 53, UDP_SERVICES),
+                    #                ShortEnumField("dport", 53, UDP_SERVICES),
+                    #                ShortField("len", None),
+                    #                XShortField("chksum", None), ]
+            l4_layer = pkt_bld.UDP(sport  = kwargs['udp_src_port'],
+                                   dport  = kwargs['udp_dst_port'],
+                                   len    = kwargs['udp_length'],
+                                   chksum = kwargs['udp_checksum'])
+            # UDP VM
+            if kwargs['udp_src_port_count'] != 1:
+                fix_ipv4_checksum = True
+                if kwargs['udp_src_port_step'] != 1:
+                    raise Exception('udp_src_port_step has to be 1 (TRex limitation)')
+                if kwargs['udp_src_port_count'] < 1:
+                    raise Exception('udp_src_port_count has to be at least 1')
+                if kwargs['udp_src_port_mode'] == 'increment':
+                    vm_cmds.append(CTRexVmDescFlowVar(name = 'udp_src', size = 2, op = 'inc',
+                                                      min_value = kwargs['udp_src_port'],
+                                                      max_value = kwargs['udp_src_port'] + kwargs['udp_src_port_count'] - 1))
+                elif kwargs['udp_src_port_mode'] == 'decrement':
+                    vm_cmds.append(CTRexVmDescFlowVar(name = 'udp_src', size = 2, op = 'dec',
+                                                      min_value = kwargs['udp_src_port'] - kwargs['udp_src_port_count'] +1,
+                                                      max_value = kwargs['udp_src_port']))
+                elif kwargs['udp_src_port_mode'] == 'random':
+                    vm_cmds.append(CTRexVmDescFlowVar(name = 'udp_src', size = 2, op = 'random',
+                                                      min_value = kwargs['udp_src_port'],
+                                                      max_value = kwargs['udp_src_port'] + kwargs['udp_src_port_count'] - 1))
+                else:
+                    raise Exception('udp_src_port_mode %s is not supported' % kwargs['udp_src_port_mode'])
+                vm_cmds.append(CTRexVmDescWrFlowVar(fv_name='udp_src', pkt_offset = 'UDP.sport'))
+
+            if kwargs['udp_dst_port_count'] != 1:
+                fix_ipv4_checksum = True
+                if kwargs['udp_dst_port_step'] != 1:
+                    raise Exception('udp_dst_port_step has to be 1 (TRex limitation)')
+                if kwargs['udp_dst_port_count'] < 1:
+                    raise Exception('udp_dst_port_count has to be at least 1')
+                if kwargs['udp_dst_port_mode'] == 'increment':
+                    vm_cmds.append(CTRexVmDescFlowVar(name = 'udp_dst', size = 2, op = 'inc',
+                                                      min_value = kwargs['udp_dst_port'],
+                                                      max_value = kwargs['udp_dst_port'] + kwargs['udp_dst_port_count'] - 1))
+                elif kwargs['udp_dst_port_mode'] == 'decrement':
+                    vm_cmds.append(CTRexVmDescFlowVar(name = 'udp_dst', size = 2, op = 'dec',
+                                                      min_value = kwargs['udp_dst_port'] - kwargs['udp_dst_port_count'] +1,
+                                                      max_value = kwargs['udp_dst_port']))
+                elif kwargs['udp_dst_port_mode'] == 'random':
+                    vm_cmds.append(CTRexVmDescFlowVar(name = 'udp_dst', size = 2, op = 'random',
+                                                      min_value = kwargs['udp_dst_port'],
+                                                      max_value = kwargs['udp_dst_port'] + kwargs['udp_dst_port_count'] - 1))
+                else:
+                    raise Exception('udp_dst_port_mode %s is not supported' % kwargs['udp_dst_port_mode'])
+                vm_cmds.append(CTRexVmDescWrFlowVar(fv_name='udp_dst', pkt_offset = 'UDP.dport'))
+        else:
+            raise NotImplementedError("l4_protocol '%s' is not supported by TRex yet." % kwargs['l3_protocol'])
+        base_pkt /= l4_layer
 
-if __name__ == '__main__':
-    pass
+        if kwargs['length_mode'] == 'auto':
+            payload_len = 0
+        elif kwargs['length_mode'] == 'fixed':
+            payload_len = kwargs['frame_size'] - len(base_pkt)
+        else:
+            fix_ipv4_checksum = True
+            if kwargs['frame_size_step'] != 1 or kwargs['l3_length_step'] != 1:
+                raise Exception('frame_size_step and l3_length_step has to be 1 (TRex limitation)')
+            trim_dict = {'increment': 'inc', 'decrement': 'dec'}
+            if kwargs['frame_size_min'] != 64 or kwargs['frame_size_max'] != 64: # size is determined by L2, higher priority over L3 size
+                if kwargs['frame_size_min'] < 12 or kwargs['frame_size_max'] < 12:
+                    raise Exception('frame_size_min and frame_size_max should be at least 12')
+                vm_cmds.append(CTRexVmDescFlowVar(name = 'fv_rand', size=2, op=trim_dict.get(kwargs['length_mode'], kwargs['length_mode']),
+                                                  min_value = kwargs['frame_size_min'], max_value = kwargs['frame_size_max']))
+                payload_len = kwargs['frame_size_max'] - len(base_pkt)
+            else: # size is determined by L3
+                vm_cmds.append(CTRexVmDescFlowVar(name = 'fv_rand', size=2, op=trim_dict.get(kwargs['length_mode'], kwargs['length_mode']),
+                                                  min_value = kwargs['l3_length_min'] + len(l2_layer), max_value = kwargs['l3_length_max'] + len(l2_layer)))
+                payload_len = kwargs['l3_length_max'] + len(l2_layer) - len(base_pkt)
+
+            vm_cmds.append(pkt_bld.CTRexVmDescTrimPktSize('fv_rand'))
+            if l3_layer.name == 'IP' or l4_layer.name == 'UDP': # add here other things need to fix due to size change
+                if l3_layer.name == 'IP':
+                    vm_cmds.append(CTRexVmDescWrFlowVar(fv_name = 'fv_rand', pkt_offset = 'IP.len', add_val = -len(l2_layer)))
+                if l4_layer.name == 'UDP':
+                    vm_cmds.append(CTRexVmDescWrFlowVar(fv_name = 'fv_rand', pkt_offset = 'UDP.len', add_val = -len(l2_layer) - len(l3_layer)))
+
+        if payload_len < 0:
+            raise Exception('Packet length is bigger than defined by frame_size* or l3_length*')
+        base_pkt /= '!' * payload_len
+
+        pkt.set_packet(base_pkt)
+        if fix_ipv4_checksum and l3_layer.name == 'IP' and kwargs['ip_checksum'] is None:
+            vm_cmds.append(pkt_bld.CTRexVmDescFixIpv4(offset = 'IP'))
+        if vm_cmds:
+            pkt.add_command(pkt_bld.CTRexScRaw(vm_cmds))
+
+        # debug (only the base packet, without VM)
+        debug_filename = kwargs.get('save_to_pcap')
+        if type(debug_filename) is str:
+            pkt.dump_pkt_to_pcap(debug_filename)
+        #pkt.compile()
+        #pkt.dump_scripts()
+        return pkt
 
diff --git a/scripts/automation/trex_control_plane/client_utils/scapy_packet_builder.py b/scripts/automation/trex_control_plane/client_utils/scapy_packet_builder.py
index 30383469..b1b181c6 100644
--- a/scripts/automation/trex_control_plane/client_utils/scapy_packet_builder.py
+++ b/scripts/automation/trex_control_plane/client_utils/scapy_packet_builder.py
@@ -618,7 +618,7 @@ class CScapyTRexPktBuilder(CTrexPktBuilderInterface):
         # process VM
         if vm != None:
             if not isinstance(vm, (CTRexScRaw, list)):
-                raise CTRexPacketBuildException(-14, "bad value for variable pkt")
+                raise CTRexPacketBuildException(-14, "bad value for variable vm")
 
             self.add_command(vm if isinstance(vm, CTRexScRaw) else CTRexScRaw(vm))
 
diff --git a/scripts/automation/trex_control_plane/common/trex_streams.py b/scripts/automation/trex_control_plane/common/trex_streams.py
index 238c89f3..c1f1bfa6 100755
--- a/scripts/automation/trex_control_plane/common/trex_streams.py
+++ b/scripts/automation/trex_control_plane/common/trex_streams.py
@@ -510,7 +510,7 @@ class STLStream(object):
         x = yaml.dump(yaml_lst, default_flow_style=False)
         with open(yaml_file, 'w') as f:
             f.write(x)
-
+        return x
 
 
 # REMOVE ME when can - convert from stream pack to a simple stream