merge from master

19 files changed, 568 insertions, 193 deletions

diff --git a/scripts/automation/regression/stateful_tests/trex_general_test.py b/scripts/automation/regression/stateful_tests/trex_general_test.py
index 42720f70..010ceb36 100755
--- a/scripts/automation/regression/stateful_tests/trex_general_test.py
+++ b/scripts/automation/regression/stateful_tests/trex_general_test.py
@@ -165,10 +165,13 @@ class CTRexGeneral_Test(unittest.TestCase):
 
             # report benchmarks
             if self.GAManager:
-                setup_test = '%s.%s' % (CTRexScenario.setup_name, self.get_name())
-                self.GAManager.gaAddAction(Event = 'stateful_test', action = setup_test, label = 'bw_per_core', value = int(test_norm_cpu))
-                self.GAManager.gaAddAction(Event = 'stateful_test', action = setup_test, label = 'bw_per_core_exp', value = int(expected_norm_cpu))
-                self.GAManager.emptyAndReportQ()
+                try:
+                    setup_test = '%s.%s' % (CTRexScenario.setup_name, self.get_name())
+                    self.GAManager.gaAddAction(Event = 'stateful_test', action = setup_test, label = 'bw_per_core', value = int(test_norm_cpu))
+                    self.GAManager.gaAddAction(Event = 'stateful_test', action = setup_test, label = 'bw_per_core_exp', value = int(expected_norm_cpu))
+                    self.GAManager.emptyAndReportQ()
+                except Exception as e:
+                    print('Sending GA failed: %s' % e)
 
     def check_results_gt (self, res, name, val):
         if res is None:
diff --git a/scripts/automation/regression/stateless_tests/stl_benchmark_test.py b/scripts/automation/regression/stateless_tests/stl_benchmark_test.py
index ef4c435f..c2c11cc7 100755
--- a/scripts/automation/regression/stateless_tests/stl_benchmark_test.py
+++ b/scripts/automation/regression/stateless_tests/stl_benchmark_test.py
@@ -51,15 +51,18 @@ class STLBenchmark_Test(CStlGeneral_Test):
 
             # report benchmarks
             if self.GAManager:
-                profile_repr = '%s.%s %s' % (CTRexScenario.setup_name,
-                                             os.path.basename(profile_bench['name']),
-                                             repr(kwargs).replace("'", ''))
-                self.GAManager.gaAddAction(Event = 'stateless_test', action = profile_repr,
-                                           label = 'bw_per_core', value = int(agv_bw_per_core))
-                # TODO: report expected once acquired
-                #self.GAManager.gaAddAction(Event = 'stateless_test', action = profile_repr,
-                #                           label = 'bw_per_core_exp', value = int(expected_norm_cpu))
-                self.GAManager.emptyAndReportQ()
+                try:
+                    profile_repr = '%s.%s %s' % (CTRexScenario.setup_name,
+                                                os.path.basename(profile_bench['name']),
+                                                repr(kwargs).replace("'", ''))
+                    self.GAManager.gaAddAction(Event = 'stateless_test', action = profile_repr,
+                                            label = 'bw_per_core', value = int(agv_bw_per_core))
+                    # TODO: report expected once acquired
+                    #self.GAManager.gaAddAction(Event = 'stateless_test', action = profile_repr,
+                    #                           label = 'bw_per_core_exp', value = int(expected_norm_cpu))
+                    self.GAManager.emptyAndReportQ()
+                except Exception as e:
+                    print('Sending GA failed: %s' % e)
 
     def tearDown(self):
         self.stl_trex.reset()
diff --git a/scripts/automation/regression/stateless_tests/stl_client_test.py b/scripts/automation/regression/stateless_tests/stl_client_test.py
index e7c9bb5d..10f56b3f 100644
--- a/scripts/automation/regression/stateless_tests/stl_client_test.py
+++ b/scripts/automation/regression/stateless_tests/stl_client_test.py
@@ -84,9 +84,6 @@ class STLClient_Test(CStlGeneral_Test):
                 self.c.clear_stats()
                 self.c.start(ports = [self.tx_port, self.rx_port])
 
-                assert self.c.ports[self.tx_port].is_transmitting(), 'port should be active'
-                assert self.c.ports[self.rx_port].is_transmitting(), 'port should be active'
-
                 self.c.wait_on_traffic(ports = [self.tx_port, self.rx_port])
                 stats = self.c.get_stats()
 
@@ -124,9 +121,6 @@ class STLClient_Test(CStlGeneral_Test):
                 self.c.clear_stats()
                 self.c.start(ports = [self.tx_port, self.rx_port])
 
-                assert self.c.ports[self.tx_port].is_transmitting(), 'port should be active'
-                assert self.c.ports[self.rx_port].is_transmitting(), 'port should be active'
-
                 self.c.wait_on_traffic(ports = [self.tx_port, self.rx_port])
                 stats = self.c.get_stats()
 
diff --git a/scripts/automation/trex_control_plane/stl/console/trex_console.py b/scripts/automation/trex_control_plane/stl/console/trex_console.py
index f8161dcb..ab70d357 100755
--- a/scripts/automation/trex_control_plane/stl/console/trex_console.py
+++ b/scripts/automation/trex_control_plane/stl/console/trex_console.py
@@ -319,9 +319,13 @@ class TRexConsole(TRexGeneralCmd):
         return self.do_history(line)
 
     def do_push (self, line):
-        '''Push a PCAP file\n'''
+        '''Push a local PCAP file\n'''
         return self.stateless_client.push_line(line)
 
+    #def do_push_remote (self, line):
+    #    '''Push a remote accessible PCAP file\n'''
+    #    return self.stateless_client.push_remote_line(line)
+
     def help_push (self):
         return self.do_push("-h")
 
diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_pcap.py b/scripts/automation/trex_control_plane/stl/examples/stl_pcap.py
index 45ddc24b..eae0f18b 100644
--- a/scripts/automation/trex_control_plane/stl/examples/stl_pcap.py
+++ b/scripts/automation/trex_control_plane/stl/examples/stl_pcap.py
@@ -3,62 +3,52 @@ from trex_stl_lib.api import *
 import argparse
 import sys
 
-def create_vm (ip_start, ip_end):
-     vm =[
 
-            # dest
-            STLVmFlowVar(name="dst", min_value = ip_start, max_value = ip_end, size = 4, op = "inc"),
-            STLVmWrFlowVar(fv_name="dst",pkt_offset= "IP.dst"),
+def packet_hook_generator (remove_fcs, vlan_id):
 
-            # checksum
-            STLVmFixIpv4(offset = "IP")
+    def packet_hook (packet):
+        packet = Ether(packet)
 
-            ]
-
-     return vm
-
-# warning: might make test slow
-def alter_streams(streams, remove_fcs, vlan_id):
-    for stream in streams:
-        packet = Ether(stream.pkt)
         if vlan_id >= 0 and vlan_id <= 4096:
             packet_l3 = packet.payload
             packet = Ether() / Dot1Q(vlan = vlan_id) / packet_l3
+
         if remove_fcs and packet.lastlayer().name == 'Padding':
             packet.lastlayer().underlayer.remove_payload()
-        packet = STLPktBuilder(packet)
-        stream.fields['packet'] = packet.dump_pkt()
-        stream.pkt = base64.b64decode(stream.fields['packet']['binary'])
+
+        return str(packet)
+
+    return packet_hook
+
 
 def inject_pcap (pcap_file, server, port, loop_count, ipg_usec, use_vm, remove_fcs, vlan_id):
 
     # create client
     c = STLClient(server = server)
-    
-    try:
-        if use_vm:
-            vm = create_vm("10.0.0.1", "10.0.0.254")
-        else:
-            vm = None
 
-        profile = STLProfile.load_pcap(pcap_file, ipg_usec = ipg_usec, loop_count = loop_count, vm = vm)
+    if remove_fcs or vlan_id:
+        packet_hook = packet_hook_generator(remove_fcs, vlan_id)
+    else:
+        packet_hook = None
 
-        print("Loaded pcap {0} with {1} packets...\n".format(pcap_file, len(profile)))
-        streams = profile.get_streams()
-        if remove_fcs or (vlan_id >= 0 and vlan_id <= 4096):
-            alter_streams(streams, remove_fcs, vlan_id)
+    try:
 
-        # uncomment this for simulator run
-        #STLSim().run(profile.get_streams(), outfile = '/auto/srg-sce-swinfra-usr/emb/users/ybrustin/out.pcap')
+        vm = STLIPRange(dst = {'start': '10.0.0.1', 'end': '10.0.0.254', 'step' : 1}) if use_vm else None
 
         c.connect()
         c.reset(ports = [port])
-        stream_ids = c.add_streams(streams, ports = [port])
 
         c.clear_stats()
+        d = c.push_pcap(pcap_file,
+                        ipg_usec = ipg_usec,
+                        count = loop_count,
+                        vm = vm,
+                        packet_hook = packet_hook)
+
+        STLSim().run(d, outfile = 'test.cap')
+
+        c.wait_on_traffic()
 
-        c.start()
-        c.wait_on_traffic(ports = [port])
 
         stats = c.get_stats()
         opackets = stats[port]['opackets']
diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_pcap_remote.py b/scripts/automation/trex_control_plane/stl/examples/stl_pcap_remote.py
new file mode 100644
index 00000000..c47eee31
--- /dev/null
+++ b/scripts/automation/trex_control_plane/stl/examples/stl_pcap_remote.py
@@ -0,0 +1,123 @@
+import stl_path
+from trex_stl_lib.api import *
+import argparse
+import sys
+
+
+def inject_pcap (c, pcap_file, port, loop_count, ipg_usec, duration):
+
+    pcap_file = os.path.abspath(pcap_file)
+
+    c.reset(ports = [port])
+    c.push_remote(pcap_file, ports = [port], ipg_usec = ipg_usec, speedup = 1.0, count = loop_count, duration = duration)
+    # assume 100 seconds is enough - but can be more
+    c.wait_on_traffic(ports = [port], timeout = 100)
+
+    stats = c.get_stats()
+    opackets = stats[port]['opackets']
+
+    return opackets
+    #print("{0} packets were Tx on port {1}\n".format(opackets, port))
+
+    
+
+def setParserOptions():
+    parser = argparse.ArgumentParser(prog="stl_pcap.py")
+
+    parser.add_argument("-f", "--file", help = "pcap file to inject",
+                        dest = "pcap",
+                        required = True,
+                        type = str)
+
+    parser.add_argument("-s", "--server", help = "TRex server address",
+                        dest = "server",
+                        default = 'localhost',
+                        type = str)
+
+    parser.add_argument("-p", "--port", help = "port to inject on",
+                        dest = "port",
+                        required = True,
+                        type = int)
+
+    parser.add_argument("-n", "--number", help = "How many times to inject pcap [default is 1, 0 means forever]",
+                        dest = "loop_count",
+                        default = 1,
+                        type = int)
+
+    parser.add_argument("-i", help = "IPG in usec",
+                        dest = "ipg",
+                        default = None,
+                        type = float)
+
+    parser.add_argument("-d", help = "duration in seconds",
+                        dest = "duration",
+                        default = -1,
+                        type = float)
+
+    return parser
+
+def sizeof_fmt(num, suffix='B'):
+    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
+        if abs(num) < 1024.0:
+            return "%3.1f%s%s" % (num, unit, suffix)
+        num /= 1024.0
+    return "%.1f%s%s" % (num, 'Yi', suffix)
+
+
+def read_txt_file (filename):
+
+    with open(filename) as f:
+        lines = f.readlines()
+
+    caps = []
+    for raw in lines:
+       raw = raw.rstrip()
+       if raw[0] == '#':
+           continue
+       ext=os.path.splitext(raw)[1]
+       if ext not in ['.cap', '.pcap', '.erf']:
+           # skip unknown format
+           continue
+    
+       caps.append(raw)
+
+    return caps
+
+
+def start (args):
+    
+    parser = setParserOptions()
+    options = parser.parse_args(args)
+    
+    ext = os.path.splitext(options.pcap)[1]
+    if ext == '.txt':
+        caps = read_txt_file(options.pcap)
+    elif ext in ['.cap', '.pcap']:
+        caps = [options.pcap]
+    else:
+        print("unknown file extension for file {0}".format(options.pcap))
+        return
+
+    c = STLClient(server = options.server)
+    try:
+        c.connect()
+        for i, cap in enumerate(caps, start = 1):
+            before = time.time()
+            print ("{:} CAP {:} @ {:} - ".format(i, cap, sizeof_fmt(os.path.getsize(cap)))),
+            injected = inject_pcap(c, cap, options.port, options.loop_count, options.ipg, options.duration)
+            print("took {:.2f} seconds for {:} packets").format(time.time() - before, injected)
+
+    except STLError as e:
+        print(e)
+        return
+
+    finally:
+        c.disconnect()
+
+def main ():
+    start(sys.argv[1:])
+
+# inject pcap
+if __name__ == '__main__':
+    main()
+
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_async_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_async_client.py
index 022077a9..5c9faf0f 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_async_client.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_async_client.py
@@ -194,11 +194,13 @@ class CTRexAsyncClient():
         if not self.connected:
             return
 
+        # mark for join
+        self.active = False
+
         # signal that the context was destroyed (exit the thread loop)
         self.context.term()
 
-        # mark for join and join
-        self.active = False
+        # join
         self.t.join()
 
         # done
@@ -239,6 +241,7 @@ class CTRexAsyncClient():
 
             except zmq.ContextTerminated:
                 # outside thread signaled us to exit
+                assert(not self.active)
                 break
 
             msg = json.loads(line)
@@ -256,6 +259,8 @@ class CTRexAsyncClient():
         # closing of socket must be from the same thread
         self.socket.close(linger = 0)
 
+    def is_thread_alive (self):
+        return self.t.is_alive()
 
     # did we get info for the last 3 seconds ?
     def is_alive (self):
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
index 862a9979..010d966c 100755
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
@@ -313,6 +313,10 @@ class EventsHandler(object):
             if session_id != self.client.session_id:
                 self.__async_event_port_released(port_id)
 
+        elif (type == 7):
+            port_id = int(data['port_id'])
+            ev = "port {0} job failed".format(port_id)
+            show_event = True
 
         # server stopped
         elif (type == 100):
@@ -331,30 +335,37 @@ class EventsHandler(object):
 
     # private functions
 
+    # on rare cases events may come on a non existent prot
+    # (server was re-run with different config)
     def __async_event_port_job_done (self, port_id):
-        self.client.ports[port_id].async_event_port_job_done()
+        if port_id in self.client.ports:
+            self.client.ports[port_id].async_event_port_job_done()
 
     def __async_event_port_stopped (self, port_id):
-        self.client.ports[port_id].async_event_port_stopped()
+        if port_id in self.client.ports:
+            self.client.ports[port_id].async_event_port_stopped()
 
 
     def __async_event_port_started (self, port_id):
-        self.client.ports[port_id].async_event_port_started()
-
+        if port_id in self.client.ports:
+            self.client.ports[port_id].async_event_port_started()
 
     def __async_event_port_paused (self, port_id):
-        self.client.ports[port_id].async_event_port_paused()
+        if port_id in self.client.ports:
+            self.client.ports[port_id].async_event_port_paused()
 
 
     def __async_event_port_resumed (self, port_id):
-        self.client.ports[port_id].async_event_port_resumed()
-
+        if port_id in self.client.ports:
+            self.client.ports[port_id].async_event_port_resumed()
 
     def __async_event_port_acquired (self, port_id, who):
-        self.client.ports[port_id].async_event_acquired(who)
+        if port_id in self.client.ports:
+            self.client.ports[port_id].async_event_acquired(who)
 
     def __async_event_port_released (self, port_id):
-        self.client.ports[port_id].async_event_released()
+        if port_id in self.client.ports:
+            self.client.ports[port_id].async_event_released()
 
     def __async_event_server_stopped (self):
         self.client.connected = False
@@ -711,6 +722,17 @@ class STLClient(object):
         return rc
 
 
+    def __push_remote (self, pcap_filename, port_id_list, ipg_usec, speedup, count, duration):
+
+        port_id_list = self.__ports(port_id_list)
+        rc = RC()
+
+        for port_id in port_id_list:
+            rc.add(self.ports[port_id].push_remote(pcap_filename, ipg_usec, speedup, count, duration))
+
+        return rc
+
+
     def __validate (self, port_id_list = None):
         port_id_list = self.__ports(port_id_list)
 
@@ -832,6 +854,9 @@ class STLClient(object):
     # clear stats
     def __clear_stats(self, port_id_list, clear_global, clear_flow_stats):
 
+        # we must be sync with the server
+        self.async_client.barrier()
+
         for port_id in port_id_list:
             self.ports[port_id].clear_stats()
 
@@ -1852,7 +1877,136 @@ class STLClient(object):
 
 
     @__api_check(True)
-    def validate (self, ports = None, mult = "1", duration = "-1", total = False):
+    def push_remote (self,
+                     pcap_filename,
+                     ports = None,
+                     ipg_usec = None,
+                     speedup = 1.0,
+                     count = 1,
+                     duration = -1):
+        """
+            Push a remote server-reachable PCAP file
+            the path must be fullpath accessible to the server
+
+            :parameters:
+                pcap_filename : str
+                    PCAP file name in full path and accessible to the server
+
+                ports : list
+                    Ports on which to execute the command
+
+                ipg_usec : float
+                    Inter-packet gap in microseconds
+
+                speedup : float
+                    A factor to adjust IPG. effectively IPG = IPG / speedup
+
+                count: int
+                    How many times to transmit the cap
+
+                duration: float
+                    Limit runtime by duration in seconds
+            :raises:
+                + :exc:`STLError`
+
+        """
+        ports = ports if ports is not None else self.get_acquired_ports()
+        ports = self._validate_port_list(ports)
+
+        validate_type('pcap_filename', pcap_filename, str)
+        validate_type('ipg_usec', ipg_usec, (float, int, type(None)))
+        validate_type('speedup',  speedup, (float, int))
+        validate_type('count',  count, int)
+        validate_type('duration', duration, (float, int))
+
+        self.logger.pre_cmd("Pushing remote PCAP on port(s) {0}:".format(ports))
+        rc = self.__push_remote(pcap_filename, ports, ipg_usec, speedup, count, duration)
+        self.logger.post_cmd(rc)
+
+        if not rc:
+            raise STLError(rc)
+
+
+    @__api_check(True)
+    def push_pcap (self,
+                   pcap_filename,
+                   ports = None,
+                   ipg_usec = None,
+                   speedup = 1.0,
+                   count = 1,
+                   duration = -1,
+                   force = False,
+                   vm = None,
+                   packet_hook = None):
+        """
+            Push a local PCAP to the server
+            This is equivalent to loading a PCAP file to a profile
+            and attaching the profile to port(s)
+            
+            file size is limited to 1MB
+
+            :parameters:
+                pcap_filename : str
+                    PCAP filename (accessible locally)
+
+                ports : list
+                    Ports on which to execute the command
+
+                ipg_usec : float
+                    Inter-packet gap in microseconds
+
+                speedup : float
+                    A factor to adjust IPG. effectively IPG = IPG / speedup
+
+                count: int
+                    How many times to transmit the cap
+
+                duration: float
+                    Limit runtime by duration in seconds
+
+                force: bool
+                    Ignore file size limit - push any file size to the server
+
+                vm: list of VM instructions
+                    VM instructions to apply for every packet
+
+                packet_hook : Callable or function
+                    Will be applied to every packet
+
+            :raises:
+                + :exc:`STLError`
+
+        """
+        ports = ports if ports is not None else self.get_acquired_ports()
+        ports = self._validate_port_list(ports)
+
+        validate_type('pcap_filename', pcap_filename, str)
+        validate_type('ipg_usec', ipg_usec, (float, int, type(None)))
+        validate_type('speedup',  speedup, (float, int))
+        validate_type('count',  count, int)
+        validate_type('duration', duration, (float, int))
+        validate_type('vm', vm, (list, type(None)))
+        
+        # no support for > 1MB PCAP - use push remote
+        if not force and os.path.getsize(pcap_filename) > (1024 * 1024):
+            raise STLError("PCAP size of {:} is too big for local push - consider using remote push or provide 'force'".format(format_num(os.path.getsize(pcap_filename), suffix = 'B')))
+
+        self.remove_all_streams(ports = ports)
+
+        profile = STLProfile.load_pcap(pcap_filename,
+                                       ipg_usec,
+                                       speedup,
+                                       count,
+                                       vm = vm,
+                                       packet_hook = packet_hook)
+
+        id_list = self.add_streams(profile.get_streams(), ports)
+
+        return self.start(ports = ports, duration = duration)
+
+
+    @__api_check(True)
+    def validate (self, ports = None, mult = "1", duration = -1, total = False):
         """
             Validate port(s) configuration
 
@@ -1899,6 +2053,8 @@ class STLClient(object):
         rc = self.__validate(ports)
         self.logger.post_cmd(rc)
 
+        if not rc:
+            raise STLError(rc)
 
         for port in ports:
             self.ports[port].print_profile(mult_obj, duration)
@@ -1931,7 +2087,6 @@ class STLClient(object):
         if not type(clear_global) is bool:
             raise STLArgumentError('clear_global', clear_global)
 
-
         rc = self.__clear_stats(ports, clear_global, clear_flow_stats)
         if not rc:
             raise STLError(rc)
@@ -1995,6 +2150,11 @@ class STLClient(object):
 
         # wait while any of the required ports are active
         while set(self.get_active_ports()).intersection(ports):
+
+            # make sure ASYNC thread is still alive - otherwise we will be stuck forever
+            if not self.async_client.is_thread_alive():
+                raise STLError("subscriber thread is dead")
+
             time.sleep(0.01)
             if time.time() > expr:
                 raise STLTimeoutError(timeout)
@@ -2519,6 +2679,7 @@ class STLClient(object):
                                          "push",
                                          self.push_line.__doc__,
                                          parsing_opts.FILE_PATH,
+                                         parsing_opts.REMOTE_FILE,
                                          parsing_opts.PORT_LIST_WITH_ALL,
                                          parsing_opts.COUNT,
                                          parsing_opts.DURATION,
@@ -2540,16 +2701,25 @@ class STLClient(object):
             else:
                 self.stop(active_ports)
 
-        # pcap injection removes all previous streams from the ports
-        self.remove_all_streams(ports = opts.ports)
-            
-        profile = STLProfile.load_pcap(opts.file[0],
-                                       opts.ipg_usec,
-                                       opts.speedup,
-                                       opts.count)
 
-        id_list = self.add_streams(profile.get_streams(), opts.ports)
-        self.start(ports = opts.ports, duration = opts.duration, force = opts.force)
+        if opts.remote:
+            self.push_remote(opts.file[0],
+                             ports     = opts.ports,
+                             ipg_usec  = opts.ipg_usec,
+                             speedup   = opts.speedup,
+                             count     = opts.count,
+                             duration  = opts.duration)
+
+        else:
+            self.push_pcap(opts.file[0],
+                           ports     = opts.ports,
+                           ipg_usec  = opts.ipg_usec,
+                           speedup   = opts.speedup,
+                           count     = opts.count,
+                           duration  = opts.duration,
+                           force     = opts.force)
+
+        
 
         return True
 
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py
index 0afeff20..33a7b3af 100755
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py
@@ -23,7 +23,9 @@ traffic_config_kwargs = {
     'split_by_cores': 'split',              # ( split | duplicate | single ) TRex extention: split = split traffic by cores, duplicate = duplicate traffic for all cores, single = run only with sinle core (not implemented yet)
     'load_profile': None,                   # TRex extention: path to filename with stream profile (stream builder parameters will be ignored, limitation: modify)
     'consistent_random': False,             # TRex extention: False (default) = random sequence will be different every run, True = random sequence will be same every run
-    'ignore_macs': False,                   # TRex extention: True = use MACs from server configuration , no MAC VM (workaround on lack of ARP)
+    'ignore_macs': False,                   # TRex extention: True = use MACs from server configuration, no MAC VM (workaround on lack of ARP)
+    'disable_flow_stats': False,            # TRex extention: True = don't use flow stats for this stream, (workaround for limitation on type of packet for flow_stats)
+    'flow_stats_id': None,                  # TRex extention: uint, for use of STLHltStream, specifies id for flow stats (see stateless manual for flow_stats details)
     'port_handle': None,
     'port_handle2': None,
     'bidirectional': False,
@@ -446,7 +448,7 @@ class CTRexHltApi(object):
         kwargs = merge_kwargs(traffic_config_kwargs, user_kwargs)
         stream_id = kwargs['stream_id']
         mode = kwargs['mode']
-        pg_id = None
+        pg_id = kwargs['flow_stats_id']
         port_handle = port_list = self._parse_port_list(kwargs['port_handle'])
 
         ALLOWED_MODES = ['create', 'modify', 'remove', 'enable', 'disable', 'reset']
@@ -864,7 +866,10 @@ def STLHltStream(**user_kwargs):
         raise STLError('Could not create transmit_mode object %s: %s' % (transmit_mode, e if isinstance(e, STLError) else traceback.format_exc()))
 
     try:
-        pg_id = kwargs.get('pg_id')
+        if kwargs['l3_protocol'] == 'ipv4' and not kwargs['disable_flow_stats']:
+            pg_id = kwargs.get('pg_id', kwargs.get('flow_stats_id'))
+        else:
+            pg_id = None
         stream = STLStream(packet = packet,
                            random_seed = 1 if is_true(kwargs['consistent_random']) else 0,
                            #enabled = True,
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py
index f06d5d70..e12efaf9 100755
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py
@@ -1492,4 +1492,27 @@ class STLPktBuilder(CTrexPktBuilderInterface):
         pass;
 
 
+def STLIPRange (src = None,
+                dst = None,
+                fix_chksum = True):
+
+    vm = []
+
+    if src:
+        vm += [
+               STLVmFlowVar(name="src", min_value = src['start'], max_value = src['end'], size = 4, op = "inc", step = src['step']),
+               STLVmWrFlowVar(fv_name="src",pkt_offset= "IP.src")
+              ]
+
+    if dst:
+        vm += [
+                STLVmFlowVar(name="dst", min_value = dst['start'], max_value = dst['end'], size = 4, op = "inc", step = dst['step']),
+                STLVmWrFlowVar(fv_name="dst",pkt_offset= "IP.dst")
+              ]
+
+    if fix_chksum:
+        vm.append( STLVmFixIpv4(offset = "IP"))
+
+
+    return vm
 
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py
index e8f89b27..391b2076 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py
@@ -29,17 +29,20 @@ def mult_to_factor (mult, max_bps_l2, max_pps, line_util):
 
 # describes a single port
 class Port(object):
-    STATE_DOWN       = 0
-    STATE_IDLE       = 1
-    STATE_STREAMS    = 2
-    STATE_TX         = 3
-    STATE_PAUSE      = 4
+    STATE_DOWN         = 0
+    STATE_IDLE         = 1
+    STATE_STREAMS      = 2
+    STATE_TX           = 3
+    STATE_PAUSE        = 4
+    STATE_PCAP_TX      = 5
+
     PortState = namedtuple('PortState', ['state_id', 'state_name'])
     STATES_MAP = {STATE_DOWN: "DOWN",
                   STATE_IDLE: "IDLE",
                   STATE_STREAMS: "IDLE",
                   STATE_TX: "ACTIVE",
-                  STATE_PAUSE: "PAUSE"}
+                  STATE_PAUSE: "PAUSE",
+                  STATE_PCAP_TX : "ACTIVE"}
 
 
     def __init__ (self, port_id, user, comm_link, session_id, info):
@@ -67,6 +70,54 @@ class Port(object):
         self.owner = ''
 
 
+    # decorator to verify port is up
+    def up(func):
+        def func_wrapper(*args):
+            port = args[0]
+
+            if not port.is_up():
+                return port.err("{0} - port is down".format(func.__name__))
+
+            return func(*args)
+
+        return func_wrapper
+
+    # owned
+    def owned(func):
+        def func_wrapper(*args):
+            port = args[0]
+
+            if not port.is_up():
+                return port.err("{0} - port is down".format(func.__name__))
+
+            if not port.is_acquired():
+                return port.err("{0} - port is not owned".format(func.__name__))
+
+            return func(*args)
+
+        return func_wrapper
+
+
+    # decorator to check server is readable (port not down and etc.)
+    def writeable(func):
+        def func_wrapper(*args):
+            port = args[0]
+
+            if not port.is_up():
+                return port.err("{0} - port is down".format(func.__name__))
+
+            if not port.is_acquired():
+                return port.err("{0} - port is not owned".format(func.__name__))
+
+            if not port.is_writeable():
+                return port.err("{0} - port is not in a writeable state".format(func.__name__))
+
+            return func(*args)
+
+        return func_wrapper
+
+
+
     def err(self, msg):
         return RC_ERR("port {0} : {1}\n".format(self.port_id, msg))
 
@@ -79,7 +130,39 @@ class Port(object):
     def get_formatted_speed (self):
         return "{0} Gbps".format(self.info['speed'])
 
+    def is_acquired(self):
+        return (self.handler != None)
+
+    def is_up (self):
+        return (self.state != self.STATE_DOWN)
+
+    def is_active(self):
+        return (self.state == self.STATE_TX ) or (self.state == self.STATE_PAUSE) or (self.state == self.STATE_PCAP_TX)
+
+    def is_transmitting (self):
+        return (self.state == self.STATE_TX) or (self.state == self.STATE_PCAP_TX)
+
+    def is_paused (self):
+        return (self.state == self.STATE_PAUSE)
+
+    def is_writeable (self):
+        # operations on port can be done on state idle or state streams
+        return ((self.state == self.STATE_IDLE) or (self.state == self.STATE_STREAMS))
+
+    def get_owner (self):
+        if self.is_acquired():
+            return self.user
+        else:
+            return self.owner
+
+    def __allocate_stream_id (self):
+        id = self.next_available_id
+        self.next_available_id += 1
+        return id
+
+
     # take the port
+    @up
     def acquire(self, force = False, sync_streams = True):
         params = {"port_id":     self.port_id,
                   "user":        self.user,
@@ -99,6 +182,7 @@ class Port(object):
 
       
     # sync all the streams with the server
+    @up
     def sync_streams (self):
         params = {"port_id": self.port_id}
 
@@ -114,6 +198,7 @@ class Port(object):
         return self.ok()
 
     # release the port
+    @up
     def release(self):
         params = {"port_id": self.port_id,
                   "handler": self.handler}
@@ -129,25 +214,11 @@ class Port(object):
         else:
             return self.err(rc.err())
 
-    def is_acquired(self):
-        return (self.handler != None)
-
-    def is_active(self):
-        return(self.state == self.STATE_TX ) or (self.state == self.STATE_PAUSE)
-
-    def is_transmitting (self):
-        return (self.state == self.STATE_TX)
-
-    def is_paused (self):
-        return (self.state == self.STATE_PAUSE)
-
-    def get_owner (self):
-        if self.is_acquired():
-            return self.user
-        else:
-            return self.owner
+ 
 
+    @up
     def sync(self):
+
         params = {"port_id": self.port_id}
 
         rc = self.transmit("get_port_status", params)
@@ -167,6 +238,8 @@ class Port(object):
             self.state = self.STATE_TX
         elif port_state == "PAUSE":
             self.state = self.STATE_PAUSE
+        elif port_state == "PCAP_TX":
+            self.state = self.STATE_PCAP_TX
         else:
             raise Exception("port {0}: bad state received from server '{1}'".format(self.port_id, port_state))
 
@@ -181,27 +254,11 @@ class Port(object):
         return self.ok()
 
 
-    # return TRUE if write commands
-    def is_port_writable (self):
-        # operations on port can be done on state idle or state streams
-        return ((self.state == self.STATE_IDLE) or (self.state == self.STATE_STREAMS))
-
-
-    def __allocate_stream_id (self):
-        id = self.next_available_id
-        self.next_available_id += 1
-        return id
-
 
     # add streams
+    @writeable
     def add_streams (self, streams_list):
 
-        if not self.is_acquired():
-            return self.err("port is not owned")
-
-        if not self.is_port_writable():
-            return self.err("Please stop port before attempting to add streams")
-
         # listify
         streams_list = streams_list if isinstance(streams_list, list) else [streams_list]
         
@@ -273,14 +330,9 @@ class Port(object):
 
 
     # remove stream from port
+    @writeable
     def remove_streams (self, stream_id_list):
 
-        if not self.is_acquired():
-            return self.err("port is not owned")
-
-        if not self.is_port_writable():
-            return self.err("Please stop port before attempting to remove streams")
-
         # single element to list
         stream_id_list = stream_id_list if isinstance(stream_id_list, list) else [stream_id_list]
 
@@ -314,14 +366,9 @@ class Port(object):
 
 
     # remove all the streams
+    @writeable
     def remove_all_streams (self):
 
-        if not self.is_acquired():
-            return self.err("port is not owned")
-
-        if not self.is_port_writable():
-            return self.err("Please stop port before attempting to remove streams")
-
         params = {"handler": self.handler,
                   "port_id": self.port_id}
 
@@ -348,19 +395,11 @@ class Port(object):
         return self.streams
 
 
-    # start traffic
+    @writeable
     def start (self, mul, duration, force):
-        if not self.is_acquired():
-            return self.err("port is not owned")
-
-        if self.state == self.STATE_DOWN:
-            return self.err("Unable to start traffic - port is down")
 
         if self.state == self.STATE_IDLE:
-            return self.err("Unable to start traffic - no streams attached to port")
-
-        if self.state == self.STATE_TX:
-            return self.err("Unable to start traffic - port is already transmitting")
+            return self.err("unable to start traffic - no streams attached to port")
 
         params = {"handler":  self.handler,
                   "port_id":  self.port_id,
@@ -378,15 +417,12 @@ class Port(object):
 
     # stop traffic
     # with force ignores the cached state and sends the command
+    @owned
     def stop (self, force = False):
         
-        if not self.is_acquired():
-            return self.err("port is not owned")
-
-        # port is already stopped
-        if not force:
-            if (self.state == self.STATE_IDLE) or (self.state == self.state == self.STATE_STREAMS):
-                return self.ok()
+        # if not is not active and not force - go back
+        if not self.is_active() and not force:
+            return self.ok()
 
         params = {"handler": self.handler,
                   "port_id": self.port_id}
@@ -420,19 +456,10 @@ class Port(object):
         return not self.tx_stopped_ts or (datetime.now() - self.tx_stopped_ts) > timedelta(milliseconds = rx_delay_ms)
 
 
-
+    @writeable
     def remove_rx_filters (self):
         assert(self.has_rx_enabled())
 
-        if not self.is_acquired():
-            return self.err("port is not owned")
-
-        if self.state == self.STATE_DOWN:
-            return self.err("Unable to remove RX filters - port is down")
-
-        if self.state == self.STATE_TX:
-            return self.err("Unable to remove RX filters - port is transmitting")
-
         if self.state == self.STATE_IDLE:
             return self.ok()
 
@@ -446,11 +473,11 @@ class Port(object):
 
         return self.ok()
 
-
+    @owned
     def pause (self):
 
-        if not self.is_acquired():
-            return self.err("port is not owned")
+        if (self.state == self.STATE_PCAP_TX) :
+            return self.err("pause is not supported during PCAP TX")
 
         if (self.state != self.STATE_TX) :
             return self.err("port is not transmitting")
@@ -466,12 +493,9 @@ class Port(object):
 
         return self.ok()
 
-
+    @owned
     def resume (self):
 
-        if not self.is_acquired():
-            return self.err("port is not owned")
-
         if (self.state != self.STATE_PAUSE) :
             return self.err("port is not in pause mode")
 
@@ -488,11 +512,11 @@ class Port(object):
 
         return self.ok()
 
-
+    @owned
     def update (self, mul, force):
 
-        if not self.is_acquired():
-            return self.err("port is not owned")
+        if (self.state == self.STATE_PCAP_TX) :
+            return self.err("update is not supported during PCAP TX")
 
         if (self.state != self.STATE_TX) :
             return self.err("port is not transmitting")
@@ -508,15 +532,9 @@ class Port(object):
 
         return self.ok()
 
-
+    @owned
     def validate (self):
 
-        if not self.is_acquired():
-            return self.err("port is not owned")
-
-        if (self.state == self.STATE_DOWN):
-            return self.err("port is down")
-
         if (self.state == self.STATE_IDLE):
             return self.err("no streams attached to port")
 
@@ -532,12 +550,8 @@ class Port(object):
         return self.ok()
 
 
+    @owned
     def set_attr (self, attr_dict):
-        if not self.is_acquired():
-            return self.err("port is not owned")
-
-        if (self.state == self.STATE_DOWN):
-            return self.err("port is down")
 
         params = {"handler": self.handler,
                   "port_id": self.port_id,
@@ -552,6 +566,24 @@ class Port(object):
 
         return self.ok()
 
+    @writeable
+    def push_remote (self, pcap_filename, ipg_usec, speedup, count, duration):
+
+        params = {"handler": self.handler,
+                  "port_id": self.port_id,
+                  "pcap_filename": pcap_filename,
+                  "ipg_usec": ipg_usec if ipg_usec is not None else -1,
+                  "speedup": speedup,
+                  "count": count,
+                  "duration": duration}
+
+        rc = self.transmit("push_remote", params)
+        if rc.bad():
+            return self.err(rc.err())
+
+        self.state = self.STATE_PCAP_TX
+        return self.ok()
+
 
     def get_profile (self):
         return self.profile
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
index c7513144..61122e79 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
@@ -28,6 +28,9 @@ SS_COMPAT = [GLOBAL_STATS, STREAMS_STATS]
 
 ExportableStats = namedtuple('ExportableStats', ['raw_data', 'text_table'])
 
+def round_float (f):
+    return float("%.2f" % f)
+
 # deep mrege of dicts dst = src + dst
 def deep_merge_dicts (dst, src):
     for k, v in src.items():
@@ -633,10 +636,10 @@ class CGlobalStats(CTRexStats):
                              ("version", "{ver}, UUID: {uuid}".format(ver=self.server_version.get("version", "N/A"),
                                                                       uuid="N/A")),
 
-                             ("cpu_util", "{0}% {1}".format( format_threshold(self.get("m_cpu_util"), [85, 100], [0, 85]),
+                             ("cpu_util", "{0}% {1}".format( format_threshold(round_float(self.get("m_cpu_util")), [85, 100], [0, 85]),
                                                               self.get_trend_gui("m_cpu_util", use_raw = True))),
 
-                             ("rx_cpu_util", "{0}% {1}".format( format_threshold(self.get("m_rx_cpu_util"), [85, 100], [0, 85]),
+                             ("rx_cpu_util", "{0}% {1}".format( format_threshold(round_float(self.get("m_rx_cpu_util")), [85, 100], [0, 85]),
                                                                 self.get_trend_gui("m_rx_cpu_util", use_raw = True))),
 
                              (" ", ""),
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py
index 165942d8..a7fd3026 100755
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py
@@ -7,6 +7,7 @@ from .trex_stl_packet_builder_scapy import STLPktBuilder, Ether, IP, UDP, TCP, R
 from collections import OrderedDict, namedtuple
 
 from scapy.utils import ltoa
+from scapy.error import Scapy_Exception
 import random
 import yaml
 import base64
@@ -927,7 +928,7 @@ class STLProfile(object):
     
     # loop_count = 0 means loop forever
     @staticmethod
-    def load_pcap (pcap_file, ipg_usec = None, speedup = 1.0, loop_count = 1, vm = None):
+    def load_pcap (pcap_file, ipg_usec = None, speedup = 1.0, loop_count = 1, vm = None, packet_hook = None):
         """ Convert a pcap file with a number of packets to a list of connected streams.  
 
         packet1->packet2->packet3 etc 
@@ -938,7 +939,7 @@ class STLProfile(object):
                        Name of the pcap file 
 
                   ipg_usec   : float
-                       Inter packet gap in usec. If IPG=0, IPG is taken from pcap file
+                       Inter packet gap in usec. If IPG is None, IPG is taken from pcap file
 
                   speedup   : float 
                        When reading the pcap file, divide IPG by this "speedup" factor. Resulting IPG is sped up by this factor. 
@@ -949,6 +950,9 @@ class STLProfile(object):
                   vm        :  list 
                         List of Field engine instructions 
 
+                  packet_hook : Callable or function
+                        will be applied to every packet
+
                  :return: STLProfile
 
         """
@@ -958,8 +962,8 @@ class STLProfile(object):
             raise STLError("file '{0}' does not exists".format(pcap_file))
 
         # make sure IPG is not less than 1 usec
-        if ipg_usec is not None and ipg_usec < 1:
-            raise STLError("ipg_usec cannot be less than 1 usec: '{0}'".format(ipg_usec))
+        if ipg_usec is not None and ipg_usec < 0.001:
+            raise STLError("ipg_usec cannot be less than 0.001 usec: '{0}'".format(ipg_usec))
 
         if loop_count < 0:
             raise STLError("'loop_count' cannot be negative")
@@ -967,8 +971,15 @@ class STLProfile(object):
         streams = []
         last_ts_usec = 0
 
-        pkts = RawPcapReader(pcap_file).read_all()
-        
+        try:
+            pkts = RawPcapReader(pcap_file).read_all()
+        except Scapy_Exception as e:
+            raise STLError("failed to open PCAP file '{0}'".format(pcap_file))
+
+        if packet_hook:
+            pkts = [(packet_hook(cap), meta) for (cap, meta) in pkts]
+
+
         for i, (cap, meta) in enumerate(pkts, start = 1):
             # IPG - if not provided, take from cap
             if ipg_usec == None:
@@ -984,7 +995,6 @@ class STLProfile(object):
                 next = i + 1
                 action_count = 0
 
-            
             streams.append(STLStream(name = i,
                                      packet = STLPktBuilder(pkt_buffer = cap, vm = vm),
                                      mode = STLTXSingleBurst(total_pkts = 1, percentage = 100),
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py
index ad46625b..98e3ca6a 100755
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py
@@ -32,6 +32,7 @@ PROMISCUOUS = 19
 NO_PROMISCUOUS = 20
 PROMISCUOUS_SWITCH = 21
 TUNABLES = 22
+REMOTE_FILE = 23
 
 GLOBAL_STATS = 50
 PORT_STATS = 51
@@ -290,6 +291,11 @@ OPTIONS_DB = {MULTIPLIER: ArgumentPack(['-m', '--multiplier'],
                                          'default': False,
                                          'help': "Set if you want to stop active ports before appyling command."}),
 
+              REMOTE_FILE: ArgumentPack(['-r', '--remote'],
+                                        {"action": "store_true",
+                                         'default': False,
+                                         'help': "file path should be interpeted by the server (remote file)"}),
+
               FILE_PATH: ArgumentPack(['-f'],
                                       {'metavar': 'FILE',
                                        'dest': 'file',
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py
index 5c0dfb14..7e0bf9e4 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py
@@ -150,11 +150,15 @@ def format_text(text, *args):
 
 
 def format_threshold (value, red_zone, green_zone):
-    if value >= red_zone[0] and value <= red_zone[1]:
-        return format_text("{0}".format(value), 'red')
+    try:
+        if value >= red_zone[0] and value <= red_zone[1]:
+            return format_text("{0}".format(value), 'red')
 
-    if value >= green_zone[0] and value <= green_zone[1]:
-        return format_text("{0}".format(value), 'green')
+        if value >= green_zone[0] and value <= green_zone[1]:
+            return format_text("{0}".format(value), 'green')
+    except TypeError:
+        # if value is not comparable or not a number - skip this
+        pass
 
     return "{0}".format(value)
 
diff --git a/scripts/exp/pcap_remote_basic-0-ex.erf b/scripts/exp/pcap_remote_basic-0-ex.erf
new file mode 100644
index 00000000..3c626419
--- /dev/null
+++ b/scripts/exp/pcap_remote_basic-0-ex.erf
diff --git a/scripts/exp/pcap_remote_duration-0-ex.erf b/scripts/exp/pcap_remote_duration-0-ex.erf
new file mode 100644
index 00000000..3731f735
--- /dev/null
+++ b/scripts/exp/pcap_remote_duration-0-ex.erf
diff --git a/scripts/exp/pcap_remote_loop-0-ex.erf b/scripts/exp/pcap_remote_loop-0-ex.erf
new file mode 100644
index 00000000..c5f3d8be
--- /dev/null
+++ b/scripts/exp/pcap_remote_loop-0-ex.erf
diff --git a/scripts/exp/remote_test.cap b/scripts/exp/remote_test.cap
new file mode 100644
index 00000000..05462b28
--- /dev/null
+++ b/scripts/exp/remote_test.cap