Merge branch 'master' into cpu_per_core

Conflicts:
	scripts/automation/trex_control_plane/server/singleton_daemon.py

61 files changed, 982 insertions, 361 deletions

diff --git a/scripts/automation/regression/functional_tests/stl_basic_tests.py b/scripts/automation/regression/functional_tests/stl_basic_tests.py
index dbbf2530..a4e28ca9 100644
--- a/scripts/automation/regression/functional_tests/stl_basic_tests.py
+++ b/scripts/automation/regression/functional_tests/stl_basic_tests.py
@@ -119,9 +119,9 @@ class CStlBasic_Test(functional_general_test.CGeneralFunctional_Test):
 
     def run_sim (self, yaml, output, options = "", silent = False, obj = None):
         if output:
-            user_cmd = "-f {0} -o {1} {2}".format(yaml, output, options)
+            user_cmd = "-f {0} -o {1} {2} -p {3}".format(yaml, output, options, self.scripts_path)
         else:
-            user_cmd = "-f {0} {1}".format(yaml, options)
+            user_cmd = "-f {0} {1} -p {2}".format(yaml, options, self.scripts_path)
 
         if silent:
             user_cmd += " --silent"
@@ -201,6 +201,8 @@ class CStlBasic_Test(functional_general_test.CGeneralFunctional_Test):
              ["multi_burst_2st_1000pkt.py","-m 1 -l 100",True],
              ["pcap.py", "-m 1", True],
              ["pcap_with_vm.py", "-m 1", True],
+             ["flow_stats.py", "-m 1 -l 1", True],
+             ["flow_stats_latency.py", "-m 1 -l 1", True],
 
             # YAML test
              ["yaml/burst_1000_pkt.yaml","-m 1 -l 100",True],
diff --git a/scripts/automation/regression/stateful_tests/trex_general_test.py b/scripts/automation/regression/stateful_tests/trex_general_test.py
index 86fe93e7..82b1d9d1 100755
--- a/scripts/automation/regression/stateful_tests/trex_general_test.py
+++ b/scripts/automation/regression/stateful_tests/trex_general_test.py
@@ -80,9 +80,9 @@ class CTRexGeneral_Test(unittest.TestCase):
                 device_cfg.set_tftp_config(CTRexScenario.router_cfg['tftp_config_dict'])
                 CTRexScenario.router.load_platform_data_from_file(device_cfg)
                 CTRexScenario.router.launch_connection(device_cfg)
-                running_image = CTRexScenario.router.get_running_image_details()['image']
-                print('Current router image: %s' % running_image)
                 if CTRexScenario.router_cfg['forceImageReload']:
+                    running_image = CTRexScenario.router.get_running_image_details()['image']
+                    print('Current router image: %s' % running_image)
                     needed_image = device_cfg.get_image_name()
                     if not CTRexScenario.router.is_image_matches(needed_image):
                         print('Setting router image: %s' % needed_image)
@@ -96,7 +96,7 @@ class CTRexGeneral_Test(unittest.TestCase):
                             self.fail('Unable to set router image: %s, current image is: %s' % (needed_image, running_image))
                     else:
                         print('Matches needed image: %s' % needed_image)
-                CTRexScenario.router_image = running_image
+                    CTRexScenario.router_image = running_image
 
             if self.modes:
                 print(termstyle.green('\t!!!\tRunning with modes: %s, not suitable tests will be skipped.\t!!!' % list(self.modes)))
diff --git a/scripts/automation/regression/stateless_tests/stl_client_test.py b/scripts/automation/regression/stateless_tests/stl_client_test.py
index c5e0c3c6..ed125cde 100644
--- a/scripts/automation/regression/stateless_tests/stl_client_test.py
+++ b/scripts/automation/regression/stateless_tests/stl_client_test.py
@@ -46,6 +46,8 @@ class STLClient_Test(CStlGeneral_Test):
 
     @classmethod
     def tearDownClass(cls):
+        if CTRexScenario.stl_init_error:
+            return
         # connect back at end of tests
         if not cls.is_connected():
             CTRexScenario.stl_trex.connect()
diff --git a/scripts/automation/regression/stateless_tests/stl_general_test.py b/scripts/automation/regression/stateless_tests/stl_general_test.py
index 4ff1eec9..82738f96 100644
--- a/scripts/automation/regression/stateless_tests/stl_general_test.py
+++ b/scripts/automation/regression/stateless_tests/stl_general_test.py
@@ -85,7 +85,8 @@ class STLBasic_Test(CStlGeneral_Test):
     @nottest
     def test_connectivity(self):
         if not self.is_loopback:
-            CTRexScenario.router.load_clean_config()
+            if CTRexScenario.router_cfg['forceImageReload']:
+                CTRexScenario.router.load_clean_config()
             CTRexScenario.router.configure_basic_interfaces()
             CTRexScenario.router.config_pbr(mode = "config")
 
diff --git a/scripts/automation/regression/stateless_tests/stl_rx_test.py b/scripts/automation/regression/stateless_tests/stl_rx_test.py
index 9725e821..84f32081 100644
--- a/scripts/automation/regression/stateless_tests/stl_rx_test.py
+++ b/scripts/automation/regression/stateless_tests/stl_rx_test.py
@@ -3,6 +3,8 @@ from .stl_general_test import CStlGeneral_Test, CTRexScenario
 from trex_stl_lib.api import *
 import os, sys
 
+ERROR_LATENCY_TOO_HIGH = 1
+
 class STLRX_Test(CStlGeneral_Test):
     """Tests for RX feature"""
 
@@ -11,8 +13,8 @@ class STLRX_Test(CStlGeneral_Test):
         #    self.skip('This test makes trex08 and trex09 sick. Fix those ASAP.')
         if self.is_virt_nics:
             self.skip('Skip this for virtual NICs for now')
-        per_driver_params = {"rte_vmxnet3_pmd": [1, 50, 1], "rte_ixgbe_pmd": [30, 5000, 1], "rte_i40e_pmd": [80, 5000, 1],
-                             "rte_igb_pmd": [80, 500, 1], "rte_em_pmd": [1, 50, 1], "rte_virtio_pmd": [1, 50, 1]}
+        per_driver_params = {"rte_vmxnet3_pmd": [1, 50, 1,False], "rte_ixgbe_pmd": [30, 5000, 1,True,200,400], "rte_i40e_pmd": [80, 5000, 1,True,100,250],
+                             "rte_igb_pmd": [80, 500, 1,False], "rte_em_pmd": [1, 50, 1,False], "rte_virtio_pmd": [1, 50, 1,False]}
 
         CStlGeneral_Test.setUp(self)
         assert 'bi' in CTRexScenario.stl_ports_map
@@ -22,6 +24,9 @@ class STLRX_Test(CStlGeneral_Test):
         self.tx_port, self.rx_port = CTRexScenario.stl_ports_map['bi'][0]
 
         port_info = self.c.get_port_info(ports = self.rx_port)[0]
+        self.speed = port_info['speed']
+
+
         cap = port_info['rx']['caps']
         if "flow_stats" not in cap or "latency" not in cap:
             self.skip('port {0} does not support RX'.format(self.rx_port))
@@ -38,14 +43,55 @@ class STLRX_Test(CStlGeneral_Test):
 
         self.pkt = STLPktBuilder(pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)/('Your_paylaod_comes_here'))
         self.large_pkt = STLPktBuilder(pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)/('a'*1000))
+        self.pkt_9k = STLPktBuilder(pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)/('a'*9000))
+
+
+        drv_name=port_info['driver']
+        self.latency_9k_enable=per_driver_params[drv_name][3]
+        if self.latency_9k_enable:
+            self.latency_9k_max_average = per_driver_params[drv_name][4]
+            self.latency_9k_max_latency = per_driver_params[drv_name][5]
+
 
     @classmethod
     def tearDownClass(cls):
+        if CTRexScenario.stl_init_error:
+            return
         # connect back at end of tests
         if not cls.is_connected():
             CTRexScenario.stl_trex.connect()
 
 
+    def __verify_latency (self, latency_stats,max_latency,max_average):
+
+        error=0;
+        err_latency = latency_stats['err_cntrs']
+        latency     = latency_stats['latency']
+
+        for key in err_latency :
+            error +=err_latency[key]
+        if error !=0 :
+            pprint.pprint(err_latency)
+            tmp = 'RX pkts ERROR - one of the error is on'
+            print(tmp)
+            #assert False, tmp
+
+        if latency['average']> max_average:
+            pprint.pprint(latency_stats)
+            tmp = 'Average latency is too high {0} {1} '.format(latency['average'], max_average)
+            print(tmp)
+            return ERROR_LATENCY_TOO_HIGH
+
+        if latency['total_max']> max_latency:
+            pprint.pprint(latency_stats)
+            tmp = 'Max latency is too high {0} {1} '.format(latency['total_max'], max_latency)
+            print(tmp)
+            return ERROR_LATENCY_TOO_HIGH
+
+        return 0
+
+
+
     def __verify_flow (self, pg_id, total_pkts, pkt_len, stats):
         flow_stats = stats['flow_stats'].get(pg_id)
         latency_stats = stats['latency'].get(pg_id)
@@ -78,7 +124,7 @@ class STLRX_Test(CStlGeneral_Test):
             tmp = 'TX pkts mismatch - got: {0}, expected: {1}'.format(tx_pkts, total_pkts)
             assert False, tmp
 
-        if tx_bytes != (total_pkts * pkt_len):
+        if tx_bytes != (total_pkts * (pkt_len + 4)): # + 4 for ethernet CRC
             pprint.pprint(flow_stats)
             tmp = 'TX bytes mismatch - got: {0}, expected: {1}'.format(tx_bytes, (total_pkts * pkt_len))
             assert False, tmp
@@ -90,7 +136,7 @@ class STLRX_Test(CStlGeneral_Test):
 
         if "rx_bytes" in self.cap:
             rx_bytes = flow_stats['rx_bytes'].get(self.rx_port, 0)
-            if rx_bytes != (total_pkts * pkt_len) and not self.drops_expected:
+            if rx_bytes != (total_pkts * (pkt_len + 4)) and not self.drops_expected: # +4 for ethernet CRC
                 pprint.pprint(flow_stats)
                 tmp = 'RX bytes mismatch - got: {0}, expected: {1}'.format(rx_bytes, (total_pkts * pkt_len))
                 assert False, tmp
@@ -214,6 +260,104 @@ class STLRX_Test(CStlGeneral_Test):
             assert False , '{0}'.format(e)
 
 
+
+    def __test_9k_stream(self,pgid,ports,precet,max_latency,avg_latency,duration,pkt_size):
+        my_pg_id=pgid
+        s_ports=ports;
+        all_ports=list(CTRexScenario.stl_ports_map['map'].keys());
+        if ports == None:
+            s_ports=all_ports
+        assert( type(s_ports)==list)
+
+        stream_pkt = STLPktBuilder(pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)/('a'*pkt_size))
+
+        try:
+            # reset all ports
+            self.c.reset(ports = all_ports)
+
+
+            for pid in s_ports:
+                s1  = STLStream(name = 'rx',
+                               packet = self.pkt,
+                               flow_stats = STLFlowLatencyStats(pg_id = my_pg_id+pid),
+                               mode = STLTXCont(pps = 1000))
+
+                s2 = STLStream(name = 'bulk',
+                               packet = stream_pkt,
+                               mode = STLTXCont(percentage =precet))
+
+
+                # add both streams to ports
+                self.c.add_streams([s1,s2], ports = [pid])
+
+            self.c.clear_stats()
+
+            self.c.start(ports = s_ports,duration = duration)
+            self.c.wait_on_traffic(ports = s_ports,timeout = duration+10,rx_delay_ms = 100)
+            stats = self.c.get_stats()
+
+            for pid in s_ports:
+                latency_stats = stats['latency'].get(my_pg_id+pid)
+                #pprint.pprint(latency_stats)
+                if self.__verify_latency (latency_stats,max_latency,avg_latency) !=0:
+                    return (ERROR_LATENCY_TOO_HIGH);
+
+            return 0
+
+        except STLError as e:
+            assert False , '{0}'.format(e)
+
+
+
+
+
+    # check low latency when you have stream of 9K stream 
+    def test_9k_stream(self):
+
+        if self.latency_9k_enable == False:
+           print("SKIP")
+           return
+
+        for i in range(0,5):
+            print("Iteration {0}".format(i));
+            duration=random.randint(10, 70);
+            pgid=random.randint(1, 65000);
+            pkt_size=random.randint(1000, 9000);
+            all_ports = list(CTRexScenario.stl_ports_map['map'].keys());
+
+
+            s_port=random.sample(all_ports, random.randint(1, len(all_ports)) )
+            s_port=sorted(s_port)
+            if self.speed == 40 :
+                # the NIC does not support all full rate in case both port works let's filter odd ports  
+                tmp_l=[]
+                for port  in s_port:
+                    if ((int(port) % 2) ==0):
+                        tmp_l.append(port);
+                s_port=tmp_l;
+                if len(s_port)==0:
+                    s_port=[0];
+
+            error=1;
+            for j in range(0,5):
+               print(" {4} - duration {0} pgid {1} pkt_size {2} s_port {3} ".format(duration,pgid,pkt_size,s_port,j));
+               if self.__test_9k_stream(pgid,
+                                        s_port,90,
+                                        self.latency_9k_max_latency,
+                                        self.latency_9k_max_average,
+                                        duration,
+                                        pkt_size)==0:
+                   error=0;
+                   break;
+
+            if error:
+                assert False , "Latency too high"
+            else:
+                print("===>Iteration {0} PASS {1}".format(i,j));
+
+
+
+
     
     # this test adds more and more latency streams and re-test with incremental
     def test_incremental_latency_streams (self):
diff --git a/scripts/automation/regression/trex.py b/scripts/automation/regression/trex.py
index a0a1d42d..44f2faba 100644
--- a/scripts/automation/regression/trex.py
+++ b/scripts/automation/regression/trex.py
@@ -38,6 +38,8 @@ class CTRexScenario:
     is_copied        = False
     GAManager        = None
     no_daemon        = False
+    router_image     = None
+    debug_image      = False
 
 class CTRexRunner:
     """This is an instance for generating a CTRexRunner"""
diff --git a/scripts/automation/regression/trex_unit_test.py b/scripts/automation/regression/trex_unit_test.py
index 0762fc95..915cd682 100755
--- a/scripts/automation/regression/trex_unit_test.py
+++ b/scripts/automation/regression/trex_unit_test.py
@@ -48,6 +48,17 @@ import re
 import time
 from distutils.dir_util import mkpath
 
+# override nose's strange representation of setUpClass errors
+def __suite_repr__(self):
+    if hasattr(self.context, '__module__'): # inside class, setUpClass etc.
+        class_repr = nose.suite._strclass(self.context)
+    else:                                   # outside of class, setUpModule etc.
+        class_repr = nose.suite._strclass(self.__class__)
+    return '%s.%s' % (class_repr, getattr(self.context, '__name__', self.context))
+
+nose.suite.ContextSuite.__repr__ = __suite_repr__
+nose.suite.ContextSuite.__str__  = __suite_repr__
+
 def check_trex_path(trex_path):
     if os.path.isfile('%s/trex_daemon_server' % trex_path):
         return os.path.abspath(trex_path)
@@ -132,6 +143,12 @@ class CTRexTestConfiguringPlugin(Plugin):
         parser.add_option('--no-daemon', action="store_true", default = False,
                             dest="no_daemon",
                             help="Flag that specifies to use running stl server, no need daemons.")
+        parser.add_option('--debug-image', action="store_true", default = False,
+                            dest="debug_image",
+                            help="Flag that specifies to use t-rex-64-debug as TRex executable.")
+        parser.add_option('--trex-args', action='store', default = '',
+                            dest="trex_args",
+                            help="Additional TRex arguments (--no-watchdog etc.).")
 
 
     def configure(self, options, conf):
@@ -168,9 +185,12 @@ class CTRexTestConfiguringPlugin(Plugin):
         CTRexScenario.benchmark     = self.benchmark
         CTRexScenario.modes         = set(self.modes)
         CTRexScenario.server_logs   = self.server_logs
+        CTRexScenario.debug_image   = options.debug_image
         if not self.no_daemon:
-            CTRexScenario.trex          = CTRexClient(trex_host = self.configuration.trex['trex_name'],
-                                                      verbose   = self.json_verbose)
+            CTRexScenario.trex      = CTRexClient(trex_host   = self.configuration.trex['trex_name'],
+                                                  verbose     = self.json_verbose,
+                                                  debug_image = options.debug_image,
+                                                  trex_args   = options.trex_args)
             if not CTRexScenario.trex.check_master_connectivity():
                 print('Could not connect to master daemon')
                 sys.exit(-1)
@@ -202,12 +222,12 @@ class CTRexTestConfiguringPlugin(Plugin):
             if not res:
                 print('Could not restart TRex daemon server')
                 sys.exit(-1)
+            print('Restarted.')
 
-            trex_cmds = CTRexScenario.trex.get_trex_cmds()
-            if trex_cmds:
-                if self.kill_running:
-                    CTRexScenario.trex.kill_all_trexes()
-                else:
+            if self.kill_running:
+                CTRexScenario.trex.kill_all_trexes()
+            else:
+                if CTRexScenario.trex.get_trex_cmds():
                     print('TRex is already running')
                     sys.exit(-1)
 
@@ -238,11 +258,11 @@ class CTRexTestConfiguringPlugin(Plugin):
         if self.stateful:
             CTRexScenario.trex = None
         if self.stateless:
-            if not self.no_daemon:
+            if self.no_daemon:
+                if CTRexScenario.stl_trex and CTRexScenario.stl_trex.is_connected():
+                    CTRexScenario.stl_trex.disconnect()
+            else:
                 CTRexScenario.trex.force_kill(False)
-            if CTRexScenario.stl_trex and CTRexScenario.stl_trex.is_connected():
-                CTRexScenario.stl_trex.disconnect()
-                #time.sleep(3)
             CTRexScenario.stl_trex = None
 
 
@@ -256,6 +276,9 @@ def save_setup_info():
             setup_info += 'Server: %s, Modes: %s' % (cfg.trex.get('trex_name'), cfg.trex.get('modes'))
             if cfg.router:
                 setup_info += '\nRouter: Model: %s, Image: %s' % (cfg.router.get('model'), CTRexScenario.router_image)
+            if CTRexScenario.debug_image:
+                setup_info += '\nDebug image: %s' % CTRexScenario.debug_image
+                
             with open('%s/report_%s.info' % (CTRexScenario.report_dir, CTRexScenario.setup_name), 'w') as f:
                 f.write(setup_info)
     except Exception as err:
diff --git a/scripts/automation/trex_control_plane/common/text_opts.py b/scripts/automation/trex_control_plane/common/text_opts.py
index 78a0ab1f..ab0fd2f2 100755
--- a/scripts/automation/trex_control_plane/common/text_opts.py
+++ b/scripts/automation/trex_control_plane/common/text_opts.py
@@ -61,6 +61,9 @@ def format_time (t_sec):
     if t_sec < 0:
         return "infinite"
 
+    if t_sec == 0:
+        return "zero"
+
     if t_sec < 1:
         # low numbers
         for unit in ['ms', 'usec', 'ns']:
diff --git a/scripts/automation/trex_control_plane/doc_stl/conf.py b/scripts/automation/trex_control_plane/doc_stl/conf.py
index 45738b6e..c8788ca7 100644
--- a/scripts/automation/trex_control_plane/doc_stl/conf.py
+++ b/scripts/automation/trex_control_plane/doc_stl/conf.py
@@ -15,6 +15,21 @@
 import sys
 import os
 import shlex
+import functools
+
+def no_op_wraps(func):
+    """Replaces functools.wraps in order to undo wrapping.
+
+    Can be used to preserve the decorated function's signature
+    in the documentation generated by Sphinx.
+
+    """
+    def wrapper(decorator):
+        return func
+    return wrapper
+
+functools.wraps = no_op_wraps
+
 
 
 # If extensions (or modules to document with autodoc) are in another directory,
@@ -309,4 +324,5 @@ autoclass_content = "both"
 
 # A workaround for the responsive tables always having annoying scrollbars.
 def setup(app):
-  app.add_stylesheet("no_scrollbars.css")
\ No newline at end of file
+  app.add_stylesheet("no_scrollbars.css")
+
diff --git a/scripts/automation/trex_control_plane/server/CCustomLogger.py b/scripts/automation/trex_control_plane/server/CCustomLogger.py
index a8823cea..6d3974a6 100755
--- a/scripts/automation/trex_control_plane/server/CCustomLogger.py
+++ b/scripts/automation/trex_control_plane/server/CCustomLogger.py
@@ -3,15 +3,13 @@ import sys
 import os

 import logging

 

+def prepare_dir(log_path):

+    log_dir = os.path.dirname(log_path)

+    if not os.path.exists(log_dir):

+        os.makedirs(log_dir)

 

 def setup_custom_logger(name, log_path = None):

     # first make sure path availabe

-#   if log_path is None:

-#       log_path = os.getcwd()+'/trex_log.log'

-#   else:

-#       directory = os.path.dirname(log_path)

-#       if not os.path.exists(directory):

-#           os.makedirs(directory)

     logging.basicConfig(level   = logging.INFO, 

                         format  = '%(asctime)s %(name)-10s %(module)-20s %(levelname)-8s %(message)s',

                         datefmt = '%m-%d %H:%M')

@@ -31,6 +29,7 @@ def setup_custom_logger(name, log_path = None):
 

 def setup_daemon_logger (name, log_path = None):

     # first make sure path availabe

+    prepare_dir(log_path)

     try:

         os.unlink(log_path)

     except:

diff --git a/scripts/automation/trex_control_plane/server/singleton_daemon.py b/scripts/automation/trex_control_plane/server/singleton_daemon.py
index 0a3b9c09..1784cc42 100755
--- a/scripts/automation/trex_control_plane/server/singleton_daemon.py
+++ b/scripts/automation/trex_control_plane/server/singleton_daemon.py
@@ -2,14 +2,17 @@ import errno
 import os
 import shlex
 import socket
+import signal
 import tempfile
 import types
 from subprocess import Popen
 from time import sleep
+import outer_packages
+import jsonrpclib
 
 # uses Unix sockets for determine running process.
 # (assumes used daemons will register proper socket)
-# all daemons should use -p argument as listening tcp port
+# all daemons should use -p argument as listening tcp port and check_connectivity RPC method
 class SingletonDaemon(object):
 
     # run_cmd can be function of how to run daemon or a str to run at subprocess
@@ -28,14 +31,14 @@ class SingletonDaemon(object):
 
     # returns True if daemon is running
     def is_running(self):
-        lock_socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
         try:
-            lock_socket.bind('\0' + self.tag) # the check is ~200000 faster and more reliable than checking via 'netstat' or 'ps' etc.
+            lock_socket = register_socket(self.tag) # the check is ~200000 faster and more reliable than checking via 'netstat' or 'ps' etc.
+            lock_socket.shutdown(socket.SHUT_RDWR)
             lock_socket.close()
         except socket.error: # Unix socket in use
             return True
         # Unix socket is not used, but maybe it's old version of daemon not using socket
-        return bool(self.get_pid())
+        return bool(self.get_pid_by_listening_port())
 
 
     # get pid of running daemon by registered Unix socket (most robust way)
@@ -71,59 +74,70 @@ class SingletonDaemon(object):
         if pid:
             return pid
 
-
-    # kill daemon
-    def kill(self, timeout = 5):
-        pid = self.get_pid()
-        if not pid:
-            return False
-        ret_code, stdout, stderr = run_command('kill %s' % pid) # usual kill
-        if ret_code:
-            raise Exception('Failed to run kill command for %s: %s' % (self.name, [ret_code, stdout, stderr]))
+    def kill_by_signal(self, pid, signal_name, timeout):
+        os.kill(pid, signal_name)
         poll_rate = 0.1
         for i in range(int(timeout / poll_rate)):
             if not self.is_running():
                 return True
             sleep(poll_rate)
-        ret_code, stdout, stderr = run_command('kill -9 %s' % pid) # unconditional kill
-        if ret_code:
-            raise Exception('Failed to run kill -9 command for %s: %s' % (self.name, [ret_code, stdout, stderr]))
-        for i in range(int(timeout / poll_rate)):
-            if not self.is_running():
+
+    # kill daemon, with verification
+    def kill(self, timeout = 15):
+        pid = self.get_pid()
+        if not pid:
+            raise Exception('%s is not running' % self.name)
+        # try Ctrl+C, usual kill, kill -9
+        for signal_name in [signal.SIGINT, signal.SIGTERM, signal.SIGKILL]:
+            if self.kill_by_signal(pid, signal_name, timeout):
                 return True
-            sleep(poll_rate)
         raise Exception('Could not kill %s, even with -9' % self.name)
 
+    # try connection as RPC client, return True upon success, False if fail
+    def check_connectivity(self, timeout = 15):
+        daemon = jsonrpclib.Server('http://127.0.0.1:%s/' % self.port)
+        poll_rate = 0.1
+        for i in range(int(timeout/poll_rate)):
+            try:
+                daemon.check_connectivity()
+                return True
+            except socket.error: # daemon is not up yet
+                sleep(poll_rate)
+        return False
 
     # start daemon
     # returns True if success, False if already running
-    def start(self, timeout = 5):
+    def start(self, timeout = 20):
         if self.is_running():
             raise Exception('%s is already running' % self.name)
         if not self.run_cmd:
             raise Exception('No starting command registered for %s' % self.name)
         if type(self.run_cmd) is types.FunctionType:
             self.run_cmd()
-        else:
-            with tempfile.TemporaryFile() as stdout_file, tempfile.TemporaryFile() as stderr_file:
-                proc = Popen(shlex.split('%s -p %s' % (self.run_cmd, self.port)), cwd = self.dir, close_fds = True,
-                            stdout = stdout_file, stderr = stderr_file)
-                if timeout > 0:
-                    poll_rate = 0.1
-                    for i in range(int(timeout/poll_rate)):
-                        sleep(poll_rate)
-                        if bool(proc.poll()): # process ended with error
-                            stdout_file.seek(0)
-                            stderr_file.seek(0)
-                            raise Exception('Run of %s ended unexpectfully: %s' % (self.name, [proc.returncode, stdout_file.read().decode(errors = 'replace'), stderr_file.read().decode(errors = 'replace')]))
-                        elif proc.poll() == 0: # process runs other process, and ended
-                            break
-        if self.is_running():
-            return True
-        raise Exception('%s failed to run.' % self.name)
+            return
+        with tempfile.TemporaryFile() as stdout_file, tempfile.TemporaryFile() as stderr_file:
+            proc = Popen(shlex.split('%s -p %s' % (self.run_cmd, self.port)), cwd = self.dir, close_fds = True,
+                         stdout = stdout_file, stderr = stderr_file)
+            if timeout > 0:
+                poll_rate = 0.1
+                for i in range(int(timeout/poll_rate)):
+                    if self.is_running():
+                        break
+                    sleep(poll_rate)
+                    if bool(proc.poll()): # process ended with error
+                        stdout_file.seek(0)
+                        stderr_file.seek(0)
+                        raise Exception('Run of %s ended unexpectfully: %s' % (self.name, [proc.returncode, stdout_file.read().decode(errors = 'replace'), stderr_file.read().decode(errors = 'replace')]))
+                    elif proc.poll() == 0: # process runs other process, and ended
+                        break
+            if self.is_running():
+                if self.check_connectivity():
+                    return True
+                raise Exception('Daemon process is running, but no connectivity')
+            raise Exception('%s failed to run.' % self.name)
 
     # restart the daemon
-    def restart(self, timeout = 5):
+    def restart(self, timeout = 15):
         if self.is_running():
             self.kill(timeout)
         return self.start(timeout)
@@ -135,8 +149,9 @@ def register_socket(tag):
     lock_socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
     try:
         lock_socket.bind('\0%s' % tag)
+        return lock_socket
     except socket.error:
-        raise Exception('Error: process with tag %s is already running.' % tag)
+        raise socket.error('Error: process with tag %s is already running.' % tag)
 
 # runs command
 def run_command(command, timeout = 15, cwd = None):
@@ -152,6 +167,8 @@ def run_command(command, timeout = 15, cwd = None):
             if proc.poll() is None:
                 proc.kill() # timeout
                 return (errno.ETIME, '', 'Timeout on running: %s' % command)
+        else:
+            proc.wait()
         stdout_file.seek(0)
         stderr_file.seek(0)
         return (proc.returncode, stdout_file.read().decode(errors = 'replace'), stderr_file.read().decode(errors = 'replace'))
diff --git a/scripts/automation/trex_control_plane/server/trex_launch_thread.py b/scripts/automation/trex_control_plane/server/trex_launch_thread.py
index 74ce1750..22606753 100755
--- a/scripts/automation/trex_control_plane/server/trex_launch_thread.py
+++ b/scripts/automation/trex_control_plane/server/trex_launch_thread.py
@@ -6,6 +6,7 @@ import signal
 import socket

 from common.trex_status_e import TRexStatus

 import subprocess

+import shlex

 import time

 import threading

 import logging

@@ -29,49 +30,51 @@ class AsynchronousTRexSession(threading.Thread):
         self.trexObj.zmq_dump                       = {}

 

     def run (self):

-            

-        with open(os.devnull, 'w') as DEVNULL:

-            self.time_stamps['start'] = self.time_stamps['run_time'] = time.time()

-            self.session   = subprocess.Popen("exec "+self.cmd, cwd = self.launch_path, shell=True, stdin = DEVNULL, stderr = subprocess.PIPE, preexec_fn=os.setsid)

-            logger.info("TRex session initialized successfully, Parent process pid is {pid}.".format( pid = self.session.pid ))

-            while self.session.poll() is None:  # subprocess is NOT finished

-                time.sleep(0.5)

-                if self.stoprequest.is_set():

-                    logger.debug("Abort request received by handling thread. Terminating TRex session." )

-                    os.killpg(self.session.pid, signal.SIGUSR1)

-                    self.trexObj.set_status(TRexStatus.Idle)

-                    self.trexObj.set_verbose_status("TRex is Idle")

-                    break

-

-            self.time_stamps['run_time'] = time.time() - self.time_stamps['start']

-

-            try:

-                if self.time_stamps['run_time'] < 5:

-                    logger.error("TRex run failed due to wrong input parameters, or due to readability issues.")

-                    self.trexObj.set_verbose_status("TRex run failed due to wrong input parameters, or due to readability issues.\n\nTRex command: {cmd}\n\nRun output:\n{output}".format(

-                        cmd = self.cmd, output = self.load_trex_output(self.export_path)))

-                    self.trexObj.errcode = -11

-                elif (self.session.returncode is not None and self.session.returncode != 0) or ( (self.time_stamps['run_time'] < self.duration) and (not self.stoprequest.is_set()) ):

-                    if (self.session.returncode is not None and self.session.returncode != 0):

-                        logger.debug("Failed TRex run due to session return code ({ret_code})".format( ret_code = self.session.returncode ) )

-                    elif ( (self.time_stamps['run_time'] < self.duration) and not self.stoprequest.is_set()):

-                        logger.debug("Failed TRex run due to running time ({runtime}) combined with no-stopping request.".format( runtime = self.time_stamps['run_time'] ) )

-

-                    logger.warning("TRex run was terminated unexpectedly by outer process or by the hosting OS")

-                    self.trexObj.set_verbose_status("TRex run was terminated unexpectedly by outer process or by the hosting OS.\n\nRun output:\n{output}".format(

-                        output = self.load_trex_output(self.export_path)))

-                    self.trexObj.errcode = -15

-                else:

-                    logger.info("TRex run session finished.")

-                    self.trexObj.set_verbose_status('TRex finished.')

-                    self.trexObj.errcode = None

-                    

-            finally:

-                self.trexObj.set_status(TRexStatus.Idle)

-                logger.info("TRex running state changed to 'Idle'.")

-                self.trexObj.expect_trex.clear()

-                logger.debug("Finished handling a single run of TRex.")

-                self.trexObj.zmq_dump   = None

+        try:

+            with open(self.export_path, 'w') as output_file:

+                self.time_stamps['start'] = self.time_stamps['run_time'] = time.time()

+                self.session   = subprocess.Popen(shlex.split(self.cmd), cwd = self.launch_path, stdout = output_file, preexec_fn=os.setsid, close_fds = True)

+                logger.info("TRex session initialized successfully, Parent process pid is {pid}.".format( pid = self.session.pid ))

+                while self.session.poll() is None:  # subprocess is NOT finished

+                    time.sleep(0.5)

+                    if self.stoprequest.is_set():

+                        logger.debug("Abort request received by handling thread. Terminating TRex session." )

+                        os.killpg(self.session.pid, signal.SIGUSR1)

+                        self.trexObj.set_status(TRexStatus.Idle)

+                        self.trexObj.set_verbose_status("TRex is Idle")

+                        break

+        except Exception as e:

+            logger.error(e)

+

+        self.time_stamps['run_time'] = time.time() - self.time_stamps['start']

+

+        try:

+            if self.time_stamps['run_time'] < 5:

+                logger.error("TRex run failed due to wrong input parameters, or due to readability issues.")

+                self.trexObj.set_verbose_status("TRex run failed due to wrong input parameters, or due to readability issues.\n\nTRex command: {cmd}\n\nRun output:\n{output}".format(

+                    cmd = self.cmd, output = self.load_trex_output(self.export_path)))

+                self.trexObj.errcode = -11

+            elif (self.session.returncode is not None and self.session.returncode != 0) or ( (self.time_stamps['run_time'] < self.duration) and (not self.stoprequest.is_set()) ):

+                if (self.session.returncode is not None and self.session.returncode != 0):

+                    logger.debug("Failed TRex run due to session return code ({ret_code})".format( ret_code = self.session.returncode ) )

+                elif ( (self.time_stamps['run_time'] < self.duration) and not self.stoprequest.is_set()):

+                    logger.debug("Failed TRex run due to running time ({runtime}) combined with no-stopping request.".format( runtime = self.time_stamps['run_time'] ) )

+

+                logger.warning("TRex run was terminated unexpectedly by outer process or by the hosting OS")

+                self.trexObj.set_verbose_status("TRex run was terminated unexpectedly by outer process or by the hosting OS.\n\nRun output:\n{output}".format(

+                    output = self.load_trex_output(self.export_path)))

+                self.trexObj.errcode = -15

+            else:

+                logger.info("TRex run session finished.")

+                self.trexObj.set_verbose_status('TRex finished.')

+                self.trexObj.errcode = None

+

+        finally:

+            self.trexObj.set_status(TRexStatus.Idle)

+            logger.info("TRex running state changed to 'Idle'.")

+            self.trexObj.expect_trex.clear()

+            logger.debug("Finished handling a single run of TRex.")

+            self.trexObj.zmq_dump   = None

 

     def join (self, timeout = None):

         self.stoprequest.set()

diff --git a/scripts/automation/trex_control_plane/server/trex_server.py b/scripts/automation/trex_control_plane/server/trex_server.py
index 45ef9ac1..9fe7d70b 100755
--- a/scripts/automation/trex_control_plane/server/trex_server.py
+++ b/scripts/automation/trex_control_plane/server/trex_server.py
@@ -30,9 +30,9 @@ import shlex
 import tempfile
 
 try:
-    from .singleton_daemon import register_socket
+    from .singleton_daemon import register_socket, run_command
 except:
-    from singleton_daemon import register_socket
+    from singleton_daemon import register_socket, run_command
 
 
 # setup the logger
@@ -134,6 +134,7 @@ class CTRexServer(object):
         self.server.register_function(self.add)
         self.server.register_function(self.cancel_reservation)
         self.server.register_function(self.connectivity_check)
+        self.server.register_function(self.connectivity_check, 'check_connectivity') # alias
         self.server.register_function(self.force_trex_kill)
         self.server.register_function(self.get_file)
         self.server.register_function(self.get_files_list)
@@ -164,16 +165,6 @@ class CTRexServer(object):
             self.server.shutdown()
             #self.server.server_close()
 
-    def _run_command(self, command, timeout = 15, cwd = None):
-        if timeout:
-            command = 'timeout %s %s' % (timeout, command)
-        # pipes might stuck, even with timeout
-        with tempfile.TemporaryFile() as stdout_file, tempfile.TemporaryFile() as stderr_file:
-            proc = subprocess.Popen(shlex.split(command), stdout=stdout_file, stderr=stderr_file, cwd = cwd)
-            proc.wait()
-            stdout_file.seek(0)
-            stderr_file.seek(0)
-            return (proc.returncode, stdout_file.read().decode(errors = 'replace'), stderr_file.read().decode(errors = 'replace'))
 
     # get files from Trex server and return their content (mainly for logs)
     @staticmethod
@@ -234,7 +225,7 @@ class CTRexServer(object):
         try:
             logger.info("Processing get_trex_version() command.")
             if not self.trex_version:
-                ret_code, stdout, stderr = self._run_command('./t-rex-64 --help', cwd = self.TREX_PATH, timeout = 0)
+                ret_code, stdout, stderr = run_command('./t-rex-64 --help', cwd = self.TREX_PATH)
                 search_result = re.search('\n\s*(Version\s*:.+)', stdout, re.DOTALL)
                 if not search_result:
                     raise Exception('Could not determine version from ./t-rex-64 --help')
@@ -319,7 +310,7 @@ class CTRexServer(object):
                 return False
 
             
-    def start_trex(self, trex_cmd_options, user, block_to_success = True, timeout = 40, stateless = False):
+    def start_trex(self, trex_cmd_options, user, block_to_success = True, timeout = 40, stateless = False, debug_image = False, trex_args = ''):
         with self.start_lock:
             logger.info("Processing start_trex() command.")
             if self.is_reserved():
@@ -332,7 +323,7 @@ class CTRexServer(object):
                 return Fault(-13, '')  # raise at client TRexInUseError
             
             try:
-                server_cmd_data = self.generate_run_cmd(stateless = stateless, **trex_cmd_options)
+                server_cmd_data = self.generate_run_cmd(stateless = stateless, debug_image = debug_image, trex_args = trex_args, **trex_cmd_options)
                 self.zmq_monitor.first_dump = True
                 self.trex.start_trex(self.TREX_PATH, server_cmd_data)
                 logger.info("TRex session has been successfully initiated.")
@@ -383,7 +374,7 @@ class CTRexServer(object):
     # returns list of tuples (pid, command line) of running TRex(es)
     def get_trex_cmds(self):
         logger.info('Processing get_trex_cmds() command.')
-        ret_code, stdout, stderr = self._run_command('ps -u root --format pid,comm,cmd')
+        ret_code, stdout, stderr = run_command('ps -u root --format pid,comm,cmd')
         if ret_code:
             raise Exception('Failed to determine running processes, stderr: %s' % stderr)
         trex_cmds_list = []
@@ -396,23 +387,14 @@ class CTRexServer(object):
         return trex_cmds_list
 
 
-    def kill_all_trexes(self):
+    # Silently tries to kill TRexes with given signal.
+    # Responsibility of client to verify with get_trex_cmds.
+    def kill_all_trexes(self, signal_name):
         logger.info('Processing kill_all_trexes() command.')
         trex_cmds_list = self.get_trex_cmds()
-        if not trex_cmds_list:
-            return False
         for pid, cmd in trex_cmds_list:
-            logger.info('Killing process %s %s' % (pid, cmd))
-            self._run_command('kill %s' % pid)
-            ret_code_ps, _, _ = self._run_command('ps -p %s' % pid)
-            if not ret_code_ps:
-                logger.info('Killing with -9.')
-                self._run_command('kill -9 %s' % pid)
-                ret_code_ps, _, _ = self._run_command('ps -p %s' % pid)
-                if not ret_code_ps:
-                    logger.info('Could not kill process.')
-                    raise Exception('Could not kill process %s %s' % (pid, cmd))
-        return True
+            logger.info('Killing with signal %s process %s %s' % (signal_name, pid, cmd))
+            os.kill(int(pid), signal_name)
 
 
     def wait_until_kickoff_finish (self, timeout = 40):
@@ -431,7 +413,7 @@ class CTRexServer(object):
         return self.trex.get_running_info()
 
 
-    def generate_run_cmd (self, iom = 0, export_path="/tmp/trex.txt", stateless = False, **kwargs):
+    def generate_run_cmd (self, iom = 0, export_path="/tmp/trex.txt", stateless = False, debug_image = False, trex_args = '', **kwargs):
         """ generate_run_cmd(self, iom, export_path, kwargs) -> str
 
         Generates a custom running command for the kick-off of the TRex traffic generator.
@@ -468,6 +450,8 @@ class CTRexServer(object):
                 continue
             else:
                 trex_cmd_options += (dash + '{k} {val}'.format( k = tmp_key, val =  value ))
+        if trex_args:
+            trex_cmd_options += ' %s' % trex_args
 
         if not stateless:
             if 'f' not in kwargs:
@@ -475,12 +459,12 @@ class CTRexServer(object):
             if 'd' not in kwargs:
                 raise Exception('Argument -d should be specified in stateful command')
 
-        cmd = "{nice}{run_command} --iom {io} {cmd_options} --no-key > {export}".format( # -- iom 0 disables the periodic log to the screen (not needed)
+        cmd = "{nice}{run_command}{debug_image} --iom {io} {cmd_options} --no-key".format( # -- iom 0 disables the periodic log to the screen (not needed)
             nice = '' if self.trex_nice == 0 else 'nice -n %s ' % self.trex_nice,
             run_command = self.TREX_START_CMD,
+            debug_image = '-debug' if debug_image else '',
             cmd_options = trex_cmd_options,
-            io          = iom, 
-            export = export_path )
+            io          = iom)
 
         logger.info("TREX FULL COMMAND: {command}".format(command = cmd) )
 
diff --git a/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py
index 3b01560a..a13fe31f 100755
--- a/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py
+++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py
@@ -13,6 +13,7 @@ from distutils.util import strtobool
 from collections import deque, OrderedDict
 from json import JSONDecoder
 import traceback
+import signal
 
 try:
     from . import outer_packages
@@ -39,7 +40,7 @@ class CTRexClient(object):
     This class defines the client side of the RESTfull interaction with TRex
     """
 
-    def __init__(self, trex_host, max_history_size = 100, filtered_latency_amount = 0.001, trex_daemon_port = 8090, master_daemon_port = 8091, trex_zmq_port = 4500, verbose = False):
+    def __init__(self, trex_host, max_history_size = 100, filtered_latency_amount = 0.001, trex_daemon_port = 8090, master_daemon_port = 8091, trex_zmq_port = 4500, verbose = False, debug_image = False, trex_args = ''):
         """ 
         Instantiate a TRex client object, and connecting it to listening daemon-server
 
@@ -92,42 +93,11 @@ class CTRexClient(object):
         self.decoder            = JSONDecoder()
         self.history            = jsonrpclib.history.History()
         self.master_daemon_path = "http://{hostname}:{port}/".format( hostname = self.trex_host, port = master_daemon_port )
+        self.master_daemon      = jsonrpclib.Server(self.master_daemon_path, history = self.history)
         self.trex_server_path   = "http://{hostname}:{port}/".format( hostname = self.trex_host, port = trex_daemon_port )
-        self.connect_master()
-        self.connect_server()
-
-
-    def connect_master(self):
-        '''
-        Connects to Master daemon via JsonRPC.
-        This daemon controls TRex daemon server.
-        Return true if success, false if fail
-        '''
-        try:
-            print('Connecting to Master daemon @ %s ...' % self.master_daemon_path)
-            self.master_daemon      = jsonrpclib.Server(self.master_daemon_path, history = self.history)
-            self.check_master_connectivity()
-            print('Connected to Master daemon.')
-            return True
-        except Exception as e:
-            print(e)
-            return False
-
-    def connect_server(self):
-        '''
-        Connects to TRex daemon server via JsonRPC.
-        This daemon controls TRex. (start/stop)
-        Return true if success, false if fail
-        '''
-        try:
-            print('Connecting to TRex daemon server @ %s ...' % self.trex_server_path)
-            self.server             = jsonrpclib.Server(self.trex_server_path, history = self.history)
-            self.check_server_connectivity()
-            print('Connected TRex server daemon.')
-            return True
-        except Exception as e:
-            print(e)
-            return False
+        self.server             = jsonrpclib.Server(self.trex_server_path, history = self.history)
+        self.debug_image        = debug_image
+        self.trex_args          = trex_args
 
 
     def add (self, x, y):
@@ -191,7 +161,7 @@ class CTRexClient(object):
         self.result_obj.clear_results()
         try:
             issue_time = time.time()
-            retval = self.server.start_trex(trex_cmd_options, user, block_to_success, timeout)
+            retval = self.server.start_trex(trex_cmd_options, user, block_to_success, timeout, False, self.debug_image, self.trex_args)
         except AppError as err:
             self._handle_AppError_exception(err.args[0])
         except ProtocolError:
@@ -237,7 +207,7 @@ class CTRexClient(object):
         """
         try:
             user = user or self.__default_user
-            retval = self.server.start_trex(trex_cmd_options, user, block_to_success, timeout, True)
+            retval = self.server.start_trex(trex_cmd_options, user, block_to_success, timeout, True, self.debug_image, self.trex_args)
         except AppError as err:
             self._handle_AppError_exception(err.args[0])
         except ProtocolError:
@@ -322,18 +292,28 @@ class CTRexClient(object):
         finally:
             self.prompt_verbose_data()
 
-    def kill_all_trexes(self):
+    def kill_all_trexes(self, timeout = 15):
         """
         Kills running TRex processes (if exists) on the server, not only owned by current daemon.
         Raises exception upon error killing.
 
         :return: 
-            + **True** if any process killed 
+            + **True** if processes killed/not running
             + **False** otherwise.
 
         """
         try:
-            return self.server.kill_all_trexes()
+            poll_rate = 0.1
+            # try Ctrl+C, usual kill, -9
+            for signal_name in [signal.SIGINT, signal.SIGTERM, signal.SIGKILL]:
+                self.server.kill_all_trexes(signal_name)
+                for i in range(int(timeout / poll_rate)):
+                    if not self.get_trex_cmds():
+                        return True
+                    sleep(poll_rate)
+            if self.get_trex_cmds():
+                return False
+            return True
         except AppError as err:
             self._handle_AppError_exception(err.args[0])
         finally:
@@ -1518,8 +1498,15 @@ class CTRexResult(object):
         return result
 
 
-
-
 if __name__ == "__main__":
-    pass
+    c = CTRexClient('127.0.0.1')
+    print('restarting daemon')
+    c.restart_trex_daemon()
+    print('kill any running')
+    c.kill_all_trexes()
+    print('start')
+    c.start_stateless()
+    print('sleep')
+    time.sleep(5)
+    print('done')
 
diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_flow_latency_stats.py b/scripts/automation/trex_control_plane/stl/examples/stl_flow_latency_stats.py
index 86b91728..ac0e212b 100644
--- a/scripts/automation/trex_control_plane/stl/examples/stl_flow_latency_stats.py
+++ b/scripts/automation/trex_control_plane/stl/examples/stl_flow_latency_stats.py
@@ -114,7 +114,7 @@ def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len, bw):
     else:
         print("TX pkts match   - {0}".format(tx_pkts))
 
-    if tx_bytes != (total_pkts * pkt_len):
+    if tx_bytes != (total_pkts * (pkt_len + 4)): # +4 for ethernet CRC
         print("TX bytes mismatch - got: {0}, expected: {1}".format(tx_bytes, (total_pkts * pkt_len)))
         pprint.pprint(flow_stats)
         return False
diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_pcap.py b/scripts/automation/trex_control_plane/stl/examples/stl_pcap.py
index eae0f18b..98af6134 100644
--- a/scripts/automation/trex_control_plane/stl/examples/stl_pcap.py
+++ b/scripts/automation/trex_control_plane/stl/examples/stl_pcap.py
@@ -39,14 +39,12 @@ def inject_pcap (pcap_file, server, port, loop_count, ipg_usec, use_vm, remove_f
         c.reset(ports = [port])
 
         c.clear_stats()
-        d = c.push_pcap(pcap_file,
+        c.push_pcap(pcap_file,
                         ipg_usec = ipg_usec,
                         count = loop_count,
                         vm = vm,
                         packet_hook = packet_hook)
 
-        STLSim().run(d, outfile = 'test.cap')
-
         c.wait_on_traffic()
 
 
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/__init__.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/__init__.py
index ba9459c1..c6e14df3 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/__init__.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/__init__.py
@@ -1,7 +1,7 @@
 import sys
 
 if sys.version_info < (2, 7):
-    print("\n**** TRex STL pacakge requires Python version >= 2.7 ***\n")
+    print("\n**** TRex STL package requires Python version >= 2.7 ***\n")
     exit(-1)
 
 from . import trex_stl_ext
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
index 70f38bb0..38a18d16 100755
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
@@ -1257,18 +1257,161 @@ class STLClient(object):
 
 
     # get stats
-    def get_stats (self, ports = None, async_barrier = True):
+    def get_stats (self, ports = None, sync_now = True):
+        """
+        Return dictionary containing statistics information gathered from the server.
+
+        :parameters:
+
+          ports - List of ports to retreive stats on.
+                  If None, assume the request is for all acquired ports.
+
+          sync_now - Boolean - If true, create a call to the server to get latest stats, and wait for result to arrive. Otherwise, return last stats saved in client cache.
+                            Downside of putting True is a slight delay (few 10th msecs) in getting the result. For practical uses, value should be True.
+        :return:
+            Statistics dictionary of dictionaries with the following format:
+
+            ===============================  ===============
+            key                               Meaning
+            ===============================  ===============
+            :ref:`numbers (0,1,..<total>`    Statistcs per port number
+            :ref:`total <total>`             Sum of port statistics
+            :ref:`flow_stats <flow_stats>`   Per flow statistics
+            :ref:`global <global>`           Global statistics
+            :ref:`latency <latency>`         Per flow statistics regarding flow latency
+            ===============================  ===============
+
+            Below is description of each of the inner dictionaries.
+
+            .. _total:
+
+            **total** and per port statistics contain dictionary with following format.
+
+            ===============================  ===============
+            key                               Meaning
+            ===============================  ===============
+            ibytes                           Number of input bytes
+            ierrors                          Number of input errors
+            ipackets                         Number of input packets
+            obytes                           Number of output bytes
+            oerrors                          Number of output errors
+            opackets                         Number of output packets
+            rx_bps                           Receive bytes per second rate
+            rx_pps                           Receive packet per second rate
+            tx_bps                           Transmit bytes per second rate
+            tx_pps                           Transmit packet per second rate
+            ===============================  ===============
+
+            .. _flow_stats:
+
+            **flow_stats** contains dictionaries per packet group id (pg id). Each one with the following structure.
+
+            =================   ===============
+            key                 Meaning
+            =================   ===============
+            rx_bps              Receivd bytes per second rate
+            rx_bps_l1           Receivd bytes per second rate, including layer one
+            rx_bytes            Total number of received bytes
+            rx_pkts             Total number of received packets
+            rx_pps              Received packets per second
+            tx_bps              Transmitted bytes per second rate
+            tx_bps_l1           Transmitted bytes per second rate, including layer one
+            tx_bytes            Total number of sent bytes
+            tx_pkts             Total number of sent packets
+            tx_pps              Transmit packets per second
+            =================   ===============
+
+            .. _global:
+
+            **global**
+
+            =================   ===============
+            key                 Meaning
+            =================   ===============
+            bw_per_core         Estimated byte rate Trex can support per core. This is calculated by extrapolation of current rate and load on transmitting cores.
+            cpu_util            Estimate of the average utilization percentage of the transimitting cores
+            queue_full          Total number of packets we could not transmit because NIC TX queue was full. This usually indicates that the rate we trying to transmit is too high for this port
+            rx_cpu_util         Estimate of the utilization percentage of the core handling RX traffic
+            rx_drop_bps         Received bytes per second drop rate
+            rx_bps              Received bytes per second rate
+            rx_pps              Received packets per second rate
+            tx_bps              Transmit bytes per second rate
+            tx_pps              Transmit packets per second rate
+            =================   ===============
+
+            .. _latency:
+
+            **latency** contains dictionary per packet group id (pg id). Each one with the following structure.
+
+            ===========================          ===============
+            key                                  Meaning
+            ===========================          ===============
+            :ref:`err_cntrs<err-cntrs>`          Counters describing errors that occured with this pg id
+            :ref:`latency<lat_inner>`            Information regarding packet latency
+            ===========================          ===============
+
+            Following are the inner dictionaries of latency
+
+            .. _err-cntrs:
+
+            **err-cntrs**
+
+            =================   ===============
+            key                 Meaning (see better explanation below the table)
+            =================   ===============
+            dropped             How many packets were dropped.
+            dup                 How many packets were duplicated.
+            out_of_order        How many packets we received out of order.
+            seq_too_high        How many events of packet with sequence number too high we saw.
+            seq_too_low         How many events of packet with sequence number too low we saw.
+            =================   ===============
+
+            For calculating packet error events, we add sequence number to each packet's payload. We decide what went wrong only according to sequence number
+            of last packet received and that of the previous packet. 'seq_too_low' and 'seq_too_high' count events we see. 'dup', 'out_of_order' and 'dropped'
+            are heuristics we apply to try and understand what happened. They will be accurate in common error scenarios.
+            We describe few scenarios below to help understand this.
+
+            Scenario 1: Received packet with seq num 10, and another one with seq num 10. We increment 'dup' and 'seq_too_low' by 1.
+
+            Scenario 2: Received pacekt with seq num 10 and then packet with seq num 15. We assume 4 packets were dropped, and increment 'dropped' by 4, and 'seq_too_high' by 1.
+            We expect next packet to arrive with sequence number 16.
+
+            Scenario 2 continue: Received packet with seq num 11. We increment 'seq_too_low' by 1. We increment 'out_of_order' by 1. We *decrement* 'dropped' by 1.
+            (We assume here that one of the packets we considered as dropped before, actually arrived out of order).
+
+
+            .. _lat_inner:
+
+            **latency**
+
+            =================   ===============
+            key                 Meaning
+            =================   ===============
+            average             Average latency over the stream lifetime (usec). Total average is computed each sampling period by following formula: <average> = <prev average>/2 + <last sampling period average>/2
+            histogram           Dictionary describing logarithmic distribution histogram of packet latencies. Keys in the dictionary represent range of latencies (in usec). Values are the total number of packets received in this latency range. For example, an entry {100:13} would mean that we saw 13 packets with latency in the range between 100 and 200 usec.
+            jitter              Jitter of latency samples, computed as described in :rfc:`3550#appendix-A.8`
+            last_max            Maximum latency measured between last two data reads from server.
+            total_max           Maximum latency measured over the stream lifetime (in usec).
+            total_min           Minimum latency measured over the stream lifetime (in usec).
+            =================   ===============
+
+
+
+        :raises:
+          None
+
+        """
         # by default use all acquired ports
         ports = ports if ports is not None else self.get_acquired_ports()
         ports = self._validate_port_list(ports)
 
         # check async barrier
-        if not type(async_barrier) is bool:
-            raise STLArgumentError('async_barrier', async_barrier)
+        if not type(sync_now) is bool:
+            raise STLArgumentError('sync_now', sync_now)
 
 
         # if the user requested a barrier - use it
-        if async_barrier:
+        if sync_now:
             rc = self.async_client.barrier()
             if not rc:
                 raise STLError(rc)
@@ -1282,7 +1425,7 @@ class STLClient(object):
 
         :parameters:
           ev_type_filter - 'info', 'warning' or a list of those
-                           default is no filter
+                           default: no filter
 
         :return:
             logged events
@@ -1720,6 +1863,11 @@ class STLClient(object):
         ports = self._validate_port_list(ports)
 
 
+        validate_type('mult', mult, basestring)
+        validate_type('force', force, bool)
+        validate_type('duration', duration, (int, float))
+        validate_type('total', total, bool)
+
         # verify multiplier
         mult_obj = parsing_opts.decode_multiplier(mult,
                                                   allow_update = False,
@@ -1727,17 +1875,6 @@ class STLClient(object):
         if not mult_obj:
             raise STLArgumentError('mult', mult)
 
-        # some type checkings
-
-        if not type(force) is bool:
-            raise STLArgumentError('force', force)
-
-        if not isinstance(duration, (int, float)):
-            raise STLArgumentError('duration', duration)
-
-        if not type(total) is bool:
-            raise STLArgumentError('total', total)
-
 
         # verify ports are stopped or force stop them
         active_ports = list(set(self.get_active_ports()).intersection(ports))
@@ -1780,11 +1917,12 @@ class STLClient(object):
 
         """
 
-        ports = ports if ports is not None else self.get_active_ports()
-        ports = self._validate_port_list(ports)
+        if ports is None:
+            ports = self.get_active_ports()
+            if not ports:
+                return
 
-        if not ports:
-            return
+        ports = self._validate_port_list(ports)
 
         self.logger.pre_cmd("Stopping traffic on port(s) {0}:".format(ports))
         rc = self.__stop(ports)
@@ -1833,6 +1971,9 @@ class STLClient(object):
         ports = ports if ports is not None else self.get_active_ports()
         ports = self._validate_port_list(ports)
 
+        validate_type('mult', mult, basestring)
+        validate_type('force', force, bool)
+        validate_type('total', total, bool)
 
         # verify multiplier
         mult_obj = parsing_opts.decode_multiplier(mult,
@@ -1841,10 +1982,6 @@ class STLClient(object):
         if not mult_obj:
             raise STLArgumentError('mult', mult)
 
-        # verify total
-        if not type(total) is bool:
-            raise STLArgumentError('total', total)
-
 
         # call low level functions
         self.logger.pre_cmd("Updating traffic on port(s) {0}:".format(ports))
@@ -2068,6 +2205,10 @@ class STLClient(object):
         ports = ports if ports is not None else self.get_acquired_ports()
         ports = self._validate_port_list(ports)
 
+        validate_type('mult', mult, basestring)
+        validate_type('duration', duration, (int, float))
+        validate_type('total', total, bool)
+
 
         # verify multiplier
         mult_obj = parsing_opts.decode_multiplier(mult,
@@ -2076,11 +2217,6 @@ class STLClient(object):
         if not mult_obj:
             raise STLArgumentError('mult', mult)
 
-
-        if not isinstance(duration, (int, float)):
-            raise STLArgumentError('duration', duration)
-
-
         self.logger.pre_cmd("Validating streams on port(s) {0}:".format(ports))
         rc = self.__validate(ports)
         self.logger.post_cmd(rc)
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_exceptions.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_exceptions.py
index 3c443ba6..b6fad865 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_exceptions.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_exceptions.py
@@ -4,6 +4,11 @@ import linecache
 
 from .utils.text_opts import *
 
+try:
+    basestring
+except NameError:
+    basestring = str
+
 # basic error for API
 class STLError(Exception):
     def __init__ (self, msg):
@@ -56,7 +61,8 @@ class STLArgumentError(STLError):
 # raised when argument type is not valid for operation
 class STLTypeError(STLError):
     def __init__ (self, arg_name, arg_type, valid_types):
-        self.msg = "Argument: '%s' invalid type: %s, expecting type(s): %s." % (arg_name, arg_type, valid_types)
+        self.msg = "Argument: '%s' invalid type: '%s', expecting type(s): %s." % (arg_name, arg_type.__name__,
+            [t.__name__ for t in valid_types] if isinstance(valid_types, tuple) else valid_types.__name__)
 
 # raised when timeout occurs
 class STLTimeoutError(STLError):
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py
index 11e80b9a..62724e64 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py
@@ -40,18 +40,11 @@ class BpSimException(Exception):
 
 # stateless simulation
 class STLSim(object):
-    def __init__ (self, bp_sim_path = None, handler = 0, port_id = 0, api_h = "dummy"):
+    def __init__ (self, bp_sim_path, handler = 0, port_id = 0, api_h = "dummy"):
 
-        if not bp_sim_path:
-            # auto find scripts
-            m = re.match(".*/trex-core", os.getcwd())
-            if not m:
-                raise STLError('cannot find BP sim path, please provide it')
-
-            self.bp_sim_path = os.path.join(m.group(0), 'scripts')
-
-        else:
-            self.bp_sim_path = bp_sim_path
+        self.bp_sim_path = os.path.abspath(bp_sim_path)
+        if not os.path.exists(self.bp_sim_path):
+            raise STLError('BP sim path %s does not exist' % self.bp_sim_path)
 
         # dummies
         self.handler = handler
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
index 990a40da..8e8388fd 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
@@ -1119,6 +1119,7 @@ class CLatencyStats(CTRexStats):
                     output[int_pg_id]['latency']['total_min'] = min_val
                 else:
                     output[int_pg_id]['latency']['total_min'] = StatNotAvailable('total_min')
+                    output[int_pg_id]['latency']['histogram'] = {}
 
         self.latest_stats = output
         return True
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py
index b4903e81..6835ea5f 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py
@@ -64,3 +64,4 @@ def list_difference (l1, l2):
 
 def is_sub_list (l1, l2):
     return set(l1) <= set(l2)
+
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py
index a435e54e..ceaf1af8 100755
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py
@@ -92,10 +92,6 @@ match_multiplier_help = """Multiplier should be passed in the following format:
 # value should be divided
 def decode_multiplier(val, allow_update = False, divide_count = 1):
 
-    # must be string
-    if not isinstance(val, str):
-        return None
-
     # do we allow updates ?  +/-
     if not allow_update:
         match = re.match("^(\d+(\.\d+)?)(bps|kbps|mbps|gbps|pps|kpps|mpps|%?)$", val)
@@ -246,26 +242,24 @@ OPTIONS_DB = {MULTIPLIER: ArgumentPack(['-m', '--multiplier'],
                                    'type': int}),
 
               PROMISCUOUS: ArgumentPack(['--prom'],
-                                        {'help': "sets port promiscuous on",
+                                        {'help': "Sets port promiscuous on",
                                          'dest': "prom",
                                          'default': None,
                                          'action': "store_true"}),
 
-
               TUNABLES: ArgumentPack(['-t'],
-                                     {'help': "sets tunable for a profile",
+                                     {'help': "Sets tunables for a profile. Example: '-t fsize=100,pg_id=7'",
+                                      'metavar': 'T1=VAL[,T2=VAL ...]',
                                       'dest': "tunables",
                                       'default': None,
                                       'type': decode_tunables}),
 
-
               NO_PROMISCUOUS: ArgumentPack(['--no_prom'],
-                                           {'help': "sets port promiscuous off",
+                                           {'help': "Sets port promiscuous off",
                                             'dest': "prom",
                                             'default': None,
                                             'action': "store_false"}),
 
-
               PORT_LIST: ArgumentPack(['--port', '-p'],
                                         {"nargs": '+',
                                          'dest':'ports',
@@ -481,4 +475,4 @@ def gen_parser(stateless_client, op_name, description, *args):
 
 
 if __name__ == "__main__":
-    pass
\ No newline at end of file
+    pass
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py
index 7e0bf9e4..26e64dae 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py
@@ -61,6 +61,9 @@ def format_time (t_sec):
     if t_sec < 0:
         return "infinite"
 
+    if t_sec == 0:
+        return "zero"
+
     if t_sec < 1:
         # low numbers
         for unit in ['ms', 'usec', 'ns']:
diff --git a/scripts/cfg/trex_advanced_cfg-10g.yaml b/scripts/cfg/trex_advanced_cfg-10g.yaml
new file mode 100644
index 00000000..4450bb5b
--- /dev/null
+++ b/scripts/cfg/trex_advanced_cfg-10g.yaml
@@ -0,0 +1,15 @@
+- port_limit    : 2         
+  version       : 2
+  #interfaces    : ["04:00.0", "04:00.1", "06:00.0", "06:00.1"]   # list of the interfaces to bind run ./dpdk_nic_bind.py --status 
+  interfaces    : ["04:00.0", "04:00.1"]   # list of the interfaces to bind run ./dpdk_nic_bind.py --status 
+  port_info       :  # set eh mac addr 
+  prefix          : setup1   # WARNING,WARNING,WARNING DON'T USE THIS IF YOU DON'T KNOW WHAT YOU ARE DOING
+  limit_memory    : 1024     # WARNING,WARNING,WARNING DON'T USE THIS IF YOU DON'T KNOW WHAT YOU ARE DOING
+  port_info       :  # set eh mac addr
+         - dest_mac        :   [0x00, 0xe0, 0xed, 0x5d, 0x84, 0x65]
+           src_mac         :   [0x00, 0xe0, 0xed, 0x5d, 0x84, 0x64]
+         - dest_mac        :   [0x00, 0xe0, 0xed, 0x5d, 0x84, 0x64]
+           src_mac         :   [0x00, 0xe0, 0xed, 0x5d, 0x84, 0x65]
+
+
+
diff --git a/scripts/cfg/trex_advanced_dont_use_x710-card1.yaml b/scripts/cfg/trex_advanced_dont_use_x710-card1.yaml
new file mode 100644
index 00000000..2a9f1c94
--- /dev/null
+++ b/scripts/cfg/trex_advanced_dont_use_x710-card1.yaml
@@ -0,0 +1,23 @@
+- port_limit    : 2
+  version       : 2
+  interfaces    : ["06:00.0", "06:00.1"]   
+  prefix          : setup2 # WARNING,WARNING,WARNING DON'T USE THIS IF YOU DON'T KNOW WHAT YOU ARE DOING
+  limit_memory    : 1024   # WARNING,WARNING,WARNING DON'T USE THIS IF YOU DON'T KNOW WHAT YOU ARE DOING
+  zmq_pub_port    : 4510   # WARNING,WARNING,WARNING DON'T USE THIS IF YOU DON'T KNOW WHAT YOU ARE DOING
+  zmq_rpc_port    : 4511   # WARNING,WARNING,WARNING DON'T USE THIS IF YOU DON'T KNOW WHAT YOU ARE DOING
+  port_info       :  # set eh mac addr
+         - dest_mac        :   [0x00, 0xe0, 0xed, 0x5d, 0x84, 0x65]
+           src_mac         :   [0x00, 0xe0, 0xed, 0x5d, 0x84, 0x64]
+         - dest_mac        :   [0x00, 0xe0, 0xed, 0x5d, 0x84, 0x64]
+           src_mac         :   [0x00, 0xe0, 0xed, 0x5d, 0x84, 0x65]
+         - dest_mac        :   [0x00, 0xe0, 0xed, 0x5d, 0x84, 0x67]
+           src_mac         :   [0x00, 0xe0, 0xed, 0x5d, 0x84, 0x66]
+         - dest_mac        :   [0x00, 0xe0, 0xed, 0x5d, 0x84, 0x66]
+           src_mac         :   [0x00, 0xe0, 0xed, 0x5d, 0x84, 0x67]
+  platform :
+        master_thread_id  : 4
+        latency_thread_id : 7
+        dual_if   :
+             - socket   : 0
+               threads  : [5,6]
+
diff --git a/scripts/exp/flow_stats.pcap b/scripts/exp/flow_stats.pcap
new file mode 100644
index 00000000..267eeaab
--- /dev/null
+++ b/scripts/exp/flow_stats.pcap
diff --git a/scripts/exp/flow_stats_latency.pcap b/scripts/exp/flow_stats_latency.pcap
new file mode 100644
index 00000000..89d27701
--- /dev/null
+++ b/scripts/exp/flow_stats_latency.pcap
diff --git a/scripts/gdb_script.txt b/scripts/gdb_script.txt
new file mode 100644
index 00000000..d6a7d22c
--- /dev/null
+++ b/scripts/gdb_script.txt
@@ -0,0 +1,3 @@
+run
+bt full
+
diff --git a/scripts/gdb_script1.txt b/scripts/gdb_script1.txt
new file mode 100644
index 00000000..b6854743
--- /dev/null
+++ b/scripts/gdb_script1.txt
@@ -0,0 +1,16 @@
+run
+set logging overwrite on
+set logging file gdb.bt
+set logging on 
+set pagination off 
+echo backtrace
+bt 
+bt full
+gcore
+info registers
+set logging off
+quit
+
+
+
+
diff --git a/scripts/master_daemon.py b/scripts/master_daemon.py
index 0b1b7363..aa49f207 100755
--- a/scripts/master_daemon.py
+++ b/scripts/master_daemon.py
@@ -9,15 +9,15 @@ from collections import OrderedDict
 from argparse import *
 from time import time, sleep
 from glob import glob
+import signal
 
 sys.path.append(os.path.join('automation', 'trex_control_plane', 'server'))
+import CCustomLogger
 import outer_packages
 from singleton_daemon import SingletonDaemon, register_socket, run_command
 from jsonrpclib.SimpleJSONRPCServer import SimpleJSONRPCServer
 import termstyle
 
-logging.basicConfig(level = logging.FATAL) # keep quiet
-
 ### Server functions ###
 
 def check_connectivity():
@@ -83,33 +83,52 @@ def start_master_daemon():
     proc.start()
     for i in range(50):
         if master_daemon.is_running():
-            print(termstyle.green('Master daemon is started'))
-            os._exit(0)
+            return True
         sleep(0.1)
-    fail(termstyle.red('Master daemon failed to run'))
-
+    fail(termstyle.red('Master daemon failed to run. Please look in log: %s' % logging_file))
+
+def set_logger():
+    log_dir = os.path.dirname(logging_file)
+    if not os.path.exists(log_dir):
+        os.makedirs(log_dir)
+    if os.path.exists(logging_file):
+        if os.path.exists(logging_file_bu):
+            os.unlink(logging_file_bu)
+        os.rename(logging_file, logging_file_bu)
+    CCustomLogger.setup_daemon_logger('Master daemon', logging_file)
 
 def start_master_daemon_func():
-    register_socket(master_daemon.tag)
-    server = SimpleJSONRPCServer(('0.0.0.0', master_daemon.port))
-    print('Started master daemon (port %s)' % master_daemon.port)
-    server.register_function(add)
-    server.register_function(check_connectivity)
-    server.register_function(get_trex_path)
-    server.register_function(update_trex)
-    # trex_daemon_server
-    server.register_function(trex_daemon_server.is_running, 'is_trex_daemon_running')
-    server.register_function(trex_daemon_server.restart, 'restart_trex_daemon')
-    server.register_function(trex_daemon_server.start, 'start_trex_daemon')
-    server.register_function(trex_daemon_server.stop, 'stop_trex_daemon')
-    # stl rpc proxy
-    server.register_function(stl_rpc_proxy.is_running, 'is_stl_rpc_proxy_running')
-    server.register_function(stl_rpc_proxy.restart, 'restart_stl_rpc_proxy')
-    server.register_function(stl_rpc_proxy.start, 'start_stl_rpc_proxy')
-    server.register_function(stl_rpc_proxy.stop, 'stop_stl_rpc_proxy')
-    server.register_function(server.funcs.keys, 'get_methods') # should be last
-    server.serve_forever()
+    try:
+        set_logger()
+        register_socket(master_daemon.tag)
+        server = SimpleJSONRPCServer(('0.0.0.0', master_daemon.port))
+        logging.info('Started master daemon (port %s)' % master_daemon.port)
+        server.register_function(add)
+        server.register_function(check_connectivity)
+        server.register_function(get_trex_path)
+        server.register_function(update_trex)
+        # trex_daemon_server
+        server.register_function(trex_daemon_server.is_running, 'is_trex_daemon_running')
+        server.register_function(trex_daemon_server.restart, 'restart_trex_daemon')
+        server.register_function(trex_daemon_server.start, 'start_trex_daemon')
+        server.register_function(trex_daemon_server.stop, 'stop_trex_daemon')
+        # stl rpc proxy
+        server.register_function(stl_rpc_proxy.is_running, 'is_stl_rpc_proxy_running')
+        server.register_function(stl_rpc_proxy.restart, 'restart_stl_rpc_proxy')
+        server.register_function(stl_rpc_proxy.start, 'start_stl_rpc_proxy')
+        server.register_function(stl_rpc_proxy.stop, 'stop_stl_rpc_proxy')
+        server.register_function(server.funcs.keys, 'get_methods') # should be last
+        signal.signal(signal.SIGTSTP, stop_handler) # ctrl+z
+        signal.signal(signal.SIGTERM, stop_handler) # kill
+        server.serve_forever()
+    except KeyboardInterrupt:
+        logging.info('Ctrl+C')
+    except Exception as e:
+        logging.error('Closing due to error: %s' % e)
 
+def stop_handler(signalnum, *args, **kwargs):
+    logging.info('Got signal %s, exiting.' % signalnum)
+    sys.exit(0)
 
 # returns True if given path is under current dir or /tmp
 def _check_path_under_current_or_temp(path):
@@ -170,8 +189,10 @@ stl_rpc_proxy      = SingletonDaemon('Stateless RPC proxy', 'trex_stl_rpc_proxy'
 trex_daemon_server = SingletonDaemon('TRex daemon server', 'trex_daemon_server', args.trex_daemon_port, './trex_daemon_server start', args.trex_dir)
 master_daemon      = SingletonDaemon('Master daemon', 'trex_master_daemon', args.master_port, start_master_daemon) # add ourself for easier check if running, kill etc.
 
-daemons_by_name = {}
 tmp_dir = '/tmp/trex-tmp'
+logging_file = '/var/log/trex/master_daemon.log'
+logging_file_bu = '/var/log/trex/master_daemon.log_bu'
+os.chdir('/')
 
 if not _check_path_under_current_or_temp(args.trex_dir):
     raise Exception('Only allowed to use path under /tmp or current directory')
@@ -182,6 +203,7 @@ if not os.path.exists(args.trex_dir):
     os.makedirs(args.trex_dir)
     os.chmod(args.trex_dir, 0o777)
 elif args.allow_update:
+    print('Due to allow updates flag, setting mode 777 on given directory')
     os.chmod(args.trex_dir, 0o777)
 
 if not os.path.exists(tmp_dir):
@@ -203,9 +225,13 @@ if args.action != 'show':
         print(termstyle.red(e))
         sys.exit(1)
 
-# prints running status
-if daemon.is_running():
-    print(termstyle.green('%s is running' % daemon.name))
+passive = {'start': 'started', 'restart': 'restarted', 'stop': 'stopped', 'show': 'running'}
+
+if args.action in ('show', 'start', 'restart') and daemon.is_running() or \
+    args.action == 'stop' and not daemon.is_running():
+    print(termstyle.green('%s is %s' % (daemon.name, passive[args.action])))
+    os._exit(0)
 else:
-    print(termstyle.red('%s is NOT running' % daemon.name))
+    print(termstyle.red('%s is NOT %s' % (daemon.name, passive[args.action])))
+    os._exit(-1)
 
diff --git a/scripts/simple_start_server.py b/scripts/simple_start_server.py
new file mode 100644
index 00000000..2a3908cb
--- /dev/null
+++ b/scripts/simple_start_server.py
@@ -0,0 +1,150 @@
+#!/usr/bin/python
+import os
+import sys
+from time import time, sleep
+import shlex
+import threading
+import subprocess
+import multiprocessing
+import tempfile
+import fnmatch
+
+
+sys.path.append('automation/trex_control_plane/stl')
+from trex_stl_lib.api import *
+
+def run_server(command):
+    return subprocess.Popen(shlex.split(command), stdout = subprocess.PIPE, stderr = subprocess.PIPE, close_fds = True)
+
+
+def run_command(command, timeout = 15, cwd = None):
+    # pipes might stuck, even with timeout
+    with tempfile.TemporaryFile() as stdout_file, tempfile.TemporaryFile() as stderr_file:
+        proc = subprocess.Popen(shlex.split(command), stdout = stdout_file, stderr = stderr_file, cwd = cwd, close_fds = True)
+        if timeout > 0:
+            poll_rate = 0.1
+            for i in range(int(timeout/poll_rate)):
+                sleep(poll_rate)
+                if proc.poll() is not None: # process stopped
+                    break
+            if proc.poll() is None:
+                proc.kill() # timeout
+                return (errno.ETIME, '', 'Timeout on running: %s' % command)
+        else:
+            proc.wait()
+        stdout_file.seek(0)
+        stderr_file.seek(0)
+        return (proc.returncode, stdout_file.read().decode(errors = 'replace'), stderr_file.read().decode(errors = 'replace'))
+
+def get_trex_cmds():
+    ret_code, stdout, stderr = run_command('ps -u root --format pid,comm,cmd')
+    if ret_code:
+        raise Exception('Failed to determine running processes, stderr: %s' % stderr)
+    trex_cmds_list = []
+    for line in stdout.splitlines():
+        pid, proc_name, full_cmd = line.strip().split(' ', 2)
+        pid = pid.strip()
+        full_cmd = full_cmd.strip()
+        if proc_name.find('t-rex-64') >= 0:
+            trex_cmds_list.append((pid, full_cmd))
+        else:
+            if full_cmd.find('t-rex-64') >= 0:
+                trex_cmds_list.append((pid, full_cmd))
+
+    return trex_cmds_list
+
+def is_any_core ():
+    ret_code, stdout, stderr = run_command('ls')
+    assert(ret_code==0);
+    l= stdout.split()
+    for file in l:
+         if fnmatch.fnmatch(file, 'core.*'):
+            return True
+    return False
+
+
+def kill_all_trexes():
+    trex_cmds_list = get_trex_cmds()
+    if not trex_cmds_list:
+        return False
+    for pid, cmd in trex_cmds_list:
+        run_command('kill %s' % pid)
+        ret_code_ps, _, _ = run_command('ps -p %s' % pid)
+        if not ret_code_ps:
+            run_command('kill -9 %s' % pid)
+            ret_code_ps, _, _ = run_command('ps -p %s' % pid)
+            if not ret_code_ps:
+                pass;
+    return True
+
+def term_all_trexes():
+    trex_cmds_list = get_trex_cmds()
+    if not trex_cmds_list:
+        return False
+    for pid, cmd in trex_cmds_list:
+        print pid
+        run_command('kill -INT %s' % pid)
+    return True
+
+
+
+def run_one_iter ():
+    try:
+       server = run_server('./t-rex-64-debug-gdb-bt  -i -c 4 --iom 0')
+    
+       print "sleep 1 sec"
+       time.sleep(1);
+       crash=True;
+    
+       if True:
+           c = STLClient()
+           print 'Connecting to server'
+           c.connect()
+           print 'Connected'
+            
+           print 'Mapping'
+           print 'Map: %s' % stl_map_ports(c)
+           c.disconnect()
+           crash=False;
+
+    except Exception as e:
+       print(e)
+    finally :
+        if crash:
+            print "Crash seen, wait for the info"
+            # wait the process to make the core file 
+            loop=0;
+            while True:
+                if server.poll() is not None: # server ended
+                      print 'Server stopped.\nReturn code: %s\nStderr: %s\nStdout: %s' % (server.returncode, server.stdout.read().decode(errors = 'replace'), server.stderr.read().decode(errors = 'replace'))
+                      break;
+                time.sleep(1);
+                loop=loop+1;
+                if loop >600:
+                    print "Timeout on crash!!"
+                    break;
+            return 1
+        else:
+           print "kill process ",server.pid
+           term_all_trexes();
+           kill_all_trexes();
+           return 0
+
+
+def loop_inter ():
+    kill_all_trexes()
+    cnt=0;
+    while True:
+
+        print (time.strftime("%H:%M:%S")),
+        print "Iter",cnt
+        ret=run_one_iter ()
+        if ret==1:
+            break;
+        cnt=cnt+1;
+        if is_any_core ():
+            print "stop due to core file"
+            break;
+
+loop_inter ()
+
diff --git a/scripts/stl/burst_3st_1000pkt.py b/scripts/stl/burst_3st_1000pkt.py
index 8fcdca57..88a30c84 100644
--- a/scripts/stl/burst_3st_1000pkt.py
+++ b/scripts/stl/burst_3st_1000pkt.py
@@ -10,27 +10,27 @@ class STLS1(object):
 
     def create_stream (self):
 
-        # create a base packet and pad it to size
-        size = self.fsize - 4; # no FCS
+        # Create base packet and pad it to size
+        size = self.fsize - 4; # HW will add 4 bytes ethernet FCS
         base_pkt =  Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)
         base_pkt1 =  Ether()/IP(src="16.0.0.2",dst="48.0.0.1")/UDP(dport=12,sport=1025)
         base_pkt2 =  Ether()/IP(src="16.0.0.3",dst="48.0.0.1")/UDP(dport=12,sport=1025)
         pad = max(0, size - len(base_pkt)) * 'x'
 
 
-        return STLProfile( [ STLStream( isg = 10.0, # star in delay 
+        return STLProfile( [ STLStream( isg = 10.0, # start in delay 
                                         name    ='S0',
                                         packet = STLPktBuilder(pkt = base_pkt/pad),
                                         mode = STLTXSingleBurst( pps = 10, total_pkts = self.burst_size),
                                         next = 'S1'), # point to next stream 
 
-                             STLStream( self_start = False, # stream is  disabled enable trow S0
+                             STLStream( self_start = False,  # Stream is disabled. Will run because it is pointed from S0
                                         name    ='S1',
                                         packet  = STLPktBuilder(pkt = base_pkt1/pad),
                                         mode    = STLTXSingleBurst( pps = 10, total_pkts = self.burst_size),
                                         next    = 'S2' ),
 
-                             STLStream(  self_start = False, # stream is  disabled enable trow S0
+                             STLStream(  self_start = False, # Stream is disabled. Will run because it is pointed from S1
                                          name   ='S2',
                                          packet = STLPktBuilder(pkt = base_pkt2/pad),
                                          mode = STLTXSingleBurst( pps = 10, total_pkts = self.burst_size )
diff --git a/scripts/stl/burst_3st_600pkt.py b/scripts/stl/burst_3st_600pkt.py
index 978c8920..b81f256b 100644
--- a/scripts/stl/burst_3st_600pkt.py
+++ b/scripts/stl/burst_3st_600pkt.py
@@ -9,27 +9,27 @@ class STLS1(object):
 
     def create_stream (self):
 
-        # create a base packet and pad it to size
-        size = self.fsize - 4; # no FCS
+        # Create base packet and pad it to size
+        size = self.fsize - 4; # HW will add 4 bytes ethernet FCS
         base_pkt =  Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)
         base_pkt1 =  Ether()/IP(src="16.0.0.2",dst="48.0.0.1")/UDP(dport=12,sport=1025)
         base_pkt2 =  Ether()/IP(src="16.0.0.3",dst="48.0.0.1")/UDP(dport=12,sport=1025)
         pad = max(0, size - len(base_pkt)) * 'x'
 
 
-        return STLProfile( [ STLStream( isg = 10.0, # star in delay 
+        return STLProfile( [ STLStream( isg = 10.0, # start in delay 
                                         name    ='S0',
                                         packet = STLPktBuilder(pkt = base_pkt/pad),
                                         mode = STLTXSingleBurst( pps = 10, total_pkts = 10),
                                         next = 'S1'), # point to next stream 
 
-                             STLStream( self_start = False, # stream is  disabled enable trow S0
+                             STLStream( self_start = False, # Stream is disabled. Will run because it is pointed from S0
                                         name    ='S1',
                                         packet  = STLPktBuilder(pkt = base_pkt1/pad),
                                         mode    = STLTXSingleBurst( pps = 10, total_pkts = 20),
                                         next    = 'S2' ),
 
-                             STLStream(  self_start = False, # stream is  disabled enable trow S0
+                             STLStream(  self_start = False, # Stream is disabled. Will run because it is pointed from S1
                                          name   ='S2',
                                          packet = STLPktBuilder(pkt = base_pkt2/pad),
                                          mode = STLTXSingleBurst( pps = 10, total_pkts = 30 )
diff --git a/scripts/stl/burst_3st_loop_x_times.py b/scripts/stl/burst_3st_loop_x_times.py
index 175b8315..ec217e9f 100644
--- a/scripts/stl/burst_3st_loop_x_times.py
+++ b/scripts/stl/burst_3st_loop_x_times.py
@@ -9,27 +9,27 @@ class STLS1(object):
 
     def create_stream (self):
 
-        # create a base packet and pad it to size
-        size = self.fsize - 4; # no FCS
+        # Create base packet and pad it to size
+        size = self.fsize - 4; # HW will add 4 bytes ethernet FCS
         base_pkt =  Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)
         base_pkt1 =  Ether()/IP(src="16.0.0.2",dst="48.0.0.1")/UDP(dport=12,sport=1025)
         base_pkt2 =  Ether()/IP(src="16.0.0.3",dst="48.0.0.1")/UDP(dport=12,sport=1025)
         pad = max(0, size - len(base_pkt)) * 'x'
 
 
-        return STLProfile( [ STLStream( isg = 10.0, # star in delay 
+        return STLProfile( [ STLStream( isg = 10.0, # start in delay 
                                         name    ='S0',
                                         packet = STLPktBuilder(pkt = base_pkt/pad),
                                         mode = STLTXSingleBurst( pps = 10, total_pkts = 1),
                                         next = 'S1'), # point to next stream 
 
-                             STLStream( self_start = False, # stream is  disabled enable trow S0
+                             STLStream( self_start = False, # Stream is disabled. Will run because it is pointed from S0
                                         name    ='S1',
                                         packet  = STLPktBuilder(pkt = base_pkt1/pad),
                                         mode    = STLTXSingleBurst( pps = 10, total_pkts = 2),
                                         next    = 'S2' ),
 
-                             STLStream(  self_start = False, # stream is  disabled enable trow S0
+                             STLStream(  self_start = False, # Stream is disabled. Will run because it is pointed from S1
                                          name   ='S2',
                                          packet = STLPktBuilder(pkt = base_pkt2/pad),
                                          mode = STLTXSingleBurst( pps = 10, total_pkts = 3 ),
diff --git a/scripts/stl/flow_stats.py b/scripts/stl/flow_stats.py
index cbb5ac21..a50ba848 100644
--- a/scripts/stl/flow_stats.py
+++ b/scripts/stl/flow_stats.py
@@ -1,21 +1,29 @@
 from trex_stl_lib.api import *
-import os
-
-# stream from pcap file. continues pps 10 in sec 
-CP = os.path.join(os.path.dirname(__file__))
 
 class STLS1(object):
-
-    def get_streams (self, direction = 0, **kwargs):
-        return [STLStream(packet = STLPktBuilder(pkt = os.path.join(CP, "yaml/udp_64B_no_crc.pcap")), # path relative to pwd 
-                          mode = STLTXCont(pps=1000),
-                          flow_stats = STLFlowStats(pg_id = 7)),
-
-                STLStream(packet = STLPktBuilder(pkt = os.path.join(CP, "yaml/udp_594B_no_crc.pcap")), # path relative to pwd 
-                          mode = STLTXCont(pps=5000),
-                          flow_stats = STLFlowStats(pg_id = 12))
+    """
+    Create flow stat stream of UDP packet.
+    Can specify using tunables the packet length (fsize) and packet group id (pg_id)
+    """
+    def __init__ (self):
+        self.fsize = 64
+        self.pg_id = 0
+
+    def _create_stream (self):
+        size = self.fsize - 4; # HW will add 4 bytes ethernet CRC
+        base_pkt = Ether() / IP(src = "16.0.0.1", dst = "48.0.0.1") / UDP(dport = 12, sport = 1025)
+        pad = max(0, size - len(base_pkt)) * 'x'
+        pkt = STLPktBuilder(pkt = base_pkt/pad)
+
+        return [STLStream(packet = pkt,
+                          mode = STLTXCont(pps=1),
+                          flow_stats = STLFlowStats(pg_id = self.pg_id))
                ]
 
+    def get_streams (self, fsize = 64, pg_id = 7, **kwargs):
+        self.fsize = fsize
+        self.pg_id = pg_id
+        return self._create_stream()
 
 # dynamic load - used for trex console or simulator
 def register():
diff --git a/scripts/stl/flow_stats_latency.py b/scripts/stl/flow_stats_latency.py
index e1541272..e053549e 100644
--- a/scripts/stl/flow_stats_latency.py
+++ b/scripts/stl/flow_stats_latency.py
@@ -1,21 +1,35 @@
 from trex_stl_lib.api import *
-import os
-
-# stream from pcap file. continues pps 10 in sec 
-CP = os.path.join(os.path.dirname(__file__))
 
 class STLS1(object):
-
-    def get_streams (self, direction = 0, **kwargs):
-        return [STLStream(packet = STLPktBuilder(pkt = os.path.join(CP, "yaml/udp_64B_no_crc.pcap")), # path relative to pwd 
+    """
+    Create flow stat latency stream of UDP packet.
+    Can specify using tunables the packet length (fsize) and packet group id (pg_id)
+    Since we can't have two latency streams with same pg_id, in order to be able to start this profile
+    on more than one port, we add port_id to the pg_id
+    Notice that for perfomance reasons, latency streams are not affected by -m flag, so
+    you can only change the pps value by editing the code.
+    """
+
+    def __init__ (self):
+        self.fsize = 64
+        self.pg_id = 0
+
+    def _create_stream (self):
+        size = self.fsize - 4; # HW will add 4 bytes ethernet FCS
+        base_pkt = Ether() / IP(src = "16.0.0.1", dst = "48.0.0.1") / UDP(dport = 12, sport = 1025)
+        pad = max(0, size - len(base_pkt)) * 'x'
+        pkt = STLPktBuilder(pkt = base_pkt/pad)
+
+        return [STLStream(packet = pkt,
                           mode = STLTXCont(pps=1000),
-                          flow_stats = STLFlowLatencyStats(pg_id = 1 + kwargs['port_id'])),
-
-                STLStream(packet = STLPktBuilder(pkt = os.path.join(CP, "yaml/udp_594B_no_crc.pcap")), # path relative to pwd 
-                          mode = STLTXCont(pps=5000),
-                          flow_stats = STLFlowLatencyStats(pg_id = 50 + kwargs['port_id']))
+                          flow_stats = STLFlowLatencyStats(pg_id = self.pg_id))
                ]
 
+    def get_streams (self, fsize = 64, pg_id = 7, **kwargs):
+        self.fsize = fsize
+        self.pg_id = pg_id + kwargs['port_id']
+        return self._create_stream()
+
 
 # dynamic load - used for trex console or simulator
 def register():
diff --git a/scripts/stl/imix.py b/scripts/stl/imix.py
index 82edbfa5..c9b1ff17 100644
--- a/scripts/stl/imix.py
+++ b/scripts/stl/imix.py
@@ -18,7 +18,7 @@ class STLImix(object):
 
 
     def create_stream (self, size, pps, isg, vm ):
-        # create a base packet and pad it to size
+        # Create base packet and pad it to size
         base_pkt = Ether()/IP()/UDP()
         pad = max(0, size - len(base_pkt)) * 'x'
 
diff --git a/scripts/stl/multi_burst_2st_1000pkt.py b/scripts/stl/multi_burst_2st_1000pkt.py
index 1a43ae96..fe4b4eac 100644
--- a/scripts/stl/multi_burst_2st_1000pkt.py
+++ b/scripts/stl/multi_burst_2st_1000pkt.py
@@ -10,20 +10,20 @@ class STLS1(object):
 
     def create_stream (self):
 
-        # create a base packet and pad it to size
-        size = self.fsize - 4; # no FCS
+        # Create base packet and pad it to size
+        size = self.fsize - 4; # HW will add 4 bytes ethernet FCS
         base_pkt =  Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)
         base_pkt1 =  Ether()/IP(src="16.0.0.2",dst="48.0.0.1")/UDP(dport=12,sport=1025)
         pad = max(0, size - len(base_pkt)) * 'x'
 
 
-        return STLProfile( [ STLStream( isg = 10.0, # star in delay 
+        return STLProfile( [ STLStream( isg = 10.0, # start in delay 
                                         name    ='S0',
                                         packet = STLPktBuilder(pkt = base_pkt/pad),
                                         mode = STLTXSingleBurst( pps = 10, total_pkts = self.burst_size),
                                         next = 'S1'), # point to next stream 
 
-                             STLStream( self_start = False, # stream is  disabled enable trow S0
+                             STLStream( self_start = False, # Stream is disabled. Will run because it is pointed from S0
                                         name    ='S1',
                                         packet  = STLPktBuilder(pkt = base_pkt1/pad),
                                         mode    = STLTXMultiBurst( pps = 1000,pkts_per_burst = 4,ibg = 1000000.0,count = 5)
diff --git a/scripts/stl/simple_3st.py b/scripts/stl/simple_3st.py
index 8979057c..ae388f13 100644
--- a/scripts/stl/simple_3st.py
+++ b/scripts/stl/simple_3st.py
@@ -8,15 +8,15 @@ class STLS1(object):
 
     def create_stream (self):
 
-        # create a base packet and pad it to size
-        size = self.fsize - 4; # no FCS
+        # Create base packet and pad it to size
+        size = self.fsize - 4; # HW will add 4 bytes ethernet FCS
         base_pkt =  Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)
         base_pkt1 =  Ether()/IP(src="16.0.0.2",dst="48.0.0.1")/UDP(dport=12,sport=1025)
         base_pkt2 =  Ether()/IP(src="16.0.0.3",dst="48.0.0.1")/UDP(dport=12,sport=1025)
         pad = max(0, size - len(base_pkt)) * 'x'
 
 
-        return STLProfile( [ STLStream( isg = 1.0, # star in delay in usec 
+        return STLProfile( [ STLStream( isg = 1.0, # start in delay in usec 
                                         packet = STLPktBuilder(pkt = base_pkt/pad),
                                         mode = STLTXCont( pps = 10),
                                         ), 
diff --git a/scripts/stl/udp_1pkt.py b/scripts/stl/udp_1pkt.py
index 13516ecd..f2601d79 100644
--- a/scripts/stl/udp_1pkt.py
+++ b/scripts/stl/udp_1pkt.py
@@ -18,8 +18,8 @@ class STLS1(object):
         return t[self.mode]
 
     def create_stream (self):
-        # create a base packet and pad it to size
-        size = self.fsize - 4; # no FCS
+        # Create base packet and pad it to size
+        size = self.fsize - 4; # HW will add 4 bytes ethernet FCS
 
         base_pkt = self.create_pkt_base ()
 
diff --git a/scripts/stl/udp_1pkt_1mac.py b/scripts/stl/udp_1pkt_1mac.py
index 4adffd7a..ade5b592 100644
--- a/scripts/stl/udp_1pkt_1mac.py
+++ b/scripts/stl/udp_1pkt_1mac.py
@@ -8,8 +8,8 @@ class STLS1(object):
 
     def create_stream (self):
 
-        # create a base packet and pad it to size
-        size = self.fsize - 4; # no FCS
+        # Create base packet and pad it to size
+        size = self.fsize - 4; # HW will add 4 bytes ethernet FCS
         base_pkt =  Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)
         pad = max(0, size - len(base_pkt)) * 'x'
 
diff --git a/scripts/stl/udp_1pkt_1mac_override.py b/scripts/stl/udp_1pkt_1mac_override.py
index 04700420..410c2630 100644
--- a/scripts/stl/udp_1pkt_1mac_override.py
+++ b/scripts/stl/udp_1pkt_1mac_override.py
@@ -10,8 +10,8 @@ class STLS1(object):
 
     def create_stream (self):
 
-        # create a base packet and pad it to size
-        size = self.fsize - 4; # no FCS
+        # Create base packet and pad it to size
+        size = self.fsize - 4; # HW will add 4 bytes ethernet FCS
 
         # Ether(src="00:bb:12:34:56:01") this will tell TRex to take the src-mac from packet and not from config file
         base_pkt =  Ether(src="00:bb:12:34:56:01")/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)  
diff --git a/scripts/stl/udp_1pkt_1mac_step.py b/scripts/stl/udp_1pkt_1mac_step.py
index 1e5e4bd8..69a84d67 100644
--- a/scripts/stl/udp_1pkt_1mac_step.py
+++ b/scripts/stl/udp_1pkt_1mac_step.py
@@ -9,8 +9,8 @@ class STLS1(object):
 
     def create_stream (self):
 
-        # create a base packet and pad it to size
-        size = self.fsize - 4; # no FCS
+        # Create base packet and pad it to size
+        size = self.fsize - 4; # HW will add 4 bytes ethernet FCS
         base_pkt =  Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)
         pad = max(0, size - len(base_pkt)) * 'x'
 
diff --git a/scripts/stl/udp_1pkt_mac.py b/scripts/stl/udp_1pkt_mac.py
index 598e2074..93376aff 100644
--- a/scripts/stl/udp_1pkt_mac.py
+++ b/scripts/stl/udp_1pkt_mac.py
@@ -9,8 +9,8 @@ class STLS1(object):
 
     def create_stream (self):
 
-        # create a base packet and pad it to size
-        size = self.fsize - 4; # no FCS
+        # Create base packet and pad it to size
+        size = self.fsize - 4; # HW will add 4 bytes ethernet FCS
         base_pkt =  Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)
         pad = max(0, size - len(base_pkt)) * 'x'
 
diff --git a/scripts/stl/udp_1pkt_mac_mask1.py b/scripts/stl/udp_1pkt_mac_mask1.py
index efb45da7..9a4862a9 100644
--- a/scripts/stl/udp_1pkt_mac_mask1.py
+++ b/scripts/stl/udp_1pkt_mac_mask1.py
@@ -9,8 +9,8 @@ class STLS1(object):
 
     def create_stream (self):
 
-        # create a base packet and pad it to size
-        size = self.fsize - 4; # no FCS
+        # Create base packet and pad it to size
+        size = self.fsize - 4; # HW will add 4 bytes ethernet FCS
         base_pkt =  Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)
         pad = max(0, size - len(base_pkt)) * 'x'
 
diff --git a/scripts/stl/udp_1pkt_mac_mask2.py b/scripts/stl/udp_1pkt_mac_mask2.py
index b95a32e3..748ddbb1 100644
--- a/scripts/stl/udp_1pkt_mac_mask2.py
+++ b/scripts/stl/udp_1pkt_mac_mask2.py
@@ -9,8 +9,8 @@ class STLS1(object):
 
     def create_stream (self):
 
-        # create a base packet and pad it to size
-        size = self.fsize - 4; # no FCS
+        # Create base packet and pad it to size
+        size = self.fsize - 4; # HW will add 4 bytes ethernet FCS
         base_pkt =  Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)
         pad = max(0, size - len(base_pkt)) * 'x'
 
diff --git a/scripts/stl/udp_1pkt_mac_mask3.py b/scripts/stl/udp_1pkt_mac_mask3.py
index 7a5d2864..f3593ccb 100644
--- a/scripts/stl/udp_1pkt_mac_mask3.py
+++ b/scripts/stl/udp_1pkt_mac_mask3.py
@@ -9,8 +9,8 @@ class STLS1(object):
 
     def create_stream (self):
 
-        # create a base packet and pad it to size
-        size = self.fsize - 4; # no FCS
+        # Create base packet and pad it to size
+        size = self.fsize - 4; # HW will add 4 bytes ethernet FCS
         base_pkt =  Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)
         pad = max(0, size - len(base_pkt)) * 'x'
 
diff --git a/scripts/stl/udp_1pkt_mac_mask5.py b/scripts/stl/udp_1pkt_mac_mask5.py
index 75f9bbf1..901c2d98 100644
--- a/scripts/stl/udp_1pkt_mac_mask5.py
+++ b/scripts/stl/udp_1pkt_mac_mask5.py
@@ -9,8 +9,8 @@ class STLS1(object):
 
     def create_stream (self):
 
-        # create a base packet and pad it to size
-        size = self.fsize - 4; # no FCS
+        # Create base packet and pad it to size
+        size = self.fsize - 4; # HW will add 4 bytes ethernet FCS
         base_pkt =  Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)
         pad = max(0, size - len(base_pkt)) * 'x'
 
diff --git a/scripts/stl/udp_1pkt_mac_step.py b/scripts/stl/udp_1pkt_mac_step.py
index 0ebd035d..a2444905 100644
--- a/scripts/stl/udp_1pkt_mac_step.py
+++ b/scripts/stl/udp_1pkt_mac_step.py
@@ -9,8 +9,8 @@ class STLS1(object):
 
     def create_stream (self):
 
-        # create a base packet and pad it to size
-        size = self.fsize - 4; # no FCS
+        # Create base packet and pad it to size
+        size = self.fsize - 4; # HW will add 4 bytes ethernet FCS
         base_pkt =  Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)
         pad = max(0, size - len(base_pkt)) * 'x'
 
diff --git a/scripts/stl/udp_1pkt_range_clients.py b/scripts/stl/udp_1pkt_range_clients.py
index 9bd3c335..f1fc57f4 100644
--- a/scripts/stl/udp_1pkt_range_clients.py
+++ b/scripts/stl/udp_1pkt_range_clients.py
@@ -16,8 +16,8 @@ class STLS1(object):
 
     def create_stream (self):
 
-        # create a base packet and pad it to size
-        size = self.fsize - 4; # no FCS
+        # Create base packet and pad it to size
+        size = self.fsize - 4; # HW will add 4 bytes ethernet FCS
         base_pkt =  Ether(src="00:00:dd:dd:00:01")/IP(src="55.55.1.1",dst="58.0.0.1")/UDP(dport=12,sport=1025)
         pad = max(0, size - len(base_pkt)) * 'x'
 
diff --git a/scripts/stl/udp_1pkt_range_clients_split.py b/scripts/stl/udp_1pkt_range_clients_split.py
index a8c71c0a..9bf09ba4 100644
--- a/scripts/stl/udp_1pkt_range_clients_split.py
+++ b/scripts/stl/udp_1pkt_range_clients_split.py
@@ -16,8 +16,8 @@ class STLS1(object):
 
     def create_stream (self):
 
-        # create a base packet and pad it to size
-        size = self.fsize - 4; # no FCS
+        # Create base packet and pad it to size
+        size = self.fsize - 4; # HW will add 4 bytes ethernet FCS
         base_pkt =  Ether(src="00:00:dd:dd:00:01")/IP(src="55.55.1.1",dst="58.0.0.1")/UDP(dport=12,sport=1025)
         pad = max(0, size - len(base_pkt)) * 'x'
 
diff --git a/scripts/stl/udp_1pkt_range_clients_split_garp.py b/scripts/stl/udp_1pkt_range_clients_split_garp.py
index d7f48ed7..4bad8afd 100644
--- a/scripts/stl/udp_1pkt_range_clients_split_garp.py
+++ b/scripts/stl/udp_1pkt_range_clients_split_garp.py
@@ -10,7 +10,7 @@ class STLS1(object):
         self.num_clients  =3000; # max is 16bit
 
     def create_stream (self):
-        # create a base packet and pad it to size
+        # Create base packet and pad it to size
         base_pkt =  Ether(src="00:00:dd:dd:00:01",dst="ff:ff:ff:ff:ff:ff")/ARP(psrc="55.55.1.1",hwsrc="00:00:dd:dd:00:01", hwdst="00:00:dd:dd:00:01", pdst="55.55.1.1")
 
         vm = STLScVmRaw( [ STLVmFlowVar(name="mac_src", min_value=1, max_value=self.num_clients, size=2, op="inc"),
diff --git a/scripts/stl/udp_1pkt_src_ip_split.py b/scripts/stl/udp_1pkt_src_ip_split.py
index 778ccf54..48e02433 100644
--- a/scripts/stl/udp_1pkt_src_ip_split.py
+++ b/scripts/stl/udp_1pkt_src_ip_split.py
@@ -9,8 +9,8 @@ class STLS1(object):
         self.fsize  =64;
 
     def create_stream (self):
-        # create a base packet and pad it to size
-        size = self.fsize - 4; # no FCS
+        # Create base packet and pad it to size
+        size = self.fsize - 4; # HW will add 4 bytes ethernet FCS
 
         base_pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)
 
diff --git a/scripts/stl/udp_1pkt_src_ip_split_latency.py b/scripts/stl/udp_1pkt_src_ip_split_latency.py
index af8d4cd7..4b297d70 100644
--- a/scripts/stl/udp_1pkt_src_ip_split_latency.py
+++ b/scripts/stl/udp_1pkt_src_ip_split_latency.py
@@ -2,15 +2,22 @@ from trex_stl_lib.api import *
 
 
 # split the range of IP to cores 
+# add tunable by fsize to change the size of the frame 
+# latency frame is always 64 
+# trex>start -f stl/udp_1pkt_src_ip_split_latency.py -t fsize=64 -m 30% --port 0 --force
 #
+#
+
 class STLS1(object):
 
     def __init__ (self):
         self.fsize  =64;
+        self.lfsize  =64;
+
 
     def create_stream (self, dir,port_id):
-        # create a base packet and pad it to size
-        size = self.fsize - 4; # no FCS
+        # Create base packet and pad it to size
+        size = self.fsize - 4; # HW will add 4 bytes ethernet FCS
 
         if dir==0:
             src_ip="16.0.0.1"
@@ -22,6 +29,7 @@ class STLS1(object):
         base_pkt = Ether()/IP(src=src_ip,dst=dst_ip)/UDP(dport=12,sport=1025)
 
         pad = max(0, size - len(base_pkt)) * 'x'
+        pad_latency = max(0, (self.lfsize-4) - len(base_pkt)) * 'x'
                              
         vm = STLScVmRaw( [   STLVmFlowVar ( "ip_src",  min_value="10.0.0.1",
                                             max_value="10.0.0.255", size=4, step=1,op="inc"),
@@ -39,8 +47,7 @@ class STLS1(object):
 
 
                   # latency stream   
-                  STLStream(packet = STLPktBuilder(pkt = "yaml/udp_64B_no_crc.pcap", 
-                            path_relative_to_profile = True), 
+                  STLStream(packet = STLPktBuilder(pkt = base_pkt/pad_latency),
                             mode = STLTXCont(pps=1000),
                             flow_stats = STLFlowLatencyStats(pg_id = 12+port_id))
   
@@ -48,7 +55,9 @@ class STLS1(object):
         return stream
 
 
-    def get_streams (self, direction = 0, **kwargs):
+    def get_streams (self, direction = 0, fsize = 64,lfsize = 64, **kwargs):
+        self.fsize  =fsize;
+        self.lfsize =lfsize
         return self.create_stream(direction,kwargs['port_id'])
 
 
diff --git a/scripts/stl/udp_1pkt_tuple_gen.py b/scripts/stl/udp_1pkt_tuple_gen.py
index 4e9ab12d..733d511b 100644
--- a/scripts/stl/udp_1pkt_tuple_gen.py
+++ b/scripts/stl/udp_1pkt_tuple_gen.py
@@ -3,7 +3,7 @@ from trex_stl_lib.api import *
 class STLS1(object):
 
     def create_stream (self, packet_len):
-        # create a base packet and pad it to size
+        # Create base packet and pad it to size
 
         base_pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)
 
diff --git a/scripts/stl/udp_1pkt_tuple_gen_split.py b/scripts/stl/udp_1pkt_tuple_gen_split.py
index e7a33b22..cc9eb5fc 100644
--- a/scripts/stl/udp_1pkt_tuple_gen_split.py
+++ b/scripts/stl/udp_1pkt_tuple_gen_split.py
@@ -9,8 +9,8 @@ class STLS1(object):
         self.fsize  =64;
 
     def create_stream (self):
-        # create a base packet and pad it to size
-        size = self.fsize - 4; # no FCS
+        # Create base packet and pad it to size
+        size = self.fsize - 4; # HW will add 4 bytes ethernet FCS
 
         base_pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)
 
diff --git a/scripts/stl/udp_3pkt_pcap.py b/scripts/stl/udp_3pkt_pcap.py
index 19ff46bc..2983f9a1 100644
--- a/scripts/stl/udp_3pkt_pcap.py
+++ b/scripts/stl/udp_3pkt_pcap.py
@@ -9,19 +9,19 @@ class STLS1(object):
 
     def create_stream (self):
 
-        return STLProfile( [ STLStream( isg = 10.0, # star in delay 
+        return STLProfile( [ STLStream( isg = 10.0, # start in delay 
                                         name    ='S0',
                                         packet = STLPktBuilder(pkt = os.path.join(CP, "yaml/udp_64B_no_crc.pcap")),
                                         mode = STLTXSingleBurst( pps = 10, total_pkts = 10),
                                         next = 'S1'), # point to next stream 
 
-                             STLStream( self_start = False, # stream is  disabled enable trow S0
+                             STLStream( self_start = False, # Stream is disabled. Will run because it is pointed from S0
                                         name    ='S1',
                                         packet  = STLPktBuilder(pkt = os.path.join(CP, "yaml/udp_594B_no_crc.pcap")),
                                         mode    = STLTXSingleBurst( pps = 10, total_pkts = 20),
                                         next    = 'S2' ),
 
-                             STLStream(  self_start = False, # stream is  disabled enable trow S0
+                             STLStream(  self_start = False, # Stream is disabled. Will run because it is pointed from S1
                                          name   ='S2',
                                          packet = STLPktBuilder(pkt = os.path.join(CP, "yaml/udp_1518B_no_crc.pcap")),
                                          mode = STLTXSingleBurst( pps = 10, total_pkts = 30 )
diff --git a/scripts/stl/udp_for_benchmarks.py b/scripts/stl/udp_for_benchmarks.py
index 2033e80e..956498d0 100644
--- a/scripts/stl/udp_for_benchmarks.py
+++ b/scripts/stl/udp_for_benchmarks.py
@@ -1,5 +1,23 @@
 from trex_stl_lib.api import *
 
+# Tunable example 
+#
+#trex>profile -f stl/udp_for_benchmarks.py
+#
+#Profile Information:
+#
+#
+#General Information:
+#Filename:         stl/udp_for_benchmarks.py
+#Stream count:          1      
+#
+#Specific Information:
+#Type:             Python Module
+#Tunables:         ['stream_count = 1', 'direction = 0', 'packet_len = 64']
+#
+#trex>start -f stl/udp_for_benchmarks.py -t  packet_len=128 --port 0
+#
+
 class STLS1(object):
     '''
     Generalization of udp_1pkt_simple, can specify number of streams and packet length
diff --git a/scripts/t-rex-64-debug-gdb-bt b/scripts/t-rex-64-debug-gdb-bt
new file mode 100644
index 00000000..2d64ce62
--- /dev/null
+++ b/scripts/t-rex-64-debug-gdb-bt
@@ -0,0 +1,12 @@
+#! /bin/bash
+export LD_LIBRARY_PATH=`pwd`
+/bin/gdb --batch --command=gdb_script1.txt --args ./_t-rex-64-debug $@ 
+RESULT=$?
+
+if [ $RESULT -ne 0 ]; then
+  exit $RESULT
+fi
+
+
+
+
diff --git a/scripts/t-rex-64-debug-gdb-core b/scripts/t-rex-64-debug-gdb-core
new file mode 100644
index 00000000..cc790448
--- /dev/null
+++ b/scripts/t-rex-64-debug-gdb-core
@@ -0,0 +1,12 @@
+#! /bin/bash
+export LD_LIBRARY_PATH=`pwd`
+/bin/gdb ./_t-rex-64-debug $@ 
+RESULT=$?
+
+if [ $RESULT -ne 0 ]; then
+  exit $RESULT
+fi
+
+
+
+