Merge branch 'master' into scapy_server

author: itraviv <itraviv@cisco.com> 2016-07-31 11:56:41 +0300
committer: itraviv <itraviv@cisco.com> 2016-07-31 11:56:41 +0300
commit: 893d0feef9ba6fa3fb36c49f4b5bcad47cb2bf60 (patch)
tree: 689a09fa656f990672d2d62143dc173a46fe0316 /scripts/automation/trex_control_plane
parent: abf329075bd14f5f41c3753d560260ac809ec4f3 (diff)
parent: dceb010b01e9f8a0e9c905370d39f149f01cab7e (diff)
20 files changed, 1539 insertions, 522 deletions
diff --git a/scripts/automation/trex_control_plane/common/text_opts.py b/scripts/automation/trex_control_plane/common/text_opts.py
index 78a0ab1f..c9ab7ca8 100755
--- a/scripts/automation/trex_control_plane/common/text_opts.py
+++ b/scripts/automation/trex_control_plane/common/text_opts.py
@@ -27,7 +27,10 @@ class TextCodesStripper:
     def strip (s):
         return re.sub(TextCodesStripper.pattern, '', s)
 
-def format_num (size, suffix = "", compact = True, opts = ()):
+def format_num (size, suffix = "", compact = True, opts = None):
+    if opts is None:
+        opts = ()
+
     txt = "NaN"
 
     if type(size) == str:
@@ -61,6 +64,9 @@ def format_time (t_sec):
     if t_sec < 0:
         return "infinite"
 
+    if t_sec == 0:
+        return "zero"
+
     if t_sec < 1:
         # low numbers
         for unit in ['ms', 'usec', 'ns']:
diff --git a/scripts/automation/trex_control_plane/doc_stl/conf.py b/scripts/automation/trex_control_plane/doc_stl/conf.py
index 45738b6e..c8788ca7 100644
--- a/scripts/automation/trex_control_plane/doc_stl/conf.py
+++ b/scripts/automation/trex_control_plane/doc_stl/conf.py
@@ -15,6 +15,21 @@
 import sys
 import os
 import shlex
+import functools
+
+def no_op_wraps(func):
+    """Replaces functools.wraps in order to undo wrapping.
+
+    Can be used to preserve the decorated function's signature
+    in the documentation generated by Sphinx.
+
+    """
+    def wrapper(decorator):
+        return func
+    return wrapper
+
+functools.wraps = no_op_wraps
+
 
 
 # If extensions (or modules to document with autodoc) are in another directory,
@@ -309,4 +324,5 @@ autoclass_content = "both"
 
 # A workaround for the responsive tables always having annoying scrollbars.
 def setup(app):
-  app.add_stylesheet("no_scrollbars.css")
-\ No newline at end of file
+  app.add_stylesheet("no_scrollbars.css")
+
diff --git a/scripts/automation/trex_control_plane/server/CCustomLogger.py b/scripts/automation/trex_control_plane/server/CCustomLogger.py
index a8823cea..6d3974a6 100755
--- a/scripts/automation/trex_control_plane/server/CCustomLogger.py
+++ b/scripts/automation/trex_control_plane/server/CCustomLogger.py
@@ -3,15 +3,13 @@ import sys
 import os
 import logging
 
+def prepare_dir(log_path):
+    log_dir = os.path.dirname(log_path)
+    if not os.path.exists(log_dir):
+        os.makedirs(log_dir)
 
 def setup_custom_logger(name, log_path = None):
     # first make sure path availabe
-#   if log_path is None:
-#       log_path = os.getcwd()+'/trex_log.log'
-#   else:
-#       directory = os.path.dirname(log_path)
-#       if not os.path.exists(directory):
-#           os.makedirs(directory)
     logging.basicConfig(level   = logging.INFO, 
                         format  = '%(asctime)s %(name)-10s %(module)-20s %(levelname)-8s %(message)s',
                         datefmt = '%m-%d %H:%M')
@@ -31,6 +29,7 @@ def setup_custom_logger(name, log_path = None):
 
 def setup_daemon_logger (name, log_path = None):
     # first make sure path availabe
+    prepare_dir(log_path)
     try:
         os.unlink(log_path)
     except:
diff --git a/scripts/automation/trex_control_plane/server/singleton_daemon.py b/scripts/automation/trex_control_plane/server/singleton_daemon.py
index 8fdedc6e..1784cc42 100755
--- a/scripts/automation/trex_control_plane/server/singleton_daemon.py
+++ b/scripts/automation/trex_control_plane/server/singleton_daemon.py
@@ -2,6 +2,7 @@ import errno
 import os
 import shlex
 import socket
+import signal
 import tempfile
 import types
 from subprocess import Popen
@@ -73,31 +74,27 @@ class SingletonDaemon(object):
         if pid:
             return pid
 
-
-    # kill daemon
-    def kill(self, timeout = 10):
-        pid = self.get_pid()
-        if not pid:
-            return False
-        ret_code, stdout, stderr = run_command('kill %s' % pid) # usual kill
-        if ret_code:
-            raise Exception('Failed to run kill command for %s: %s' % (self.name, [ret_code, stdout, stderr]))
+    def kill_by_signal(self, pid, signal_name, timeout):
+        os.kill(pid, signal_name)
         poll_rate = 0.1
         for i in range(int(timeout / poll_rate)):
             if not self.is_running():
                 return True
             sleep(poll_rate)
-        ret_code, stdout, stderr = run_command('kill -9 %s' % pid) # unconditional kill
-        if ret_code:
-            raise Exception('Failed to run kill -9 command for %s: %s' % (self.name, [ret_code, stdout, stderr]))
-        for i in range(int(timeout / poll_rate)):
-            if not self.is_running():
+
+    # kill daemon, with verification
+    def kill(self, timeout = 15):
+        pid = self.get_pid()
+        if not pid:
+            raise Exception('%s is not running' % self.name)
+        # try Ctrl+C, usual kill, kill -9
+        for signal_name in [signal.SIGINT, signal.SIGTERM, signal.SIGKILL]:
+            if self.kill_by_signal(pid, signal_name, timeout):
                 return True
-            sleep(poll_rate)
         raise Exception('Could not kill %s, even with -9' % self.name)
 
     # try connection as RPC client, return True upon success, False if fail
-    def check_connectivity(self, timeout = 5):
+    def check_connectivity(self, timeout = 15):
         daemon = jsonrpclib.Server('http://127.0.0.1:%s/' % self.port)
         poll_rate = 0.1
         for i in range(int(timeout/poll_rate)):
@@ -140,7 +137,7 @@ class SingletonDaemon(object):
             raise Exception('%s failed to run.' % self.name)
 
     # restart the daemon
-    def restart(self, timeout = 5):
+    def restart(self, timeout = 15):
         if self.is_running():
             self.kill(timeout)
         return self.start(timeout)
diff --git a/scripts/automation/trex_control_plane/server/trex_launch_thread.py b/scripts/automation/trex_control_plane/server/trex_launch_thread.py
index 82a7f996..22606753 100755
--- a/scripts/automation/trex_control_plane/server/trex_launch_thread.py
+++ b/scripts/automation/trex_control_plane/server/trex_launch_thread.py
@@ -30,49 +30,51 @@ class AsynchronousTRexSession(threading.Thread):
         self.trexObj.zmq_dump                       = {}
 
     def run (self):
-            
-        with open(os.devnull, 'w') as DEVNULL:
-            self.time_stamps['start'] = self.time_stamps['run_time'] = time.time()
-            self.session   = subprocess.Popen(shlex.split(self.cmd), cwd = self.launch_path, stdin = DEVNULL, stderr = subprocess.PIPE, preexec_fn=os.setsid, close_fds = True)
-            logger.info("TRex session initialized successfully, Parent process pid is {pid}.".format( pid = self.session.pid ))
-            while self.session.poll() is None:  # subprocess is NOT finished
-                time.sleep(0.5)
-                if self.stoprequest.is_set():
-                    logger.debug("Abort request received by handling thread. Terminating TRex session." )
-                    os.killpg(self.session.pid, signal.SIGUSR1)
-                    self.trexObj.set_status(TRexStatus.Idle)
-                    self.trexObj.set_verbose_status("TRex is Idle")
-                    break
-
-            self.time_stamps['run_time'] = time.time() - self.time_stamps['start']
-
-            try:
-                if self.time_stamps['run_time'] < 5:
-                    logger.error("TRex run failed due to wrong input parameters, or due to readability issues.")
-                    self.trexObj.set_verbose_status("TRex run failed due to wrong input parameters, or due to readability issues.\n\nTRex command: {cmd}\n\nRun output:\n{output}".format(
-                        cmd = self.cmd, output = self.load_trex_output(self.export_path)))
-                    self.trexObj.errcode = -11
-                elif (self.session.returncode is not None and self.session.returncode != 0) or ( (self.time_stamps['run_time'] < self.duration) and (not self.stoprequest.is_set()) ):
-                    if (self.session.returncode is not None and self.session.returncode != 0):
-                        logger.debug("Failed TRex run due to session return code ({ret_code})".format( ret_code = self.session.returncode ) )
-                    elif ( (self.time_stamps['run_time'] < self.duration) and not self.stoprequest.is_set()):
-                        logger.debug("Failed TRex run due to running time ({runtime}) combined with no-stopping request.".format( runtime = self.time_stamps['run_time'] ) )
-
-                    logger.warning("TRex run was terminated unexpectedly by outer process or by the hosting OS")
-                    self.trexObj.set_verbose_status("TRex run was terminated unexpectedly by outer process or by the hosting OS.\n\nRun output:\n{output}".format(
-                        output = self.load_trex_output(self.export_path)))
-                    self.trexObj.errcode = -15
-                else:
-                    logger.info("TRex run session finished.")
-                    self.trexObj.set_verbose_status('TRex finished.')
-                    self.trexObj.errcode = None
-                    
-            finally:
-                self.trexObj.set_status(TRexStatus.Idle)
-                logger.info("TRex running state changed to 'Idle'.")
-                self.trexObj.expect_trex.clear()
-                logger.debug("Finished handling a single run of TRex.")
-                self.trexObj.zmq_dump   = None
+        try:
+            with open(self.export_path, 'w') as output_file:
+                self.time_stamps['start'] = self.time_stamps['run_time'] = time.time()
+                self.session   = subprocess.Popen(shlex.split(self.cmd), cwd = self.launch_path, stdout = output_file, preexec_fn=os.setsid, close_fds = True)
+                logger.info("TRex session initialized successfully, Parent process pid is {pid}.".format( pid = self.session.pid ))
+                while self.session.poll() is None:  # subprocess is NOT finished
+                    time.sleep(0.5)
+                    if self.stoprequest.is_set():
+                        logger.debug("Abort request received by handling thread. Terminating TRex session." )
+                        os.killpg(self.session.pid, signal.SIGUSR1)
+                        self.trexObj.set_status(TRexStatus.Idle)
+                        self.trexObj.set_verbose_status("TRex is Idle")
+                        break
+        except Exception as e:
+            logger.error(e)
+
+        self.time_stamps['run_time'] = time.time() - self.time_stamps['start']
+
+        try:
+            if self.time_stamps['run_time'] < 5:
+                logger.error("TRex run failed due to wrong input parameters, or due to readability issues.")
+                self.trexObj.set_verbose_status("TRex run failed due to wrong input parameters, or due to readability issues.\n\nTRex command: {cmd}\n\nRun output:\n{output}".format(
+                    cmd = self.cmd, output = self.load_trex_output(self.export_path)))
+                self.trexObj.errcode = -11
+            elif (self.session.returncode is not None and self.session.returncode != 0) or ( (self.time_stamps['run_time'] < self.duration) and (not self.stoprequest.is_set()) ):
+                if (self.session.returncode is not None and self.session.returncode != 0):
+                    logger.debug("Failed TRex run due to session return code ({ret_code})".format( ret_code = self.session.returncode ) )
+                elif ( (self.time_stamps['run_time'] < self.duration) and not self.stoprequest.is_set()):
+                    logger.debug("Failed TRex run due to running time ({runtime}) combined with no-stopping request.".format( runtime = self.time_stamps['run_time'] ) )
+
+                logger.warning("TRex run was terminated unexpectedly by outer process or by the hosting OS")
+                self.trexObj.set_verbose_status("TRex run was terminated unexpectedly by outer process or by the hosting OS.\n\nRun output:\n{output}".format(
+                    output = self.load_trex_output(self.export_path)))
+                self.trexObj.errcode = -15
+            else:
+                logger.info("TRex run session finished.")
+                self.trexObj.set_verbose_status('TRex finished.')
+                self.trexObj.errcode = None
+
+        finally:
+            self.trexObj.set_status(TRexStatus.Idle)
+            logger.info("TRex running state changed to 'Idle'.")
+            self.trexObj.expect_trex.clear()
+            logger.debug("Finished handling a single run of TRex.")
+            self.trexObj.zmq_dump   = None
 
     def join (self, timeout = None):
         self.stoprequest.set()
diff --git a/scripts/automation/trex_control_plane/server/trex_server.py b/scripts/automation/trex_control_plane/server/trex_server.py
index 8f7e99f0..9fe7d70b 100755
--- a/scripts/automation/trex_control_plane/server/trex_server.py
+++ b/scripts/automation/trex_control_plane/server/trex_server.py
@@ -310,7 +310,7 @@ class CTRexServer(object):
                 return False
 
             
-    def start_trex(self, trex_cmd_options, user, block_to_success = True, timeout = 40, stateless = False):
+    def start_trex(self, trex_cmd_options, user, block_to_success = True, timeout = 40, stateless = False, debug_image = False, trex_args = ''):
         with self.start_lock:
             logger.info("Processing start_trex() command.")
             if self.is_reserved():
@@ -323,7 +323,7 @@ class CTRexServer(object):
                 return Fault(-13, '')  # raise at client TRexInUseError
             
             try:
-                server_cmd_data = self.generate_run_cmd(stateless = stateless, **trex_cmd_options)
+                server_cmd_data = self.generate_run_cmd(stateless = stateless, debug_image = debug_image, trex_args = trex_args, **trex_cmd_options)
                 self.zmq_monitor.first_dump = True
                 self.trex.start_trex(self.TREX_PATH, server_cmd_data)
                 logger.info("TRex session has been successfully initiated.")
@@ -387,23 +387,14 @@ class CTRexServer(object):
         return trex_cmds_list
 
 
-    def kill_all_trexes(self):
+    # Silently tries to kill TRexes with given signal.
+    # Responsibility of client to verify with get_trex_cmds.
+    def kill_all_trexes(self, signal_name):
         logger.info('Processing kill_all_trexes() command.')
         trex_cmds_list = self.get_trex_cmds()
-        if not trex_cmds_list:
-            return False
         for pid, cmd in trex_cmds_list:
-            logger.info('Killing process %s %s' % (pid, cmd))
-            run_command('kill %s' % pid)
-            ret_code_ps, _, _ = run_command('ps -p %s' % pid)
-            if not ret_code_ps:
-                logger.info('Killing with -9.')
-                run_command('kill -9 %s' % pid)
-                ret_code_ps, _, _ = run_command('ps -p %s' % pid)
-                if not ret_code_ps:
-                    logger.info('Could not kill process.')
-                    raise Exception('Could not kill process %s %s' % (pid, cmd))
-        return True
+            logger.info('Killing with signal %s process %s %s' % (signal_name, pid, cmd))
+            os.kill(int(pid), signal_name)
 
 
     def wait_until_kickoff_finish (self, timeout = 40):
@@ -422,7 +413,7 @@ class CTRexServer(object):
         return self.trex.get_running_info()
 
 
-    def generate_run_cmd (self, iom = 0, export_path="/tmp/trex.txt", stateless = False, **kwargs):
+    def generate_run_cmd (self, iom = 0, export_path="/tmp/trex.txt", stateless = False, debug_image = False, trex_args = '', **kwargs):
         """ generate_run_cmd(self, iom, export_path, kwargs) -> str
 
         Generates a custom running command for the kick-off of the TRex traffic generator.
@@ -459,6 +450,8 @@ class CTRexServer(object):
                 continue
             else:
                 trex_cmd_options += (dash + '{k} {val}'.format( k = tmp_key, val =  value ))
+        if trex_args:
+            trex_cmd_options += ' %s' % trex_args
 
         if not stateless:
             if 'f' not in kwargs:
@@ -466,12 +459,12 @@ class CTRexServer(object):
             if 'd' not in kwargs:
                 raise Exception('Argument -d should be specified in stateful command')
 
-        cmd = "{nice}{run_command} --iom {io} {cmd_options} --no-key > {export}".format( # -- iom 0 disables the periodic log to the screen (not needed)
+        cmd = "{nice}{run_command}{debug_image} --iom {io} {cmd_options} --no-key".format( # -- iom 0 disables the periodic log to the screen (not needed)
             nice = '' if self.trex_nice == 0 else 'nice -n %s ' % self.trex_nice,
             run_command = self.TREX_START_CMD,
+            debug_image = '-debug' if debug_image else '',
             cmd_options = trex_cmd_options,
-            io          = iom, 
-            export = export_path )
+            io          = iom)
 
         logger.info("TREX FULL COMMAND: {command}".format(command = cmd) )
 
diff --git a/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py
index 3b01560a..ecf6083b 100755
--- a/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py
+++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py
@@ -13,6 +13,7 @@ from distutils.util import strtobool
 from collections import deque, OrderedDict
 from json import JSONDecoder
 import traceback
+import signal
 
 try:
     from . import outer_packages
@@ -39,7 +40,7 @@ class CTRexClient(object):
     This class defines the client side of the RESTfull interaction with TRex
     """
 
-    def __init__(self, trex_host, max_history_size = 100, filtered_latency_amount = 0.001, trex_daemon_port = 8090, master_daemon_port = 8091, trex_zmq_port = 4500, verbose = False):
+    def __init__(self, trex_host, max_history_size = 100, filtered_latency_amount = 0.001, trex_daemon_port = 8090, master_daemon_port = 8091, trex_zmq_port = 4500, verbose = False, debug_image = False, trex_args = ''):
         """ 
         Instantiate a TRex client object, and connecting it to listening daemon-server
 
@@ -72,6 +73,8 @@ class CTRexClient(object):
                 sets a verbose output on supported class method.
 
                 default value : **False**
+             trex_args : string
+                additional arguments passed to TRex. For example, "-w 3 --no-watchdog"
 
         :raises:
             socket errors, in case server could not be reached.
@@ -81,53 +84,23 @@ class CTRexClient(object):
             self.trex_host          = socket.gethostbyname(trex_host)
         except: # give it another try
             self.trex_host          = socket.gethostbyname(trex_host)
-        self.trex_daemon_port   = trex_daemon_port
-        self.master_daemon_port = master_daemon_port
-        self.trex_zmq_port      = trex_zmq_port
-        self.seq                = None
-        self._last_sample       = time.time()
-        self.__default_user     = get_current_user()
-        self.verbose            = verbose
-        self.result_obj         = CTRexResult(max_history_size, filtered_latency_amount)
-        self.decoder            = JSONDecoder()
-        self.history            = jsonrpclib.history.History()
-        self.master_daemon_path = "http://{hostname}:{port}/".format( hostname = self.trex_host, port = master_daemon_port )
-        self.trex_server_path   = "http://{hostname}:{port}/".format( hostname = self.trex_host, port = trex_daemon_port )
-        self.connect_master()
-        self.connect_server()
-
-
-    def connect_master(self):
-        '''
-        Connects to Master daemon via JsonRPC.
-        This daemon controls TRex daemon server.
-        Return true if success, false if fail
-        '''
-        try:
-            print('Connecting to Master daemon @ %s ...' % self.master_daemon_path)
-            self.master_daemon      = jsonrpclib.Server(self.master_daemon_path, history = self.history)
-            self.check_master_connectivity()
-            print('Connected to Master daemon.')
-            return True
-        except Exception as e:
-            print(e)
-            return False
-
-    def connect_server(self):
-        '''
-        Connects to TRex daemon server via JsonRPC.
-        This daemon controls TRex. (start/stop)
-        Return true if success, false if fail
-        '''
-        try:
-            print('Connecting to TRex daemon server @ %s ...' % self.trex_server_path)
-            self.server             = jsonrpclib.Server(self.trex_server_path, history = self.history)
-            self.check_server_connectivity()
-            print('Connected TRex server daemon.')
-            return True
-        except Exception as e:
-            print(e)
-            return False
+        self.trex_daemon_port       = trex_daemon_port
+        self.master_daemon_port     = master_daemon_port
+        self.trex_zmq_port          = trex_zmq_port
+        self.seq                    = None
+        self._last_sample           = time.time()
+        self.__default_user         = get_current_user()
+        self.verbose                = verbose
+        self.result_obj             = CTRexResult(max_history_size, filtered_latency_amount)
+        self.decoder                = JSONDecoder()
+        self.history                = jsonrpclib.history.History()
+        self.master_daemon_path     = "http://{hostname}:{port}/".format( hostname = self.trex_host, port = master_daemon_port )
+        self.master_daemon          = jsonrpclib.Server(self.master_daemon_path, history = self.history)
+        self.trex_server_path       = "http://{hostname}:{port}/".format( hostname = self.trex_host, port = trex_daemon_port )
+        self.server                 = jsonrpclib.Server(self.trex_server_path, history = self.history)
+        self.debug_image            = debug_image
+        self.trex_args              = trex_args
+        self.sample_to_run_finish   = self.sample_until_finish # alias for legacy
 
 
     def add (self, x, y):
@@ -191,7 +164,7 @@ class CTRexClient(object):
         self.result_obj.clear_results()
         try:
             issue_time = time.time()
-            retval = self.server.start_trex(trex_cmd_options, user, block_to_success, timeout)
+            retval = self.server.start_trex(trex_cmd_options, user, block_to_success, timeout, False, self.debug_image, self.trex_args)
         except AppError as err:
             self._handle_AppError_exception(err.args[0])
         except ProtocolError:
@@ -237,7 +210,7 @@ class CTRexClient(object):
         """
         try:
             user = user or self.__default_user
-            retval = self.server.start_trex(trex_cmd_options, user, block_to_success, timeout, True)
+            retval = self.server.start_trex(trex_cmd_options, user, block_to_success, timeout, True, self.debug_image, self.trex_args)
         except AppError as err:
             self._handle_AppError_exception(err.args[0])
         except ProtocolError:
@@ -322,18 +295,28 @@ class CTRexClient(object):
         finally:
             self.prompt_verbose_data()
 
-    def kill_all_trexes(self):
+    def kill_all_trexes(self, timeout = 15):
         """
         Kills running TRex processes (if exists) on the server, not only owned by current daemon.
         Raises exception upon error killing.
 
         :return: 
-            + **True** if any process killed 
+            + **True** if processes killed/not running
             + **False** otherwise.
 
         """
         try:
-            return self.server.kill_all_trexes()
+            poll_rate = 0.1
+            # try Ctrl+C, usual kill, -9
+            for signal_name in [signal.SIGINT, signal.SIGTERM, signal.SIGKILL]:
+                self.server.kill_all_trexes(signal_name)
+                for i in range(int(timeout / poll_rate)):
+                    if not self.get_trex_cmds():
+                        return True
+                    time.sleep(poll_rate)
+            if self.get_trex_cmds():
+                return False
+            return True
         except AppError as err:
             self._handle_AppError_exception(err.args[0])
         finally:
@@ -555,7 +538,7 @@ class CTRexClient(object):
             finally:
                 self.prompt_verbose_data()
 
-    def sample_until_condition (self, condition_func, time_between_samples = 5):
+    def sample_until_condition (self, condition_func, time_between_samples = 1):
         """
         Automatically sets ongoing sampling of TRex data, with sampling rate described by time_between_samples.
 
@@ -569,7 +552,7 @@ class CTRexClient(object):
             time_between_samples : int
                 determines the time between each sample of the server
 
-                default value : **5**
+                default value : **1**
 
         :return: 
             the first result object (see :class:`CTRexResult` for further details) of the TRex run on which the condition has been met.
@@ -599,15 +582,15 @@ class CTRexClient(object):
             # this could come from provided method 'condition_func'
             raise
 
-    def sample_to_run_finish (self, time_between_samples = 5):
+    def sample_until_finish (self, time_between_samples = 1):
         """
-        Automatically sets automatically sampling of TRex data with sampling rate described by time_between_samples until TRex run finished.
+        Automatically samples TRex data with sampling rate described by time_between_samples until TRex run finishes.
 
         :parameters:        
             time_between_samples : int
                 determines the time between each sample of the server
 
-                default value : **5**
+                default value : **1**
 
         :return: 
             the latest result object (see :class:`CTRexResult` for further details) with sampled data.
@@ -629,7 +612,7 @@ class CTRexClient(object):
         results = self.get_result_obj()
         return results
             
-    def sample_x_seconds (self, sample_time, time_between_samples = 5):
+    def sample_x_seconds (self, sample_time, time_between_samples = 1):
         """
         Automatically sets ongoing sampling of TRex data for sample_time seconds, with sampling rate described by time_between_samples.
         Does not stop the TRex afterwards!
@@ -643,7 +626,7 @@ class CTRexClient(object):
             time_between_samples : int
                 determines the time between each sample of the server
 
-                default value : **5**
+                default value : **1**
 
         :return:
             the first result object (see :class:`CTRexResult` for further details) of the TRex run after given sample_time.
@@ -657,12 +640,12 @@ class CTRexClient(object):
         """
         # make sure TRex is running. raise exceptions here if any
         self.wait_until_kickoff_finish()
-        elapsed_time = 0
+        end_time = time.time() + sample_time
         while self.is_running():
-            if elapsed_time >= sample_time:
+            if time.time() < end_time:
+                time.sleep(time_between_samples)
+            else:
                 return self.get_result_obj()
-            time.sleep(time_between_samples)
-            elapsed_time += time_between_samples
         raise UserWarning("TRex has stopped at %s seconds (before expected %s seconds)\nTry increasing test duration or decreasing sample_time" % (elapsed_time, sample_time))
 
     def get_result_obj (self, copy_obj = True):
@@ -1291,7 +1274,7 @@ class CTRexResult(object):
                 .. tip:: | Use '.' to enter one level deeper in dictionary hierarchy. 
                          | Use '[i]' to access the i'th indexed object of an array.
 
-            tree_path_to_key : regex
+            regex : regex
                 apply a regex to filter results out from a multiple results set.
 
                 Filter applies only on keys of dictionary type.
@@ -1319,7 +1302,7 @@ class CTRexResult(object):
                 .. tip:: | Use '.' to enter one level deeper in dictionary hierarchy. 
                          | Use '[i]' to access the i'th indexed object of an array.
 
-            tree_path_to_key : regex
+            regex : regex
                 apply a regex to filter results out from a multiple results set.
 
                 Filter applies only on keys of dictionary type.
@@ -1372,7 +1355,7 @@ class CTRexResult(object):
 
         if not len(self._history):
             return -1
-        return len(self.__get_value_by_path(self._history[-1], 'trex-global.data', 'opackets-\d+'))
+        return len(self.get_last_value('trex-global.data', 'opackets-\d+'))
 
 
     def update_result_data (self, latest_dump):
@@ -1403,14 +1386,21 @@ class CTRexResult(object):
                 # check for up to 2% change between expected and actual
                 if (self._current_tx_rate['m_tx_bps'] > 0.98 * self._expected_tx_rate['m_tx_expected_bps']):
                     self._done_warmup = True
+                    latest_dump['warmup_barrier'] = True
 
             # handle latency data
             if self.latency_checked:
-                latency_per_port         = self.get_last_value("trex-latecny-v2.data", "port-")
+                # fix typos, by "pointer"
+                if 'trex-latecny-v2' in latest_dump and 'trex-latency-v2' not in latest_dump:
+                    latest_dump['trex-latency-v2'] = latest_dump['trex-latecny-v2']
+                if 'trex-latecny' in latest_dump and 'trex-latency' not in latest_dump:
+                    latest_dump['trex-latency'] = latest_dump['trex-latecny']
+
+                latency_per_port         = self.get_last_value("trex-latency-v2.data", "port-")
                 self._max_latency        = self.__get_filtered_max_latency(latency_per_port, self.filtered_latency_amount)
-                avg_latency              = self.get_last_value("trex-latecny.data", "avg-")
+                avg_latency              = self.get_last_value("trex-latency.data", "avg-")
                 self._avg_latency        = CTRexResult.__avg_all_and_rename_keys(avg_latency)
-                avg_win_latency_list     = self.get_value_list("trex-latecny.data", "avg-")
+                avg_win_latency_list     = self.get_value_list("trex-latency.data", "avg-")
                 self._avg_window_latency = CTRexResult.__calc_latency_win_stats(avg_win_latency_list)
 
             tx_pkts = CTRexResult.__get_value_by_path(latest_dump, "trex-global.data.m_total_tx_pkts")
@@ -1447,12 +1437,12 @@ class CTRexResult(object):
             for i, p in re.findall(r'(\d+)|([\w|-]+)', tree_path):
                 dct = dct[p or int(i)]
             if regex is not None and isinstance(dct, dict):
-            	res = {}
-            	for key,val in dct.items():
-            		match = re.match(regex, key)
-            		if match:
-            			res[key]=val
-            	return res
+                res = {}
+                for key,val in dct.items():
+                    match = re.match(regex, key)
+                    if match:
+                        res[key]=val
+                return res
             else:
                return dct
         except (KeyError, TypeError):
@@ -1500,26 +1490,70 @@ class CTRexResult(object):
     @staticmethod
     def __get_filtered_max_latency (src_dict, filtered_latency_amount = 0.001):
         result = {}
-        for port, data in src_dict.items():
-            if not port.startswith('port-'):
-                continue
-            max_port = 'max-%s' % port[5:]
-            res = data['hist']
-            if not len(res['histogram']):
-                result[max_port] = 0
-                continue
-            result[max_port] = 5 # if sum below will not get to filtered amount, use this value
-            sum_high = 0.0
-            for elem in reversed(res['histogram']):
-                sum_high += elem['val']
-                if sum_high >= filtered_latency_amount * res['cnt']:
-                    result[max_port] = elem['key'] + int('5' + repr(elem['key'])[2:])
-                    break
+        if src_dict:
+            for port, data in src_dict.items():
+                if not port.startswith('port-'):
+                    continue
+                max_port = 'max-%s' % port[5:]
+                res = data['hist']
+                if not len(res['histogram']):
+                    result[max_port] = 0
+                    continue
+                result[max_port] = 5 # if sum below will not get to filtered amount, use this value
+                sum_high = 0.0
+                for elem in reversed(res['histogram']):
+                    sum_high += elem['val']
+                    if sum_high >= filtered_latency_amount * res['cnt']:
+                        result[max_port] = elem['key'] + int('5' + repr(elem['key'])[2:])
+                        break
         return result
 
 
+    # history iterator after warmup period
+    def _get_steady_state_history_iterator(self):
+        if not self.is_done_warmup():
+            raise Exception('Warm-up period not finished')
+        for index, res in enumerate(self._history):
+            if 'warmup_barrier' in res:
+                for steady_state_index in range(index, max(index, len(self._history) - 1)):
+                    yield self._history[steady_state_index]
+                return
+        for index in range(len(self._history) - 1):
+            yield self._history[index]
+
+
+    def get_avg_steady_state_value(self, tree_path_to_key):
+        '''
+        Gets average value after warmup period.
+        For example: <result object>.get_avg_steady_state_value('trex-global.data.m_tx_bps')
+        Usually more accurate than latest history value.
+
+        :parameters:
+            tree_path_to_key : str
+                defines a path to desired data.
+
+        :return: 
+            average value at steady state
+
+        :raises:
+            Exception in case steady state period was not reached or tree_path_to_key was not found in result.
+        '''
+        values_arr = [self.__get_value_by_path(res, tree_path_to_key) for res in self._get_steady_state_history_iterator()]
+        values_arr = list(filter(lambda x: x is not None, values_arr))
+        if not values_arr:
+            raise Exception('All the keys are None, probably wrong tree_path_to_key: %s' % tree_path_to_key)
+        return sum(values_arr) / float(len(values_arr))
 
 
 if __name__ == "__main__":
-    pass
+    c = CTRexClient('127.0.0.1')
+    print('restarting daemon')
+    c.restart_trex_daemon()
+    print('kill any running')
+    c.kill_all_trexes()
+    print('start')
+    c.start_stateless()
+    print('sleep')
+    time.sleep(5)
+    print('done')
 
diff --git a/scripts/automation/trex_control_plane/stl/console/trex_console.py b/scripts/automation/trex_control_plane/stl/console/trex_console.py
index ab70d357..110457d6 100755
--- a/scripts/automation/trex_control_plane/stl/console/trex_console.py
+++ b/scripts/automation/trex_control_plane/stl/console/trex_console.py
@@ -241,20 +241,7 @@ class TRexConsole(TRexGeneralCmd):
 
 
     def postcmd(self, stop, line):
-
-        if not self.stateless_client.is_connected():
-            self.prompt = "trex(offline)>"
-            self.supported_rpc = None
-
-        elif not self.stateless_client.get_acquired_ports():
-            self.prompt = "trex(read-only)>"
-
-        elif self.stateless_client.is_all_ports_acquired():
-            self.prompt = "trex>"
-
-        else:
-            self.prompt = "trex {0}>".format(self.stateless_client.get_acquired_ports())
-
+        self.prompt = self.stateless_client.generate_prompt(prefix = 'trex')
         return stop
 
 
@@ -292,6 +279,11 @@ class TRexConsole(TRexGeneralCmd):
         self.stateless_client.ping_line(line)
 
 
+    @verify_connected
+    def do_shutdown (self, line):
+        '''Sends the server a shutdown request\n'''
+        self.stateless_client.shutdown_line(line)
+
     # set verbose on / off
     def do_verbose(self, line):
         '''Shows or set verbose mode\n'''
@@ -316,25 +308,21 @@ class TRexConsole(TRexGeneralCmd):
         self.do_history("-h")
 
     def do_shell (self, line):
-        return self.do_history(line)
+        self.do_history(line)
 
     def do_push (self, line):
         '''Push a local PCAP file\n'''
-        return self.stateless_client.push_line(line)
-
-    #def do_push_remote (self, line):
-    #    '''Push a remote accessible PCAP file\n'''
-    #    return self.stateless_client.push_remote_line(line)
+        self.stateless_client.push_line(line)
 
     def help_push (self):
-        return self.do_push("-h")
+        self.do_push("-h")
 
     def do_portattr (self, line):
         '''Change/show port(s) attributes\n'''
-        return self.stateless_client.set_port_attr_line(line)
+        self.stateless_client.set_port_attr_line(line)
 
     def help_portattr (self):
-        return self.do_portattr("-h")
+        self.do_portattr("-h")
 
     @verify_connected
     def do_map (self, line):
@@ -459,7 +447,6 @@ class TRexConsole(TRexGeneralCmd):
 
         self.stateless_client.start_line(line)
 
-        
 
 
     def help_start(self):
@@ -549,7 +536,7 @@ class TRexConsole(TRexGeneralCmd):
 
     def do_events (self, line):
         '''shows events recieved from server\n'''
-        return self.stateless_client.get_events_line(line)
+        self.stateless_client.get_events_line(line)
 
 
     def complete_profile(self, text, line, begidx, endidx):
@@ -563,11 +550,11 @@ class TRexConsole(TRexGeneralCmd):
     @verify_connected
     def do_tui (self, line):
         '''Shows a graphical console\n'''
-
         parser = parsing_opts.gen_parser(self,
                                          "tui",
                                          self.do_tui.__doc__,
-                                         parsing_opts.XTERM)
+                                         parsing_opts.XTERM,
+                                         parsing_opts.LOCKED)
 
         opts = parser.parse_args(line.split())
         if opts is None:
@@ -581,16 +568,20 @@ class TRexConsole(TRexGeneralCmd):
             info = self.stateless_client.get_connection_info()
 
             exe = './trex-console --top -t -q -s {0} -p {1} --async_port {2}'.format(info['server'], info['sync_port'], info['async_port'])
-            cmd = ['/usr/bin/xterm', '-geometry', '111x49', '-sl', '0', '-title', 'trex_tui', '-e', exe]
+            cmd = ['/usr/bin/xterm', '-geometry', '{0}x{1}'.format(self.tui.MIN_COLS, self.tui.MIN_ROWS), '-sl', '0', '-title', 'trex_tui', '-e', exe]
 
             # detach child
             self.terminal = subprocess.Popen(cmd, preexec_fn = os.setpgrp)
 
             return
 
+        
+        try:
+            with self.stateless_client.logger.supress():
+                self.tui.show(self.stateless_client, self.save_console_history, locked = opts.locked)
 
-        with self.stateless_client.logger.supress():
-            self.tui.show()
+        except self.tui.ScreenSizeException as e:
+            print(format_text(str(e) + "\n", 'bold'))
 
 
     def help_tui (self):
@@ -872,7 +863,8 @@ def main():
 
         # TUI
         if options.tui:
-            console.do_tui("-x" if options.xtui else "")
+            console.do_tui("-x" if options.xtui else "-l")
+
         else:
             console.start()
             
diff --git a/scripts/automation/trex_control_plane/stl/console/trex_tui.py b/scripts/automation/trex_control_plane/stl/console/trex_tui.py
index d3be4435..a69c4165 100644
--- a/scripts/automation/trex_control_plane/stl/console/trex_tui.py
+++ b/scripts/automation/trex_control_plane/stl/console/trex_tui.py
@@ -4,6 +4,8 @@ import os
 import time
 from collections import OrderedDict, deque
 import datetime
+import readline
+from texttable import ansi_len
 
 if sys.version_info > (3,0):
     from io import StringIO
@@ -15,9 +17,23 @@ from trex_stl_lib.utils import text_tables
 from trex_stl_lib import trex_stl_stats
 from trex_stl_lib.utils.filters import ToggleFilter
 
+class TUIQuit(Exception):
+    pass
+
+
 # for STL exceptions
 from trex_stl_lib.api import *
 
+def ascii_split (s):
+    output = []
+
+    lines = s.split('\n')
+    for elem in lines:
+        if ansi_len(elem) > 0:
+            output.append(elem)
+
+    return output
+
 class SimpleBar(object):
     def __init__ (self, desc, pattern):
         self.desc = desc
@@ -210,7 +226,7 @@ class TrexTUILatencyStats(TrexTUIPanel):
         super(TrexTUILatencyStats, self).__init__(mng, "lstats")
         self.key_actions = OrderedDict()
         self.key_actions['c'] = {'action': self.action_clear,  'legend': 'clear', 'show': True}
-        self.key_actions['t'] = {'action': self.action_toggle_histogram,  'legend': 'toggle histogram', 'show': True}
+        self.key_actions['h'] = {'action': self.action_toggle_histogram,  'legend': 'histogram toggle', 'show': True}
         self.is_histogram = False
 
 
@@ -238,6 +254,23 @@ class TrexTUILatencyStats(TrexTUIPanel):
          self.stateless_client.latency_stats.clear_stats()
          return ""
 
+
+# utilization stats
+class TrexTUIUtilizationStats(TrexTUIPanel):
+    def __init__ (self, mng):
+        super(TrexTUIUtilizationStats, self).__init__(mng, "ustats")
+        self.key_actions = {}
+
+    def show (self):
+        stats = self.stateless_client._get_formatted_stats(port_id_list = None, stats_mask = trex_stl_stats.UT_COMPAT)
+        # print stats to screen
+        for stat_type, stat_data in stats.items():
+            text_tables.print_table_with_header(stat_data.text_table, stat_type)
+
+    def get_key_actions (self):
+        return self.key_actions 
+
+
 # log
 class TrexTUILog():
     def __init__ (self):
@@ -258,24 +291,42 @@ class TrexTUILog():
             print(msg)
 
 
+# a predicate to wrap function as a bool
+class Predicate(object):
+    def __init__ (self, func):
+        self.func = func
+
+    def __nonzero__ (self):
+        return True if self.func() else False
+    def __bool__ (self):
+        return True if self.func() else False
+
+
 # Panels manager (contains server panels)
 class TrexTUIPanelManager():
     def __init__ (self, tui):
         self.tui = tui
         self.stateless_client = tui.stateless_client
         self.ports = self.stateless_client.get_all_ports()
-        
+        self.locked = False
 
         self.panels = {}
         self.panels['dashboard'] = TrexTUIDashBoard(self)
         self.panels['sstats']    = TrexTUIStreamsStats(self)
         self.panels['lstats']    = TrexTUILatencyStats(self)
+        self.panels['ustats']    = TrexTUIUtilizationStats(self)
 
         self.key_actions = OrderedDict()
-        self.key_actions['q'] = {'action': self.action_quit, 'legend': 'quit', 'show': True}
+
+        # we allow console only when ports are acquired
+        self.key_actions['ESC'] = {'action': self.action_none, 'legend': 'console', 'show': Predicate(lambda : not self.locked)}
+
+        self.key_actions['q'] = {'action': self.action_none, 'legend': 'quit', 'show': True}
         self.key_actions['d'] = {'action': self.action_show_dash, 'legend': 'dashboard', 'show': True}
         self.key_actions['s'] = {'action': self.action_show_sstats, 'legend': 'streams', 'show': True}
         self.key_actions['l'] = {'action': self.action_show_lstats, 'legend': 'latency', 'show': True}
+        self.key_actions['u'] = {'action': self.action_show_ustats, 'legend': 'util', 'show': True}
+        
 
         # start with dashboard
         self.main_panel = self.panels['dashboard']
@@ -290,7 +341,6 @@ class TrexTUIPanelManager():
         self.show_log = False
 
         
-
     def generate_legend (self):
 
         self.legend = "\n{:<12}".format("browse:")
@@ -327,14 +377,18 @@ class TrexTUIPanelManager():
 
 
     # on window switch or turn on / off of the TUI we call this
-    def init (self, show_log = False):
+    def init (self, show_log = False, locked = False):
         self.show_log = show_log
+        self.locked = locked
         self.generate_legend()
 
-    def show (self):
+    def show (self, show_legend):
         self.main_panel.show()
         self.print_connection_status()
-        self.print_legend()
+
+        if show_legend:
+            self.generate_legend()
+            self.print_legend()
 
         if self.show_log:
             self.log.show()
@@ -350,21 +404,22 @@ class TrexTUIPanelManager():
             msg = self.main_panel.get_key_actions()[ch]['action']()
 
         else:
-            msg = ""
+            return False
 
         self.generate_legend()
-
-        if msg == None:
-            return False
-        else:
-            if msg:
-                self.log.add_event(msg)
-            return True
+        return True
+
+        #if msg == None:
+        #    return False
+        #else:
+        #    if msg:
+        #        self.log.add_event(msg)
+        #    return True
             
 
     # actions
 
-    def action_quit (self):
+    def action_none (self):
         return None
 
     def action_show_dash (self):
@@ -392,6 +447,11 @@ class TrexTUIPanelManager():
         self.init(self.show_log)
         return ""
 
+    def action_show_ustats(self):
+        self.main_panel = self.panels['ustats']
+        self.init(self.show_log)
+        return ""
+
 # shows a textual top style window
 class TrexTUI():
 
@@ -400,52 +460,71 @@ class TrexTUI():
     STATE_RECONNECT  = 2
     is_graph = False
 
+    MIN_ROWS = 50
+    MIN_COLS = 111
+
+    class ScreenSizeException(Exception):
+        def __init__ (self, cols, rows):
+            msg = "TUI requires console screen size of at least {0}x{1}, current is {2}x{3}".format(TrexTUI.MIN_COLS,
+                                                                                                    TrexTUI.MIN_ROWS,
+                                                                                                    cols,
+                                                                                                    rows)
+            super(TrexTUI.ScreenSizeException, self).__init__(msg)
+
     def __init__ (self, stateless_client):
         self.stateless_client = stateless_client
 
         self.pm = TrexTUIPanelManager(self)
+                    
 
+    def clear_screen (self, lines = 50):
+        # reposition the cursor
+        sys.stdout.write("\x1b[0;0H")
 
+        # clear all lines
+        for i in range(lines):
+            sys.stdout.write("\x1b[0K")
+            if i < (lines - 1):
+                sys.stdout.write("\n")
 
-    def handle_key_input (self):
-        # try to read a single key
-        ch = os.read(sys.stdin.fileno(), 1).decode()
-        if ch != None and len(ch) > 0:
-            return (self.pm.handle_key(ch), True)
+        # reposition the cursor
+        sys.stdout.write("\x1b[0;0H")
 
-        else:
-            return (True, False)
-            
+        #sys.stdout.write("\x1b[2J\x1b[H")
 
-    def clear_screen (self):
-        #os.system('clear')
-        # maybe this is faster ?
-        sys.stdout.write("\x1b[2J\x1b[H")
 
+    def show (self, client, save_console_history, show_log = False, locked = False):
+        
+        rows, cols = os.popen('stty size', 'r').read().split()
+        if (int(rows) < TrexTUI.MIN_ROWS) or (int(cols) < TrexTUI.MIN_COLS):
+            raise self.ScreenSizeException(rows = rows, cols = cols)
 
+        with AsyncKeys(client, save_console_history, locked) as async_keys:
+            sys.stdout.write("\x1bc")
+            self.async_keys = async_keys
+            self.show_internal(show_log, locked)
 
-    def show (self, show_log = False):
-        # init termios
-        old_settings = termios.tcgetattr(sys.stdin)
-        new_settings = termios.tcgetattr(sys.stdin)
-        new_settings[3] = new_settings[3] & ~(termios.ECHO | termios.ICANON) # lflags
-        new_settings[6][termios.VMIN] = 0  # cc
-        new_settings[6][termios.VTIME] = 0 # cc
-        termios.tcsetattr(sys.stdin, termios.TCSADRAIN, new_settings)
 
-        self.pm.init(show_log)
+
+    def show_internal (self, show_log, locked):
+
+        self.pm.init(show_log, locked)
 
         self.state = self.STATE_ACTIVE
-        self.draw_policer = 0
+        self.last_redraw_ts = 0
 
         try:
             while True:
                 # draw and handle user input
-                cont, force_draw = self.handle_key_input()
-                self.draw_screen(force_draw)
-                if not cont:
-                    break
-                time.sleep(0.1)
+                status = self.async_keys.tick(self.pm)
+
+                self.draw_screen(status)
+
+                # speedup for keys, slower for no keys
+                if status == AsyncKeys.STATUS_NONE:
+                    time.sleep(0.01)
+                else:
+                    time.sleep(0.001)
 
                 # regular state
                 if self.state == self.STATE_ACTIVE:
@@ -473,34 +552,510 @@ class TrexTUI():
                         self.state = self.STATE_LOST_CONT
 
 
-        finally:
-            # restore
-            termios.tcsetattr(sys.stdin, termios.TCSADRAIN, old_settings)
+        except TUIQuit:
+            print("\nExiting TUI...")
 
         print("")
 
 
     # draw once
-    def draw_screen (self, force_draw = False):
-
-        if (self.draw_policer >= 5) or (force_draw):
+    def draw_screen (self, status):
+        t = time.time() - self.last_redraw_ts
+        redraw = (t >= 0.5) or (status == AsyncKeys.STATUS_REDRAW_ALL)
 
+        if redraw:
             # capture stdout to a string
             old_stdout = sys.stdout
             sys.stdout = mystdout = StringIO()
-            self.pm.show()
+            self.pm.show(show_legend = self.async_keys.is_legend_mode())
+            self.last_snap = mystdout.getvalue()
+
+            self.async_keys.draw()
             sys.stdout = old_stdout
 
             self.clear_screen()
 
-            print(mystdout.getvalue())
+            sys.stdout.write(mystdout.getvalue())
+           
+            sys.stdout.flush()
+            self.last_redraw_ts = time.time()
+
+        elif status == AsyncKeys.STATUS_REDRAW_KEYS:
+            sys.stdout.write("\x1b[4A")
 
+            self.async_keys.draw()
             sys.stdout.flush()
 
-            self.draw_policer = 0
-        else:
-            self.draw_policer += 1 
+        return
+
+
 
     def get_state (self):
         return self.state
 
+
+
+
+
+# handles async IO
+class AsyncKeys:
+
+    MODE_LEGEND  = 1
+    MODE_CONSOLE = 2
+
+    STATUS_NONE        = 0
+    STATUS_REDRAW_KEYS = 1
+    STATUS_REDRAW_ALL  = 2
+
+    def __init__ (self, client, save_console_history, locked = False):
+        self.engine_console = AsyncKeysEngineConsole(self, client, save_console_history)
+        self.engine_legend  = AsyncKeysEngineLegend(self)
+        self.locked = locked
+
+        if locked:
+            self.engine = self.engine_legend
+            self.locked = True
+        else:
+            self.engine = self.engine_console
+            self.locked = False
+
+    def __enter__ (self):
+        # init termios
+        self.old_settings = termios.tcgetattr(sys.stdin)
+        new_settings = termios.tcgetattr(sys.stdin)
+        new_settings[3] = new_settings[3] & ~(termios.ECHO | termios.ICANON) # lflags
+        new_settings[6][termios.VMIN] = 0  # cc
+        new_settings[6][termios.VTIME] = 0 # cc
+        termios.tcsetattr(sys.stdin, termios.TCSADRAIN, new_settings)
+
+        # huge buffer - no print without flush
+        sys.stdout = open('/dev/stdout', 'w', TrexTUI.MIN_COLS * TrexTUI.MIN_COLS * 2)
+        return self
+
+    def __exit__ (self, type, value, traceback):
+        termios.tcsetattr(sys.stdin, termios.TCSADRAIN, self.old_settings)
+
+        # restore sys.stdout
+        sys.stdout.close()
+        sys.stdout = sys.__stdout__
+
+
+    def is_legend_mode (self):
+        return self.engine.get_type() == AsyncKeys.MODE_LEGEND
+
+    def is_console_mode (self):
+        return self.engine.get_type == AsyncKeys.MODE_CONSOLE
+
+    def switch (self):
+        if self.is_legend_mode():
+            self.engine = self.engine_console
+        else:
+            self.engine = self.engine_legend
+
+
+    def tick (self, pm):
+        seq = ''
+        # drain all chars
+        while True:
+            ch = os.read(sys.stdin.fileno(), 1).decode()
+            if not ch:
+                break
+            seq += ch
+
+        if not seq:
+            return self.STATUS_NONE
+
+        # ESC for switch
+        if seq == '\x1b':
+            if not self.locked:
+                self.switch()
+            return self.STATUS_REDRAW_ALL
+
+        # EOF (ctrl + D)
+        if seq == '\x04':
+            raise TUIQuit()
+
+        # pass tick to engine
+        return self.engine.tick(seq, pm)
+
+
+    def draw (self):
+        self.engine.draw()
+ 
+
+    
+# Legend engine
+class AsyncKeysEngineLegend:
+    def __init__ (self, async):
+        self.async = async
+
+    def get_type (self):
+        return self.async.MODE_LEGEND
+
+    def tick (self, seq, pm):
+
+        if seq == 'q':
+            raise TUIQuit()
+
+        # ignore escapes
+        if len(seq) > 1:
+            return AsyncKeys.STATUS_NONE
+
+        rc = pm.handle_key(seq)
+        return AsyncKeys.STATUS_REDRAW_ALL if rc else AsyncKeys.STATUS_NONE
+
+    def draw (self):
+        pass
+
+
+# console engine
+class AsyncKeysEngineConsole:
+    def __init__ (self, async, client, save_console_history):
+        self.async = async
+        self.lines = deque(maxlen = 100)
+
+        self.generate_prompt       = client.generate_prompt
+        self.save_console_history  = save_console_history
+
+        self.ac = {'start'        : client.start_line,
+                   'stop'         : client.stop_line,
+                   'pause'        : client.pause_line,
+                   'push'         : client.push_line,
+                   'resume'       : client.resume_line,
+                   'update'       : client.update_line,
+                   'connect'      : client.connect_line,
+                   'disconnect'   : client.disconnect_line,
+                   'acquire'      : client.acquire_line,
+                   'release'      : client.release_line,
+                   'quit'         : self.action_quit,
+                   'q'            : self.action_quit,
+                   'exit'         : self.action_quit,
+                   'help'         : self.action_help,
+                   '?'            : self.action_help}
+
+        # fetch readline history and add relevants
+        for i in range(0, readline.get_current_history_length()):
+            cmd = readline.get_history_item(i)
+            if cmd and cmd.split()[0] in self.ac:
+                self.lines.appendleft(CmdLine(cmd))
+
+        # new line
+        self.lines.appendleft(CmdLine(''))
+        self.line_index = 0
+        self.last_status = ''
+
+    def action_quit (self, _):
+        raise TUIQuit()
+
+    def action_help (self, _):
+        return ' '.join([format_text(cmd, 'bold') for cmd in self.ac.keys()])
+
+    def get_type (self):
+        return self.async.MODE_CONSOLE
+
+
+    def handle_escape_char (self, seq):
+        # up
+        if seq == '\x1b[A':
+            self.line_index = min(self.line_index + 1, len(self.lines) - 1)
+
+        # down
+        elif seq == '\x1b[B':
+            self.line_index = max(self.line_index - 1, 0)
+
+        # left
+        elif seq == '\x1b[D':
+            self.lines[self.line_index].go_left()
+
+        # right
+        elif seq == '\x1b[C':
+            self.lines[self.line_index].go_right()
+
+        # del
+        elif seq == '\x1b[3~':
+            self.lines[self.line_index].del_key()
+
+        # home
+        elif seq == '\x1b[H':
+            self.lines[self.line_index].home_key()
+
+        # end
+        elif seq == '\x1b[F':
+            self.lines[self.line_index].end_key()
+            return True
+
+        # unknown key
+        else:
+            return AsyncKeys.STATUS_NONE
+
+        return AsyncKeys.STATUS_REDRAW_KEYS
+
+
+    def tick (self, seq, _):
+    
+        # handle escape chars
+        if len(seq) > 1:
+            return self.handle_escape_char(seq)
+
+        # handle each char
+        for ch in seq:
+            return self.handle_single_key(ch)
+
+
+    
+    def handle_single_key (self, ch):
+
+        # newline
+        if ch == '\n':
+            self.handle_cmd()
+
+        # backspace
+        elif ch == '\x7f':
+            self.lines[self.line_index].backspace()
+
+        # TAB
+        elif ch == '\t':
+            tokens = self.lines[self.line_index].get().split()
+            if not tokens:
+                return
+
+            if len(tokens) == 1:
+                self.handle_tab_names(tokens[0])
+            else:
+                self.handle_tab_files(tokens)
+
+
+        # simple char
+        else:
+            self.lines[self.line_index] += ch
+
+        return AsyncKeys.STATUS_REDRAW_KEYS
+
+
+    # handle TAB key for completing function names
+    def handle_tab_names (self, cur):
+        matching_cmds = [x for x in self.ac if x.startswith(cur)]
+
+        common = os.path.commonprefix([x for x in self.ac if x.startswith(cur)])
+        if common:
+            if len(matching_cmds) == 1:
+                self.lines[self.line_index].set(common + ' ')
+                self.last_status = ''
+            else:
+                self.lines[self.line_index].set(common)
+                self.last_status = 'ambigious: '+ ' '.join([format_text(cmd, 'bold') for cmd in matching_cmds])
+
+
+    # handle TAB for completing filenames
+    def handle_tab_files (self, tokens):
+
+        # only commands with files
+        if tokens[0] not in {'start', 'push'}:
+            return
+
+        # '-f' with no paramters - no partial and use current dir
+        if tokens[-1] == '-f':
+            partial = ''
+            d = '.'
+
+        # got a partial path
+        elif tokens[-2] == '-f':
+            partial = tokens.pop()
+
+            # check for dirs
+            dirname, basename = os.path.dirname(partial), os.path.basename(partial)
+            if os.path.isdir(dirname):
+                d = dirname
+                partial = basename
+            else:
+                d = '.'
+        else:
+            return
+
+        # fetch all dirs and files matching wildcard
+        files = []
+        for x in os.listdir(d):
+            if os.path.isdir(os.path.join(d, x)):
+                files.append(x + '/')
+            elif x.endswith('.py') or x.endswith('yaml') or x.endswith('pcap') or x.endswith('cap'):
+                files.append(x)
+
+        # dir might not have the files
+        if not files:
+            self.last_status = format_text('no loadble files under path', 'bold')
+            return
+
+
+        # find all the matching files
+        matching_files = [x for x in files if x.startswith(partial)] if partial else files
+
+        # do we have a longer common than partial ?
+        common = os.path.commonprefix([x for x in files if x.startswith(partial)])
+        if not common:
+            common = partial
+
+        tokens.append(os.path.join(d, common) if d is not '.' else common)
+
+        # reforge the line
+        newline = ' '.join(tokens)
+
+        if len(matching_files) == 1:
+            if os.path.isfile(tokens[-1]):
+                newline += ' '
+
+            self.lines[self.line_index].set(newline)
+            self.last_status = ''
+        else:
+            self.lines[self.line_index].set(newline)
+            self.last_status = '    '.join([format_text(f, 'bold') for f in matching_files[:5]])
+            if len(matching_files) > 5:
+                self.last_status += ' ... [{0} more matches]'.format(len(matching_files) - 5)
+
+
+
+    def split_cmd (self, cmd):
+        s = cmd.split(' ', 1)
+        op = s[0]
+        param = s[1] if len(s) == 2 else ''
+        return op, param
+
+
+    def handle_cmd (self):
+        cmd = self.lines[self.line_index].get().strip()
+        if not cmd:
+            return
+
+        op, param = self.split_cmd(cmd)
+        
+        func = self.ac.get(op)
+        if func:
+            func_rc = func(param)
+
+        # take out the empty line
+        empty_line = self.lines.popleft()
+        assert(empty_line.ro_line == '')
+
+        if not self.lines or self.lines[0].ro_line != cmd:
+            self.lines.appendleft(CmdLine(cmd))
+        
+        # back in
+        self.lines.appendleft(empty_line)
+        self.line_index = 0
+        readline.add_history(cmd)
+        self.save_console_history()
+
+        # back to readonly
+        for line in self.lines:
+            line.invalidate()
+
+        assert(self.lines[0].modified == False)
+        color = None
+        if not func:
+            self.last_status = "unknown command: '{0}'".format(format_text(cmd.split()[0], 'bold'))
+        else:
+            # internal commands
+            if isinstance(func_rc, str):
+                self.last_status = func_rc
+
+            # RC response
+            else:
+                # success
+                if func_rc:
+                    self.last_status = format_text("[OK]", 'green')
+
+                # errors
+                else:
+                    err_msgs = ascii_split(str(func_rc))
+                    self.last_status = format_text(err_msgs[0], 'red')
+                    if len(err_msgs) > 1:
+                        self.last_status += " [{0} more errors messages]".format(len(err_msgs) - 1)
+                    color = 'red'
+
+        # trim too long lines
+        if ansi_len(self.last_status) > TrexTUI.MIN_COLS:
+            self.last_status = format_text(self.last_status[:TrexTUI.MIN_COLS] + "...", color, 'bold')
+
+
+    def draw (self):
+        sys.stdout.write("\nPress 'ESC' for navigation panel...\n")
+        sys.stdout.write("status: \x1b[0K{0}\n".format(self.last_status))
+        sys.stdout.write("\n{0}\x1b[0K".format(self.generate_prompt(prefix = 'tui')))
+        self.lines[self.line_index].draw()
+
+
+# a readline alike command line - can be modified during edit
+class CmdLine(object):
+    def __init__ (self, line):
+        self.ro_line      = line
+        self.w_line       = None
+        self.modified     = False
+        self.cursor_index = len(line)
+
+    def get (self):
+        if self.modified:
+            return self.w_line
+        else:
+            return self.ro_line
+
+    def set (self, line, cursor_pos = None):
+        self.w_line = line
+        self.modified = True
+
+        if cursor_pos is None:
+            self.cursor_index = len(self.w_line)
+        else:
+            self.cursor_index = cursor_pos
+
+
+    def __add__ (self, other):
+        assert(0)
+
+
+    def __str__ (self):
+        return self.get()
+
+
+    def __iadd__ (self, other):
+
+        self.set(self.get()[:self.cursor_index] + other + self.get()[self.cursor_index:],
+                 cursor_pos = self.cursor_index + len(other))
+
+        return self
+
+
+    def backspace (self):
+        if self.cursor_index == 0:
+            return
+
+        self.set(self.get()[:self.cursor_index - 1] + self.get()[self.cursor_index:],
+                 self.cursor_index - 1)
+
+
+    def del_key (self):
+        if self.cursor_index == len(self.get()):
+            return
+
+        self.set(self.get()[:self.cursor_index] + self.get()[self.cursor_index + 1:],
+                 self.cursor_index)
+
+    def home_key (self):
+        self.cursor_index = 0
+
+    def end_key (self):
+        self.cursor_index = len(self.get())
+
+    def invalidate (self):
+        self.modified = False
+        self.w_line = None
+        self.cursor_index = len(self.ro_line)
+
+    def go_left (self):
+        self.cursor_index = max(0, self.cursor_index - 1)
+
+    def go_right (self):
+        self.cursor_index = min(len(self.get()), self.cursor_index + 1)
+
+    def draw (self):
+        sys.stdout.write(self.get())
+        sys.stdout.write('\b' * (len(self.get()) - self.cursor_index))
+
diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_flow_latency_stats.py b/scripts/automation/trex_control_plane/stl/examples/stl_flow_latency_stats.py
index 86b91728..d8a99479 100644
--- a/scripts/automation/trex_control_plane/stl/examples/stl_flow_latency_stats.py
+++ b/scripts/automation/trex_control_plane/stl/examples/stl_flow_latency_stats.py
@@ -1,10 +1,12 @@
+# Example showing how to define stream for latency measurement, and how to parse the latency information
+
 import stl_path
 from trex_stl_lib.api import *
 
 import time
 import pprint
 
-def rx_example (tx_port, rx_port, burst_size, bw):
+def rx_example (tx_port, rx_port, burst_size, pps):
 
     print("\nGoing to inject {0} packets on port {1} - checking RX stats on port {2}\n".format(burst_size, tx_port, rx_port))
 
@@ -19,7 +21,7 @@ def rx_example (tx_port, rx_port, burst_size, bw):
                        packet = pkt,
                        flow_stats = STLFlowLatencyStats(pg_id = 5),
                        mode = STLTXSingleBurst(total_pkts = total_pkts,
-                                               percentage = bw))
+                                               pps = pps))
 
         # connect to server
         c.connect()
@@ -32,7 +34,7 @@ def rx_example (tx_port, rx_port, burst_size, bw):
 
         print("\nInjecting {0} packets on port {1}\n".format(total_pkts, tx_port))
 
-        rc = rx_iteration(c, tx_port, rx_port, total_pkts, pkt.get_pkt_len(), bw)
+        rc = rx_iteration(c, tx_port, rx_port, total_pkts, pkt.get_pkt_len())
         if not rc:
             passed = False
 
@@ -44,12 +46,12 @@ def rx_example (tx_port, rx_port, burst_size, bw):
         c.disconnect()
 
     if passed:
-        print("\nTest has passed :-)\n")
+        print("\nTest passed :-)\n")
     else:
-        print("\nTest has failed :-(\n")
+        print("\nTest failed :-(\n")
 
 # RX one iteration
-def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len, bw):
+def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len):
     
     c.clear_stats()
 
@@ -58,7 +60,8 @@ def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len, bw):
 
     stats = c.get_stats()
     flow_stats = stats['flow_stats'].get(5)
-    lat_stats = stats['latency'].get(5)
+    global_lat_stats = stats['latency']
+    lat_stats = global_lat_stats.get(5)
     if not flow_stats:
         print("no flow stats available")
         return False
@@ -74,6 +77,8 @@ def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len, bw):
     dup = lat_stats['err_cntrs']['dup']
     sth = lat_stats['err_cntrs']['seq_too_high']
     stl = lat_stats['err_cntrs']['seq_too_low']
+    old_flow = global_lat_stats['global']['old_flow']
+    bad_hdr = global_lat_stats['global']['bad_hdr']
     lat = lat_stats['latency']
     jitter = lat['jitter']
     avg = lat['average']
@@ -89,6 +94,10 @@ def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len, bw):
             return False
 
     print('Error counters: dropped:{0}, ooo:{1} dup:{2} seq too high:{3} seq too low:{4}'.format(drops, ooo, dup, sth, stl))
+    if old_flow:
+        print ('Packets arriving too late after flow stopped: {0}'.format(old_flow))
+    if bad_hdr:
+        print ('Latency packets with corrupted info: {0}'.format(bad_hdr))
     print('Latency info:')
     print("  Maximum latency(usec): {0}".format(tot_max))
     print("  Minimum latency(usec): {0}".format(tot_min))
@@ -114,7 +123,7 @@ def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len, bw):
     else:
         print("TX pkts match   - {0}".format(tx_pkts))
 
-    if tx_bytes != (total_pkts * pkt_len):
+    if tx_bytes != (total_pkts * (pkt_len + 4)): # +4 for ethernet CRC
         print("TX bytes mismatch - got: {0}, expected: {1}".format(tx_bytes, (total_pkts * pkt_len)))
         pprint.pprint(flow_stats)
         return False
@@ -131,5 +140,5 @@ def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len, bw):
     return True
 
 # run the tests
-rx_example(tx_port = 1, rx_port = 0, burst_size = 500000, bw = 50)
+rx_example(tx_port = 0, rx_port = 1, burst_size = 1000, pps = 1000)
 
diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_flow_stats.py b/scripts/automation/trex_control_plane/stl/examples/stl_flow_stats.py
index ed4902fa..3c630ece 100644
--- a/scripts/automation/trex_control_plane/stl/examples/stl_flow_stats.py
+++ b/scripts/automation/trex_control_plane/stl/examples/stl_flow_stats.py
@@ -1,3 +1,5 @@
+# Example showing how to define stream for getting per flow statistics, and how to parse the received statistics
+
 import stl_path
 from trex_stl_lib.api import *
 
@@ -27,18 +29,14 @@ def rx_example (tx_port, rx_port, burst_size, bw):
         # prepare our ports
         c.reset(ports = [tx_port, rx_port])
 
-        # add both streams to ports
+        # add stream to port
         c.add_streams([s1], ports = [tx_port])
 
-        print("\ninjecting {0} packets on port {1}\n".format(total_pkts, tx_port))
-
-        for i in range(0, 10):
-            print("\nStarting iteration: {0}:".format(i))
-            rc = rx_iteration(c, tx_port, rx_port, total_pkts, pkt.get_pkt_len(), bw)
-            if not rc:
-                passed = False
-                break
+        print("\ngoing to inject {0} packets on port {1}\n".format(total_pkts, tx_port))
 
+        rc = rx_iteration(c, tx_port, rx_port, total_pkts, s1.get_pkt_len())
+        if not rc:
+            passed = False
 
     except STLError as e:
         passed = False
@@ -48,19 +46,21 @@ def rx_example (tx_port, rx_port, burst_size, bw):
         c.disconnect()
 
     if passed:
-        print("\nTest has passed :-)\n")
+        print("\nTest passed :-)\n")
     else:
-        print("\nTest has failed :-(\n")
+        print("\nTest failed :-(\n")
 
 # RX one iteration
-def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len, bw):
-    
+def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len):
+    ret = True
+
     c.clear_stats()
 
     c.start(ports = [tx_port])
     c.wait_on_traffic(ports = [tx_port])
 
-    flow_stats = c.get_stats()['flow_stats'].get(5)
+    global_flow_stats = c.get_stats()['flow_stats']
+    flow_stats = global_flow_stats.get(5)
     if not flow_stats:
         print("no flow stats available")
         return False
@@ -78,26 +78,33 @@ def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len, bw):
     if tx_pkts != total_pkts:
         print("TX pkts mismatch - got: {0}, expected: {1}".format(tx_pkts, total_pkts))
         pprint.pprint(flow_stats)
-        return False
+        ret = False
     else:
         print("TX pkts match   - {0}".format(tx_pkts))
 
     if tx_bytes != (total_pkts * pkt_len):
         print("TX bytes mismatch - got: {0}, expected: {1}".format(tx_bytes, (total_pkts * pkt_len)))
         pprint.pprint(flow_stats)
-        return False
+        ret = False
     else:
         print("TX bytes match  - {0}".format(tx_bytes))
 
     if rx_pkts != total_pkts:
         print("RX pkts mismatch - got: {0}, expected: {1}".format(rx_pkts, total_pkts))
         pprint.pprint(flow_stats)
-        return False
+        ret = False
     else:
         print("RX pkts match   - {0}".format(rx_pkts))
 
-    return True
+
+    for field in ['rx_err', 'tx_err']:
+        for port in global_flow_stats['global'][field].keys():
+            if global_flow_stats['global'][field][port] != 0:
+                print ("\n{0} on port {1}: {2} - You should consider increasing rx_delay_ms value in wait_on_traffic"
+                       .format(field, port, global_flow_stats['global'][field][port]))
+
+    return ret
 
 # run the tests
-rx_example(tx_port = 1, rx_port = 2, burst_size = 500000, bw = 50)
+rx_example(tx_port = 0, rx_port = 1, burst_size = 500, bw = 50)
 
diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_imix.py b/scripts/automation/trex_control_plane/stl/examples/stl_imix.py
index 46d86b2b..875186ba 100644
--- a/scripts/automation/trex_control_plane/stl/examples/stl_imix.py
+++ b/scripts/automation/trex_control_plane/stl/examples/stl_imix.py
@@ -5,6 +5,7 @@ import time
 import json
 from pprint import pprint
 import argparse
+import sys
 
 # IMIX test
 # it maps the ports to sides
@@ -97,6 +98,7 @@ def imix_test (server, mult):
     except STLError as e:
         passed = False
         print(e)
+        sys.exit(1)
 
     finally:
         c.disconnect()
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/__init__.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/__init__.py
index ba9459c1..c6e14df3 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/__init__.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/__init__.py
@@ -1,7 +1,7 @@
 import sys
 
 if sys.version_info < (2, 7):
-    print("\n**** TRex STL pacakge requires Python version >= 2.7 ***\n")
+    print("\n**** TRex STL package requires Python version >= 2.7 ***\n")
     exit(-1)
 
 from . import trex_stl_ext
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
index 22895a75..4e3d3092 100755
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
@@ -511,8 +511,7 @@ class STLClient(object):
         self.connected = False
 
         # API classes
-        self.api_vers = [ {'type': 'core', 'major': 1, 'minor':2 }
-                        ]
+        self.api_vers = [ {'type': 'core', 'major': 1, 'minor': 3 } ]
         self.api_h = {'core': None}
 
         # logger
@@ -555,14 +554,17 @@ class STLClient(object):
 
         self.latency_stats = trex_stl_stats.CLatencyStats(self.ports)
 
+        self.util_stats = trex_stl_stats.CUtilStats(self)
+
         self.stats_generator = trex_stl_stats.CTRexInfoGenerator(self.global_stats,
                                                                  self.ports,
                                                                  self.flow_stats,
                                                                  self.latency_stats,
+                                                                 self.util_stats,
                                                                  self.async_client.monitor)
 
 
-       
+
 
     ############# private functions - used by the class itself ###########
 
@@ -1254,18 +1256,187 @@ class STLClient(object):
 
 
     # get stats
-    def get_stats (self, ports = None, async_barrier = True):
+    def get_stats (self, ports = None, sync_now = True):
+        """
+        Return dictionary containing statistics information gathered from the server.
+
+        :parameters:
+
+          ports - List of ports to retreive stats on.
+                  If None, assume the request is for all acquired ports.
+
+          sync_now - Boolean - If true, create a call to the server to get latest stats, and wait for result to arrive. Otherwise, return last stats saved in client cache.
+                            Downside of putting True is a slight delay (few 10th msecs) in getting the result. For practical uses, value should be True.
+        :return:
+            Statistics dictionary of dictionaries with the following format:
+
+            ===============================  ===============
+            key                               Meaning
+            ===============================  ===============
+            :ref:`numbers (0,1,..<total>`    Statistcs per port number
+            :ref:`total <total>`             Sum of port statistics
+            :ref:`flow_stats <flow_stats>`   Per flow statistics
+            :ref:`global <global>`           Global statistics
+            :ref:`latency <latency>`         Per flow statistics regarding flow latency
+            ===============================  ===============
+
+            Below is description of each of the inner dictionaries.
+
+            .. _total:
+
+            **total** and per port statistics contain dictionary with following format.
+
+            Most of the bytes counters (unless specified otherwise) are in L2 layer, including the Ethernet FCS. e.g. minimum packet size is 64 bytes
+
+            ===============================  ===============
+            key                               Meaning
+            ===============================  ===============
+            ibytes                           Number of input bytes 
+            ierrors                          Number of input errors
+            ipackets                         Number of input packets 
+            obytes                           Number of output bytes  
+            oerrors                          Number of output errors
+            opackets                         Number of output packets
+            rx_bps                           Receive bytes per second rate (L2 layer)
+            rx_pps                           Receive packet per second rate
+            tx_bps                           Transmit bytes per second rate (L2 layer)
+            tx_pps                           Transmit packet per second rate
+            ===============================  ===============
+
+            .. _flow_stats:
+
+            **flow_stats** contains :ref:`global dictionary <flow_stats_global>`, and dictionaries per packet group id (pg id). See structures below.
+
+            **per pg_id flow stat** dictionaries have following structure:
+
+            =================   ===============
+            key                 Meaning
+            =================   ===============
+            rx_bps              Received bytes per second rate
+            rx_bps_l1           Received bytes per second rate, including layer one
+            rx_bytes            Total number of received bytes
+            rx_pkts             Total number of received packets
+            rx_pps              Received packets per second
+            tx_bps              Transmit bytes per second rate
+            tx_bps_l1           Transmit bytes per second rate, including layer one
+            tx_bytes            Total number of sent bytes
+            tx_pkts             Total number of sent packets
+            tx_pps              Transmit packets per second rate
+            =================   ===============
+
+            .. _flow_stats_global:
+
+            **global flow stats** dictionary has the following structure:
+
+            =================   ===============
+            key                 Meaning
+            =================   ===============
+            rx_err              Number of flow statistics packets received that we could not associate to any pg_id. This can happen if latency on the used setup is large. See :ref:`wait_on_traffic <wait_on_traffic>` rx_delay_ms parameter for details.
+            tx_err              Number of flow statistics packets transmitted that we could not associate to any pg_id. This is never expected. If you see this different than 0, please report.
+            =================   ===============
+
+            .. _global:
+
+            **global**
+
+            =================   ===============
+            key                 Meaning
+            =================   ===============
+            bw_per_core         Estimated byte rate Trex can support per core. This is calculated by extrapolation of current rate and load on transmitting cores.
+            cpu_util            Estimate of the average utilization percentage of the transimitting cores
+            queue_full          Total number of packets transmitted while the NIC TX queue was full. The packets will be transmitted, eventually, but will create high CPU%due to polling the queue.  This usually indicates that the rate we trying to transmit is too high for this port. 
+            rx_cpu_util         Estimate of the utilization percentage of the core handling RX traffic. Too high value of this CPU utilization could cause drop of latency streams. 
+            rx_drop_bps         Received bytes per second drop rate
+            rx_bps              Received bytes per second rate
+            rx_pps              Received packets per second rate
+            tx_bps              Transmit bytes per second rate
+            tx_pps              Transmit packets per second rate
+            =================   ===============
+
+            .. _latency:
+
+            **latency** contains :ref:`global dictionary <lat_stats_global>`, and dictionaries per packet group id (pg id). Each one with the following structure.
+
+            **per pg_id latency stat** dictionaries have following structure:
+
+            ===========================          ===============
+            key                                  Meaning
+            ===========================          ===============
+            :ref:`err_cntrs<err-cntrs>`          Counters describing errors that occured with this pg id
+            :ref:`latency<lat_inner>`            Information regarding packet latency
+            ===========================          ===============
+
+            Following are the inner dictionaries of latency
+
+            .. _err-cntrs:
+
+            **err-cntrs**
+
+            =================   ===============
+            key                 Meaning (see better explanation below the table)
+            =================   ===============
+            dropped             How many packets were dropped (estimation)
+            dup                 How many packets were duplicated.
+            out_of_order        How many packets we received out of order.
+            seq_too_high        How many events of packet with sequence number too high we saw.
+            seq_too_low         How many events of packet with sequence number too low we saw.
+            =================   ===============
+
+            For calculating packet error events, we add sequence number to each packet's payload. We decide what went wrong only according to sequence number
+            of last packet received and that of the previous packet. 'seq_too_low' and 'seq_too_high' count events we see. 'dup', 'out_of_order' and 'dropped'
+            are heuristics we apply to try and understand what happened. They will be accurate in common error scenarios.
+            We describe few scenarios below to help understand this.
+
+            Scenario 1: Received packet with seq num 10, and another one with seq num 10. We increment 'dup' and 'seq_too_low' by 1.
+
+            Scenario 2: Received pacekt with seq num 10 and then packet with seq num 15. We assume 4 packets were dropped, and increment 'dropped' by 4, and 'seq_too_high' by 1.
+            We expect next packet to arrive with sequence number 16.
+
+            Scenario 2 continue: Received packet with seq num 11. We increment 'seq_too_low' by 1. We increment 'out_of_order' by 1. We *decrement* 'dropped' by 1.
+            (We assume here that one of the packets we considered as dropped before, actually arrived out of order).
+
+
+            .. _lat_inner:
+
+            **latency**
+
+            =================   ===============
+            key                 Meaning
+            =================   ===============
+            average             Average latency over the stream lifetime (usec).Low pass filter is applied to the last window average.It is computed each sampling period by following formula: <average> = <prev average>/2 + <last sampling period average>/2
+            histogram           Dictionary describing logarithmic distribution histogram of packet latencies. Keys in the dictionary represent range of latencies (in usec). Values are the total number of packets received in this latency range. For example, an entry {100:13} would mean that we saw 13 packets with latency in the range between 100 and 200 usec.
+            jitter              Jitter of latency samples, computed as described in :rfc:`3550#appendix-A.8`
+            last_max            Maximum latency measured between last two data reads from server (0.5 sec window).
+            total_max           Maximum latency measured over the stream lifetime (in usec).
+            total_min           Minimum latency measured over the stream lifetime (in usec).
+            =================   ===============
+
+            .. _lat_stats_global:
+
+            **global latency stats** dictionary has the following structure:
+
+            =================   ===============
+            key                 Meaning
+            =================   ===============
+            old_flow            Number of latency statistics packets received that we could not associate to any pg_id. This can happen if latency on the used setup is large. See :ref:`wait_on_traffic <wait_on_traffic>` rx_delay_ms parameter for details.
+            bad_hdr             Number of latency packets received with bad latency data. This can happen becuase of garbage packets in the network, or if the DUT causes packet corruption.
+            =================   ===============
+
+        :raises:
+          None
+
+        """
         # by default use all acquired ports
         ports = ports if ports is not None else self.get_acquired_ports()
         ports = self._validate_port_list(ports)
 
         # check async barrier
-        if not type(async_barrier) is bool:
-            raise STLArgumentError('async_barrier', async_barrier)
+        if not type(sync_now) is bool:
+            raise STLArgumentError('sync_now', sync_now)
 
 
         # if the user requested a barrier - use it
-        if async_barrier:
+        if sync_now:
             rc = self.async_client.barrier()
             if not rc:
                 raise STLError(rc)
@@ -1279,7 +1450,7 @@ class STLClient(object):
 
         :parameters:
           ev_type_filter - 'info', 'warning' or a list of those
-                           default is no filter
+                           default: no filter
 
         :return:
             logged events
@@ -1487,8 +1658,8 @@ class STLClient(object):
 
         """
 
-        self.logger.pre_cmd( "Pinging the server on '{0}' port '{1}': ".format(self.connection_info['server'],
-                                                                               self.connection_info['sync_port']))
+        self.logger.pre_cmd("Pinging the server on '{0}' port '{1}': ".format(self.connection_info['server'],
+                                                                              self.connection_info['sync_port']))
         rc = self._transmit("ping", api_class = None)
         
         self.logger.post_cmd(rc)
@@ -1497,6 +1668,30 @@ class STLClient(object):
             raise STLError(rc)
 
     @__api_check(True)
+    def server_shutdown (self, force = False):
+        """
+            Sends the server a request for total shutdown
+
+            :parameters:
+                force - shutdown server even if some ports are owned by another
+                        user
+
+            :raises:
+                + :exc:`STLError`
+
+        """
+
+        self.logger.pre_cmd("Sending shutdown request for the server")
+
+        rc = self._transmit("shutdown", params = {'force': force, 'user': self.username})
+
+        self.logger.post_cmd(rc)
+
+        if not rc:
+            raise STLError(rc)
+
+
+    @__api_check(True)
     def get_active_pgids(self):
         """
             Get active group IDs
@@ -1519,6 +1714,23 @@ class STLClient(object):
         if not rc:
             raise STLError(rc)
 
+    @__api_check(True)
+    def get_util_stats(self):
+        """
+            Get utilization stats:
+            History of TRex CPU utilization per thread (list of lists)
+            MBUFs memory consumption per CPU socket.
+
+            :parameters:
+                None
+
+            :raises:
+                + :exc:`STLError`
+
+        """
+        self.logger.pre_cmd('Getting Utilization stats')
+        return self.util_stats.get_stats()
+
 
     @__api_check(True)
     def reset(self, ports = None):
@@ -1700,6 +1912,11 @@ class STLClient(object):
         ports = self._validate_port_list(ports)
 
 
+        validate_type('mult', mult, basestring)
+        validate_type('force', force, bool)
+        validate_type('duration', duration, (int, float))
+        validate_type('total', total, bool)
+
         # verify multiplier
         mult_obj = parsing_opts.decode_multiplier(mult,
                                                   allow_update = False,
@@ -1707,17 +1924,6 @@ class STLClient(object):
         if not mult_obj:
             raise STLArgumentError('mult', mult)
 
-        # some type checkings
-
-        if not type(force) is bool:
-            raise STLArgumentError('force', force)
-
-        if not isinstance(duration, (int, float)):
-            raise STLArgumentError('duration', duration)
-
-        if not type(total) is bool:
-            raise STLArgumentError('total', total)
-
 
         # verify ports are stopped or force stop them
         active_ports = list(set(self.get_active_ports()).intersection(ports))
@@ -1739,8 +1945,6 @@ class STLClient(object):
             raise STLError(rc)
 
 
-
-    
     @__api_check(True)
     def stop (self, ports = None, rx_delay_ms = 10):
         """
@@ -1762,11 +1966,12 @@ class STLClient(object):
 
         """
 
-        ports = ports if ports is not None else self.get_active_ports()
-        ports = self._validate_port_list(ports)
+        if ports is None:
+            ports = self.get_active_ports()
+            if not ports:
+                return
 
-        if not ports:
-            return
+        ports = self._validate_port_list(ports)
 
         self.logger.pre_cmd("Stopping traffic on port(s) {0}:".format(ports))
         rc = self.__stop(ports)
@@ -1815,6 +2020,9 @@ class STLClient(object):
         ports = ports if ports is not None else self.get_active_ports()
         ports = self._validate_port_list(ports)
 
+        validate_type('mult', mult, basestring)
+        validate_type('force', force, bool)
+        validate_type('total', total, bool)
 
         # verify multiplier
         mult_obj = parsing_opts.decode_multiplier(mult,
@@ -1823,10 +2031,6 @@ class STLClient(object):
         if not mult_obj:
             raise STLArgumentError('mult', mult)
 
-        # verify total
-        if not type(total) is bool:
-            raise STLArgumentError('total', total)
-
 
         # call low level functions
         self.logger.pre_cmd("Updating traffic on port(s) {0}:".format(ports))
@@ -1927,7 +2131,7 @@ class STLClient(object):
         ports = ports if ports is not None else self.get_acquired_ports()
         ports = self._validate_port_list(ports)
 
-        validate_type('pcap_filename', pcap_filename, str)
+        validate_type('pcap_filename', pcap_filename, basestring)
         validate_type('ipg_usec', ipg_usec, (float, int, type(None)))
         validate_type('speedup',  speedup, (float, int))
         validate_type('count',  count, int)
@@ -1994,7 +2198,7 @@ class STLClient(object):
         ports = ports if ports is not None else self.get_acquired_ports()
         ports = self._validate_port_list(ports)
 
-        validate_type('pcap_filename', pcap_filename, str)
+        validate_type('pcap_filename', pcap_filename, basestring)
         validate_type('ipg_usec', ipg_usec, (float, int, type(None)))
         validate_type('speedup',  speedup, (float, int))
         validate_type('count',  count, int)
@@ -2050,6 +2254,10 @@ class STLClient(object):
         ports = ports if ports is not None else self.get_acquired_ports()
         ports = self._validate_port_list(ports)
 
+        validate_type('mult', mult, basestring)
+        validate_type('duration', duration, (int, float))
+        validate_type('total', total, bool)
+
 
         # verify multiplier
         mult_obj = parsing_opts.decode_multiplier(mult,
@@ -2058,11 +2266,6 @@ class STLClient(object):
         if not mult_obj:
             raise STLArgumentError('mult', mult)
 
-
-        if not isinstance(duration, (int, float)):
-            raise STLArgumentError('duration', duration)
-
-
         self.logger.pre_cmd("Validating streams on port(s) {0}:".format(ports))
         rc = self.__validate(ports)
         self.logger.post_cmd(rc)
@@ -2136,6 +2339,8 @@ class STLClient(object):
     @__api_check(True)
     def wait_on_traffic (self, ports = None, timeout = 60, rx_delay_ms = 10):
         """
+            .. _wait_on_traffic:
+
             Block until traffic on specified port(s) has ended
 
             :parameters:
@@ -2146,12 +2351,11 @@ class STLClient(object):
                     timeout in seconds
 
                 rx_delay_ms : int
-                    time to wait until RX filters are removed
-                    this value should reflect the time it takes
-                    packets which were transmitted to arrive
+                    Time to wait (in milliseconds) after last packet was sent, until RX filters used for
+                    measuring flow statistics and latency are removed.
+                    This value should reflect the time it takes packets which were transmitted to arrive
                     to the destination.
-                    after this time the RX filters will be removed
-
+                    After this time, RX filters will be removed, and packets arriving for per flow statistics feature and latency flows will be counted as errors.
 
             :raises:
                 + :exc:`STLTimeoutError` - in case timeout has expired
@@ -2247,13 +2451,14 @@ class STLClient(object):
                 rc = f(*args)
             except STLError as e:
                 client.logger.log("Log:\n" + format_text(e.brief() + "\n", 'bold'))
-                return
+                return RC_ERR(e.brief())
 
             # if got true - print time
             if rc:
                 delta = time.time() - time1
                 client.logger.log(format_time(delta) + "\n")
 
+            return rc
 
         return wrap
 
@@ -2261,7 +2466,22 @@ class STLClient(object):
     def ping_line (self, line):
         '''pings the server'''
         self.ping()
-        return True
+        return RC_OK()
+
+    @__console
+    def shutdown_line (self, line):
+        '''shutdown the server'''
+        parser = parsing_opts.gen_parser(self,
+                                         "shutdown",
+                                         self.shutdown_line.__doc__,
+                                         parsing_opts.FORCE)
+
+        opts = parser.parse_args(line.split())
+        if not opts:
+            return opts
+
+        self.server_shutdown(force = opts.force)    
+        return RC_OK()
 
     @__console
     def connect_line (self, line):
@@ -2273,14 +2493,13 @@ class STLClient(object):
                                          parsing_opts.FORCE)
 
         opts = parser.parse_args(line.split(), default_ports = self.get_all_ports())
-        if opts is None:
-            return
+        if not opts:
+            return opts
 
         self.connect()
         self.acquire(ports = opts.ports, force = opts.force)
 
-        # true means print time
-        return True
+        return RC_OK()
 
 
     @__console
@@ -2295,19 +2514,19 @@ class STLClient(object):
                                          parsing_opts.FORCE)
 
         opts = parser.parse_args(line.split(), default_ports = self.get_all_ports())
-        if opts is None:
-            return
+        if not opts:
+            return opts
 
         # filter out all the already owned ports
         ports = list_difference(opts.ports, self.get_acquired_ports())
         if not ports:
-            self.logger.log("acquire - all port(s) {0} are already acquired".format(opts.ports))
-            return
+            msg = "acquire - all of port(s) {0} are already acquired".format(opts.ports)
+            self.logger.log(format_text(msg, 'bold'))
+            return RC_ERR(msg)
 
         self.acquire(ports = ports, force = opts.force)
 
-        # true means print time
-        return True
+        return RC_OK()
 
 
     #
@@ -2321,23 +2540,24 @@ class STLClient(object):
                                          parsing_opts.PORT_LIST_WITH_ALL)
 
         opts = parser.parse_args(line.split(), default_ports = self.get_acquired_ports())
-        if opts is None:
-            return
+        if not opts:
+            return opts
 
         ports = list_intersect(opts.ports, self.get_acquired_ports())
         if not ports:
             if not opts.ports:
-                self.logger.log("release - no acquired ports")
-                return
+                msg = "release - no acquired ports"
+                self.logger.log(format_text(msg, 'bold'))
+                return RC_ERR(msg)
             else:
-                self.logger.log("release - none of port(s) {0} are acquired".format(opts.ports))
-                return
+                msg = "release - none of port(s) {0} are acquired".format(opts.ports)
+                self.logger.log(format_text(msg, 'bold'))
+                return RC_ERR(msg)
 
         
         self.release(ports = ports)
 
-        # true means print time
-        return True
+        return RC_OK()
 
 
     @__console
@@ -2349,23 +2569,23 @@ class STLClient(object):
                                          self.reacquire_line.__doc__)
 
         opts = parser.parse_args(line.split())
-        if opts is None:
-            return
+        if not opts:
+            return opts
 
         # find all the on-owned ports under your name
         my_unowned_ports = list_difference([k for k, v in self.ports.items() if v.get_owner() == self.username], self.get_acquired_ports())
         if not my_unowned_ports:
-            self.logger.log("reacquire - no unowned ports under '{0}'".format(self.username))
-            return
+            msg = "reacquire - no unowned ports under '{0}'".format(self.username)
+            self.logger.log(msg)
+            return RC_ERR(msg)
 
         self.acquire(ports = my_unowned_ports, force = True)
-        return True
+        return RC_OK()
 
 
     @__console
     def disconnect_line (self, line):
         self.disconnect()
-        
 
 
     @__console
@@ -2378,13 +2598,12 @@ class STLClient(object):
                                          parsing_opts.PORT_LIST_WITH_ALL)
 
         opts = parser.parse_args(line.split(), default_ports = self.get_acquired_ports(), verify_acquired = True)
-        if opts is None:
-            return
+        if not opts:
+            return opts
 
         self.reset(ports = opts.ports)
 
-        # true means print time
-        return True
+        return RC_OK()
 
 
 
@@ -2405,15 +2624,15 @@ class STLClient(object):
                                          parsing_opts.DRY_RUN)
 
         opts = parser.parse_args(line.split(), default_ports = self.get_acquired_ports(), verify_acquired = True)
-        if opts is None:
-            return
+        if not opts:
+            return opts
 
         active_ports = list_intersect(self.get_active_ports(), opts.ports)
         if active_ports:
             if not opts.force:
                 msg = "Port(s) {0} are active - please stop them or add '--force'\n".format(active_ports)
                 self.logger.log(format_text(msg, 'bold'))
-                return
+                return RC_ERR(msg)
             else:
                 self.stop(active_ports)
 
@@ -2431,8 +2650,10 @@ class STLClient(object):
             else:
                 # must be exact
                 if len(opts.ports) != len(opts.tunables):
-                    self.logger.log('tunables section count must be 1 or exactly as the number of ports: got {0}'.format(len(opts.tunables)))
-                    return
+                    msg = 'tunables section count must be 1 or exactly as the number of ports: got {0}'.format(len(opts.tunables))
+                    self.logger.log(msg)
+                    return RC_ERR(msg)
+
                 tunables = opts.tunables
             
 
@@ -2452,9 +2673,10 @@ class STLClient(object):
                 self.add_streams(profile.get_streams(), ports = port)
 
         except STLError as e:
-            self.logger.log(format_text("\nError while loading profile '{0}'\n".format(opts.file[0]), 'bold'))
+            msg = format_text("\nError while loading profile '{0}'\n".format(opts.file[0]), 'bold')
+            self.logger.log(msg)
             self.logger.log(e.brief() + "\n")
-            return
+            return RC_ERR(msg)
 
 
         if opts.dry:
@@ -2466,8 +2688,7 @@ class STLClient(object):
                        opts.duration,
                        opts.total)
 
-        # true means print time
-        return True
+        return RC_OK()
 
 
 
@@ -2480,24 +2701,25 @@ class STLClient(object):
                                          parsing_opts.PORT_LIST_WITH_ALL)
 
         opts = parser.parse_args(line.split(), default_ports = self.get_active_ports(), verify_acquired = True)
-        if opts is None:
-            return
+        if not opts:
+            return opts
 
 
         # find the relevant ports
         ports = list_intersect(opts.ports, self.get_active_ports())
         if not ports:
             if not opts.ports:
-                self.logger.log('stop - no active ports')
+                msg = 'stop - no active ports'
             else:
-                self.logger.log('stop - no active traffic on ports {0}'.format(opts.ports))
-            return
+                msg = 'stop - no active traffic on ports {0}'.format(opts.ports)
+
+            self.logger.log(msg)
+            return RC_ERR(msg)
 
         # call API
         self.stop(ports)
 
-        # true means print time
-        return True
+        return RC_OK()
 
 
     @__console
@@ -2512,23 +2734,24 @@ class STLClient(object):
                                          parsing_opts.FORCE)
 
         opts = parser.parse_args(line.split(), default_ports = self.get_active_ports(), verify_acquired = True)
-        if opts is None:
-            return
+        if not opts:
+            return opts
 
 
         # find the relevant ports
         ports = list_intersect(opts.ports, self.get_active_ports())
         if not ports:
             if not opts.ports:
-                self.logger.log('update - no active ports')
+                msg = 'update - no active ports'
             else:
-                self.logger.log('update - no active traffic on ports {0}'.format(opts.ports))
-            return
+                msg = 'update - no active traffic on ports {0}'.format(opts.ports)
+
+            self.logger.log(msg)
+            return RC_ERR(msg)
 
         self.update(ports, opts.mult, opts.total, opts.force)
 
-        # true means print time
-        return True
+        return RC_OK()
 
 
     @__console
@@ -2540,27 +2763,29 @@ class STLClient(object):
                                          parsing_opts.PORT_LIST_WITH_ALL)
 
         opts = parser.parse_args(line.split(), default_ports = self.get_transmitting_ports(), verify_acquired = True)
-        if opts is None:
-            return
+        if not opts:
+            return opts
 
         # check for already paused case
         if opts.ports and is_sub_list(opts.ports, self.get_paused_ports()):
-            self.logger.log('pause - all of port(s) {0} are already paused'.format(opts.ports))
-            return
+            msg = 'pause - all of port(s) {0} are already paused'.format(opts.ports)
+            self.logger.log(msg)
+            return RC_ERR(msg)
 
         # find the relevant ports
         ports = list_intersect(opts.ports, self.get_transmitting_ports())
         if not ports:
             if not opts.ports:
-                self.logger.log('pause - no transmitting ports')
+                msg = 'pause - no transmitting ports'
             else:
-                self.logger.log('pause - none of ports {0} are transmitting'.format(opts.ports))
-            return
+                msg = 'pause - none of ports {0} are transmitting'.format(opts.ports)
+
+            self.logger.log(msg)
+            return RC_ERR(msg)
 
         self.pause(ports)
 
-        # true means print time
-        return True
+        return RC_OK()
 
 
     @__console
@@ -2572,23 +2797,25 @@ class STLClient(object):
                                          parsing_opts.PORT_LIST_WITH_ALL)
 
         opts = parser.parse_args(line.split(), default_ports = self.get_paused_ports(), verify_acquired = True)
-        if opts is None:
-            return
+        if not opts:
+            return opts
 
         # find the relevant ports
         ports = list_intersect(opts.ports, self.get_paused_ports())
         if not ports:
             if not opts.ports:
-                self.logger.log('resume - no paused ports')
+                msg = 'resume - no paused ports'
             else:
-                self.logger.log('resume - none of ports {0} are paused'.format(opts.ports))
-            return
+                msg = 'resume - none of ports {0} are paused'.format(opts.ports)
+                
+            self.logger.log(msg)
+            return RC_ERR(msg)
 
 
         self.resume(ports)
 
         # true means print time
-        return True
+        return RC_OK()
 
    
     @__console
@@ -2602,8 +2829,8 @@ class STLClient(object):
 
         opts = parser.parse_args(line.split())
 
-        if opts is None:
-            return
+        if not opts:
+            return opts
 
         self.clear_stats(opts.ports)
 
@@ -2622,8 +2849,8 @@ class STLClient(object):
 
         opts = parser.parse_args(line.split())
 
-        if opts is None:
-            return
+        if not opts:
+            return opts
 
         # determine stats mask
         mask = self.__get_mask_keys(**self.__filter_namespace_args(opts, trex_stl_stats.ALL_STATS_OPTS))
@@ -2653,8 +2880,8 @@ class STLClient(object):
 
         opts = parser.parse_args(line.split())
 
-        if opts is None:
-            return
+        if not opts:
+            return opts
 
         streams = self._get_streams(opts.ports, set(opts.streams))
         if not streams:
@@ -2680,8 +2907,8 @@ class STLClient(object):
                                          parsing_opts.PORT_LIST_WITH_ALL)
 
         opts = parser.parse_args(line.split())
-        if opts is None:
-            return
+        if not opts:
+            return opts
 
         self.validate(opts.ports)
 
@@ -2705,8 +2932,8 @@ class STLClient(object):
                                          parsing_opts.FORCE)
 
         opts = parser.parse_args(line.split())
-        if opts is None:
-            return
+        if not opts:
+            return opts
 
         active_ports = list(set(self.get_active_ports()).intersection(opts.ports))
 
@@ -2714,7 +2941,7 @@ class STLClient(object):
             if not opts.force:
                 msg = "Port(s) {0} are active - please stop them or add '--force'\n".format(active_ports)
                 self.logger.log(format_text(msg, 'bold'))
-                return
+                return RC_ERR(msg)
             else:
                 self.stop(active_ports)
 
@@ -2738,7 +2965,7 @@ class STLClient(object):
 
         
 
-        return True
+        return RC_OK()
 
 
 
@@ -2753,8 +2980,8 @@ class STLClient(object):
                                          parsing_opts.PROMISCUOUS_SWITCH)
 
         opts = parser.parse_args(line.split(), default_ports = self.get_acquired_ports(), verify_acquired = True)
-        if opts is None:
-            return
+        if not opts:
+            return opts
 
         # if no attributes - fall back to printing the status
         if opts.prom is None:
@@ -2762,7 +2989,7 @@ class STLClient(object):
             return
 
         self.set_port_attr(opts.ports, opts.prom)
-
+        return RC_OK()
     
 
     @__console
@@ -2775,8 +3002,8 @@ class STLClient(object):
                                          parsing_opts.FILE_PATH)
 
         opts = parser.parse_args(line.split())
-        if opts is None:
-            return
+        if not opts:
+            return opts
 
         info = STLProfile.get_info(opts.file[0])
 
@@ -2793,7 +3020,7 @@ class STLClient(object):
 
         if profile_type == 'python':
             self.logger.log('Type:             {:^12}'.format('Python Module'))
-            self.logger.log('Tunables:         {:^12}'.format(['{0} = {1}'.format(k ,v) for k, v in info['tunables'].items()]))
+            self.logger.log('Tunables:         {:^12}'.format(str(['{0} = {1}'.format(k ,v) for k, v in info['tunables'].items()])))
 
         elif profile_type == 'yaml':
             self.logger.log('Type:             {:^12}'.format('YAML'))
@@ -2832,8 +3059,8 @@ class STLClient(object):
                                          *x)
 
         opts = parser.parse_args(line.split())
-        if opts is None:
-            return
+        if not opts:
+            return opts
 
 
         ev_type_filter = []
@@ -2854,3 +3081,15 @@ class STLClient(object):
         if opts.clear:
             self.clear_events()
          
+    def generate_prompt (self, prefix = 'trex'):
+        if not self.is_connected():
+            return "{0}(offline)>".format(prefix)
+
+        elif not self.get_acquired_ports():
+            return "{0}(read-only)>".format(prefix)
+
+        elif self.is_all_ports_acquired():
+            return "{0}>".format(prefix)
+
+        else:
+            return "{0} {1}>".format(prefix, self.get_acquired_ports())
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_exceptions.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_exceptions.py
index 3c443ba6..b6fad865 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_exceptions.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_exceptions.py
@@ -4,6 +4,11 @@ import linecache
 
 from .utils.text_opts import *
 
+try:
+    basestring
+except NameError:
+    basestring = str
+
 # basic error for API
 class STLError(Exception):
     def __init__ (self, msg):
@@ -56,7 +61,8 @@ class STLArgumentError(STLError):
 # raised when argument type is not valid for operation
 class STLTypeError(STLError):
     def __init__ (self, arg_name, arg_type, valid_types):
-        self.msg = "Argument: '%s' invalid type: %s, expecting type(s): %s." % (arg_name, arg_type, valid_types)
+        self.msg = "Argument: '%s' invalid type: '%s', expecting type(s): %s." % (arg_name, arg_type.__name__,
+            [t.__name__ for t in valid_types] if isinstance(valid_types, tuple) else valid_types.__name__)
 
 # raised when timeout occurs
 class STLTimeoutError(STLError):
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py
index be46e95f..d239fc57 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py
@@ -68,7 +68,7 @@ class Port(object):
         self.has_rx_streams = False
 
         self.owner = ''
-
+        self.last_factor_type = None
 
     # decorator to verify port is up
     def up(func):
@@ -417,6 +417,9 @@ class Port(object):
             self.state = last_state
             return self.err(rc.err())
 
+        # save this for TUI
+        self.last_factor_type = mul['type']
+
         return self.ok()
 
 
@@ -424,7 +427,7 @@ class Port(object):
     # with force ignores the cached state and sends the command
     @owned
     def stop (self, force = False):
-        
+
         # if not is not active and not force - go back
         if not self.is_active() and not force:
             return self.ok()
@@ -437,10 +440,11 @@ class Port(object):
             return self.err(rc.err())
 
         self.state = self.STATE_STREAMS
+        self.last_factor_type = None
 
         # timestamp for last tx
         self.tx_stopped_ts = datetime.now()
-
+        
         return self.ok()
 
 
@@ -535,6 +539,9 @@ class Port(object):
         if rc.bad():
             return self.err(rc.err())
 
+        # save this for TUI
+        self.last_factor_type = mul['type']
+
         return self.ok()
 
     @owned
@@ -712,6 +719,7 @@ class Port(object):
         # until thread is locked - order is important
         self.tx_stopped_ts = datetime.now()
         self.state = self.STATE_STREAMS
+        self.last_factor_type = None
 
     # rest of the events are used for TUI / read only sessions
     def async_event_port_stopped (self):
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
index 0ec98a0d..1bf0a9a4 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
@@ -11,7 +11,6 @@ import datetime
 import time
 import re
 import math
-import copy
 import threading
 import pprint
 
@@ -22,13 +21,16 @@ PORT_STATUS = 'ps'
 STREAMS_STATS = 's'
 LATENCY_STATS = 'ls'
 LATENCY_HISTOGRAM = 'lh'
+CPU_STATS = 'c'
+MBUF_STATS = 'm'
 
-ALL_STATS_OPTS = [GLOBAL_STATS, PORT_STATS, PORT_STATUS, STREAMS_STATS, LATENCY_STATS, PORT_GRAPH, LATENCY_HISTOGRAM]
+ALL_STATS_OPTS = [GLOBAL_STATS, PORT_STATS, PORT_STATUS, STREAMS_STATS, LATENCY_STATS, PORT_GRAPH, LATENCY_HISTOGRAM, CPU_STATS, MBUF_STATS]
 COMPACT = [GLOBAL_STATS, PORT_STATS]
 GRAPH_PORT_COMPACT = [GLOBAL_STATS, PORT_GRAPH]
 SS_COMPAT = [GLOBAL_STATS, STREAMS_STATS] # stream stats
 LS_COMPAT = [GLOBAL_STATS, LATENCY_STATS] # latency stats
 LH_COMPAT = [GLOBAL_STATS, LATENCY_HISTOGRAM] # latency histogram
+UT_COMPAT = [GLOBAL_STATS, CPU_STATS, MBUF_STATS] # utilization
 
 ExportableStats = namedtuple('ExportableStats', ['raw_data', 'text_table'])
 
@@ -104,6 +106,13 @@ def calculate_diff_raw (samples):
 
     return total
 
+get_number_of_bytes_cache = {}
+# get number of bytes: '64b'->64, '9kb'->9000 etc.
+def get_number_of_bytes(val):
+    if val not in get_number_of_bytes_cache:
+        get_number_of_bytes_cache[val] = int(val[:-1].replace('k', '000'))
+    return get_number_of_bytes_cache[val]
+
 # a simple object to keep a watch over a field
 class WatchedField(object):
 
@@ -138,11 +147,12 @@ class CTRexInfoGenerator(object):
     STLClient and the ports.
     """
 
-    def __init__(self, global_stats_ref, ports_dict_ref, rx_stats_ref, latency_stats_ref, async_monitor):
+    def __init__(self, global_stats_ref, ports_dict_ref, rx_stats_ref, latency_stats_ref, util_stats_ref, async_monitor):
         self._global_stats = global_stats_ref
         self._ports_dict = ports_dict_ref
         self._rx_stats_ref = rx_stats_ref
         self._latency_stats_ref = latency_stats_ref
+        self._util_stats_ref = util_stats_ref
         self._async_monitor = async_monitor
 
     def generate_single_statistic(self, port_id_list, statistic_type):
@@ -167,6 +177,12 @@ class CTRexInfoGenerator(object):
         elif statistic_type == LATENCY_HISTOGRAM:
             return self._generate_latency_histogram()
 
+        elif statistic_type == CPU_STATS:
+            return self._generate_cpu_util_stats()
+
+        elif statistic_type == MBUF_STATS:
+            return self._generate_mbuf_util_stats()
+
         else:
             # ignore by returning empty object
             return {}
@@ -404,6 +420,73 @@ class CTRexInfoGenerator(object):
         stats_table.header(header)
         return {"latency_histogram": ExportableStats(None, stats_table)}
 
+    def _generate_cpu_util_stats(self):
+        util_stats = self._util_stats_ref.get_stats(use_1sec_cache = True)
+        stats_table = text_tables.TRexTextTable()
+        if util_stats:
+            if 'cpu' not in util_stats:
+                raise Exception("Excepting 'cpu' section in stats %s" % util_stats)
+            cpu_stats = util_stats['cpu']
+            hist_len = len(cpu_stats[0])
+            avg_len = min(5, hist_len)
+            show_len = min(15, hist_len)
+            stats_table.header(['Thread', 'Avg', 'Latest'] + list(range(-1, 0 - show_len, -1)))
+            stats_table.set_cols_align(['l'] + ['r'] * (show_len + 1))
+            stats_table.set_cols_width([8, 3, 6] + [3] * (show_len - 1))
+            stats_table.set_cols_dtype(['t'] * (show_len + 2))
+            for i in range(min(14, len(cpu_stats))):
+                avg = int(round(sum(cpu_stats[i][:avg_len]) / avg_len))
+                stats_table.add_row([i, avg] + cpu_stats[i][:show_len])
+        else:
+            stats_table.add_row(['No Data.'])
+        return {'cpu_util(%)': ExportableStats(None, stats_table)}
+
+    def _generate_mbuf_util_stats(self):
+        util_stats = self._util_stats_ref.get_stats(use_1sec_cache = True)
+        stats_table = text_tables.TRexTextTable()
+        if util_stats:
+            if 'mbuf_stats' not in util_stats:
+                raise Exception("Excepting 'mbuf_stats' section in stats %s" % util_stats)
+            mbuf_stats = util_stats['mbuf_stats']
+            for mbufs_per_socket in mbuf_stats.values():
+                first_socket_mbufs = mbufs_per_socket
+                break
+            if not self._util_stats_ref.mbuf_types_list:
+                mbuf_keys = list(first_socket_mbufs.keys())
+                mbuf_keys.sort(key = get_number_of_bytes)
+                self._util_stats_ref.mbuf_types_list = mbuf_keys
+            types_len = len(self._util_stats_ref.mbuf_types_list)
+            stats_table.set_cols_align(['l'] + ['r'] * (types_len + 1))
+            stats_table.set_cols_width([10] + [7] * (types_len + 1))
+            stats_table.set_cols_dtype(['t'] * (types_len + 2))
+            stats_table.header([''] + self._util_stats_ref.mbuf_types_list + ['RAM(MB)'])
+            total_list = []
+            sum_totals = 0
+            for mbuf_type in self._util_stats_ref.mbuf_types_list:
+                sum_totals += first_socket_mbufs[mbuf_type][1] * get_number_of_bytes(mbuf_type) + 64
+                total_list.append(first_socket_mbufs[mbuf_type][1])
+            sum_totals *= len(list(mbuf_stats.values()))
+            total_list.append(int(sum_totals/1e6))
+            stats_table.add_row(['Total:'] + total_list)
+            stats_table.add_row(['Used:'] + [''] * (types_len + 1))
+            for socket_name in sorted(list(mbuf_stats.keys())):
+                mbufs = mbuf_stats[socket_name]
+                socket_show_name = socket_name.replace('cpu-', '').replace('-', ' ').capitalize() + ':'
+                sum_used = 0
+                used_list = []
+                percentage_list = []
+                for mbuf_type in self._util_stats_ref.mbuf_types_list:
+                    used = mbufs[mbuf_type][1] - mbufs[mbuf_type][0]
+                    sum_used += used * get_number_of_bytes(mbuf_type) + 64
+                    used_list.append(used)
+                    percentage_list.append('%s%%' % int(100 * used / mbufs[mbuf_type][1]))
+                used_list.append(int(sum_used/1e6))
+                stats_table.add_row([socket_show_name] + used_list)
+                stats_table.add_row(['Percent:'] + percentage_list + [''])
+        else:
+            stats_table.add_row(['No Data.'])
+        return {'mbuf_util': ExportableStats(None, stats_table)}
+
     @staticmethod
     def _get_rational_block_char(value, range_start, interval):
         # in Konsole, utf-8 is sometimes printed with artifacts, return ascii for now
@@ -693,12 +776,12 @@ class CTRexStats(object):
 
         return value
 
-    def get(self, field, format=False, suffix=""):
+    def get(self, field, format=False, suffix="", opts = None):
         value = self._get(self.latest_stats, field)
         if value == None:
             return 'N/A'
 
-        return value if not format else format_num(value, suffix)
+        return value if not format else format_num(value, suffix = suffix, opts = opts)
 
 
     def get_rel(self, field, format=False, suffix=""):
@@ -937,14 +1020,20 @@ class CPortStats(CTRexStats):
         else:
             state = format_text(state, 'bold')
 
+        # default rate format modifiers
+        rate_format = {'bpsl1': None, 'bps': None, 'pps': None, 'percentage': 'bold'}
+
         # mark owned ports by color
         if self._port_obj:
             owner = self._port_obj.get_owner()
+            rate_format[self._port_obj.last_factor_type] = ('blue', 'bold')
             if self._port_obj.is_acquired():
                 owner = format_text(owner, 'green')
+
         else:
             owner = ''
 
+
         return {"owner": owner,
                 "state": "{0}".format(state),
                 "speed": self._port_obj.get_formatted_speed() if self._port_obj else '',
@@ -955,21 +1044,19 @@ class CPortStats(CTRexStats):
                 "-----": " ",
 
                 "Tx bps L1": "{0} {1}".format(self.get_trend_gui("m_total_tx_bps_L1", show_value = False),
-                                               self.get("m_total_tx_bps_L1", format = True, suffix = "bps")),
+                                               self.get("m_total_tx_bps_L1", format = True, suffix = "bps", opts = rate_format['bpsl1'])),
 
                 "Tx bps L2": "{0} {1}".format(self.get_trend_gui("m_total_tx_bps", show_value = False),
-                                               self.get("m_total_tx_bps", format = True, suffix = "bps")),
+                                                self.get("m_total_tx_bps", format = True, suffix = "bps", opts = rate_format['bps'])),
 
-                "Line Util.": "{0} {1}".format(self.get_trend_gui("m_percentage", show_value = False),
-                                                format_text(
-                                                    self.get("m_percentage", format = True, suffix = "%") if self._port_obj else "",
-                                                    'bold')) if self._port_obj else "",
+                "Line Util.": "{0} {1}".format(self.get_trend_gui("m_percentage", show_value = False) if self._port_obj else "",
+                                               self.get("m_percentage", format = True, suffix = "%", opts = rate_format['percentage']) if self._port_obj else ""),
 
                 "Rx bps": "{0} {1}".format(self.get_trend_gui("m_total_rx_bps", show_value = False),
                                             self.get("m_total_rx_bps", format = True, suffix = "bps")),
                   
                 "Tx pps": "{0} {1}".format(self.get_trend_gui("m_total_tx_pps", show_value = False),
-                                            self.get("m_total_tx_pps", format = True, suffix = "pps")),
+                                            self.get("m_total_tx_pps", format = True, suffix = "pps", opts = rate_format['pps'])),
 
                 "Rx pps": "{0} {1}".format(self.get_trend_gui("m_total_rx_pps", show_value = False),
                                             self.get("m_total_rx_pps", format = True, suffix = "pps")),
@@ -1010,6 +1097,13 @@ class CLatencyStats(CTRexStats):
             snapshot = {}
         output = {}
 
+        output['global'] = {}
+        for field in ['bad_hdr', 'old_flow']:
+            if 'global' in snapshot and field in snapshot['global']:
+                output['global'][field] = snapshot['global'][field]
+            else:
+                output['global'][field] = 0
+
         # we care only about the current active keys
         pg_ids = list(filter(is_intable, snapshot.keys()))
 
@@ -1036,6 +1130,7 @@ class CLatencyStats(CTRexStats):
                     output[int_pg_id]['latency']['total_min'] = min_val
                 else:
                     output[int_pg_id]['latency']['total_min'] = StatNotAvailable('total_min')
+                    output[int_pg_id]['latency']['histogram'] = {}
 
         self.latest_stats = output
         return True
@@ -1046,12 +1141,7 @@ class CRxStats(CTRexStats):
     def __init__(self, ports):
         super(CRxStats, self).__init__()
         self.ports = ports
-        self.ports_speed = {}
 
-    def get_ports_speed(self):
-        for port in self.ports:
-            self.ports_speed[str(port)] = self.ports[port].get_speed_bps()
-        self.ports_speed['total'] = sum(self.ports_speed.values())
 
     # calculates a diff between previous snapshot
     # and current one
@@ -1111,6 +1201,14 @@ class CRxStats(CTRexStats):
         # copy timestamp field
         output['ts'] = current['ts']
 
+        # global (not per pg_id) error counters
+        output['global'] = {}
+        for field in ['rx_err', 'tx_err']:
+            output['global'][field] = {}
+            if 'global' in current and field in current['global']:
+                for port in current['global'][field]:
+                    output['global'][field][int(port)] = current['global'][field][port]
+
         # we care only about the current active keys
         pg_ids = list(filter(is_intable, current.keys()))
 
@@ -1169,8 +1267,8 @@ class CRxStats(CTRexStats):
             return
 
         # TX
-        self.get_ports_speed()
         for port in pg_current['tx_pkts'].keys():
+
             prev_tx_pps   = pg_prev['tx_pps'].get(port)
             now_tx_pkts   = pg_current['tx_pkts'].get(port)
             prev_tx_pkts  = pg_prev['tx_pkts'].get(port)
@@ -1179,19 +1277,20 @@ class CRxStats(CTRexStats):
             prev_tx_bps   = pg_prev['tx_bps'].get(port)
             now_tx_bytes  = pg_current['tx_bytes'].get(port)
             prev_tx_bytes = pg_prev['tx_bytes'].get(port)
+
             pg_current['tx_bps'][port], pg_current['tx_bps_lpf'][port] = self.calc_bps(prev_tx_bps, now_tx_bytes, prev_tx_bytes, diff_sec)
 
             if pg_current['tx_bps'].get(port) != None and pg_current['tx_pps'].get(port) != None:
                 pg_current['tx_bps_L1'][port] = calc_bps_L1(pg_current['tx_bps'][port], pg_current['tx_pps'][port])
                 pg_current['tx_bps_L1_lpf'][port] = calc_bps_L1(pg_current['tx_bps_lpf'][port], pg_current['tx_pps_lpf'][port])
-                pg_current['tx_line_util'][port] = 100.0 * pg_current['tx_bps_L1'][port] / self.ports_speed[port]
             else:
                 pg_current['tx_bps_L1'][port] = None
                 pg_current['tx_bps_L1_lpf'][port] = None
-                pg_current['tx_line_util'][port] = None
+
 
         # RX
         for port in pg_current['rx_pkts'].keys():
+
             prev_rx_pps   = pg_prev['rx_pps'].get(port)
             now_rx_pkts   = pg_current['rx_pkts'].get(port)
             prev_rx_pkts  = pg_prev['rx_pkts'].get(port)
@@ -1204,11 +1303,9 @@ class CRxStats(CTRexStats):
             if pg_current['rx_bps'].get(port) != None and pg_current['rx_pps'].get(port) != None:
                 pg_current['rx_bps_L1'][port] = calc_bps_L1(pg_current['rx_bps'][port], pg_current['rx_pps'][port])
                 pg_current['rx_bps_L1_lpf'][port] = calc_bps_L1(pg_current['rx_bps_lpf'][port], pg_current['rx_pps_lpf'][port])
-                pg_current['rx_line_util'][port] = 100.0 * pg_current['rx_bps_L1'][port] / self.ports_speed[port]
             else:
                 pg_current['rx_bps_L1'][port] = None
                 pg_current['rx_bps_L1_lpf'][port] = None
-                pg_current['rx_line_util'][port] = None
 
 
     def calc_pps (self, prev_bw, now, prev, diff_sec):
@@ -1259,6 +1356,9 @@ class CRxStats(CTRexStats):
         for pg_id, value in self.latest_stats.items():
             # skip non ints
             if not is_intable(pg_id):
+                # 'global' stats are in the same level of the pg_ids. We do want them to go to the user
+                if pg_id == 'global':
+                    stats[pg_id] = value
                 continue
             # bare counters
             stats[int(pg_id)] = {}
@@ -1271,7 +1371,7 @@ class CRxStats(CTRexStats):
                         stats[int(pg_id)][field][int(port)] = val if val != 'N/A' else StatNotAvailable(field)
 
             # BW values
-            for field in ['tx_pps', 'tx_bps', 'tx_bps_L1', 'rx_pps', 'rx_bps', 'rx_bps_L1', 'tx_line_util', 'rx_line_util']:
+            for field in ['tx_pps', 'tx_bps', 'tx_bps_L1', 'rx_pps', 'rx_bps', 'rx_bps_L1']:
                 val = self.get([pg_id, field, 'total'])
                 stats[int(pg_id)][field] = {'total': val if val != 'N/A' else StatNotAvailable(field)}
                 for port in value[field].keys():
@@ -1281,7 +1381,27 @@ class CRxStats(CTRexStats):
 
         return stats
 
-
+class CUtilStats(CTRexStats):
+
+    def __init__(self, client):
+        super(CUtilStats, self).__init__()
+        self.client = client
+        self.history = deque(maxlen = 1)
+        self.mbuf_types_list = None
+        self.last_update_ts = 0
+
+    def get_stats(self, use_1sec_cache = False):
+        time_now = time.time()
+        if self.last_update_ts + 1 < time_now or not self.history or not use_1sec_cache:
+            if self.client.is_connected():
+                rc = self.client._transmit('get_utilization')
+                if not rc:
+                    raise Exception(rc)
+                self.last_update_ts = time_now
+                self.history.append(rc.data())
+            else:
+                self.history.append({})
+        return self.history[-1]
 
 if __name__ == "__main__":
     pass
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py
index b4903e81..6835ea5f 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py
@@ -64,3 +64,4 @@ def list_difference (l1, l2):
 
 def is_sub_list (l1, l2):
     return set(l1) <= set(l2)
+
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py
index 98e3ca6a..af7e90c1 100755
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py
@@ -1,6 +1,9 @@
 import argparse
 from collections import namedtuple
 from .common import list_intersect, list_difference
+from .text_opts import format_text
+from ..trex_stl_types import *
+
 import sys
 import re
 import os
@@ -33,12 +36,15 @@ NO_PROMISCUOUS = 20
 PROMISCUOUS_SWITCH = 21
 TUNABLES = 22
 REMOTE_FILE = 23
+LOCKED = 24
 
 GLOBAL_STATS = 50
 PORT_STATS = 51
 PORT_STATUS = 52
 STREAMS_STATS = 53
 STATS_MASK = 54
+CPU_STATS = 55
+MBUF_STATS = 56
 
 STREAMS_MASK = 60
 # ALL_STREAMS = 61
@@ -90,82 +96,74 @@ match_multiplier_help = """Multiplier should be passed in the following format:
 # value should be divided
 def decode_multiplier(val, allow_update = False, divide_count = 1):
 
-    # must be string
-    if not isinstance(val, str):
-        return None
+    factor_table = {None: 1, 'k': 1e3, 'm': 1e6, 'g': 1e9}
+    pattern = "^(\d+(\.\d+)?)(((k|m|g)?(bpsl1|pps|bps))|%)?"
 
     # do we allow updates ?  +/-
     if not allow_update:
-        match = re.match("^(\d+(\.\d+)?)(bps|kbps|mbps|gbps|pps|kpps|mpps|%?)$", val)
+        pattern += "$"
+        match = re.match(pattern, val)
         op = None
     else:
-        match = re.match("^(\d+(\.\d+)?)(bps|kbps|mbps|gbps|pps|kpps|mpps|%?)([\+\-])?$", val)
+        pattern += "([\+\-])?$"
+        match = re.match(pattern, val)
         if match:
-            op  = match.group(4)
+            op  = match.group(7)
         else:
             op = None
 
     result = {}
 
-    if match:
+    if not match:
+        return None
 
-        value = float(match.group(1))
-        unit = match.group(3)
-        
+    # value in group 1
+    value = float(match.group(1))
 
-        
-        # raw type (factor)
-        if not unit:
-            result['type'] = 'raw'
-            result['value'] = value
+    # decode unit as whole
+    unit = match.group(3)
 
-        elif unit == 'bps':
-            result['type'] = 'bps'
-            result['value'] = value
+    # k,m,g
+    factor = match.group(5)
 
-        elif unit == 'kbps':
-            result['type'] = 'bps'
-            result['value'] = value * 1000
+    # type of multiplier
+    m_type = match.group(6)
 
-        elif unit == 'mbps':
-            result['type'] = 'bps'
-            result['value'] = value * 1000 * 1000
+    # raw type (factor)
+    if not unit:
+        result['type'] = 'raw'
+        result['value'] = value
 
-        elif unit == 'gbps':
-            result['type'] = 'bps'
-            result['value'] = value * 1000 * 1000 * 1000
+    # percentage
+    elif unit == '%':
+        result['type'] = 'percentage'
+        result['value']  = value
 
-        elif unit == 'pps':
-            result['type'] = 'pps'
-            result['value'] = value
+    elif m_type == 'bps':
+        result['type'] = 'bps'
+        result['value'] = value * factor_table[factor]
 
-        elif unit == "kpps":
-            result['type'] = 'pps'
-            result['value'] = value * 1000
+    elif m_type == 'pps':
+        result['type'] = 'pps'
+        result['value'] = value * factor_table[factor]
 
-        elif unit == "mpps":
-            result['type'] = 'pps'
-            result['value'] = value * 1000 * 1000
+    elif m_type == 'bpsl1':
+        result['type'] = 'bpsl1'
+        result['value'] = value * factor_table[factor]
 
-        elif unit == "%":
-            result['type'] = 'percentage'
-            result['value']  = value
 
+    if op == "+":
+        result['op'] = "add"
+    elif op == "-":
+        result['op'] = "sub"
+    else:
+        result['op'] = "abs"
 
-        if op == "+":
-            result['op'] = "add"
-        elif op == "-":
-            result['op'] = "sub"
-        else:
-            result['op'] = "abs"
-
-        if result['op'] != 'percentage':
-            result['value'] = result['value'] / divide_count
+    if result['op'] != 'percentage':
+        result['value'] = result['value'] / divide_count
 
-        return result
+    return result
 
-    else:
-        return None
 
 
 def match_multiplier(val):
@@ -244,26 +242,24 @@ OPTIONS_DB = {MULTIPLIER: ArgumentPack(['-m', '--multiplier'],
                                    'type': int}),
 
               PROMISCUOUS: ArgumentPack(['--prom'],
-                                        {'help': "sets port promiscuous on",
+                                        {'help': "Sets port promiscuous on",
                                          'dest': "prom",
                                          'default': None,
                                          'action': "store_true"}),
 
-
               TUNABLES: ArgumentPack(['-t'],
-                                     {'help': "sets tunable for a profile",
+                                     {'help': "Sets tunables for a profile. Example: '-t fsize=100,pg_id=7'",
+                                      'metavar': 'T1=VAL[,T2=VAL ...]',
                                       'dest': "tunables",
                                       'default': None,
                                       'type': decode_tunables}),
 
-
               NO_PROMISCUOUS: ArgumentPack(['--no_prom'],
-                                           {'help': "sets port promiscuous off",
+                                           {'help': "Sets port promiscuous off",
                                             'dest': "prom",
                                             'default': None,
                                             'action': "store_false"}),
 
-
               PORT_LIST: ArgumentPack(['--port', '-p'],
                                         {"nargs": '+',
                                          'dest':'ports',
@@ -325,6 +321,11 @@ OPTIONS_DB = {MULTIPLIER: ArgumentPack(['-m', '--multiplier'],
                                    'default': False,
                                    'help': "Starts TUI in xterm window"}),
 
+              LOCKED: ArgumentPack(['-l', '--locked'],
+                                   {'action': 'store_true',
+                                    'dest': 'locked',
+                                    'default': False,
+                                    'help': "Locks TUI on legend mode"}),
 
               FULL_OUTPUT: ArgumentPack(['--full'],
                                          {'action': 'store_true',
@@ -346,6 +347,14 @@ OPTIONS_DB = {MULTIPLIER: ArgumentPack(['-m', '--multiplier'],
                                           {'action': 'store_true',
                                            'help': "Fetch only streams stats"}),
 
+              CPU_STATS: ArgumentPack(['-c'],
+                                      {'action': 'store_true',
+                                       'help': "Fetch only CPU utilization stats"}),
+
+              MBUF_STATS: ArgumentPack(['-m'],
+                                       {'action': 'store_true',
+                                        'help': "Fetch only MBUF utilization stats"}),
+
               STREAMS_MASK: ArgumentPack(['--streams'],
                                          {"nargs": '+',
                                           'dest':'streams',
@@ -371,7 +380,9 @@ OPTIONS_DB = {MULTIPLIER: ArgumentPack(['-m', '--multiplier'],
               STATS_MASK: ArgumentGroup(MUTEX, [GLOBAL_STATS,
                                                 PORT_STATS,
                                                 PORT_STATUS,
-                                                STREAMS_STATS],
+                                                STREAMS_STATS,
+                                                CPU_STATS,
+                                                MBUF_STATS],
                                         {})
               }
 
@@ -384,6 +395,15 @@ class CCmdArgParser(argparse.ArgumentParser):
         self.cmd_name = kwargs.get('prog')
 
 
+    # hook this to the logger
+    def _print_message(self, message, file=None):
+        self.stateless_client.logger.log(message)
+
+    def error(self, message):
+        self.print_usage()
+        self._print_message(('%s: error: %s\n') % (self.prog, message))
+        raise ValueError(message)
+
     def has_ports_cfg (self, opts):
         return hasattr(opts, "all_ports") or hasattr(opts, "ports")
 
@@ -391,7 +411,7 @@ class CCmdArgParser(argparse.ArgumentParser):
         try:
             opts = super(CCmdArgParser, self).parse_args(args, namespace)
             if opts is None:
-                return None
+                return RC_ERR("'{0}' - invalid arguments".format(self.cmd_name))
 
             if not self.has_ports_cfg(opts):
                 return opts
@@ -406,8 +426,9 @@ class CCmdArgParser(argparse.ArgumentParser):
             # so maybe we have ports configured
             invalid_ports = list_difference(opts.ports, self.stateless_client.get_all_ports())
             if invalid_ports:
-                self.stateless_client.logger.log("{0}: port(s) {1} are not valid port IDs".format(self.cmd_name, invalid_ports))
-                return None
+                msg = "{0}: port(s) {1} are not valid port IDs".format(self.cmd_name, invalid_ports)
+                self.stateless_client.logger.log(format_text(msg, 'bold'))
+                return RC_ERR(msg)
 
             # verify acquired ports
             if verify_acquired:
@@ -415,21 +436,25 @@ class CCmdArgParser(argparse.ArgumentParser):
 
                 diff = list_difference(opts.ports, acquired_ports)
                 if diff:
-                    self.stateless_client.logger.log("{0} - port(s) {1} are not acquired".format(self.cmd_name, diff))
-                    return None
+                    msg = "{0} - port(s) {1} are not acquired".format(self.cmd_name, diff)
+                    self.stateless_client.logger.log(format_text(msg, 'bold'))
+                    return RC_ERR(msg)
 
                 # no acquire ports at all
                 if not acquired_ports:
-                    self.stateless_client.logger.log("{0} - no acquired ports".format(self.cmd_name))
-                    return None
-
+                    msg = "{0} - no acquired ports".format(self.cmd_name)
+                    self.stateless_client.logger.log(format_text(msg, 'bold'))
+                    return RC_ERR(msg)
 
 
             return opts
 
+        except ValueError as e:
+            return RC_ERR("'{0}' - {1}".format(self.cmd_name, str(e)))
+
         except SystemExit:
             # recover from system exit scenarios, such as "help", or bad arguments.
-            return None
+            return RC_ERR("'{0}' - {1}".format(self.cmd_name, "no action"))
 
 
 def get_flags (opt):
@@ -469,4 +494,4 @@ def gen_parser(stateless_client, op_name, description, *args):
 
 
 if __name__ == "__main__":
-    pass
-\ No newline at end of file
+    pass
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py
index 7e0bf9e4..bfb96950 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py
@@ -27,7 +27,10 @@ class TextCodesStripper:
     def strip (s):
         return re.sub(TextCodesStripper.pattern, '', s)
 
-def format_num (size, suffix = "", compact = True, opts = ()):
+def format_num (size, suffix = "", compact = True, opts = None):
+    if opts is None:
+        opts = ()
+
     txt = "NaN"
 
     if type(size) == str:
@@ -61,6 +64,9 @@ def format_time (t_sec):
     if t_sec < 0:
         return "infinite"
 
+    if t_sec == 0:
+        return "zero"
+
     if t_sec < 1:
         # low numbers
         for unit in ['ms', 'usec', 'ns']:
author	itraviv <itraviv@cisco.com>	2016-07-31 11:56:41 +0300
committer	itraviv <itraviv@cisco.com>	2016-07-31 11:56:41 +0300
commit	893d0feef9ba6fa3fb36c49f4b5bcad47cb2bf60 (patch)
tree	689a09fa656f990672d2d62143dc173a46fe0316 /scripts/automation/trex_control_plane
parent	abf329075bd14f5f41c3753d560260ac809ec4f3 (diff)
parent	dceb010b01e9f8a0e9c905370d39f149f01cab7e (diff)