Merge branch 'capture'

10 files changed, 960 insertions, 160 deletions

diff --git a/scripts/automation/trex_control_plane/stl/console/trex_capture.py b/scripts/automation/trex_control_plane/stl/console/trex_capture.py
new file mode 100644
index 00000000..2132458e
--- /dev/null
+++ b/scripts/automation/trex_control_plane/stl/console/trex_capture.py
@@ -0,0 +1,593 @@
+from trex_stl_lib.api import *
+from trex_stl_lib.utils import parsing_opts, text_tables
+import threading
+import tempfile
+import select
+
+# defines a generic monitor writer
+class CaptureMonitorWriter(object):
+
+    def deinit(self):
+        # by default - nothing to deinit
+        pass
+        
+    def handle_pkts (self, pkts):
+        raise NotImplementedError
+        
+    def periodic_check (self):
+        # by default - nothing to check
+        pass
+        
+   
+# a stdout monitor
+class CaptureMonitorWriterStdout(CaptureMonitorWriter):
+    def __init__ (self, logger, is_brief, start_ts):
+        self.logger      = logger
+        self.is_brief    = is_brief
+        self.start_ts    = start_ts
+        
+        # unicode arrows
+        self.RX_ARROW = u'\u25c0\u2500\u2500'
+        self.TX_ARROW = u'\u25b6\u2500\u2500'
+        
+        # decode issues with Python 2
+        if sys.version_info < (3,0):
+            self.RX_ARROW = self.RX_ARROW.encode('utf-8')
+            self.TX_ARROW = self.TX_ARROW.encode('utf-8')
+
+
+        self.logger.pre_cmd("Starting stdout capture monitor - verbose: '{0}'".format('low' if self.is_brief else 'high'))
+        self.logger.post_cmd(RC_OK)
+        
+        self.logger.log(format_text("\n*** use 'capture monitor stop' to abort capturing... ***\n", 'bold')) 
+        
+ 
+        
+    def get_scapy_name (self, pkt_scapy):
+        layer = pkt_scapy
+        while layer.payload and layer.payload.name not in('Padding', 'Raw'):
+            layer = layer.payload
+        
+        return layer.name
+        
+                
+    def format_origin (self, origin):
+        if origin == 'RX':
+            return '{0} {1}'.format(self.RX_ARROW, 'RX')
+        elif origin == 'TX':
+            return '{0} {1}'.format(self.TX_ARROW, 'TX')
+        else:
+            return '{0}'.format(origin)
+            
+        
+    def __handle_pkt (self, pkt):
+        pkt_bin = base64.b64decode(pkt['binary'])
+
+        pkt_scapy = Ether(pkt_bin)
+        self.logger.log(format_text('\n\n#{} Port: {} {}\n'.format(pkt['index'], pkt['port'], self.format_origin(pkt['origin'])), 'bold', ''))
+        self.logger.log(format_text('    Type: {}, Size: {} B, TS: {:.2f} [sec]\n'.format(self.get_scapy_name(pkt_scapy), len(pkt_bin), pkt['ts'] - self.start_ts), 'bold'))
+
+        if self.is_brief:
+            self.logger.log('    {0}'.format(pkt_scapy.command()))
+        else:
+            pkt_scapy.show(label_lvl = '    ')
+            self.logger.log('')
+
+        return len(pkt_bin)
+
+
+    def handle_pkts (self, pkts):
+        try:
+            byte_count = 0
+            for pkt in pkts:
+                byte_count += self.__handle_pkt(pkt)
+
+            return byte_count
+
+        finally:
+            # make sure to restore the logger
+            self.logger.prompt_redraw()
+
+
+# a pipe based monitor
+class CaptureMonitorWriterPipe(CaptureMonitorWriter):
+    def __init__ (self, logger, start_ts):
+        
+        self.logger    = logger
+        self.fifo      = None
+        self.start_ts  = start_ts
+        
+        # generate a temp fifo pipe
+        self.fifo_name = tempfile.mktemp()
+        
+        try:
+            self.logger.pre_cmd('Starting pipe capture monitor')
+            os.mkfifo(self.fifo_name)
+            self.logger.post_cmd(RC_OK)
+
+            self.logger.log(format_text("*** Please run 'wireshark -k -i {0}' ***".format(self.fifo_name), 'bold'))
+            
+            self.logger.pre_cmd("Waiting for Wireshark pipe connection")
+            
+            # blocks until pipe is connected
+            self.fifo = os.open(self.fifo_name, os.O_WRONLY)
+            self.logger.post_cmd(RC_OK())
+            
+            self.logger.log(format_text('\n*** Capture monitoring started ***\n', 'bold'))
+            
+            # open for write using a PCAP writer
+            self.writer = RawPcapWriter(self.fifo_name, linktype = 1, sync = True)
+            self.writer._write_header(None)
+            
+            # register a poller
+            self.poll = select.poll()
+            self.poll.register(self.fifo, select.EPOLLERR)
+        
+            self.is_init = True
+                
+        except KeyboardInterrupt as e:
+            self.deinit()
+            self.logger.post_cmd(RC_ERR(""))
+            raise STLError("*** pipe monitor aborted...cleaning up")
+
+        except OSError as e:
+            self.deinit()
+            self.logger.post_cmd(RC_ERR(""))
+            raise STLError("failed to create pipe {0}\n{1}".format(self.fifo_name, str(e)))
+        
+        
+        
+    def deinit (self):
+        try:
+            if self.fifo:
+                os.close(self.fifo)
+                self.fifo = None
+                
+            if self.fifo_name:
+                os.unlink(self.fifo_name)
+                self.fifo_name = None
+                
+        except OSError:
+            pass
+            
+
+       
+    def periodic_check (self):
+        self.check_pipe()
+        
+        
+    def check_pipe (self):
+        if self.poll.poll(0):
+            raise STLError('pipe has been disconnected')
+        
+        
+    def handle_pkts (self, pkts):
+        # first check the pipe is alive
+        self.check_pipe()
+
+        return self.handle_pkts_internal(pkts)
+            
+        
+    def handle_pkts_internal (self, pkts):
+        
+        byte_count = 0
+        
+        for pkt in pkts:
+            pkt_bin = base64.b64decode(pkt['binary'])
+            ts_sec, ts_usec = sec_split_usec(pkt['ts'] - self.start_ts)
+            
+            try:
+                self.writer._write_packet(pkt_bin, sec = ts_sec, usec = ts_usec)
+            except Exception as e:
+                raise STLError('fail to write packets to pipe: {}'.format(str(e)))
+                
+            byte_count += len(pkt_bin)
+               
+        return byte_count
+        
+        
+# capture monitor - a live capture
+class CaptureMonitor(object):
+    def __init__ (self, client, cmd_lock, tx_port_list, rx_port_list, rate_pps, mon_type):
+        self.client      = client
+        self.logger      = client.logger
+        self.cmd_lock    = cmd_lock
+
+        self.t           = None
+        self.writer      = None
+        self.capture_id  = None
+        
+        self.tx_port_list = tx_port_list
+        self.rx_port_list = rx_port_list
+        self.rate_pps     = rate_pps
+        self.mon_type     = mon_type
+        
+        # try to launch
+        try:
+            self.__start()
+        except Exception as e:
+            self.__stop()
+            raise
+            
+            
+    def __start (self):
+        
+        # create a capture on the server
+        with self.logger.supress():
+            data = self.client.start_capture(self.tx_port_list, self.rx_port_list, limit = self.rate_pps, mode = 'cyclic')
+
+        self.capture_id = data['id']
+        self.start_ts   = data['ts']
+        
+
+        # create a writer
+        if self.mon_type == 'compact':
+            self.writer = CaptureMonitorWriterStdout(self.logger, True, self.start_ts)
+        elif self.mon_type == 'verbose':
+            self.writer = CaptureMonitorWriterStdout(self.logger, False, self.start_ts)
+        elif self.mon_type == 'pipe':
+            self.writer = CaptureMonitorWriterPipe(self.logger, self.start_ts)
+        else:
+            raise STLError('Internal error: unknown writer type')
+        
+        # start the fetching thread
+        self.t = threading.Thread(target = self.__thread_cb)
+        self.t.setDaemon(True)
+        self.active = True
+        self.t.start()
+  
+            
+    # internal stop
+    def __stop (self):
+
+        # stop the thread
+        if self.t and self.t.is_alive():
+            self.active = False
+            self.t.join()
+            self.t = None
+            
+        # deinit the writer
+        if self.writer:
+            self.writer.deinit()
+            self.writer = None
+            
+        # take the capture ID
+        capture_id = self.capture_id
+        self.capture_id = None
+        
+        # if we are disconnected - we cannot cleanup the capture
+        if not self.client.is_connected():
+            return
+            
+        # make sure the capture is active on the server
+        captures = [x['id'] for x in self.client.get_capture_status()]
+        if capture_id not in captures:
+            return
+            
+        # remove the capture                
+        with self.logger.supress():
+            self.client.stop_capture(capture_id)
+            
+           
+    # user call for stop (adds log)
+    def stop (self):
+        self.logger.pre_cmd("Stopping capture monitor")
+        
+        try:
+            self.__stop()
+        except Exception as e:
+            self.logger.post_cmd(RC_ERR(""))
+            raise
+        
+        self.logger.post_cmd(RC_OK())
+
+
+    def get_mon_row (self):
+            
+        return [self.capture_id,
+                format_text('ACTIVE' if self.t.is_alive() else 'DEAD', 'bold'),
+                self.pkt_count,
+                format_num(self.byte_count, suffix = 'B'),
+                ', '.join([str(x) for x in self.tx_port_list] if self.tx_port_list else '-'),
+                ', '.join([str(x) for x in self.rx_port_list] if self.rx_port_list else '-')
+                ]
+        
+
+    def is_active (self):
+        return self.active
+
+
+    def get_capture_id (self):
+        return self.capture_id
+        
+
+    # sleeps with high freq checks for active
+    def __sleep (self):
+        for _ in range(5):
+            if not self.active:
+                return False
+                
+            time.sleep(0.1)
+            
+        return True
+            
+    def __lock (self):
+        while True:
+            rc = self.cmd_lock.acquire(False)
+            if rc:
+                return True
+                
+            if not self.active:
+                return False
+            time.sleep(0.1)
+        
+    def __unlock (self):
+        self.cmd_lock.release()
+        
+    
+    def __thread_cb (self):
+        try:
+            self.__thread_main_loop()
+        
+        # common errors
+        except STLError as e:
+            self.logger.log(format_text("\n\nMonitor has encountered the following error: '{}'\n".format(e.brief()), 'bold'))
+            self.logger.log(format_text("\n*** monitor is inactive - please restart the monitor ***\n", 'bold'))
+            self.logger.prompt_redraw()
+            
+        # unexpected errors
+        except Exception as e:
+            self.logger.log("\n\n*** A fatal internal error has occurred: '{}'\n".format(str(e)))
+            self.logger.log(format_text("\n*** monitor is inactive - please restart the monitor ***\n", 'bold'))
+            self.logger.prompt_redraw()
+            
+            
+    def __thread_main_loop (self):
+        self.pkt_count  = 0
+        self.byte_count = 0
+        
+        while self.active:
+            
+            # sleep - if interrupt by graceful shutdown - go out
+            if not self.__sleep():
+                return
+            
+            # check that the writer is ok
+            self.writer.periodic_check()
+
+            # try to lock - if interrupt by graceful shutdown - go out
+            if not self.__lock():
+                return
+                
+            try:
+                if not self.client.is_connected():
+                    raise STLError('client has been disconnected')
+                    
+                rc = self.client._transmit("capture", params = {'command': 'fetch', 'capture_id': self.capture_id, 'pkt_limit': 10})
+                if not rc:
+                    raise STLError(rc)
+                    
+            finally:
+                self.__unlock()
+                
+
+            # no packets - skip
+            pkts = rc.data()['pkts']
+            if not pkts:
+                continue
+            
+            byte_count = self.writer.handle_pkts(pkts)
+            
+            self.pkt_count  += len(pkts)
+            self.byte_count += byte_count
+
+
+
+
+# main class
+class CaptureManager(object):
+    def __init__ (self, client, cmd_lock):
+        self.c          = client
+        self.cmd_lock   = cmd_lock
+        self.logger     = client.logger
+        self.monitor    = None
+        
+        # install parsers
+        
+        self.parser = parsing_opts.gen_parser(self, "capture", self.parse_line_internal.__doc__)
+        self.subparsers = self.parser.add_subparsers(title = "commands", dest="commands")
+
+        self.install_record_parser()
+        self.install_monitor_parser()
+        
+        # show
+        self.show_parser = self.subparsers.add_parser('show', help = "show all active captures")
+     
+        # reset
+        self.clear_parser = self.subparsers.add_parser('clear', help = "remove all active captures")
+       
+        # register handlers
+        self.cmds = {'record': self.parse_record, 'monitor' : self.parse_monitor, 'clear': self.parse_clear, 'show' : self.parse_show} 
+        
+        
+    def install_record_parser (self):
+        # recording
+        self.record_parser = self.subparsers.add_parser('record', help = "PCAP recording")
+        record_sub = self.record_parser.add_subparsers(title = 'commands', dest = 'record_cmd')
+        self.record_start_parser = record_sub.add_parser('start', help = "starts a new buffered capture")
+        self.record_stop_parser  = record_sub.add_parser('stop', help = "stops an active buffered capture")
+
+        # start
+        self.record_start_parser.add_arg_list(parsing_opts.TX_PORT_LIST,
+                                              parsing_opts.RX_PORT_LIST,
+                                              parsing_opts.LIMIT)
+
+        # stop
+        self.record_stop_parser.add_arg_list(parsing_opts.CAPTURE_ID,
+                                             parsing_opts.OUTPUT_FILENAME)
+
+        
+        
+    def install_monitor_parser (self):
+        # monitor
+        self.monitor_parser = self.subparsers.add_parser('monitor', help = 'live monitoring')
+        monitor_sub = self.monitor_parser.add_subparsers(title = 'commands', dest = 'mon_cmd')
+        self.monitor_start_parser = monitor_sub.add_parser('start', help = 'starts a monitor')
+        self.monitor_stop_parser  = monitor_sub.add_parser('stop', help = 'stops an active monitor')
+
+        self.monitor_start_parser.add_arg_list(parsing_opts.TX_PORT_LIST,
+                                               parsing_opts.RX_PORT_LIST,
+                                               parsing_opts.MONITOR_TYPE)
+
+        
+        
+    def stop (self):
+        if self.monitor:
+            self.monitor.stop()
+            self.monitor = None
+
+        
+    # main entry point for parsing commands from console
+    def parse_line (self, line):
+        try:
+            self.parse_line_internal(line)
+        except STLError as e:
+            self.logger.log("\nAction has failed with the following error:\n\n" + format_text(e.brief() + "\n", 'bold'))
+            return RC_ERR(e.brief())
+
+
+    def parse_line_internal (self, line):
+        '''Manage PCAP recorders'''
+
+        # default
+        if not line:
+            line = "show"
+
+        opts = self.parser.parse_args(line.split())
+        if not opts:
+            return opts
+
+        # call the handler
+        self.cmds[opts.commands](opts)
+
+
+    # record methods
+    def parse_record (self, opts):
+        if opts.record_cmd == 'start':
+            self.parse_record_start(opts)
+        elif opts.record_cmd == 'stop':
+            self.parse_record_stop(opts)
+        else:
+            self.record_parser.formatted_error("too few arguments")
+            
+            
+    def parse_record_start (self, opts):
+        if not opts.tx_port_list and not opts.rx_port_list:
+            self.record_start_parser.formatted_error('please provide either --tx or --rx')
+            return
+
+        rc = self.c.start_capture(opts.tx_port_list, opts.rx_port_list, opts.limit, mode = 'fixed')
+        
+        self.logger.log(format_text("*** Capturing ID is set to '{0}' ***".format(rc['id']), 'bold'))
+        self.logger.log(format_text("*** Please call 'capture record stop --id {0} -o <out.pcap>' when done ***\n".format(rc['id']), 'bold'))
+
+        
+    def parse_record_stop (self, opts):
+        captures = self.c.get_capture_status()
+        ids = [c['id'] for c in captures]
+        
+        if self.monitor and (opts.capture_id == self.monitor.get_capture_id()):
+            self.record_stop_parser.formatted_error("'{0}' is a monitor, please use 'capture monitor stop'".format(opts.capture_id))
+            return
+            
+        if opts.capture_id not in ids:
+            self.record_stop_parser.formatted_error("'{0}' is not an active capture ID".format(opts.capture_id))
+            return
+            
+        self.c.stop_capture(opts.capture_id, opts.output_filename)
+
+            
+    # monitor methods
+    def parse_monitor (self, opts):
+        if opts.mon_cmd == 'start':
+            self.parse_monitor_start(opts)
+        elif opts.mon_cmd == 'stop':
+            self.parse_monitor_stop(opts)
+        else:
+            self.monitor_parser.formatted_error("too few arguments")
+            
+            
+    def parse_monitor_start (self, opts):
+        mon_type = 'compact'
+        
+        if opts.verbose:
+            mon_type = 'verbose'
+        elif opts.pipe:
+            mon_type = 'pipe'
+            
+        if not opts.tx_port_list and not opts.rx_port_list:
+            self.monitor_start_parser.formatted_error('please provide either --tx or --rx')
+            return
+        
+        if self.monitor:
+            self.monitor.stop()
+            self.monitor = None
+            
+        self.monitor = CaptureMonitor(self.c, self.cmd_lock, opts.tx_port_list, opts.rx_port_list, 100, mon_type)
+        
+    
+    def parse_monitor_stop (self, opts):
+        if self.monitor:
+            self.monitor.stop()
+            self.monitor = None
+            
+        
+    def parse_clear (self, opts):
+        if self.monitor:
+            self.monitor.stop()
+            self.monitor = None
+            
+        self.c.remove_all_captures()
+        
+        
+        
+    def parse_show (self, opts):
+        data = self.c.get_capture_status()
+
+        # captures
+        cap_table = text_tables.TRexTextTable()
+        cap_table.set_cols_align(["c"] * 6)
+        cap_table.set_cols_width([15] * 6)
+
+        # monitor
+        mon_table = text_tables.TRexTextTable()
+        mon_table.set_cols_align(["c"] * 6)
+        mon_table.set_cols_width([15] * 6)
+
+        for elem in data:
+            id = elem['id']
+
+            if self.monitor and (self.monitor.get_capture_id() == id):
+                row = self.monitor.get_mon_row()
+                mon_table.add_rows([row], header=False)
+
+            else:
+                row = [id,
+                       format_text(elem['state'], 'bold'),
+                       '[{0}/{1}]'.format(elem['count'], elem['limit']),
+                       format_num(elem['bytes'], suffix = 'B'),
+                       bitfield_to_str(elem['filter']['tx']),
+                       bitfield_to_str(elem['filter']['rx'])]
+
+                cap_table.add_rows([row], header=False)
+
+        cap_table.header(['ID', 'Status', 'Packets', 'Bytes', 'TX Ports', 'RX Ports'])
+        mon_table.header(['ID', 'Status', 'Packets Seen', 'Bytes Seen', 'TX Ports', 'RX Ports'])
+
+        if cap_table._rows:
+            text_tables.print_table_with_header(cap_table, '\nActive Recorders')
+
+        if mon_table._rows:
+            text_tables.print_table_with_header(mon_table, '\nActive Monitor')
+
+
diff --git a/scripts/automation/trex_control_plane/stl/console/trex_console.py b/scripts/automation/trex_control_plane/stl/console/trex_console.py
index c9956472..d36ce7b0 100755
--- a/scripts/automation/trex_control_plane/stl/console/trex_console.py
+++ b/scripts/automation/trex_control_plane/stl/console/trex_console.py
@@ -29,6 +29,8 @@ import string
 import os
 import sys
 import tty, termios
+from threading import Lock
+import threading
 
 try:
     import stl_path
@@ -39,6 +41,7 @@ from trex_stl_lib.api import *
 from trex_stl_lib.utils.text_opts import *
 from trex_stl_lib.utils.common import user_input, get_current_user
 from trex_stl_lib.utils import parsing_opts
+from .trex_capture import CaptureManager
 
 try:
     import trex_tui
@@ -172,6 +175,10 @@ class TRexConsole(TRexGeneralCmd):
 
     def __init__(self, stateless_client, verbose = False):
 
+        # cmd lock is used to make sure background job
+        # of the console is not done while the user excutes commands
+        self.cmd_lock = Lock()
+        
         self.stateless_client = stateless_client
 
         TRexGeneralCmd.__init__(self)
@@ -184,8 +191,11 @@ class TRexConsole(TRexGeneralCmd):
         self.intro  = "\n-=TRex Console v{ver}=-\n".format(ver=__version__)
         self.intro += "\nType 'help' or '?' for supported actions\n"
 
+        self.cap_mngr = CaptureManager(stateless_client, self.cmd_lock)
+
         self.postcmd(False, "")
 
+        
 
     ################### internal section ########################
 
@@ -231,6 +241,7 @@ class TRexConsole(TRexGeneralCmd):
 
         lines = line.split(';')
         try:
+            self.cmd_lock.acquire()
             for line in lines:
                 stop = self.onecmd(line)
                 stop = self.postcmd(stop, line)
@@ -238,10 +249,15 @@ class TRexConsole(TRexGeneralCmd):
                     return "quit"
     
             return ""
+            
         except STLError as e:
             print(e)
             return ''
 
+        finally:
+            self.cmd_lock.release()
+
+
 
     def postcmd(self, stop, line):
         self.prompt = self.stateless_client.generate_prompt(prefix = 'trex')
@@ -347,12 +363,12 @@ class TRexConsole(TRexGeneralCmd):
 
         
     @verify_connected
-    def do_set_rx_sniffer (self, line):
-        '''Sets a port sniffer on RX channel as PCAP recorder'''
-        self.stateless_client.set_rx_sniffer_line(line)
+    def do_capture (self, line):
+        '''Manage PCAP captures'''
+        self.cap_mngr.parse_line(line)
 
-    def help_sniffer (self):
-        self.do_set_rx_sniffer("-h")
+    def help_capture (self):
+        self.do_capture("-h")
 
     @verify_connected
     def do_resolve (self, line):
@@ -443,7 +459,9 @@ class TRexConsole(TRexGeneralCmd):
 
     def do_disconnect (self, line):
         '''Disconnect from the server\n'''
-
+        
+        # stop any monitors before disconnecting
+        self.cap_mngr.stop()
         self.stateless_client.disconnect_line(line)
 
 
@@ -688,19 +706,25 @@ class TRexConsole(TRexGeneralCmd):
              l=help.splitlines()
              print("{:<30} {:<30}".format(cmd + " - ",l[0] ))
 
+             
     # a custorm cmdloop wrapper
     def start(self):
-        while True:
-            try:
-                self.cmdloop()
-                break
-            except KeyboardInterrupt as e:
-                if not readline.get_line_buffer():
-                    raise KeyboardInterrupt
-                else:
-                    print("")
-                    self.intro = None
-                    continue
+        try:
+            while True:
+                try:
+                    self.cmdloop()
+                    break
+                except KeyboardInterrupt as e:
+                    if not readline.get_line_buffer():
+                        raise KeyboardInterrupt
+                    else:
+                        print("")
+                        self.intro = None
+                        continue
+    
+        finally:
+            # capture manager is not presistent - kill it before going out
+            self.cap_mngr.stop()
 
         if self.terminal:
             self.terminal.kill()
@@ -933,7 +957,7 @@ def main():
         with stateless_client.logger.supress():
             stateless_client.disconnect(stop_traffic = False)
 
+
 if __name__ == '__main__':
-    
     main()
 
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
index 21ae42f1..215c0253 100755
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
@@ -17,6 +17,7 @@ from .utils.text_opts import *
 from functools import wraps
 from texttable import ansi_len
 
+
 from collections import namedtuple
 from yaml import YAMLError
 import time
@@ -600,12 +601,10 @@ class STLClient(object):
                                                                  self.util_stats,
                                                                  self.xstats,
                                                                  self.async_client.monitor)
-
-
-
-
+        
     ############# private functions - used by the class itself ###########
 
+    
     # some preprocessing for port argument
     def __ports (self, port_id_list):
 
@@ -832,27 +831,6 @@ class STLClient(object):
         return rc
 
 
-    def __set_rx_sniffer (self, port_id_list, base_filename, limit):
-        port_id_list = self.__ports(port_id_list)
-        rc = RC()
-
-        for port_id in port_id_list:
-            head, tail = os.path.splitext(base_filename)
-            filename = "{0}-{1}{2}".format(head, port_id, tail)
-            rc.add(self.ports[port_id].set_rx_sniffer(filename, limit))
-
-        return rc
-
-
-    def __remove_rx_sniffer (self, port_id_list):
-        port_id_list = self.__ports(port_id_list)
-        rc = RC()
-
-        for port_id in port_id_list:
-            rc.add(self.ports[port_id].remove_rx_sniffer())
-
-        return rc
-
     def __set_rx_queue (self, port_id_list, size):
         port_id_list = self.__ports(port_id_list)
         rc = RC()
@@ -1071,7 +1049,7 @@ class STLClient(object):
 
     ############ functions used by other classes but not users ##############
 
-    def _validate_port_list (self, port_id_list):
+    def _validate_port_list (self, port_id_list, allow_empty = False):
         # listfiy single int
         if isinstance(port_id_list, int):
             port_id_list = [port_id_list]
@@ -1080,7 +1058,7 @@ class STLClient(object):
         if not isinstance(port_id_list, list):
             raise STLTypeError('port_id_list', type(port_id_list), list)
 
-        if not port_id_list:
+        if not port_id_list and not allow_empty:
             raise STLError('No ports provided')
 
         valid_ports = self.get_all_ports()
@@ -2084,9 +2062,9 @@ class STLClient(object):
                 self.set_port_attr(ports,
                                    promiscuous = False,
                                    link_up = True if restart else None)
-                self.set_service_mode(ports, False)
-                self.remove_rx_sniffer(ports)
                 self.remove_rx_queue(ports)
+                self.set_service_mode(ports, False)
+                
                 
         except STLError as e:
             self.logger.post_cmd(False)
@@ -2996,7 +2974,7 @@ class STLClient(object):
             Resolves ports (ARP resolution)
 
             :parameters:
-                ports          - for which ports to apply a unique sniffer (each port gets a unique file)
+                ports          - which ports to resolve
                 retries        - how many times to retry on each port (intervals of 100 milliseconds)
                 verbose        - log for each request the response
             :raises:
@@ -3025,57 +3003,195 @@ class STLClient(object):
             
         
     @__api_check(True)
-    def set_rx_sniffer (self, ports = None, base_filename = 'rx.pcap', limit = 1000):
+    def start_capture (self, tx_ports, rx_ports, limit = 1000, mode = 'fixed'):
         """
-            Sets a RX sniffer for port(s) written to a PCAP file
+            Starts a low rate packet capturing on the server
 
             :parameters:
-                ports          - for which ports to apply a unique sniffer (each port gets a unique file)
-                base_filename  - filename will be appended with '-<port_number>', e.g. rx.pcap --> rx-0.pcap, rx-1.pcap etc.
-                limit          - limit how many packets will be written
+                tx_ports: list
+                    on which ports to capture TX
+                    
+                rx_ports: list
+                    on which ports to capture RX
+                    
+                limit: int
+                    limit how many packets will be written memory requierment is O(9K * limit)
+                    
+                mode: str
+                    'fixed'  - when full, newer packets will be dropped
+                    'cyclic' - when full, older packets will be dropped
+                                  
+            :returns:
+                returns a dictionary:
+                {'id: <new_id>, 'ts': <starting timestamp>}
+                
+                where 'id' is the new capture ID for future commands
+                and 'ts' is that server monotonic timestamp when
+                the capture was created
+                
             :raises:
                 + :exe:'STLError'
 
         """
-        ports = ports if ports is not None else self.get_acquired_ports()
-        ports = self._validate_port_list(ports)
-
+        # TODO: remove this when TX is implemented
+        if tx_ports:
+            raise STLError('TX port capturing is not yet implemented')
+            
         # check arguments
-        validate_type('base_filename', base_filename, basestring)
+        tx_ports = self._validate_port_list(tx_ports, allow_empty = True)
+        rx_ports = self._validate_port_list(rx_ports, allow_empty = True)
+        merge_ports = set(tx_ports + rx_ports)
+        
+        # make sure at least one port to capture
+        if not merge_ports:
+            raise STLError("start_capture - must get at least one port to capture")
+            
         validate_type('limit', limit, (int))
         if limit <= 0:
             raise STLError("'limit' must be a positive value")
 
-        self.logger.pre_cmd("Setting RX sniffers on port(s) {0}:".format(ports))
-        rc = self.__set_rx_sniffer(ports, base_filename, limit)
+        if mode not in ('fixed', 'cyclic'):
+            raise STLError("'mode' must be either 'fixed' or 'cyclic'")
+        
+        # verify service mode
+        non_service_ports =  list_difference(merge_ports, self.get_service_enabled_ports())
+        if non_service_ports:
+            raise STLError("Port(s) {0} are not under service mode. packet capturing requires all ports to be in service mode".format(non_service_ports))
+        
+            
+        # actual job
+        self.logger.pre_cmd("Starting packet capturing up to {0} packets".format(limit))
+        rc = self._transmit("capture", params = {'command': 'start', 'limit': limit, 'mode': mode, 'tx': tx_ports, 'rx': rx_ports})
         self.logger.post_cmd(rc)
 
-
         if not rc:
             raise STLError(rc)
 
+        return {'id': rc.data()['capture_id'], 'ts': rc.data()['start_ts']}
 
 
+        
     @__api_check(True)
-    def remove_rx_sniffer (self, ports = None):
+    def stop_capture (self, capture_id, output_filename = None):
         """
-            Removes RX sniffer from port(s)
+            Stops an active capture and optionally save it to a PCAP file
+
+            :parameters:
+                capture_id: int
+                    an active capture ID to stop
+                    
+                output_filename: str
+                    output filename to save capture
+                    if 'None', all captured packets will be discarded
 
             :raises:
                 + :exe:'STLError'
 
         """
-        ports = ports if ports is not None else self.get_acquired_ports()
-        ports = self._validate_port_list(ports)
+        
+        # stopping a capture requires:
+        # 1. stopping
+        # 2. fetching
+        # 3. saving to file
+        
+        # stop
+        
+        self.logger.pre_cmd("Stopping packet capture {0}".format(capture_id))
+        rc = self._transmit("capture", params = {'command': 'stop', 'capture_id': capture_id})
+        self.logger.post_cmd(rc)
+        if not rc:
+            raise STLError(rc)
+        
+        # pkt count
+        pkt_count = rc.data()['pkt_count']
+        
+        # fetch packets    
+        if output_filename:
+            self.__fetch_capture_packets(capture_id, output_filename, pkt_count)
+        
+        # remove
+        self.logger.pre_cmd("Removing PCAP capture {0} from server".format(capture_id))
+        rc = self._transmit("capture", params = {'command': 'remove', 'capture_id': capture_id})
+        self.logger.post_cmd(rc)
+        if not rc:
+            raise STLError(rc)
+        
+
+            
+    # fetch packets from the server and save them to a file
+    def __fetch_capture_packets (self, capture_id, output_filename, pkt_count):
+        self.logger.pre_cmd("Writing {0} packets to '{1}'".format(pkt_count, output_filename))
+
+        # create a PCAP file
+        writer = RawPcapWriter(output_filename, linktype = 1)
+        writer._write_header(None)
+
+        pending = pkt_count
+        rc = RC_OK()
+        
+        # fetch with iteratios - each iteration up to 50 packets
+        while pending > 0:
+            rc = self._transmit("capture", params = {'command': 'fetch', 'capture_id': capture_id, 'pkt_limit': 50})
+            if not rc:
+                self.logger.post_cmd(rc)
+                raise STLError(rc)
+
+            # make sure we are getting some progress
+            assert(rc.data()['pending'] < pending)
+            
+            pkts      = rc.data()['pkts']
+            pending   = rc.data()['pending']
+            start_ts  = rc.data()['start_ts']
+            
+            # write packets
+            for pkt in pkts:
+                # split the server timestamp relative to the capture start time
+                ts_sec, ts_usec = sec_split_usec(pkt['ts'] - start_ts)
+                
+                pkt_bin = base64.b64decode(pkt['binary'])
+                writer._write_packet(pkt_bin, sec = ts_sec, usec = ts_usec)
+
+
+
 
-        self.logger.pre_cmd("Removing RX sniffers on port(s) {0}:".format(ports))
-        rc = self.__remove_rx_sniffer(ports)
         self.logger.post_cmd(rc)
 
+            
+            
+    @__api_check(True)
+    def get_capture_status (self):
+        """
+            returns a list of all active captures
+            each element in the list is an object containing
+            info about the capture
+
+        """
+        rc = self._transmit("capture", params = {'command': 'status'})
         if not rc:
             raise STLError(rc)
 
-    
+        return rc.data()
+
+        
+    @__api_check(True)
+    def remove_all_captures (self):
+        """
+            Removes any existing captures
+        """
+        captures = self.get_capture_status()
+        
+        self.logger.pre_cmd("Removing all packet captures from server")
+        
+        for c in captures:
+            # remove
+            rc = self._transmit("capture", params = {'command': 'remove', 'capture_id': c['id']})
+            if not rc:
+                raise STLError(rc)
+
+        self.logger.post_cmd(RC_OK())
+                
+
+        
     @__api_check(True)
     def set_rx_queue (self, ports = None, size = 1000):
         """
@@ -3184,7 +3300,7 @@ class STLClient(object):
             try:
                 rc = f(*args)
             except STLError as e:
-                client.logger.log("\nAction has failed with the following error:\n" + format_text(e.brief() + "\n", 'bold'))
+                client.logger.log("\nAction has failed with the following error:\n\n" + format_text(e.brief() + "\n", 'bold'))
                 return RC_ERR(e.brief())
 
             # if got true - print time
@@ -3196,6 +3312,7 @@ class STLClient(object):
 
         return wrap
 
+
     @__console
     def ping_line (self, line):
         '''pings the server / specific IP'''
@@ -3789,29 +3906,9 @@ class STLClient(object):
                                opts.link,
                                opts.led,
                                opts.flow_ctrl)
+                         
+               
              
-             
-
-             
-    @__console
-    def set_rx_sniffer_line (self, line):
-        '''Sets a port sniffer on RX channel in form of a PCAP file'''
-
-        parser = parsing_opts.gen_parser(self,
-                                         "set_rx_sniffer",
-                                         self.set_rx_sniffer_line.__doc__,
-                                         parsing_opts.PORT_LIST_WITH_ALL,
-                                         parsing_opts.OUTPUT_FILENAME,
-                                         parsing_opts.LIMIT)
-
-        opts = parser.parse_args(line.split(), default_ports = self.get_acquired_ports(), verify_acquired = True)
-        if not opts:
-            return opts
-
-        self.set_rx_sniffer(opts.ports, opts.output_filename, opts.limit)
-
-        return RC_OK()
-        
 
     @__console
     def resolve_line (self, line):
@@ -4010,3 +4107,4 @@ class STLClient(object):
             
         self.set_service_mode(ports = opts.ports, enabled = opts.enabled)
         
+
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_jsonrpc_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_jsonrpc_client.py
index db216532..405f76be 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_jsonrpc_client.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_jsonrpc_client.py
@@ -10,6 +10,7 @@ import struct
 from .trex_stl_types import *
 from .utils.common import random_id_gen
 from .utils.zipmsg import ZippedMsg
+from threading import Lock
 
 class bcolors:
     BLUE = '\033[94m'
@@ -72,6 +73,8 @@ class JsonRpcClient(object):
         self.id_gen = random_id_gen()
         self.zipper = ZippedMsg()
 
+        self.lock = Lock()
+        
     def get_connection_details (self):
         rc = {}
         rc['server'] = self.server
@@ -137,6 +140,12 @@ class JsonRpcClient(object):
 
    
     def send_msg (self, msg, retry = 0):
+        # REQ/RESP pattern in ZMQ requires no interrupts during the send
+        with self.lock:
+            return self.__send_msg(msg, retry)
+        
+        
+    def __send_msg (self, msg, retry = 0):
         # print before
         if self.logger.check_verbose(self.logger.VERBOSE_HIGH):
             self.verbose_msg("Sending Request To Server:\n\n" + self.pretty_json(msg) + "\n")
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py
index a9509ee9..1ef3a8ff 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py
@@ -56,7 +56,8 @@ class Port(object):
     def __init__ (self, port_id, user, comm_link, session_id, info):
         self.port_id = port_id
         
-        self.state = self.STATE_IDLE
+        self.state        = self.STATE_IDLE
+        self.service_mode = False
         
         self.handler = None
         self.comm_link = comm_link
@@ -247,14 +248,16 @@ class Port(object):
             raise Exception("port {0}: bad state received from server '{1}'".format(self.port_id, port_state))
 
         self.owner = rc.data()['owner']
-
+        
         self.next_available_id = int(rc.data()['max_stream_id']) + 1
 
         self.status = rc.data()
-
+        
         # replace the attributes in a thread safe manner
         self.set_ts_attr(rc.data()['attr'])
-
+        
+        self.service_mode = rc.data()['service']
+        
         return self.ok()
 
 
@@ -489,38 +492,6 @@ class Port(object):
         return self.ok()
 
     
-    @owned
-    def set_rx_sniffer (self, pcap_filename, limit):
-
-        if not self.is_service_mode_on():
-            return self.err('port service mode must be enabled for performing RX capturing. Please enable service mode')
-            
-        params = {"handler":        self.handler,
-                  "port_id":        self.port_id,
-                  "type":           "capture",
-                  "enabled":        True,
-                  "pcap_filename":  pcap_filename,
-                  "limit":          limit}
-
-        rc = self.transmit("set_rx_feature", params)
-        if rc.bad():
-            return self.err(rc.err())
-
-        return self.ok()
-
-      
-    @owned
-    def remove_rx_sniffer (self):
-        params = {"handler":        self.handler,
-                  "port_id":        self.port_id,
-                  "type":           "capture",
-                  "enabled":        False}
-
-        rc = self.transmit("set_rx_feature", params)
-        if rc.bad():
-            return self.err(rc.err())
-
-        return self.ok()
      
     @writeable
     def set_l2_mode (self, dst_mac):
@@ -719,23 +690,21 @@ class Port(object):
     
     @owned
     def set_service_mode (self, enabled):
-        rc = self.set_attr(rx_filter_mode = 'all' if enabled else 'hw')
-        if not rc:
-            return rc
-            
-        if not enabled:
-            rc = self.remove_rx_queue()
-            if not rc:
-                return rc
-                
-            rc = self.remove_rx_sniffer()
-            if not rc:
-                return rc
-                
+        params = {"handler": self.handler,
+                  "port_id": self.port_id,
+                  "enabled": enabled}
+
+        rc = self.transmit("service", params)
+        if rc.bad():
+            return self.err(rc.err())
+
+        self.service_mode = enabled
         return self.ok()
+        
 
     def is_service_mode_on (self):
-        return self.get_rx_filter_mode() == 'all'
+        return self.service_mode
+        
                 
     @writeable
     def push_remote (self, pcap_filename, ipg_usec, speedup, count, duration, is_dual, slave_handler, min_ipg_usec):
@@ -902,11 +871,6 @@ class Port(object):
         # RX info
         rx_info = self.status['rx_info']
 
-        # RX sniffer
-        sniffer = rx_info['sniffer']
-        info['rx_sniffer'] = '{0}\n[{1} / {2}]'.format(sniffer['pcap_filename'], sniffer['count'], sniffer['limit']) if sniffer['is_active'] else 'off'
-        
-
         # RX queue
         queue = rx_info['queue']
         info['rx_queue'] = '[{0} / {1}]'.format(queue['count'], queue['size']) if queue['is_active'] else 'off'
@@ -928,8 +892,6 @@ class Port(object):
     def get_layer_cfg (self):
         return self.__attr['layer_cfg']
         
-    def get_rx_filter_mode (self):
-        return self.__attr['rx_filter_mode']
 
     def is_virtual(self):
         return self.info.get('is_virtual')
@@ -1005,7 +967,6 @@ class Port(object):
                 "layer mode": format_text(info['layer_mode'], 'green' if info['layer_mode'] == 'IPv4' else 'magenta'),
                 "RX Filter Mode": info['rx_filter_mode'],
                 "RX Queueing": info['rx_queue'],
-                "RX sniffer": info['rx_sniffer'],
                 "Grat ARP": info['grat_arp'],
 
                 }
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
index 38726062..21c9af87 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
@@ -682,7 +682,6 @@ class CTRexInfoGenerator(object):
                                         ("-----", []),
                                         ("RX Filter Mode", []),
                                         ("RX Queueing", []),
-                                        ("RX sniffer", []),
                                         ("Grat ARP", []),
                                         ]
                                        )
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py
index 0230db23..7ac508a2 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py
@@ -64,7 +64,7 @@ class RC():
                 err_count += 1
                 if len(err_list) < show_count:
                     err_list.append(format_text(x, 'bold'))
-            s = '\n'
+            s = ''
             if err_count > show_count:
                 s += format_text('Occurred %s errors, showing first %s:\n' % (err_count, show_count), 'bold')
             s += '\n'.join(err_list)
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py
index cbbacb27..72d3fa9f 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py
@@ -71,6 +71,11 @@ def list_difference (l1, l2):
 def is_sub_list (l1, l2):
     return set(l1) <= set(l2)
 
+# splits a timestamp in seconds to sec/usec
+def sec_split_usec (ts):
+    return int(ts), int( (ts - int(ts)) * 1e6 )
+    
+    
 # a simple passive timer
 class PassiveTimer(object):
 
@@ -107,4 +112,18 @@ def list_remove_dup (l):
             
     return tmp
             
-        
+def bitfield_to_list (bf):
+    rc = []
+    bitpos = 0
+
+    while bf > 0:
+        if bf & 0x1:
+            rc.append(bitpos)
+        bitpos += 1
+        bf = bf >> 1
+
+    return rc
+
+def bitfield_to_str (bf):
+    lst = bitfield_to_list(bf)
+    return "-" if not lst else ', '.join([str(x) for x in lst])
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py
index f5dab30c..53db533c 100755
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py
@@ -63,9 +63,14 @@ PKT_SIZE
 
 SERVICE_OFF
 
+TX_PORT_LIST
+RX_PORT_LIST
+
 SRC_IPV4
 DST_IPV4
 
+CAPTURE_ID
+
 GLOBAL_STATS
 PORT_STATS
 PORT_STATUS
@@ -78,12 +83,18 @@ EXTENDED_INC_ZERO_STATS
 
 STREAMS_MASK
 CORE_MASK_GROUP
+CAPTURE_PORTS_GROUP
+
+MONITOR_TYPE_VERBOSE
+MONITOR_TYPE_PIPE
+MONITOR_TYPE
 
 # ALL_STREAMS
 # STREAM_LIST_WITH_ALL
 
 # list of ArgumentGroup types
 MUTEX
+NON_MUTEX
 
 '''
 
@@ -389,7 +400,6 @@ OPTIONS_DB = {MULTIPLIER: ArgumentPack(['-m', '--multiplier'],
                                             {'help': 'Output PCAP filename',
                                              'dest': 'output_filename',
                                              'default': None,
-                                             'required': True,
                                              'type': str}),
 
 
@@ -591,6 +601,45 @@ OPTIONS_DB = {MULTIPLIER: ArgumentPack(['-m', '--multiplier'],
                                          'default': True,
                                          'help': 'Deactivates services on port(s)'}),
                 
+              TX_PORT_LIST: ArgumentPack(['--tx'],
+                                         {'nargs': '+',
+                                          'dest':'tx_port_list',
+                                          'metavar': 'TX',
+                                          'action': 'merge',
+                                          'type': int,
+                                          'help': 'A list of ports to capture on the TX side',
+                                          'default': []}),
+               
+              
+              RX_PORT_LIST: ArgumentPack(['--rx'],
+                                         {'nargs': '+',
+                                          'dest':'rx_port_list',
+                                          'metavar': 'RX',
+                                          'action': 'merge',
+                                          'type': int,
+                                          'help': 'A list of ports to capture on the RX side',
+                                          'default': []}),
+              
+              
+              MONITOR_TYPE_VERBOSE: ArgumentPack(['-v', '--verbose'],
+                                                 {'action': 'store_true',
+                                                  'dest': 'verbose',
+                                                  'default': False,
+                                                  'help': 'output to screen as verbose'}),
+              
+              MONITOR_TYPE_PIPE: ArgumentPack(['-p', '--pipe'],
+                                              {'action': 'store_true',
+                                               'dest': 'pipe',
+                                               'default': False,
+                                               'help': 'forward packets to a pipe'}),
+
+
+              CAPTURE_ID: ArgumentPack(['-i', '--id'],
+                                  {'help': "capture ID to remove",
+                                   'dest': "capture_id",
+                                   'type': int,
+                                   'required': True}),
+
               # advanced options
               PORT_LIST_WITH_ALL: ArgumentGroup(MUTEX, [PORT_LIST,
                                                         ALL_PORTS],
@@ -615,6 +664,13 @@ OPTIONS_DB = {MULTIPLIER: ArgumentPack(['-m', '--multiplier'],
                                                       CORE_MASK],
                                               {'required': False}),
 
+              CAPTURE_PORTS_GROUP: ArgumentGroup(NON_MUTEX, [TX_PORT_LIST, RX_PORT_LIST], {}),
+              
+              
+              MONITOR_TYPE: ArgumentGroup(MUTEX, [MONITOR_TYPE_VERBOSE,
+                                                  MONITOR_TYPE_PIPE],
+                                          {'required': False}),
+              
               }
 
 class _MergeAction(argparse._AppendAction):
@@ -633,16 +689,42 @@ class _MergeAction(argparse._AppendAction):
 
 class CCmdArgParser(argparse.ArgumentParser):
 
-    def __init__(self, stateless_client, *args, **kwargs):
+    def __init__(self, stateless_client = None, *args, **kwargs):
         super(CCmdArgParser, self).__init__(*args, **kwargs)
         self.stateless_client = stateless_client
         self.cmd_name = kwargs.get('prog')
         self.register('action', 'merge', _MergeAction)
 
+
+        
+    def add_arg_list (self, *args):
+        populate_parser(self, *args)
+
+        
+    # a simple hook for add subparsers to add stateless client
+    def add_subparsers(self, *args, **kwargs):
+        sub = super(CCmdArgParser, self).add_subparsers(*args, **kwargs)
+
+        # save pointer to the original add parser method
+        add_parser = sub.add_parser
+        stateless_client = self.stateless_client
+
+        def add_parser_hook (self, *args, **kwargs):
+            parser = add_parser(self, *args, **kwargs)
+            parser.stateless_client = stateless_client
+            return parser
+
+        # override with the hook
+        sub.add_parser = add_parser_hook
+        
+        return sub
+
+        
     # hook this to the logger
     def _print_message(self, message, file=None):
         self.stateless_client.logger.log(message)
 
+        
     def error(self, message):
         self.print_usage()
         self._print_message(('%s: error: %s\n') % (self.prog, message))
@@ -710,12 +792,15 @@ class CCmdArgParser(argparse.ArgumentParser):
             return RC_ERR("'{0}' - {1}".format(self.cmd_name, "no action"))
 
 
+    def formatted_error (self, msg):
+        self.print_usage()
+        self._print_message(('%s: error: %s\n') % (self.prog, msg))
+
+
 def get_flags (opt):
     return OPTIONS_DB[opt].name_or_flags
 
-def gen_parser(stateless_client, op_name, description, *args):
-    parser = CCmdArgParser(stateless_client, prog=op_name, conflict_handler='resolve',
-                           description=description)
+def populate_parser (parser, *args):
     for param in args:
         try:
 
@@ -731,6 +816,12 @@ def gen_parser(stateless_client, op_name, description, *args):
                     for sub_argument in argument.args:
                         group.add_argument(*OPTIONS_DB[sub_argument].name_or_flags,
                                            **OPTIONS_DB[sub_argument].options)
+
+                elif argument.type == NON_MUTEX:
+                    group = parser.add_argument_group(**argument.options)
+                    for sub_argument in argument.args:
+                        group.add_argument(*OPTIONS_DB[sub_argument].name_or_flags,
+                                           **OPTIONS_DB[sub_argument].options)
                 else:
                     # ignore invalid objects
                     continue
@@ -743,6 +834,12 @@ def gen_parser(stateless_client, op_name, description, *args):
         except KeyError as e:
             cause = e.args[0]
             raise KeyError("The attribute '{0}' is missing as a field of the {1} option.\n".format(cause, param))
+
+def gen_parser(stateless_client, op_name, description, *args):
+    parser = CCmdArgParser(stateless_client, prog=op_name, conflict_handler='resolve',
+                           description=description)
+
+    populate_parser(parser, *args)
     return parser
 
 
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py
index 63b05bf4..477d81a6 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py
@@ -134,10 +134,10 @@ def underline(text):
 # apply attribute on each non-empty line
 def text_attribute(text, attribute):
     return '\n'.join(['{start}{txt}{end}'.format(
-            start = TEXT_CODES[attribute]['start'],
-            txt = line,
-            end = TEXT_CODES[attribute]['end'])
-                    if line else '' for line in ('%s' % text).split('\n')])
+        start = TEXT_CODES[attribute]['start'],
+        txt = line,
+        end = TEXT_CODES[attribute]['end'])
+                      if line else '' for line in ('%s' % text).split('\n')])
 
 
 FUNC_DICT = {'blue': blue,