Merge branch 'capture'

8 files changed, 325 insertions, 142 deletions

diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
index 21ae42f1..215c0253 100755
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
@@ -17,6 +17,7 @@ from .utils.text_opts import *
 from functools import wraps
 from texttable import ansi_len
 
+
 from collections import namedtuple
 from yaml import YAMLError
 import time
@@ -600,12 +601,10 @@ class STLClient(object):
                                                                  self.util_stats,
                                                                  self.xstats,
                                                                  self.async_client.monitor)
-
-
-
-
+        
     ############# private functions - used by the class itself ###########
 
+    
     # some preprocessing for port argument
     def __ports (self, port_id_list):
 
@@ -832,27 +831,6 @@ class STLClient(object):
         return rc
 
 
-    def __set_rx_sniffer (self, port_id_list, base_filename, limit):
-        port_id_list = self.__ports(port_id_list)
-        rc = RC()
-
-        for port_id in port_id_list:
-            head, tail = os.path.splitext(base_filename)
-            filename = "{0}-{1}{2}".format(head, port_id, tail)
-            rc.add(self.ports[port_id].set_rx_sniffer(filename, limit))
-
-        return rc
-
-
-    def __remove_rx_sniffer (self, port_id_list):
-        port_id_list = self.__ports(port_id_list)
-        rc = RC()
-
-        for port_id in port_id_list:
-            rc.add(self.ports[port_id].remove_rx_sniffer())
-
-        return rc
-
     def __set_rx_queue (self, port_id_list, size):
         port_id_list = self.__ports(port_id_list)
         rc = RC()
@@ -1071,7 +1049,7 @@ class STLClient(object):
 
     ############ functions used by other classes but not users ##############
 
-    def _validate_port_list (self, port_id_list):
+    def _validate_port_list (self, port_id_list, allow_empty = False):
         # listfiy single int
         if isinstance(port_id_list, int):
             port_id_list = [port_id_list]
@@ -1080,7 +1058,7 @@ class STLClient(object):
         if not isinstance(port_id_list, list):
             raise STLTypeError('port_id_list', type(port_id_list), list)
 
-        if not port_id_list:
+        if not port_id_list and not allow_empty:
             raise STLError('No ports provided')
 
         valid_ports = self.get_all_ports()
@@ -2084,9 +2062,9 @@ class STLClient(object):
                 self.set_port_attr(ports,
                                    promiscuous = False,
                                    link_up = True if restart else None)
-                self.set_service_mode(ports, False)
-                self.remove_rx_sniffer(ports)
                 self.remove_rx_queue(ports)
+                self.set_service_mode(ports, False)
+                
                 
         except STLError as e:
             self.logger.post_cmd(False)
@@ -2996,7 +2974,7 @@ class STLClient(object):
             Resolves ports (ARP resolution)
 
             :parameters:
-                ports          - for which ports to apply a unique sniffer (each port gets a unique file)
+                ports          - which ports to resolve
                 retries        - how many times to retry on each port (intervals of 100 milliseconds)
                 verbose        - log for each request the response
             :raises:
@@ -3025,57 +3003,195 @@ class STLClient(object):
             
         
     @__api_check(True)
-    def set_rx_sniffer (self, ports = None, base_filename = 'rx.pcap', limit = 1000):
+    def start_capture (self, tx_ports, rx_ports, limit = 1000, mode = 'fixed'):
         """
-            Sets a RX sniffer for port(s) written to a PCAP file
+            Starts a low rate packet capturing on the server
 
             :parameters:
-                ports          - for which ports to apply a unique sniffer (each port gets a unique file)
-                base_filename  - filename will be appended with '-<port_number>', e.g. rx.pcap --> rx-0.pcap, rx-1.pcap etc.
-                limit          - limit how many packets will be written
+                tx_ports: list
+                    on which ports to capture TX
+                    
+                rx_ports: list
+                    on which ports to capture RX
+                    
+                limit: int
+                    limit how many packets will be written memory requierment is O(9K * limit)
+                    
+                mode: str
+                    'fixed'  - when full, newer packets will be dropped
+                    'cyclic' - when full, older packets will be dropped
+                                  
+            :returns:
+                returns a dictionary:
+                {'id: <new_id>, 'ts': <starting timestamp>}
+                
+                where 'id' is the new capture ID for future commands
+                and 'ts' is that server monotonic timestamp when
+                the capture was created
+                
             :raises:
                 + :exe:'STLError'
 
         """
-        ports = ports if ports is not None else self.get_acquired_ports()
-        ports = self._validate_port_list(ports)
-
+        # TODO: remove this when TX is implemented
+        if tx_ports:
+            raise STLError('TX port capturing is not yet implemented')
+            
         # check arguments
-        validate_type('base_filename', base_filename, basestring)
+        tx_ports = self._validate_port_list(tx_ports, allow_empty = True)
+        rx_ports = self._validate_port_list(rx_ports, allow_empty = True)
+        merge_ports = set(tx_ports + rx_ports)
+        
+        # make sure at least one port to capture
+        if not merge_ports:
+            raise STLError("start_capture - must get at least one port to capture")
+            
         validate_type('limit', limit, (int))
         if limit <= 0:
             raise STLError("'limit' must be a positive value")
 
-        self.logger.pre_cmd("Setting RX sniffers on port(s) {0}:".format(ports))
-        rc = self.__set_rx_sniffer(ports, base_filename, limit)
+        if mode not in ('fixed', 'cyclic'):
+            raise STLError("'mode' must be either 'fixed' or 'cyclic'")
+        
+        # verify service mode
+        non_service_ports =  list_difference(merge_ports, self.get_service_enabled_ports())
+        if non_service_ports:
+            raise STLError("Port(s) {0} are not under service mode. packet capturing requires all ports to be in service mode".format(non_service_ports))
+        
+            
+        # actual job
+        self.logger.pre_cmd("Starting packet capturing up to {0} packets".format(limit))
+        rc = self._transmit("capture", params = {'command': 'start', 'limit': limit, 'mode': mode, 'tx': tx_ports, 'rx': rx_ports})
         self.logger.post_cmd(rc)
 
-
         if not rc:
             raise STLError(rc)
 
+        return {'id': rc.data()['capture_id'], 'ts': rc.data()['start_ts']}
 
 
+        
     @__api_check(True)
-    def remove_rx_sniffer (self, ports = None):
+    def stop_capture (self, capture_id, output_filename = None):
         """
-            Removes RX sniffer from port(s)
+            Stops an active capture and optionally save it to a PCAP file
+
+            :parameters:
+                capture_id: int
+                    an active capture ID to stop
+                    
+                output_filename: str
+                    output filename to save capture
+                    if 'None', all captured packets will be discarded
 
             :raises:
                 + :exe:'STLError'
 
         """
-        ports = ports if ports is not None else self.get_acquired_ports()
-        ports = self._validate_port_list(ports)
+        
+        # stopping a capture requires:
+        # 1. stopping
+        # 2. fetching
+        # 3. saving to file
+        
+        # stop
+        
+        self.logger.pre_cmd("Stopping packet capture {0}".format(capture_id))
+        rc = self._transmit("capture", params = {'command': 'stop', 'capture_id': capture_id})
+        self.logger.post_cmd(rc)
+        if not rc:
+            raise STLError(rc)
+        
+        # pkt count
+        pkt_count = rc.data()['pkt_count']
+        
+        # fetch packets    
+        if output_filename:
+            self.__fetch_capture_packets(capture_id, output_filename, pkt_count)
+        
+        # remove
+        self.logger.pre_cmd("Removing PCAP capture {0} from server".format(capture_id))
+        rc = self._transmit("capture", params = {'command': 'remove', 'capture_id': capture_id})
+        self.logger.post_cmd(rc)
+        if not rc:
+            raise STLError(rc)
+        
+
+            
+    # fetch packets from the server and save them to a file
+    def __fetch_capture_packets (self, capture_id, output_filename, pkt_count):
+        self.logger.pre_cmd("Writing {0} packets to '{1}'".format(pkt_count, output_filename))
+
+        # create a PCAP file
+        writer = RawPcapWriter(output_filename, linktype = 1)
+        writer._write_header(None)
+
+        pending = pkt_count
+        rc = RC_OK()
+        
+        # fetch with iteratios - each iteration up to 50 packets
+        while pending > 0:
+            rc = self._transmit("capture", params = {'command': 'fetch', 'capture_id': capture_id, 'pkt_limit': 50})
+            if not rc:
+                self.logger.post_cmd(rc)
+                raise STLError(rc)
+
+            # make sure we are getting some progress
+            assert(rc.data()['pending'] < pending)
+            
+            pkts      = rc.data()['pkts']
+            pending   = rc.data()['pending']
+            start_ts  = rc.data()['start_ts']
+            
+            # write packets
+            for pkt in pkts:
+                # split the server timestamp relative to the capture start time
+                ts_sec, ts_usec = sec_split_usec(pkt['ts'] - start_ts)
+                
+                pkt_bin = base64.b64decode(pkt['binary'])
+                writer._write_packet(pkt_bin, sec = ts_sec, usec = ts_usec)
+
+
+
 
-        self.logger.pre_cmd("Removing RX sniffers on port(s) {0}:".format(ports))
-        rc = self.__remove_rx_sniffer(ports)
         self.logger.post_cmd(rc)
 
+            
+            
+    @__api_check(True)
+    def get_capture_status (self):
+        """
+            returns a list of all active captures
+            each element in the list is an object containing
+            info about the capture
+
+        """
+        rc = self._transmit("capture", params = {'command': 'status'})
         if not rc:
             raise STLError(rc)
 
-    
+        return rc.data()
+
+        
+    @__api_check(True)
+    def remove_all_captures (self):
+        """
+            Removes any existing captures
+        """
+        captures = self.get_capture_status()
+        
+        self.logger.pre_cmd("Removing all packet captures from server")
+        
+        for c in captures:
+            # remove
+            rc = self._transmit("capture", params = {'command': 'remove', 'capture_id': c['id']})
+            if not rc:
+                raise STLError(rc)
+
+        self.logger.post_cmd(RC_OK())
+                
+
+        
     @__api_check(True)
     def set_rx_queue (self, ports = None, size = 1000):
         """
@@ -3184,7 +3300,7 @@ class STLClient(object):
             try:
                 rc = f(*args)
             except STLError as e:
-                client.logger.log("\nAction has failed with the following error:\n" + format_text(e.brief() + "\n", 'bold'))
+                client.logger.log("\nAction has failed with the following error:\n\n" + format_text(e.brief() + "\n", 'bold'))
                 return RC_ERR(e.brief())
 
             # if got true - print time
@@ -3196,6 +3312,7 @@ class STLClient(object):
 
         return wrap
 
+
     @__console
     def ping_line (self, line):
         '''pings the server / specific IP'''
@@ -3789,29 +3906,9 @@ class STLClient(object):
                                opts.link,
                                opts.led,
                                opts.flow_ctrl)
+                         
+               
              
-             
-
-             
-    @__console
-    def set_rx_sniffer_line (self, line):
-        '''Sets a port sniffer on RX channel in form of a PCAP file'''
-
-        parser = parsing_opts.gen_parser(self,
-                                         "set_rx_sniffer",
-                                         self.set_rx_sniffer_line.__doc__,
-                                         parsing_opts.PORT_LIST_WITH_ALL,
-                                         parsing_opts.OUTPUT_FILENAME,
-                                         parsing_opts.LIMIT)
-
-        opts = parser.parse_args(line.split(), default_ports = self.get_acquired_ports(), verify_acquired = True)
-        if not opts:
-            return opts
-
-        self.set_rx_sniffer(opts.ports, opts.output_filename, opts.limit)
-
-        return RC_OK()
-        
 
     @__console
     def resolve_line (self, line):
@@ -4010,3 +4107,4 @@ class STLClient(object):
             
         self.set_service_mode(ports = opts.ports, enabled = opts.enabled)
         
+
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_jsonrpc_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_jsonrpc_client.py
index db216532..405f76be 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_jsonrpc_client.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_jsonrpc_client.py
@@ -10,6 +10,7 @@ import struct
 from .trex_stl_types import *
 from .utils.common import random_id_gen
 from .utils.zipmsg import ZippedMsg
+from threading import Lock
 
 class bcolors:
     BLUE = '\033[94m'
@@ -72,6 +73,8 @@ class JsonRpcClient(object):
         self.id_gen = random_id_gen()
         self.zipper = ZippedMsg()
 
+        self.lock = Lock()
+        
     def get_connection_details (self):
         rc = {}
         rc['server'] = self.server
@@ -137,6 +140,12 @@ class JsonRpcClient(object):
 
    
     def send_msg (self, msg, retry = 0):
+        # REQ/RESP pattern in ZMQ requires no interrupts during the send
+        with self.lock:
+            return self.__send_msg(msg, retry)
+        
+        
+    def __send_msg (self, msg, retry = 0):
         # print before
         if self.logger.check_verbose(self.logger.VERBOSE_HIGH):
             self.verbose_msg("Sending Request To Server:\n\n" + self.pretty_json(msg) + "\n")
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py
index a9509ee9..1ef3a8ff 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py
@@ -56,7 +56,8 @@ class Port(object):
     def __init__ (self, port_id, user, comm_link, session_id, info):
         self.port_id = port_id
         
-        self.state = self.STATE_IDLE
+        self.state        = self.STATE_IDLE
+        self.service_mode = False
         
         self.handler = None
         self.comm_link = comm_link
@@ -247,14 +248,16 @@ class Port(object):
             raise Exception("port {0}: bad state received from server '{1}'".format(self.port_id, port_state))
 
         self.owner = rc.data()['owner']
-
+        
         self.next_available_id = int(rc.data()['max_stream_id']) + 1
 
         self.status = rc.data()
-
+        
         # replace the attributes in a thread safe manner
         self.set_ts_attr(rc.data()['attr'])
-
+        
+        self.service_mode = rc.data()['service']
+        
         return self.ok()
 
 
@@ -489,38 +492,6 @@ class Port(object):
         return self.ok()
 
     
-    @owned
-    def set_rx_sniffer (self, pcap_filename, limit):
-
-        if not self.is_service_mode_on():
-            return self.err('port service mode must be enabled for performing RX capturing. Please enable service mode')
-            
-        params = {"handler":        self.handler,
-                  "port_id":        self.port_id,
-                  "type":           "capture",
-                  "enabled":        True,
-                  "pcap_filename":  pcap_filename,
-                  "limit":          limit}
-
-        rc = self.transmit("set_rx_feature", params)
-        if rc.bad():
-            return self.err(rc.err())
-
-        return self.ok()
-
-      
-    @owned
-    def remove_rx_sniffer (self):
-        params = {"handler":        self.handler,
-                  "port_id":        self.port_id,
-                  "type":           "capture",
-                  "enabled":        False}
-
-        rc = self.transmit("set_rx_feature", params)
-        if rc.bad():
-            return self.err(rc.err())
-
-        return self.ok()
      
     @writeable
     def set_l2_mode (self, dst_mac):
@@ -719,23 +690,21 @@ class Port(object):
     
     @owned
     def set_service_mode (self, enabled):
-        rc = self.set_attr(rx_filter_mode = 'all' if enabled else 'hw')
-        if not rc:
-            return rc
-            
-        if not enabled:
-            rc = self.remove_rx_queue()
-            if not rc:
-                return rc
-                
-            rc = self.remove_rx_sniffer()
-            if not rc:
-                return rc
-                
+        params = {"handler": self.handler,
+                  "port_id": self.port_id,
+                  "enabled": enabled}
+
+        rc = self.transmit("service", params)
+        if rc.bad():
+            return self.err(rc.err())
+
+        self.service_mode = enabled
         return self.ok()
+        
 
     def is_service_mode_on (self):
-        return self.get_rx_filter_mode() == 'all'
+        return self.service_mode
+        
                 
     @writeable
     def push_remote (self, pcap_filename, ipg_usec, speedup, count, duration, is_dual, slave_handler, min_ipg_usec):
@@ -902,11 +871,6 @@ class Port(object):
         # RX info
         rx_info = self.status['rx_info']
 
-        # RX sniffer
-        sniffer = rx_info['sniffer']
-        info['rx_sniffer'] = '{0}\n[{1} / {2}]'.format(sniffer['pcap_filename'], sniffer['count'], sniffer['limit']) if sniffer['is_active'] else 'off'
-        
-
         # RX queue
         queue = rx_info['queue']
         info['rx_queue'] = '[{0} / {1}]'.format(queue['count'], queue['size']) if queue['is_active'] else 'off'
@@ -928,8 +892,6 @@ class Port(object):
     def get_layer_cfg (self):
         return self.__attr['layer_cfg']
         
-    def get_rx_filter_mode (self):
-        return self.__attr['rx_filter_mode']
 
     def is_virtual(self):
         return self.info.get('is_virtual')
@@ -1005,7 +967,6 @@ class Port(object):
                 "layer mode": format_text(info['layer_mode'], 'green' if info['layer_mode'] == 'IPv4' else 'magenta'),
                 "RX Filter Mode": info['rx_filter_mode'],
                 "RX Queueing": info['rx_queue'],
-                "RX sniffer": info['rx_sniffer'],
                 "Grat ARP": info['grat_arp'],
 
                 }
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
index 38726062..21c9af87 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
@@ -682,7 +682,6 @@ class CTRexInfoGenerator(object):
                                         ("-----", []),
                                         ("RX Filter Mode", []),
                                         ("RX Queueing", []),
-                                        ("RX sniffer", []),
                                         ("Grat ARP", []),
                                         ]
                                        )
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py
index 0230db23..7ac508a2 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py
@@ -64,7 +64,7 @@ class RC():
                 err_count += 1
                 if len(err_list) < show_count:
                     err_list.append(format_text(x, 'bold'))
-            s = '\n'
+            s = ''
             if err_count > show_count:
                 s += format_text('Occurred %s errors, showing first %s:\n' % (err_count, show_count), 'bold')
             s += '\n'.join(err_list)
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py
index cbbacb27..72d3fa9f 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py
@@ -71,6 +71,11 @@ def list_difference (l1, l2):
 def is_sub_list (l1, l2):
     return set(l1) <= set(l2)
 
+# splits a timestamp in seconds to sec/usec
+def sec_split_usec (ts):
+    return int(ts), int( (ts - int(ts)) * 1e6 )
+    
+    
 # a simple passive timer
 class PassiveTimer(object):
 
@@ -107,4 +112,18 @@ def list_remove_dup (l):
             
     return tmp
             
-        
+def bitfield_to_list (bf):
+    rc = []
+    bitpos = 0
+
+    while bf > 0:
+        if bf & 0x1:
+            rc.append(bitpos)
+        bitpos += 1
+        bf = bf >> 1
+
+    return rc
+
+def bitfield_to_str (bf):
+    lst = bitfield_to_list(bf)
+    return "-" if not lst else ', '.join([str(x) for x in lst])
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py
index f5dab30c..53db533c 100755
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py
@@ -63,9 +63,14 @@ PKT_SIZE
 
 SERVICE_OFF
 
+TX_PORT_LIST
+RX_PORT_LIST
+
 SRC_IPV4
 DST_IPV4
 
+CAPTURE_ID
+
 GLOBAL_STATS
 PORT_STATS
 PORT_STATUS
@@ -78,12 +83,18 @@ EXTENDED_INC_ZERO_STATS
 
 STREAMS_MASK
 CORE_MASK_GROUP
+CAPTURE_PORTS_GROUP
+
+MONITOR_TYPE_VERBOSE
+MONITOR_TYPE_PIPE
+MONITOR_TYPE
 
 # ALL_STREAMS
 # STREAM_LIST_WITH_ALL
 
 # list of ArgumentGroup types
 MUTEX
+NON_MUTEX
 
 '''
 
@@ -389,7 +400,6 @@ OPTIONS_DB = {MULTIPLIER: ArgumentPack(['-m', '--multiplier'],
                                             {'help': 'Output PCAP filename',
                                              'dest': 'output_filename',
                                              'default': None,
-                                             'required': True,
                                              'type': str}),
 
 
@@ -591,6 +601,45 @@ OPTIONS_DB = {MULTIPLIER: ArgumentPack(['-m', '--multiplier'],
                                          'default': True,
                                          'help': 'Deactivates services on port(s)'}),
                 
+              TX_PORT_LIST: ArgumentPack(['--tx'],
+                                         {'nargs': '+',
+                                          'dest':'tx_port_list',
+                                          'metavar': 'TX',
+                                          'action': 'merge',
+                                          'type': int,
+                                          'help': 'A list of ports to capture on the TX side',
+                                          'default': []}),
+               
+              
+              RX_PORT_LIST: ArgumentPack(['--rx'],
+                                         {'nargs': '+',
+                                          'dest':'rx_port_list',
+                                          'metavar': 'RX',
+                                          'action': 'merge',
+                                          'type': int,
+                                          'help': 'A list of ports to capture on the RX side',
+                                          'default': []}),
+              
+              
+              MONITOR_TYPE_VERBOSE: ArgumentPack(['-v', '--verbose'],
+                                                 {'action': 'store_true',
+                                                  'dest': 'verbose',
+                                                  'default': False,
+                                                  'help': 'output to screen as verbose'}),
+              
+              MONITOR_TYPE_PIPE: ArgumentPack(['-p', '--pipe'],
+                                              {'action': 'store_true',
+                                               'dest': 'pipe',
+                                               'default': False,
+                                               'help': 'forward packets to a pipe'}),
+
+
+              CAPTURE_ID: ArgumentPack(['-i', '--id'],
+                                  {'help': "capture ID to remove",
+                                   'dest': "capture_id",
+                                   'type': int,
+                                   'required': True}),
+
               # advanced options
               PORT_LIST_WITH_ALL: ArgumentGroup(MUTEX, [PORT_LIST,
                                                         ALL_PORTS],
@@ -615,6 +664,13 @@ OPTIONS_DB = {MULTIPLIER: ArgumentPack(['-m', '--multiplier'],
                                                       CORE_MASK],
                                               {'required': False}),
 
+              CAPTURE_PORTS_GROUP: ArgumentGroup(NON_MUTEX, [TX_PORT_LIST, RX_PORT_LIST], {}),
+              
+              
+              MONITOR_TYPE: ArgumentGroup(MUTEX, [MONITOR_TYPE_VERBOSE,
+                                                  MONITOR_TYPE_PIPE],
+                                          {'required': False}),
+              
               }
 
 class _MergeAction(argparse._AppendAction):
@@ -633,16 +689,42 @@ class _MergeAction(argparse._AppendAction):
 
 class CCmdArgParser(argparse.ArgumentParser):
 
-    def __init__(self, stateless_client, *args, **kwargs):
+    def __init__(self, stateless_client = None, *args, **kwargs):
         super(CCmdArgParser, self).__init__(*args, **kwargs)
         self.stateless_client = stateless_client
         self.cmd_name = kwargs.get('prog')
         self.register('action', 'merge', _MergeAction)
 
+
+        
+    def add_arg_list (self, *args):
+        populate_parser(self, *args)
+
+        
+    # a simple hook for add subparsers to add stateless client
+    def add_subparsers(self, *args, **kwargs):
+        sub = super(CCmdArgParser, self).add_subparsers(*args, **kwargs)
+
+        # save pointer to the original add parser method
+        add_parser = sub.add_parser
+        stateless_client = self.stateless_client
+
+        def add_parser_hook (self, *args, **kwargs):
+            parser = add_parser(self, *args, **kwargs)
+            parser.stateless_client = stateless_client
+            return parser
+
+        # override with the hook
+        sub.add_parser = add_parser_hook
+        
+        return sub
+
+        
     # hook this to the logger
     def _print_message(self, message, file=None):
         self.stateless_client.logger.log(message)
 
+        
     def error(self, message):
         self.print_usage()
         self._print_message(('%s: error: %s\n') % (self.prog, message))
@@ -710,12 +792,15 @@ class CCmdArgParser(argparse.ArgumentParser):
             return RC_ERR("'{0}' - {1}".format(self.cmd_name, "no action"))
 
 
+    def formatted_error (self, msg):
+        self.print_usage()
+        self._print_message(('%s: error: %s\n') % (self.prog, msg))
+
+
 def get_flags (opt):
     return OPTIONS_DB[opt].name_or_flags
 
-def gen_parser(stateless_client, op_name, description, *args):
-    parser = CCmdArgParser(stateless_client, prog=op_name, conflict_handler='resolve',
-                           description=description)
+def populate_parser (parser, *args):
     for param in args:
         try:
 
@@ -731,6 +816,12 @@ def gen_parser(stateless_client, op_name, description, *args):
                     for sub_argument in argument.args:
                         group.add_argument(*OPTIONS_DB[sub_argument].name_or_flags,
                                            **OPTIONS_DB[sub_argument].options)
+
+                elif argument.type == NON_MUTEX:
+                    group = parser.add_argument_group(**argument.options)
+                    for sub_argument in argument.args:
+                        group.add_argument(*OPTIONS_DB[sub_argument].name_or_flags,
+                                           **OPTIONS_DB[sub_argument].options)
                 else:
                     # ignore invalid objects
                     continue
@@ -743,6 +834,12 @@ def gen_parser(stateless_client, op_name, description, *args):
         except KeyError as e:
             cause = e.args[0]
             raise KeyError("The attribute '{0}' is missing as a field of the {1} option.\n".format(cause, param))
+
+def gen_parser(stateless_client, op_name, description, *args):
+    parser = CCmdArgParser(stateless_client, prog=op_name, conflict_handler='resolve',
+                           description=description)
+
+    populate_parser(parser, *args)
     return parser
 
 
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py
index 63b05bf4..477d81a6 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py
@@ -134,10 +134,10 @@ def underline(text):
 # apply attribute on each non-empty line
 def text_attribute(text, attribute):
     return '\n'.join(['{start}{txt}{end}'.format(
-            start = TEXT_CODES[attribute]['start'],
-            txt = line,
-            end = TEXT_CODES[attribute]['end'])
-                    if line else '' for line in ('%s' % text).split('\n')])
+        start = TEXT_CODES[attribute]['start'],
+        txt = line,
+        end = TEXT_CODES[attribute]['end'])
+                      if line else '' for line in ('%s' % text).split('\n')])
 
 
 FUNC_DICT = {'blue': blue,