Merge branch 'master' into cpu_per_core

Conflicts:
	scripts/automation/trex_control_plane/server/singleton_daemon.py

8 files changed, 187 insertions, 53 deletions

diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/__init__.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/__init__.py
index ba9459c1..c6e14df3 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/__init__.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/__init__.py
@@ -1,7 +1,7 @@
 import sys
 
 if sys.version_info < (2, 7):
-    print("\n**** TRex STL pacakge requires Python version >= 2.7 ***\n")
+    print("\n**** TRex STL package requires Python version >= 2.7 ***\n")
     exit(-1)
 
 from . import trex_stl_ext
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
index 70f38bb0..38a18d16 100755
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
@@ -1257,18 +1257,161 @@ class STLClient(object):
 
 
     # get stats
-    def get_stats (self, ports = None, async_barrier = True):
+    def get_stats (self, ports = None, sync_now = True):
+        """
+        Return dictionary containing statistics information gathered from the server.
+
+        :parameters:
+
+          ports - List of ports to retreive stats on.
+                  If None, assume the request is for all acquired ports.
+
+          sync_now - Boolean - If true, create a call to the server to get latest stats, and wait for result to arrive. Otherwise, return last stats saved in client cache.
+                            Downside of putting True is a slight delay (few 10th msecs) in getting the result. For practical uses, value should be True.
+        :return:
+            Statistics dictionary of dictionaries with the following format:
+
+            ===============================  ===============
+            key                               Meaning
+            ===============================  ===============
+            :ref:`numbers (0,1,..<total>`    Statistcs per port number
+            :ref:`total <total>`             Sum of port statistics
+            :ref:`flow_stats <flow_stats>`   Per flow statistics
+            :ref:`global <global>`           Global statistics
+            :ref:`latency <latency>`         Per flow statistics regarding flow latency
+            ===============================  ===============
+
+            Below is description of each of the inner dictionaries.
+
+            .. _total:
+
+            **total** and per port statistics contain dictionary with following format.
+
+            ===============================  ===============
+            key                               Meaning
+            ===============================  ===============
+            ibytes                           Number of input bytes
+            ierrors                          Number of input errors
+            ipackets                         Number of input packets
+            obytes                           Number of output bytes
+            oerrors                          Number of output errors
+            opackets                         Number of output packets
+            rx_bps                           Receive bytes per second rate
+            rx_pps                           Receive packet per second rate
+            tx_bps                           Transmit bytes per second rate
+            tx_pps                           Transmit packet per second rate
+            ===============================  ===============
+
+            .. _flow_stats:
+
+            **flow_stats** contains dictionaries per packet group id (pg id). Each one with the following structure.
+
+            =================   ===============
+            key                 Meaning
+            =================   ===============
+            rx_bps              Receivd bytes per second rate
+            rx_bps_l1           Receivd bytes per second rate, including layer one
+            rx_bytes            Total number of received bytes
+            rx_pkts             Total number of received packets
+            rx_pps              Received packets per second
+            tx_bps              Transmitted bytes per second rate
+            tx_bps_l1           Transmitted bytes per second rate, including layer one
+            tx_bytes            Total number of sent bytes
+            tx_pkts             Total number of sent packets
+            tx_pps              Transmit packets per second
+            =================   ===============
+
+            .. _global:
+
+            **global**
+
+            =================   ===============
+            key                 Meaning
+            =================   ===============
+            bw_per_core         Estimated byte rate Trex can support per core. This is calculated by extrapolation of current rate and load on transmitting cores.
+            cpu_util            Estimate of the average utilization percentage of the transimitting cores
+            queue_full          Total number of packets we could not transmit because NIC TX queue was full. This usually indicates that the rate we trying to transmit is too high for this port
+            rx_cpu_util         Estimate of the utilization percentage of the core handling RX traffic
+            rx_drop_bps         Received bytes per second drop rate
+            rx_bps              Received bytes per second rate
+            rx_pps              Received packets per second rate
+            tx_bps              Transmit bytes per second rate
+            tx_pps              Transmit packets per second rate
+            =================   ===============
+
+            .. _latency:
+
+            **latency** contains dictionary per packet group id (pg id). Each one with the following structure.
+
+            ===========================          ===============
+            key                                  Meaning
+            ===========================          ===============
+            :ref:`err_cntrs<err-cntrs>`          Counters describing errors that occured with this pg id
+            :ref:`latency<lat_inner>`            Information regarding packet latency
+            ===========================          ===============
+
+            Following are the inner dictionaries of latency
+
+            .. _err-cntrs:
+
+            **err-cntrs**
+
+            =================   ===============
+            key                 Meaning (see better explanation below the table)
+            =================   ===============
+            dropped             How many packets were dropped.
+            dup                 How many packets were duplicated.
+            out_of_order        How many packets we received out of order.
+            seq_too_high        How many events of packet with sequence number too high we saw.
+            seq_too_low         How many events of packet with sequence number too low we saw.
+            =================   ===============
+
+            For calculating packet error events, we add sequence number to each packet's payload. We decide what went wrong only according to sequence number
+            of last packet received and that of the previous packet. 'seq_too_low' and 'seq_too_high' count events we see. 'dup', 'out_of_order' and 'dropped'
+            are heuristics we apply to try and understand what happened. They will be accurate in common error scenarios.
+            We describe few scenarios below to help understand this.
+
+            Scenario 1: Received packet with seq num 10, and another one with seq num 10. We increment 'dup' and 'seq_too_low' by 1.
+
+            Scenario 2: Received pacekt with seq num 10 and then packet with seq num 15. We assume 4 packets were dropped, and increment 'dropped' by 4, and 'seq_too_high' by 1.
+            We expect next packet to arrive with sequence number 16.
+
+            Scenario 2 continue: Received packet with seq num 11. We increment 'seq_too_low' by 1. We increment 'out_of_order' by 1. We *decrement* 'dropped' by 1.
+            (We assume here that one of the packets we considered as dropped before, actually arrived out of order).
+
+
+            .. _lat_inner:
+
+            **latency**
+
+            =================   ===============
+            key                 Meaning
+            =================   ===============
+            average             Average latency over the stream lifetime (usec). Total average is computed each sampling period by following formula: <average> = <prev average>/2 + <last sampling period average>/2
+            histogram           Dictionary describing logarithmic distribution histogram of packet latencies. Keys in the dictionary represent range of latencies (in usec). Values are the total number of packets received in this latency range. For example, an entry {100:13} would mean that we saw 13 packets with latency in the range between 100 and 200 usec.
+            jitter              Jitter of latency samples, computed as described in :rfc:`3550#appendix-A.8`
+            last_max            Maximum latency measured between last two data reads from server.
+            total_max           Maximum latency measured over the stream lifetime (in usec).
+            total_min           Minimum latency measured over the stream lifetime (in usec).
+            =================   ===============
+
+
+
+        :raises:
+          None
+
+        """
         # by default use all acquired ports
         ports = ports if ports is not None else self.get_acquired_ports()
         ports = self._validate_port_list(ports)
 
         # check async barrier
-        if not type(async_barrier) is bool:
-            raise STLArgumentError('async_barrier', async_barrier)
+        if not type(sync_now) is bool:
+            raise STLArgumentError('sync_now', sync_now)
 
 
         # if the user requested a barrier - use it
-        if async_barrier:
+        if sync_now:
             rc = self.async_client.barrier()
             if not rc:
                 raise STLError(rc)
@@ -1282,7 +1425,7 @@ class STLClient(object):
 
         :parameters:
           ev_type_filter - 'info', 'warning' or a list of those
-                           default is no filter
+                           default: no filter
 
         :return:
             logged events
@@ -1720,6 +1863,11 @@ class STLClient(object):
         ports = self._validate_port_list(ports)
 
 
+        validate_type('mult', mult, basestring)
+        validate_type('force', force, bool)
+        validate_type('duration', duration, (int, float))
+        validate_type('total', total, bool)
+
         # verify multiplier
         mult_obj = parsing_opts.decode_multiplier(mult,
                                                   allow_update = False,
@@ -1727,17 +1875,6 @@ class STLClient(object):
         if not mult_obj:
             raise STLArgumentError('mult', mult)
 
-        # some type checkings
-
-        if not type(force) is bool:
-            raise STLArgumentError('force', force)
-
-        if not isinstance(duration, (int, float)):
-            raise STLArgumentError('duration', duration)
-
-        if not type(total) is bool:
-            raise STLArgumentError('total', total)
-
 
         # verify ports are stopped or force stop them
         active_ports = list(set(self.get_active_ports()).intersection(ports))
@@ -1780,11 +1917,12 @@ class STLClient(object):
 
         """
 
-        ports = ports if ports is not None else self.get_active_ports()
-        ports = self._validate_port_list(ports)
+        if ports is None:
+            ports = self.get_active_ports()
+            if not ports:
+                return
 
-        if not ports:
-            return
+        ports = self._validate_port_list(ports)
 
         self.logger.pre_cmd("Stopping traffic on port(s) {0}:".format(ports))
         rc = self.__stop(ports)
@@ -1833,6 +1971,9 @@ class STLClient(object):
         ports = ports if ports is not None else self.get_active_ports()
         ports = self._validate_port_list(ports)
 
+        validate_type('mult', mult, basestring)
+        validate_type('force', force, bool)
+        validate_type('total', total, bool)
 
         # verify multiplier
         mult_obj = parsing_opts.decode_multiplier(mult,
@@ -1841,10 +1982,6 @@ class STLClient(object):
         if not mult_obj:
             raise STLArgumentError('mult', mult)
 
-        # verify total
-        if not type(total) is bool:
-            raise STLArgumentError('total', total)
-
 
         # call low level functions
         self.logger.pre_cmd("Updating traffic on port(s) {0}:".format(ports))
@@ -2068,6 +2205,10 @@ class STLClient(object):
         ports = ports if ports is not None else self.get_acquired_ports()
         ports = self._validate_port_list(ports)
 
+        validate_type('mult', mult, basestring)
+        validate_type('duration', duration, (int, float))
+        validate_type('total', total, bool)
+
 
         # verify multiplier
         mult_obj = parsing_opts.decode_multiplier(mult,
@@ -2076,11 +2217,6 @@ class STLClient(object):
         if not mult_obj:
             raise STLArgumentError('mult', mult)
 
-
-        if not isinstance(duration, (int, float)):
-            raise STLArgumentError('duration', duration)
-
-
         self.logger.pre_cmd("Validating streams on port(s) {0}:".format(ports))
         rc = self.__validate(ports)
         self.logger.post_cmd(rc)
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_exceptions.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_exceptions.py
index 3c443ba6..b6fad865 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_exceptions.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_exceptions.py
@@ -4,6 +4,11 @@ import linecache
 
 from .utils.text_opts import *
 
+try:
+    basestring
+except NameError:
+    basestring = str
+
 # basic error for API
 class STLError(Exception):
     def __init__ (self, msg):
@@ -56,7 +61,8 @@ class STLArgumentError(STLError):
 # raised when argument type is not valid for operation
 class STLTypeError(STLError):
     def __init__ (self, arg_name, arg_type, valid_types):
-        self.msg = "Argument: '%s' invalid type: %s, expecting type(s): %s." % (arg_name, arg_type, valid_types)
+        self.msg = "Argument: '%s' invalid type: '%s', expecting type(s): %s." % (arg_name, arg_type.__name__,
+            [t.__name__ for t in valid_types] if isinstance(valid_types, tuple) else valid_types.__name__)
 
 # raised when timeout occurs
 class STLTimeoutError(STLError):
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py
index 11e80b9a..62724e64 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py
@@ -40,18 +40,11 @@ class BpSimException(Exception):
 
 # stateless simulation
 class STLSim(object):
-    def __init__ (self, bp_sim_path = None, handler = 0, port_id = 0, api_h = "dummy"):
+    def __init__ (self, bp_sim_path, handler = 0, port_id = 0, api_h = "dummy"):
 
-        if not bp_sim_path:
-            # auto find scripts
-            m = re.match(".*/trex-core", os.getcwd())
-            if not m:
-                raise STLError('cannot find BP sim path, please provide it')
-
-            self.bp_sim_path = os.path.join(m.group(0), 'scripts')
-
-        else:
-            self.bp_sim_path = bp_sim_path
+        self.bp_sim_path = os.path.abspath(bp_sim_path)
+        if not os.path.exists(self.bp_sim_path):
+            raise STLError('BP sim path %s does not exist' % self.bp_sim_path)
 
         # dummies
         self.handler = handler
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
index 990a40da..8e8388fd 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
@@ -1119,6 +1119,7 @@ class CLatencyStats(CTRexStats):
                     output[int_pg_id]['latency']['total_min'] = min_val
                 else:
                     output[int_pg_id]['latency']['total_min'] = StatNotAvailable('total_min')
+                    output[int_pg_id]['latency']['histogram'] = {}
 
         self.latest_stats = output
         return True
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py
index b4903e81..6835ea5f 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py
@@ -64,3 +64,4 @@ def list_difference (l1, l2):
 
 def is_sub_list (l1, l2):
     return set(l1) <= set(l2)
+
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py
index a435e54e..ceaf1af8 100755
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py
@@ -92,10 +92,6 @@ match_multiplier_help = """Multiplier should be passed in the following format:
 # value should be divided
 def decode_multiplier(val, allow_update = False, divide_count = 1):
 
-    # must be string
-    if not isinstance(val, str):
-        return None
-
     # do we allow updates ?  +/-
     if not allow_update:
         match = re.match("^(\d+(\.\d+)?)(bps|kbps|mbps|gbps|pps|kpps|mpps|%?)$", val)
@@ -246,26 +242,24 @@ OPTIONS_DB = {MULTIPLIER: ArgumentPack(['-m', '--multiplier'],
                                    'type': int}),
 
               PROMISCUOUS: ArgumentPack(['--prom'],
-                                        {'help': "sets port promiscuous on",
+                                        {'help': "Sets port promiscuous on",
                                          'dest': "prom",
                                          'default': None,
                                          'action': "store_true"}),
 
-
               TUNABLES: ArgumentPack(['-t'],
-                                     {'help': "sets tunable for a profile",
+                                     {'help': "Sets tunables for a profile. Example: '-t fsize=100,pg_id=7'",
+                                      'metavar': 'T1=VAL[,T2=VAL ...]',
                                       'dest': "tunables",
                                       'default': None,
                                       'type': decode_tunables}),
 
-
               NO_PROMISCUOUS: ArgumentPack(['--no_prom'],
-                                           {'help': "sets port promiscuous off",
+                                           {'help': "Sets port promiscuous off",
                                             'dest': "prom",
                                             'default': None,
                                             'action': "store_false"}),
 
-
               PORT_LIST: ArgumentPack(['--port', '-p'],
                                         {"nargs": '+',
                                          'dest':'ports',
@@ -481,4 +475,4 @@ def gen_parser(stateless_client, op_name, description, *args):
 
 
 if __name__ == "__main__":
-    pass
\ No newline at end of file
+    pass
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py
index 7e0bf9e4..26e64dae 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py
@@ -61,6 +61,9 @@ def format_time (t_sec):
     if t_sec < 0:
         return "infinite"
 
+    if t_sec == 0:
+        return "zero"
+
     if t_sec < 1:
         # low numbers
         for unit in ['ms', 'usec', 'ns']: