Merge v2.01:

8 files changed, 339 insertions, 27 deletions

diff --git a/scripts/automation/regression/stateless_tests/stl_rx_test.py b/scripts/automation/regression/stateless_tests/stl_rx_test.py
index 70c003aa..10e70b86 100644
--- a/scripts/automation/regression/stateless_tests/stl_rx_test.py
+++ b/scripts/automation/regression/stateless_tests/stl_rx_test.py
@@ -7,7 +7,9 @@ class STLRX_Test(CStlGeneral_Test):
     """Tests for RX feature"""
 
     def setUp(self):
-        per_driver_params = {"rte_vmxnet3_pmd": [1, 50], "rte_ixgbe_pmd": [30, 5000], "rte_i40e_pmd": [80, 5000],
+        if self.is_virt_nics:
+            self.skip('Skip this for virtual NICs for now')
+        per_driver_params = {"rte_vmxnet3_pmd": [1, 50], "rte_ixgbe_pmd": [30, 5000], "rte_i40e_pmd": [80, 5000, 25],
                              "rte_igb_pmd": [80, 500], "rte_em_pmd": [1, 50], "rte_virtio_pmd": [1, 50]}
 
         CStlGeneral_Test.setUp(self)
@@ -19,16 +21,21 @@ class STLRX_Test(CStlGeneral_Test):
 
         port_info = self.c.get_port_info(ports = self.rx_port)[0]
         cap = port_info['rx']['caps']
-        print(port_info)
         if "flow_stats" not in cap or "latency" not in cap:
             self.skip('port {0} does not support RX'.format(self.rx_port))
         self.cap = cap
 
         self.rate_percent = per_driver_params[port_info['driver']][0]
         self.total_pkts = per_driver_params[port_info['driver']][1]
+        if len(per_driver_params[port_info['driver']]) > 2:
+            self.rate_lat = per_driver_params[port_info['driver']][2]
+        else:
+            self.rate_lat = self.rate_percent
+        self.drops_expected = False
         self.c.reset(ports = [self.tx_port, self.rx_port])
 
-        self.pkt = STLPktBuilder(pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)/IP()/'a_payload_example')
+        self.pkt = STLPktBuilder(pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)/('Your_paylaod_comes_here'))
+        self.large_pkt = STLPktBuilder(pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)/('a'*1000))
 
     @classmethod
     def tearDownClass(cls):
@@ -54,11 +61,18 @@ class STLRX_Test(CStlGeneral_Test):
             sth = latency_stats['err_cntrs']['seq_too_high']
             stl = latency_stats['err_cntrs']['seq_too_low']
             lat = latency_stats['latency']
-            if drops != 0 or ooo != 0 or dup != 0 or stl !=0 or sth != 0:
+            if ooo != 0 or dup != 0 or sth != 0:
                 pprint.pprint(latency_stats)
-                tmp='Dropped or out of order packets - dropped:{0}, ooo:{1} dup:{2} seq too high:{3} seq too low:{4}'.format(drops, ooo, dup, sth, stl)
+                tmp='Error packets - dropped:{0}, ooo:{1} dup:{2} seq too high:{3} seq too low:{4}'.format(drops, ooo, dup, sth, stl)
                 assert False, tmp
 
+            if (drops != 0 or stl != 0) and not self.drops_expected:
+                pprint.pprint(latency_stats)
+                tmp='Error packets - dropped:{0}, ooo:{1} dup:{2} seq too high:{3} seq too low:{4}'.format(drops, ooo, dup, sth, stl)
+                assert False, tmp
+
+
+
         if tx_pkts != total_pkts:
             pprint.pprint(flow_stats)
             tmp = 'TX pkts mismatch - got: {0}, expected: {1}'.format(tx_pkts, total_pkts)
@@ -69,14 +83,14 @@ class STLRX_Test(CStlGeneral_Test):
             tmp = 'TX bytes mismatch - got: {0}, expected: {1}'.format(tx_bytes, (total_pkts * pkt_len))
             assert False, tmp
 
-        if rx_pkts != total_pkts:
+        if rx_pkts != total_pkts and not self.drops_expected:
             pprint.pprint(flow_stats)
             tmp = 'RX pkts mismatch - got: {0}, expected: {1}'.format(rx_pkts, total_pkts)
             assert False, tmp
 
         if "rx_bytes" in self.cap:
             rx_bytes = flow_stats['rx_bytes'].get(self.rx_port, 0)
-            if rx_bytes != (total_pkts * pkt_len):
+            if rx_bytes != (total_pkts * pkt_len) and not self.drops_expected:
                 pprint.pprint(flow_stats)
                 tmp = 'RX bytes mismatch - got: {0}, expected: {1}'.format(rx_bytes, (total_pkts * pkt_len))
                 assert False, tmp
@@ -123,12 +137,12 @@ class STLRX_Test(CStlGeneral_Test):
 
     # one simple stream on TX --> RX
     def test_multiple_streams(self):
-        num_latency_streams = 110
-        num_flow_stat_streams = 110
-        total_pkts = int(self.total_pkts / num_latency_streams) / 2
+        num_latency_streams = 128
+        num_flow_stat_streams = 127
+        total_pkts = int(self.total_pkts / (num_latency_streams + num_flow_stat_streams))
         if total_pkts == 0:
             total_pkts = 1
-        percent = float(self.rate_percent) / num_latency_streams / 2
+        percent = float(self.rate_percent) / (num_latency_streams + num_flow_stat_streams)
 
         try:
             streams = []
@@ -187,3 +201,53 @@ class STLRX_Test(CStlGeneral_Test):
 
         except STLError as e:
             assert False , '{0}'.format(e)
+
+
+    
+    # this test adds more and more latency streams and re-test with incremental
+    # test does not work yet
+    def test_incremental_latency_streams (self):
+        total_pkts = self.total_pkts
+        percent = 0.5
+
+        try:
+            # We run till maximum streams allowed. At some point, expecting drops, because rate is too high.
+            # then run with less streams again, to see that system is still working.
+            for num_iter in [128, 5]:
+                exp = []
+                for i in range(1, num_iter):
+                    # mix small and large packets
+                    if i % 2 != 0:
+                        my_pkt = self.pkt
+                    else:
+                        my_pkt = self.large_pkt
+                    s1 = STLStream(name = 'rx',
+                                   packet = my_pkt,
+                                   flow_stats = STLFlowLatencyStats(pg_id = i),
+                                   mode = STLTXSingleBurst(total_pkts = total_pkts,
+                                                           percentage = percent
+                                                       ))
+
+                    # add both streams to ports
+                    self.c.add_streams([s1], ports = [self.tx_port])
+                    total_percent = i * percent
+                    if total_percent > self.rate_lat:
+                        self.drops_expected = True
+                    else:
+                        self.drops_expected = False
+
+                    print("port {0} : {1} streams at {2}% of line rate\n".format(self.tx_port, i, total_percent))
+
+                    exp.append({'pg_id': i, 'total_pkts': total_pkts, 'pkt_len': my_pkt.get_pkt_len()})
+
+                    self.__rx_iteration( exp )
+
+                self.c.remove_all_streams(ports = [self.tx_port])
+
+
+
+        except STLError as e:
+            assert False , '{0}'.format(e)
+
+
+
diff --git a/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py
index 0aac043e..3b01560a 100755
--- a/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py
+++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py
@@ -101,6 +101,7 @@ class CTRexClient(object):
         '''
         Connects to Master daemon via JsonRPC.
         This daemon controls TRex daemon server.
+        Return true if success, false if fail
         '''
         try:
             print('Connecting to Master daemon @ %s ...' % self.master_daemon_path)
@@ -115,7 +116,8 @@ class CTRexClient(object):
     def connect_server(self):
         '''
         Connects to TRex daemon server via JsonRPC.
-        This daemon controls TRex. (start/stop
+        This daemon controls TRex. (start/stop)
+        Return true if success, false if fail
         '''
         try:
             print('Connecting to TRex daemon server @ %s ...' % self.trex_server_path)
@@ -359,7 +361,7 @@ class CTRexClient(object):
         Returns TRex path on server
         '''
         try:
-            return self.master_daemon.get_trex_path()
+            return str(self.master_daemon.get_trex_path())
         except AppError as err:
             self._handle_AppError_exception(err.args[0])
         finally:
diff --git a/scripts/automation/trex_control_plane/stl/examples/rpc_proxy_server.py b/scripts/automation/trex_control_plane/stl/examples/rpc_proxy_server.py
new file mode 100755
index 00000000..39d642f4
--- /dev/null
+++ b/scripts/automation/trex_control_plane/stl/examples/rpc_proxy_server.py
@@ -0,0 +1,146 @@
+#!/usr/bin/python
+
+import argparse
+import traceback
+import logging
+import sys
+import os
+import json
+logging.basicConfig(level = logging.FATAL) # keep quiet
+
+import stl_path
+from trex_stl_lib.api import *
+from trex_stl_lib.trex_stl_hltapi import CTRexHltApi, HLT_ERR
+
+# ext libs
+ext_libs = os.path.join(os.pardir, os.pardir, os.pardir, os.pardir, 'external_libs')
+sys.path.append(os.path.join(ext_libs, 'jsonrpclib-pelix-0.2.5'))
+from jsonrpclib.SimpleJSONRPCServer import SimpleJSONRPCServer
+import yaml
+
+# TODO: refactor this to class
+
+native_client = None
+hltapi_client = None
+
+def OK(res = True):
+    return[True, res]
+
+def ERR(res = 'Unknown error'):
+    return [False, res]
+
+def deunicode_json(data):
+    return yaml.safe_load(json.dumps(data))
+
+
+### Server functions ###
+
+def add(a, b): # for sanity checks
+    try:
+        return OK(a + b)
+    except:
+        return ERR(traceback.format_exc())
+
+def check_connectivity():
+    return OK()
+
+def native_proxy_init(force = False, *args, **kwargs):
+    global native_client
+    if native_client and not force:
+        return ERR('Native Client is already initiated')
+    try:
+        native_client = STLClient(*args, **kwargs)
+        return OK('Native Client initiated')
+    except:
+        return ERR(traceback.format_exc())
+
+def native_proxy_del():
+    global native_client
+    native_client = None
+    return OK()
+
+def hltapi_proxy_init(force = False, *args, **kwargs):
+    global hltapi_client
+    if hltapi_client and not force:
+        return ERR('HLTAPI Client is already initiated')
+    try:
+        hltapi_client = CTRexHltApi(*args, **kwargs)
+        return OK('HLTAPI Client initiated')
+    except:
+        return ERR(traceback.format_exc())
+
+def hltapi_proxy_del():
+    global hltapi_client
+    hltapi_client = None
+    return OK()
+
+
+# any method not listed above can be called with passing its name here
+def native_method(func_name, *args, **kwargs):
+    try:
+        func = getattr(native_client, func_name)
+        return OK(func(*deunicode_json(args), **deunicode_json(kwargs)))
+    except:
+        return ERR(traceback.format_exc())
+
+# any HLTAPI method can be called with passing its name here
+def hltapi_method(func_name, *args, **kwargs):
+    try:
+        func = getattr(hltapi_client, func_name)
+        return func(*deunicode_json(args), **deunicode_json(kwargs))
+    except:
+        return HLT_ERR(traceback.format_exc())
+
+### /Server functions ###
+
+
+### Main ###
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description = 'Runs TRex Stateless proxy for usage with any language client.')
+    parser.add_argument('-p', '--port', type=int, default = 8095, dest='port', action = 'store',
+                        help = 'Select port on which the stl proxy will run.\nDefault is 8095.')
+    args = parser.parse_args()
+    native_methods = [
+                      'acquire',
+                      'connect',
+                      'disconnect',
+                      'get_stats',
+                      'get_warnings',
+                      'push_remote',
+                      'reset',
+                      'wait_on_traffic',
+                     ]
+    hltapi_methods = [
+                      'connect',
+                      'cleanup_session',
+                      'interface_config',
+                      'traffic_config',
+                      'traffic_control',
+                      'traffic_stats',
+                     ]
+
+    try:
+        server = SimpleJSONRPCServer(('0.0.0.0', args.port))
+        server.register_function(add)
+        server.register_function(check_connectivity)
+        server.register_function(native_proxy_init)
+        server.register_function(native_proxy_del)
+        server.register_function(hltapi_proxy_init)
+        server.register_function(hltapi_proxy_del)
+
+        for method in native_methods:
+            server.register_function(lambda  method=method, *args, **kwargs: native_method(method, *args, **kwargs), method)
+        server.register_function(native_method)
+        for method in hltapi_methods:
+            if method == 'connect':
+                server.register_function(lambda  method=method, *args, **kwargs: hltapi_method(method, *args, **kwargs), 'hlt_connect')
+            else:
+                server.register_function(lambda  method=method, *args, **kwargs: hltapi_method(method, *args, **kwargs), method)
+        server.register_function(hltapi_method)
+        print('Started Stateless RPC proxy at port %s' % args.port)
+        server.serve_forever()
+    except KeyboardInterrupt:
+        print('Done')
+
diff --git a/scripts/automation/trex_control_plane/stl/examples/using_rpc_proxy.py b/scripts/automation/trex_control_plane/stl/examples/using_rpc_proxy.py
new file mode 100755
index 00000000..82bf0d0a
--- /dev/null
+++ b/scripts/automation/trex_control_plane/stl/examples/using_rpc_proxy.py
@@ -0,0 +1,101 @@
+#!/router/bin/python
+
+import argparse
+import sys
+import os
+from time import sleep
+
+# ext libs
+ext_libs = os.path.join(os.pardir, os.pardir, os.pardir, os.pardir, 'external_libs')
+sys.path.append(os.path.join(ext_libs, 'jsonrpclib-pelix-0.2.5'))
+import jsonrpclib
+
+def fail(msg):
+    print(msg)
+    sys.exit(1)
+
+def verify(res):
+    if not res[0]:
+        fail(res[1])
+    return res
+
+def verify_hlt(res):
+    if res['status'] == 0:
+        fail(res['log'])
+    return res
+
+### Main ###
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description = 'Use of Stateless through rpc_proxy. (Can be implemented in any language)')
+    parser.add_argument('-s', '--server', type=str, default = 'localhost', dest='server', action = 'store',
+                        help = 'Address of rpc proxy.')
+    parser.add_argument('-p', '--port', type=int, default = 8095, dest='port', action = 'store',
+                        help = 'Port of rpc proxy.\nDefault is 8095.')
+    args = parser.parse_args()
+
+    server = jsonrpclib.Server('http://%s:%s' % (args.server, args.port))
+
+# Native API
+
+    print('Initializing Native Client')
+    verify(server.native_proxy_init(server = args.server, force = True))
+
+    print('Connecting to TRex server')
+    verify(server.connect())
+
+    print('Resetting all ports')
+    verify(server.reset())
+
+    print('Getting ports info')
+    res = verify(server.native_method(func_name = 'get_port_info'))
+    print('Ports info is: %s' % res[1])
+    ports = [port['index'] for port in res[1]]
+
+    print('Sending pcap to ports %s' % ports)
+    verify(server.push_remote(pcap_filename = 'stl/sample.pcap'))
+
+    print('Getting stats')
+    res = verify(server.get_stats())
+    print('Stats: %s' % res[1])
+
+    print('Resetting all ports')
+    verify(server.reset())
+
+    print('Deleting Native Client instance')
+    verify(server.native_proxy_del())
+
+# HLTAPI
+
+    print('Initializing HLTAPI Client')
+    verify(server.hltapi_proxy_init(force = True))
+
+    print('HLTAPI connect')
+    verify_hlt(server.hlt_connect(device = args.server, port_list = ports, reset = True, break_locks = True))
+
+    print('Creating traffic')
+    verify_hlt(server.traffic_config(
+            mode = 'create', bidirectional = True,
+            port_handle = ports[0], port_handle2 = ports[1],
+            frame_size = 100,
+            l3_protocol = 'ipv4',
+            ip_src_addr = '10.0.0.1', ip_src_mode = 'increment', ip_src_count = 254,
+            ip_dst_addr = '8.0.0.1', ip_dst_mode = 'increment', ip_dst_count = 254,
+            l4_protocol = 'udp',
+            udp_dst_port = 12, udp_src_port = 1025,
+            rate_percent = 10, ignore_macs = True,
+            ))
+
+    print('Starting traffic for 5 sec')
+    verify_hlt(server.traffic_control(action = 'run', port_handle = ports[:2]))
+
+    sleep(5)
+    print('Stopping traffic')
+    verify_hlt(server.traffic_control(action = 'stop', port_handle = ports[:2]))
+
+    print('Getting stats')
+    res = verify_hlt(server.traffic_stats(mode = 'aggregate', port_handle = ports[:2]))
+    print(res)
+
+    print('Deleting HLTAPI Client instance')
+    verify(server.hltapi_proxy_del())
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_jsonrpc_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_jsonrpc_client.py
index fa04b9f6..065a1442 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_jsonrpc_client.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_jsonrpc_client.py
@@ -273,8 +273,8 @@ class JsonRpcClient(object):
         except zmq.error.ZMQError as e:
             return RC_ERR("ZMQ Error: Bad server or port name: " + str(e))
 
-        self.socket.setsockopt(zmq.SNDTIMEO, 1000)
-        self.socket.setsockopt(zmq.RCVTIMEO, 1000)
+        self.socket.setsockopt(zmq.SNDTIMEO, 10000)
+        self.socket.setsockopt(zmq.RCVTIMEO, 10000)
 
         self.connected = True
 
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
index 4ead255d..f9db22a1 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
@@ -307,8 +307,8 @@ class CTRexInfoGenerator(object):
         # for TUI - maximum 5 
         pg_ids = list(filter(is_intable, lat_stats.latest_stats.keys()))[:5]
         stream_count = len(pg_ids)
-        lstats_data = OrderedDict([#('TX pkts',       []),
-                                   #('RX pkts',       []),
+        lstats_data = OrderedDict([('TX pkts',       []),
+                                   ('RX pkts',       []),
                                    ('Max latency',   []),
                                    ('Avg latency',   []),
                                    ('-- Window --', [''] * stream_count),
@@ -324,8 +324,8 @@ class CTRexInfoGenerator(object):
             history = [x for x in lat_stats.history]
         flow_stats = self._rx_stats_ref.get_stats()
         for pg_id in pg_ids:
-            #lstats_data['TX pkts'].append(flow_stats[pg_id]['tx_pkts']['total'] if pg_id in flow_stats else '')
-            #lstats_data['RX pkts'].append(flow_stats[pg_id]['rx_pkts']['total'] if pg_id in flow_stats else '')
+            lstats_data['TX pkts'].append(flow_stats[pg_id]['tx_pkts']['total'] if pg_id in flow_stats else '')
+            lstats_data['RX pkts'].append(flow_stats[pg_id]['rx_pkts']['total'] if pg_id in flow_stats else '')
             lstats_data['Avg latency'].append(try_int(lat_stats.get([pg_id, 'latency', 'average'])))
             lstats_data['Max latency'].append(try_int(lat_stats.get([pg_id, 'latency', 'total_max'])))
             lstats_data['Last (max)'].append(try_int(lat_stats.get([pg_id, 'latency', 'last_max'])))
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py
index d84af22f..aa6c4218 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py
@@ -147,16 +147,15 @@ def listify (x):
         return [x]
 
 # shows as 'N/A', but does not let any compares for user to not mistake in automation
-class StatNotAvailable(object):
-    def __init__(self, stat_name):
-        self.stat_name = stat_name
-
-    def __repr__(self, *args, **kwargs):
-        return 'N/A'
+class StatNotAvailable(str):
+    def __new__(cls, value, *args, **kwargs):
+        cls.stat_name = value
+        return super(StatNotAvailable, cls).__new__(cls, 'N/A')
 
     def __cmp__(self, *args, **kwargs):
         raise Exception("Stat '%s' not available at this setup" % self.stat_name)
 
+
 class LRU_cache(OrderedDict):
     def __init__(self, maxlen = 20, *args, **kwargs):
         OrderedDict.__init__(self, *args, **kwargs)
diff --git a/src/stateless/rx/trex_stateless_rx_core.cpp b/src/stateless/rx/trex_stateless_rx_core.cpp
index 5dac4b4a..6cabc820 100644
--- a/src/stateless/rx/trex_stateless_rx_core.cpp
+++ b/src/stateless/rx/trex_stateless_rx_core.cpp
@@ -159,7 +159,7 @@ void CRxCoreStateless::handle_rx_pkt(CLatencyManagerPerPortStl *lp, rte_mbuf_t *
                                     m_rfc2544[hw_id].seq_err_events_too_low++;
                                 }
                             } else {
-                                if (unlikely (m_rfc2544[hw_id].seq - seq > 100000)) {
+                                if (unlikely (seq - m_rfc2544[hw_id].seq > 100000)) {
                                     // packet reorder while we had wrap around
                                     if (seq == (m_rfc2544[hw_id].seq - 1)) {
                                         m_rfc2544[hw_id].dup += 1;
@@ -196,7 +196,6 @@ void CRxCoreStateless::handle_rx_pkt(CLatencyManagerPerPortStl *lp, rte_mbuf_t *
             }
         }
     }
-    rte_pktmbuf_free(m);
 }
 
 // In VM setup, handle packets coming as messages from DP cores.
@@ -222,6 +221,7 @@ void CRxCoreStateless::handle_rx_queue_msgs(uint8_t thread_id, CNodeRing * r) {
             assert( rx_port_index < m_max_ports );
             lp = &m_ports[rx_port_index];
             handle_rx_pkt(lp, (rte_mbuf_t *)l_msg->m_pkt);
+            rte_pktmbuf_free((rte_mbuf_t *)l_msg->m_pkt);
             break;
         default:
             printf("ERROR latency-thread message type is not valid %d \n", msg_type);