sync streams and some other bugs

10 files changed, 122 insertions, 78 deletions

diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_bi_dir_flows.py b/scripts/automation/trex_control_plane/stl/examples/stl_bi_dir_flows.py
index 2382f2f4..7e90e264 100644
--- a/scripts/automation/trex_control_plane/stl/examples/stl_bi_dir_flows.py
+++ b/scripts/automation/trex_control_plane/stl/examples/stl_bi_dir_flows.py
@@ -20,11 +20,11 @@ def create_pkt (size, direction):
     vm = [
         # src
         STLVmFlowVar(name="src",min_value=src['start'],max_value=src['end'],size=4,op="inc"),
-        STLVmWriteFlowVar(fv_name="src",pkt_offset= "IP.src"),
+        STLVmWrFlowVar(fv_name="src",pkt_offset= "IP.src"),
 
         # dst
         STLVmFlowVar(name="dst",min_value=dst['start'],max_value=dst['end'],size=4,op="inc"),
-        STLVmWriteFlowVar(fv_name="dst",pkt_offset= "IP.dst"),
+        STLVmWrFlowVar(fv_name="dst",pkt_offset= "IP.dst"),
 
         # checksum
         STLVmFixIpv4(offset = "IP")
diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_run_udp_simple.py b/scripts/automation/trex_control_plane/stl/examples/stl_run_udp_simple.py
index 388e42e7..db005608 100644
--- a/scripts/automation/trex_control_plane/stl/examples/stl_run_udp_simple.py
+++ b/scripts/automation/trex_control_plane/stl/examples/stl_run_udp_simple.py
@@ -47,11 +47,11 @@ def create_pkt (frame_size = 9000, direction=0):
     vm = [
         # src
         STLVmFlowVar(name="src",min_value=src['start'],max_value=src['end'],size=4,op="inc"),
-        STLVmWriteFlowVar(fv_name="src",pkt_offset= "IP.src"),
+        STLVmWrFlowVar(fv_name="src",pkt_offset= "IP.src"),
 
         # dst
         STLVmFlowVar(name="dst",min_value=dst['start'],max_value=dst['end'],size=4,op="inc"),
-        STLVmWriteFlowVar(fv_name="dst",pkt_offset= "IP.dst"),
+        STLVmWrFlowVar(fv_name="dst",pkt_offset= "IP.dst"),
 
         # checksum
         STLVmFixIpv4(offset = "IP")
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py
index 29bad041..4dd07a13 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py
@@ -1,6 +1,9 @@
 
 from collections import namedtuple, OrderedDict
 
+from trex_stl_packet_builder_scapy import CScapyTRexPktBuilder
+from trex_stl_streams import STLStream
+import base64
 import trex_stl_stats
 from trex_stl_types import *
 import time
@@ -132,10 +135,23 @@ class Port(object):
         else:
             raise Exception("port {0}: bad state received from server '{1}'".format(self.port_id, port_state))
 
-        # TODO: handle syncing the streams into stream_db
 
         self.next_available_id = long(rc.data()['max_stream_id']) + 1
 
+        # sync the streams
+        params = {"port_id": self.port_id}
+
+        command = RpcCmdData("get_all_streams", params)
+        rc = self.transmit(command.method, command.params)
+        if rc.bad():
+            return self.err(rc.err())
+
+        for k, v in rc.data()['streams'].iteritems():
+            self.streams[k] = {'next_id': v['next_stream_id'],
+                               'pkt'    : base64.b64decode(v['packet']['binary']),
+                               'mode'   : v['mode']['type'],
+                               'rate'   : STLStream.get_rate_from_field(v['mode']['rate'])}
+
         return self.ok()
 
 
@@ -161,55 +177,64 @@ class Port(object):
             return self.err("Please stop port before attempting to add streams")
 
         # listify
-        streams_list = copy.deepcopy(streams_list if isinstance(streams_list, list) else [streams_list])
+        streams_list = streams_list if isinstance(streams_list, list) else [streams_list]
         
         lookup = {}
 
         # allocate IDs
         for stream in streams_list:
-            if stream.get_id() == None:
-                stream.set_id(self.__allocate_stream_id())
 
-            lookup[stream.get_name()] = stream.get_id()
+            # allocate stream id
+            stream_id = stream.get_id() if stream.get_id() is not None else self.__allocate_stream_id()
+            if stream_id in self.streams:
+                return self.err('Stream ID: {0} already exists'.format(stream_id))
 
-        batch = []
+            # name
+            name = stream.get_name() if stream.get_name() is not None else id(stream)
+            if name in lookup:
+                return self.err("multiple streams with duplicate name: '{0}'".format(name))
+            lookup[name] = stream_id
 
-        
+        batch = []
         for stream in streams_list:
 
+            name = stream.get_name() if stream.get_name() is not None else id(stream)
+            stream_id = lookup[name]
             next_id = -1
+
             next = stream.get_next()
             if next:
                 if not next in lookup:
                     return self.err("stream dependency error - unable to find '{0}'".format(next))
                 next_id = lookup[next]
 
-
-            stream.set_next_id(next_id)
-        
-
             stream_json = stream.to_json()
-            stream_json['next_stream_id'] = stream.get_next_id()
+            stream_json['next_stream_id'] = next_id
 
             params = {"handler": self.handler,
                       "port_id": self.port_id,
-                      "stream_id": stream.get_id(),
+                      "stream_id": stream_id,
                       "stream": stream_json}
 
             cmd = RpcCmdData('add_stream', params)
             batch.append(cmd)
 
-            self.streams[stream.get_id()] = stream
 
         rc = self.transmit_batch(batch)
-        if not rc:
-            return self.err(str(rc))
 
+        for i, single_rc in enumerate(rc):
+            if single_rc:
+                stream_id = batch[i].params['stream_id']
+                next_id   = batch[i].params['stream']['next_stream_id']
+                self.streams[stream_id] = {'next_id' : next_id,
+                                           'pkt'  : streams_list[i].get_pkt(),
+                                           'mode' : streams_list[i].get_mode(),
+                                           'rate' : streams_list[i].get_rate()}
 
-        # the only valid state now
-        self.state = self.STATE_STREAMS
 
-        return self.ok()
+        self.state = self.STATE_STREAMS if (len(self.streams) > 0) else self.STATE_IDLE
+
+        return self.ok() if rc else self.err(str(rc))
 
 
 
@@ -239,16 +264,16 @@ class Port(object):
             cmd = RpcCmdData('remove_stream', params)
             batch.append(cmd)
 
-            del self.streams[stream_id]
-
 
         rc = self.transmit_batch(batch)
-        if not rc:
-            return self.err(rc.err())
+        for i, single_rc in enumerate(rc):
+            if single_rc:
+                id = batch[i].params['stream_id']
+                del self.streams[stream_id]
 
         self.state = self.STATE_STREAMS if (len(self.streams) > 0) else self.STATE_IDLE
 
-        return self.ok()
+        return self.ok() if rc else self.err(rc.err())
 
 
     # remove all the streams
@@ -273,6 +298,7 @@ class Port(object):
 
         return self.ok()
 
+
     # get a specific stream
     def get_stream (self, stream_id):
         if stream_id in self.streams:
@@ -283,6 +309,7 @@ class Port(object):
     def get_all_streams (self):
         return self.streams
 
+
     # start traffic
     def start (self, mul, duration, force):
         if not self.is_acquired():
@@ -324,7 +351,6 @@ class Port(object):
                 return self.ok()
 
 
-
         params = {"handler": self.handler,
                   "port_id": self.port_id}
 
@@ -421,6 +447,7 @@ class Port(object):
 
         return self.ok()
 
+
     def get_profile (self):
         return self.profile
 
@@ -496,18 +523,24 @@ class Port(object):
             return {}
 
         data = {}
-        for id, stream in self.streams.iteritems():
+        for id, obj in self.streams.iteritems():
+
+            # lazy build scapy repr.
+            if not 'pkt_type' in obj:
+                obj['pkt_type'] = CScapyTRexPktBuilder.pkt_layers_desc_from_buffer(obj['pkt'])
+            
             data[id] = OrderedDict([ ('id',  id),
-                                     ('packet_type', stream.get_pkt_type()),
-                                     ('L2 len', stream.get_pkt_len()),
-                                     ('mode' , stream.get_mode()),
-                                     ('rate_pps', stream.get_pps()),
-                                     ('next_stream', stream.get_next_id())
+                                     ('packet_type',  obj['pkt_type']),
+                                     ('L2 len',       len(obj['pkt']) + 4),
+                                     ('mode',         obj['mode']),
+                                     ('rate',         obj['rate']),
+                                     ('next_stream',  obj['next_id'])
                                     ])
     
         return {"streams" : OrderedDict(sorted(data.items())) }
     
 
+
     ################# events handler ######################
     def async_event_port_stopped (self):
         self.state = self.STATE_STREAMS
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py
index 907125e9..54d699d8 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py
@@ -152,19 +152,29 @@ class STLSim(object):
         # load streams
         cmds_json = []
 
-        id = 1
+        id_counter = 1
 
         lookup = {}
+
         # allocate IDs
         for stream in stream_list:
-            if stream.get_id() == None:
-                stream.set_id(id)
-                id += 1
+            if stream.get_id() is not None:
+                stream_id = stream.get_id()
+            else:
+                stream_id = id_counter
+                id_counter += 1
 
-            lookup[stream.get_name()] = stream.get_id()
+            name = stream.get_name() if stream.get_name() is not None else id(stream)
+            if name in lookup:
+                raise STLError("multiple streams with name: '{0}'".format(name))
+            lookup[name] = stream_id
 
         # resolve names
         for stream in stream_list:
+
+            name = stream.get_name() if stream.get_name() is not None else id(stream)
+            stream_id = lookup[name]
+
             next_id = -1
             next = stream.get_next()
             if next:
@@ -172,19 +182,16 @@ class STLSim(object):
                     raise STLError("stream dependency error - unable to find '{0}'".format(next))
                 next_id = lookup[next]
 
-            stream.set_next_id(next_id)
-
-        for stream in stream_list:
 
             stream_json = stream.to_json()
-            stream_json['next_stream_id'] = stream.get_next_id()
+            stream_json['next_stream_id'] = next_id
 
             cmd = {"id":1,
                    "jsonrpc": "2.0",
                    "method": "add_stream",
                    "params": {"handler": self.handler,
                               "port_id": self.port_id,
-                              "stream_id": stream.get_id(),
+                              "stream_id": stream_id,
                               "stream": stream_json}
                    }
 
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
index 34c7a857..ebc686f8 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
@@ -224,13 +224,12 @@ class CTRexInfoGenerator(object):
         p_type_field_len = 0
 
         for stream_id, stream_id_sum in return_streams_data['streams'].iteritems():
-            stream_id_sum['rate_pps'] = format_num(stream_id_sum['rate_pps'], suffix='pps')
             stream_id_sum['packet_type'] = self._trim_packet_headers(stream_id_sum['packet_type'], 30)
             p_type_field_len = max(p_type_field_len, len(stream_id_sum['packet_type']))
 
         info_table = text_tables.TRexTextTable()
         info_table.set_cols_align(["c"] + ["l"] + ["r"] + ["c"] + ["r"] + ["c"])
-        info_table.set_cols_width([10]   + [p_type_field_len]  + [8]   + [16]  + [10]  + [12])
+        info_table.set_cols_width([10]   + [p_type_field_len]  + [8]   + [16]  + [15]  + [12])
         info_table.set_cols_dtype(["t"] + ["t"] + ["t"] + ["t"] + ["t"] + ["t"])
 
         info_table.add_rows([v.values()
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py
index f79d25c3..54a25606 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py
@@ -158,9 +158,7 @@ class STLStream(object):
         self.name = name
         self.next = next
 
-        # ID
-        self.set_id(stream_id)
-        self.set_next_id(None)
+        self.id = stream_id
 
 
         self.fields = {}
@@ -239,14 +237,6 @@ class STLStream(object):
     def get_id (self):
         return self.id
 
-    def set_id (self, id):
-        self.id = id
-
-    def get_next_id (self):
-        return self.next_id
-
-    def set_next_id (self, next_id):
-        self.next_id = next_id
 
     def get_name (self):
         return self.name
@@ -254,26 +244,44 @@ class STLStream(object):
     def get_next (self):
         return self.next
 
-    def get_pkt_type (self):
-        if self.packet_desc == None:
-            self.packet_desc = CScapyTRexPktBuilder.pkt_layers_desc_from_buffer(self.get_pkt())
-
-        return self.packet_desc
 
     def get_pkt (self):
         return self.pkt
 
     def get_pkt_len (self, count_crc = True):
-       pkt_len = len(base64.b64decode(self.get_pkt()))
+       pkt_len = len(self.get_pkt())
        if count_crc:
            pkt_len += 4
 
        return pkt_len
 
 
+    def get_pkt_type (self):
+        if self.packet_desc == None:
+            self.packet_desc = CScapyTRexPktBuilder.pkt_layers_desc_from_buffer(self.get_pkt())
+
+        return self.packet_desc
+
     def get_mode (self):
         return self.mode_desc
 
+    @staticmethod
+    def get_rate_from_field (rate_json):
+        t = rate_json['type']
+        v = rate_json['value']
+
+        if t == "pps":
+            return format_num(v, suffix = "pps")
+        elif t == "bps_L1":
+            return format_num(v, suffix = "bps (L1)")
+        elif t == "bps_L2":
+            return format_num(v, suffix = "bps (L2)")
+        elif t == "percentage":
+            return format_num(v, suffix = "%")
+
+    def get_rate (self):
+        return self.get_rate_from_field(self.fields['mode']['rate'])
+
 
     def to_yaml (self):
         y = {}
@@ -523,7 +531,7 @@ class STLProfile(object):
             
             streams.append(STLStream(name = i,
                                      packet = CScapyTRexPktBuilder(pkt_buffer = cap, vm = vm),
-                                     mode = STLTXSingleBurst(total_pkts = 1),
+                                     mode = STLTXSingleBurst(total_pkts = 1, percentage = 100),
                                      self_start = True if (i == 1) else False,
                                      isg = (ts_usec - last_ts_usec),  # seconds to usec
                                      action_count = action_count,
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py
index d4ad8bd2..496bea13 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py
@@ -54,6 +54,10 @@ class RC():
                 s += format_text("\n{0}".format(x.data), 'bold')
         return s
 
+    def __iter__(self):
+        return self.rc_list.__iter__()
+
+
     def prn_func (self, msg, newline = True):
         if newline:
             print msg
diff --git a/scripts/stl/yaml/imix_1pkt_2.yaml b/scripts/stl/yaml/imix_1pkt_2.yaml
index 07fd481d..dc1a457b 100644
--- a/scripts/stl/yaml/imix_1pkt_2.yaml
+++ b/scripts/stl/yaml/imix_1pkt_2.yaml
@@ -1,6 +1,6 @@
 ### Single stream UDP packet, 64B ###
 #####################################
-- name: udp_64B
+- name: udp_64B_1
   stream:
     self_start: True
     packet:
@@ -10,7 +10,7 @@
       pps: 100
     rx_stats: []
     vm: []
-- name: udp_64B
+- name: udp_64B_2
   stream:
     self_start: True
     packet:
@@ -20,7 +20,7 @@
       pps: 100
     rx_stats: []
     vm: []
-- name: udp_64B
+- name: udp_64B_3
   stream:
     self_start: True
     packet:
@@ -30,7 +30,7 @@
       pps: 100
     rx_stats: []
     vm: []
-- name: udp_64B
+- name: udp_64B_4
   stream:
     self_start: True
     packet:
@@ -39,4 +39,4 @@
       type: continuous
       pps: 100
     rx_stats: []
-    vm: []            
\ No newline at end of file
+    vm: []            
diff --git a/src/rpc-server/commands/trex_rpc_cmd_stream.cpp b/src/rpc-server/commands/trex_rpc_cmd_stream.cpp
index 50295c7c..d08c79b5 100644
--- a/src/rpc-server/commands/trex_rpc_cmd_stream.cpp
+++ b/src/rpc-server/commands/trex_rpc_cmd_stream.cpp
@@ -588,8 +588,6 @@ TrexRpcCmdGetAllStreams::_run(const Json::Value &params, Json::Value &result) {
     uint8_t port_id = parse_port(params, result);
     TrexStatelessPort *port = get_stateless_obj()->get_port_by_id(port_id);
 
-    bool    get_pkt = parse_bool(params, "get_pkt", result);
-
     std::vector <TrexStream *> streams;
     port->get_object_list(streams);
 
@@ -598,11 +596,6 @@ TrexRpcCmdGetAllStreams::_run(const Json::Value &params, Json::Value &result) {
 
         Json::Value j = stream->get_stream_json();
 
-        /* should we include the packet as well ? */
-        if (!get_pkt) {
-            j.removeMember("packet");
-        }
-
         std::stringstream ss;
         ss << stream->m_stream_id;
 
diff --git a/src/rpc-server/commands/trex_rpc_cmds.h b/src/rpc-server/commands/trex_rpc_cmds.h
index d90d880e..9545e585 100644
--- a/src/rpc-server/commands/trex_rpc_cmds.h
+++ b/src/rpc-server/commands/trex_rpc_cmds.h
@@ -107,7 +107,7 @@ void parse_vm_instr_write_mask_flow_var(const Json::Value &inst, std::unique_ptr
 
 
 TREX_RPC_CMD_DEFINE(TrexRpcCmdGetStreamList, "get_stream_list", 1, false);
-TREX_RPC_CMD_DEFINE(TrexRpcCmdGetAllStreams, "get_all_streams", 2, false);
+TREX_RPC_CMD_DEFINE(TrexRpcCmdGetAllStreams, "get_all_streams", 1, false);
 
 TREX_RPC_CMD_DEFINE(TrexRpcCmdGetStream, "get_stream", 3, false);