many bugs uncovered by the PCAP injection:

1. NamedTuple constructor
2. Scappy
3. zlib for server

6 files changed, 100 insertions, 32 deletions

diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
index 5096e33d..46d33e3e 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
@@ -2006,6 +2006,7 @@ class STLClient(object):
         try:
             # pcap injection removes all previous streams from the ports
             self.remove_all_streams(ports = opts.ports)
+            
             profile = STLProfile.load_pcap(opts.file[0],
                                            opts.ipg_usec,
                                            opts.speedup,
@@ -2018,4 +2019,5 @@ class STLClient(object):
             print e.brief()
             return
 
+        return True
 
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_jsonrpc_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_jsonrpc_client.py
index ab3c7282..96d1854d 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_jsonrpc_client.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_jsonrpc_client.py
@@ -3,10 +3,12 @@
 import zmq
 import json
 import re
-from time import sleep
 from collections import namedtuple
 from trex_stl_types import *
 from utils.common import random_id_gen
+import zlib
+import struct
+
 
 class bcolors:
     BLUE = '\033[94m'
@@ -35,12 +37,15 @@ class BatchMessage(object):
 
         msg = json.dumps(self.batch_list)
 
-        return self.rpc_client.send_raw_msg(msg)
+        return self.rpc_client.send_msg(msg)
 
 
 # JSON RPC v2.0 client
 class JsonRpcClient(object):
 
+    MSG_COMPRESS_THRESHOLD = 4096
+    MSG_COMPRESS_HEADER_MAGIC = 0xABE85CEA
+
     def __init__ (self, default_server, default_port, logger):
         self.logger = logger
         self.connected = False
@@ -109,14 +114,64 @@ class JsonRpcClient(object):
 
         id, msg = self.create_jsonrpc_v2(method_name, params)
 
-        return self.send_raw_msg(msg)
+        return self.send_msg(msg)
+
+
+    def compress_msg (self, msg):
+        # compress
+        compressed = zlib.compress(msg)
+        new_msg = struct.pack(">II", self.MSG_COMPRESS_HEADER_MAGIC, len(msg)) + compressed
+        return new_msg
+
+
+    def decompress_msg (self, msg):
+        if len(msg) < 8:
+            return None
+
+        t = struct.unpack(">II", msg[:8])
+        if (t[0] != self.MSG_COMPRESS_HEADER_MAGIC):
+            return None
+
+        x = zlib.decompress(msg[8:])
+        if len(x) != t[1]:
+            return None
+
+        return x
+
+    def send_msg (self, msg):
+        # print before
+        if self.logger.check_verbose(self.logger.VERBOSE_HIGH):
+            self.verbose_msg("Sending Request To Server:\n\n" + self.pretty_json(msg) + "\n")
+
+        if len(msg) > self.MSG_COMPRESS_THRESHOLD:
+            response = self.send_raw_msg(self.compress_msg(msg))
+            if response:
+                response = self.decompress_msg(response)
+        else:
+            response = self.send_raw_msg(msg)
+
+        if response == None:
+            return RC_ERR("*** [RPC] - Failed to decode response from server")
+
+
+        # print after
+        if self.logger.check_verbose(self.logger.VERBOSE_HIGH):
+            self.verbose_msg("Server Response:\n\n" + self.pretty_json(response) + "\n")
+
+        # process response (batch and regular)
+       
+        response_json = json.loads(response)
+
+        if isinstance(response_json, list):
+            return self.process_batch_response(response_json)
+        else:
+            return self.process_single_response(response_json)
+
 
 
     # low level send of string message
     def send_raw_msg (self, msg):
 
-        self.verbose_msg("Sending Request To Server:\n\n" + self.pretty_json(msg) + "\n")
-
         tries = 0
         while True:
             try:
@@ -141,26 +196,11 @@ class JsonRpcClient(object):
                     return RC_ERR("*** [RPC] - Failed to get server response at {0}".format(self.transport))
 
 
-        self.verbose_msg("Server Response:\n\n" + self.pretty_json(response) + "\n")
-
-        # decode
-
-        # batch ?
-        response_json = json.loads(response)
-
-        if isinstance(response_json, list):
-            rc_batch = RC()
-
-            for single_response in response_json:
-                rc = self.process_single_response(single_response)
-                rc_batch.add(rc)
-
-            return rc_batch
-
-        else:
-            return self.process_single_response(response_json)
-
+        return response
+       
+     
 
+    # processs a single response from server
     def process_single_response (self, response_json):
 
         if (response_json.get("jsonrpc") != "2.0"):
@@ -182,6 +222,17 @@ class JsonRpcClient(object):
 
   
 
+    # process a batch response
+    def process_batch_response (self, response_json):
+        rc_batch = RC()
+
+        for single_response in response_json:
+            rc = self.process_single_response(single_response)
+            rc_batch.add(rc)
+
+        return rc_batch
+
+
     def disconnect (self):
         if self.connected:
             self.socket.close(linger = 0)
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py
index 56657e22..ce7a630c 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py
@@ -3,6 +3,7 @@ from collections import namedtuple, OrderedDict
 
 import trex_stl_stats
 from trex_stl_types import *
+import time
 
 StreamOnPort = namedtuple('StreamOnPort', ['compiled_stream', 'metadata'])
 
@@ -199,14 +200,13 @@ class Port(object):
             batch.append(cmd)
 
             # meta data for show streams
-            self.streams[stream.get_id()] = StreamOnPort(stream.to_json(),
-                                                         Port._generate_stream_metadata(stream))
+            #self.streams[stream.get_id()] = StreamOnPort(stream.to_json(),
+            #                                             Port._generate_stream_metadata(stream))
 
         rc = self.transmit_batch(batch)
         if not rc:
             return self.err(rc.err())
 
-        
 
         # the only valid state now
         self.state = self.STATE_STREAMS
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py
index 4d720aac..086e46af 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py
@@ -27,7 +27,8 @@ from trex_stl_client import STLClient
 
 import re
 import json
-
+import zlib
+import struct
 
 
 import argparse
@@ -221,7 +222,12 @@ class STLSim(object):
 
         # write to temp file
         f = tempfile.NamedTemporaryFile(delete = False)
-        f.write(json.dumps(cmds_json))
+
+        msg = json.dumps(cmds_json)
+        compressed = zlib.compress(msg)
+        new_msg = struct.pack(">II", 0xABE85CEA, len(msg)) + compressed
+
+        f.write(new_msg)
         f.close()
 
         # launch bp-sim
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py
index 1a46aae7..efeb5c8a 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py
@@ -157,7 +157,10 @@ class STLStream(object):
         # packet and VM
         self.fields['packet'] = packet.dump_pkt()
         self.fields['vm']     = packet.get_vm_data()
-        self.packet_desc      = packet.pkt_layers_desc()
+
+
+        # this is heavy, calculate lazy
+        self.packet_desc = None
 
         if not rx_stats:
             self.fields['rx_stats'] = {}
@@ -194,6 +197,9 @@ class STLStream(object):
         return self.next
 
     def get_pkt_type (self):
+        if self.packet_desc == None:
+            self.packet_desc = CScapyTRexPktBuilder.pkt_layers_desc_from_buffer(base64.b64decode(self.fields['packet']['binary']))
+
         return self.packet_desc
 
     def get_pkt_len (self, count_crc = True):
@@ -432,14 +438,17 @@ class STLProfile(object):
             else:
                 next = i + 1
 
+            
             streams.append(STLStream(name = i,
                                      packet = CScapyTRexPktBuilder(pkt_buffer = cap),
                                      mode = STLTXSingleBurst(total_pkts = 1),
                                      self_start = True if (i == 1) else False,
                                      isg = (ts_usec - last_ts_usec),  # seconds to usec
                                      next = next))
+        
             last_ts_usec = ts_usec
 
+
         return STLProfile(streams)
 
       
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py
index d387ac9c..0a6e64fb 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py
@@ -4,6 +4,7 @@ from utils.text_opts import *
 from trex_stl_exceptions import *
 
 RpcCmdData = namedtuple('RpcCmdData', ['method', 'params'])
+TupleRC    = namedtuple('RC', ['rc', 'data', 'is_warn'])
 
 class RpcResponseStatus(namedtuple('RpcResponseStatus', ['success', 'id', 'msg'])):
         __slots__ = ()
@@ -19,8 +20,7 @@ class RC():
         self.rc_list = []
 
         if (rc != None):
-            tuple_rc = namedtuple('RC', ['rc', 'data', 'is_warn'])
-            self.rc_list.append(tuple_rc(rc, data, is_warn))
+            self.rc_list.append(TupleRC(rc, data, is_warn))
 
     def __nonzero__ (self):
         return self.good()