huge refactor - again

29 files changed, 871 insertions, 29 deletions

diff --git a/scripts/automation/trex_control_plane/console/__init__.py b/scripts/automation/trex_control_plane/stl/console/__init__.py
index e69de29b..e69de29b 100644
--- a/scripts/automation/trex_control_plane/console/__init__.py
+++ b/scripts/automation/trex_control_plane/stl/console/__init__.py
diff --git a/scripts/automation/trex_control_plane/console/trex_console.py b/scripts/automation/trex_control_plane/stl/console/trex_console.py
index 45428b89..789ad4ab 100755
--- a/scripts/automation/trex_control_plane/console/trex_console.py
+++ b/scripts/automation/trex_control_plane/stl/console/trex_console.py
@@ -28,11 +28,11 @@ import os
 import sys
 import tty, termios
 
-from trex_control_plane.stl.api import *
+from trex_stl_lib.api import *
 
-from trex_control_plane.common.text_opts import *
-from trex_control_plane.client_utils.general_utils import user_input, get_current_user
-from trex_control_plane.client_utils import parsing_opts
+from trex_stl_lib.utils.text_opts import *
+from trex_stl_lib.utils.common import user_input, get_current_user
+from trex_stl_lib.utils import parsing_opts
 
 
 import trex_tui
diff --git a/scripts/automation/trex_control_plane/console/trex_root_path.py b/scripts/automation/trex_control_plane/stl/console/trex_root_path.py
index de4ec03b..de4ec03b 100755
--- a/scripts/automation/trex_control_plane/console/trex_root_path.py
+++ b/scripts/automation/trex_control_plane/stl/console/trex_root_path.py
diff --git a/scripts/automation/trex_control_plane/console/trex_tui.py b/scripts/automation/trex_control_plane/stl/console/trex_tui.py
index 1ecf0868..f972b905 100644
--- a/scripts/automation/trex_control_plane/console/trex_tui.py
+++ b/scripts/automation/trex_control_plane/stl/console/trex_tui.py
@@ -6,11 +6,11 @@ from collections import OrderedDict
 import datetime
 from cStringIO import StringIO
 
-from common.text_opts import *
-from client_utils import text_tables
+from trex_stl_lib.utils.text_opts import *
+from trex_stl_lib.utils import text_tables
 
 # for STL exceptions
-from trex_control_plane.stl.api import *
+from trex_stl_lib.api import *
 
 class SimpleBar(object):
     def __init__ (self, desc, pattern):
diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_bi_dir_flows.py b/scripts/automation/trex_control_plane/stl/examples/stl_bi_dir_flows.py
new file mode 100644
index 00000000..46b84c6e
--- /dev/null
+++ b/scripts/automation/trex_control_plane/stl/examples/stl_bi_dir_flows.py
@@ -0,0 +1,114 @@
+import stl_path
+from trex_control_plane.stl.api import *
+
+import time
+import json
+
+# simple packet creation
+def create_pkt (size, direction):
+
+    ip_range = {'src': {'start': "10.0.0.1", 'end': "10.0.0.254"},
+                'dst': {'start': "8.0.0.1",  'end': "8.0.0.254"}}
+
+    if (direction == 0):
+        src = ip_range['src']
+        dst = ip_range['dst']
+    else:
+        src = ip_range['dst']
+        dst = ip_range['src']
+
+    vm = [
+        # src
+        STLVmFlowVar(name="src",min_value=src['start'],max_value=src['end'],size=4,op="inc"),
+        STLVmWriteFlowVar(fv_name="src",pkt_offset= "IP.src"),
+
+        # dst
+        STLVmFlowVar(name="dst",min_value=dst['start'],max_value=dst['end'],size=4,op="inc"),
+        STLVmWriteFlowVar(fv_name="dst",pkt_offset= "IP.dst"),
+
+        # checksum
+        STLVmFixIpv4(offset = "IP")
+        ]
+
+
+    base = Ether()/IP()/UDP()
+    pad = max(0, len(base)) * 'x'
+
+    return STLPktBuilder(pkt = base/pad,
+                         vm  = vm)
+
+
+def simple_burst ():
+   
+  
+    # create client
+    c = STLClient()
+    passed = True
+
+    try:
+        # turn this on for some information
+        #c.set_verbose("high")
+
+        # create two streams
+        s1 = STLStream(packet = create_pkt(200, 0),
+                       mode = STLTXCont(pps = 100))
+
+        # second stream with a phase of 1ms (inter stream gap)
+        s2 = STLStream(packet = create_pkt(200, 1),
+                       isg = 1000,
+                       mode = STLTXCont(pps = 100))
+
+
+        # connect to server
+        c.connect()
+
+        # prepare our ports (my machine has 0 <--> 1 with static route)
+        c.reset(ports = [0, 1])
+
+        # add both streams to ports
+        c.add_streams(s1, ports = [0])
+        c.add_streams(s2, ports = [1])
+
+        # clear the stats before injecting
+        c.clear_stats()
+
+        # choose rate and start traffic for 10 seconds on 5 mpps
+        print "Running 5 Mpps on ports 0, 1 for 10 seconds..."
+        c.start(ports = [0, 1], mult = "5mpps", duration = 10)
+
+        # block until done
+        c.wait_on_traffic(ports = [0, 1])
+
+        # read the stats after the test
+        stats = c.get_stats()
+
+        print json.dumps(stats[0], indent = 4, separators=(',', ': '), sort_keys = True)
+        print json.dumps(stats[1], indent = 4, separators=(',', ': '), sort_keys = True)
+
+        lost_a = stats[0]["opackets"] - stats[1]["ipackets"]
+        lost_b = stats[1]["opackets"] - stats[0]["ipackets"]
+
+        print "\npackets lost from 0 --> 1:   {0} pkts".format(lost_a)
+        print "packets lost from 1 --> 0:   {0} pkts".format(lost_b)
+
+        if (lost_a == 0) and (lost_b == 0):
+            passed = True
+        else:
+            passed = False
+
+    except STLError as e:
+        passed = False
+        print e
+
+    finally:
+        c.disconnect()
+
+    if passed:
+        print "\nTest has passed :-)\n"
+    else:
+        print "\nTest has failed :-(\n"
+
+
+# run the tests
+simple_burst()
+
diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_imix.py b/scripts/automation/trex_control_plane/stl/examples/stl_imix.py
new file mode 100644
index 00000000..c083a207
--- /dev/null
+++ b/scripts/automation/trex_control_plane/stl/examples/stl_imix.py
@@ -0,0 +1,101 @@
+import stl_path
+from trex_control_plane.stl.api import *
+
+import time
+import json
+from pprint import pprint
+
+# IMIX test
+# it maps the ports to sides
+# then it load a predefind profile 'IMIX'
+# and attach it to both sides and inject
+# at a certain rate for some time
+# finally it checks that all packets arrived
+def imix_test ():
+    
+
+    # create client
+    c = STLClient()
+    passed = True
+
+
+    try:
+
+        # connect to server
+        c.connect()
+
+        # take all the ports
+        c.reset()
+
+        # map ports - identify the routes
+        table = stl_map_ports(c)
+
+        print "Mapped ports to sides {0} <--> {1}".format(table['dir'][0], table['dir'][1])
+        dir_0 = table['dir'][0]
+        dir_1 = table['dir'][1]
+
+        # load IMIX profile
+        streams = c.load_profile('../../../stl/profiles/imix.py')
+
+        # add both streams to ports
+        c.add_streams(streams, ports = dir_0)
+        c.add_streams(streams, ports = dir_1)
+        
+        # clear the stats before injecting
+        c.clear_stats()
+
+        # choose rate and start traffic for 10 seconds on 5 mpps
+        duration = 10
+        mult = "5mpps"
+        print "Injecting {0} <--> {1} on total rate of '{2}' for {3} seconds".format(dir_0, dir_1, mult, duration)
+
+        c.start(ports = (dir_0 + dir_1), mult = mult, duration = duration, total = True)
+        
+        # block until done
+        c.wait_on_traffic(ports = (dir_0 + dir_1))
+
+        # read the stats after the test
+        stats = c.get_stats()
+
+        # use this for debug info on all the stats
+        #pprint(stats)
+
+        # sum dir 0
+        dir_0_opackets = sum([stats[i]["opackets"] for i in dir_0])
+        dir_0_ipackets = sum([stats[i]["ipackets"] for i in dir_0])
+
+        # sum dir 1
+        dir_1_opackets = sum([stats[i]["opackets"] for i in dir_1])
+        dir_1_ipackets = sum([stats[i]["ipackets"] for i in dir_1])
+
+
+        lost_0 = dir_0_opackets - dir_1_ipackets
+        lost_1 = dir_1_opackets - dir_0_ipackets
+
+        print "\nPackets injected from {0}: {1:,}".format(dir_0, dir_0_opackets)
+        print "Packets injected from {0}: {1:,}".format(dir_1, dir_1_opackets)
+
+        print "\npackets lost from {0} --> {1}:   {2:,} pkts".format(dir_0, dir_0, lost_0)
+        print "packets lost from {0} --> {1}:   {2:,} pkts".format(dir_0, dir_0, lost_0)
+
+        if (lost_0 == 0) and (lost_0 == 0):
+            passed = True
+        else:
+            passed = False
+
+    except STLError as e:
+        passed = False
+        print e
+
+    finally:
+        c.disconnect()
+
+    if passed:
+        print "\nTest has passed :-)\n"
+    else:
+        print "\nTest has failed :-(\n"
+
+
+# run the tests
+imix_test()
+
diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_path.py b/scripts/automation/trex_control_plane/stl/examples/stl_path.py
new file mode 100644
index 00000000..a5b8102b
--- /dev/null
+++ b/scripts/automation/trex_control_plane/stl/examples/stl_path.py
@@ -0,0 +1,4 @@
+import sys
+
+# FIXME to the write path for trex_control_plane
+sys.path.insert(0, "../../../automation")
diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_run_udp_simple.py b/scripts/automation/trex_control_plane/stl/examples/stl_run_udp_simple.py
new file mode 100644
index 00000000..47db1b5a
--- /dev/null
+++ b/scripts/automation/trex_control_plane/stl/examples/stl_run_udp_simple.py
@@ -0,0 +1,219 @@
+#!/usr/bin/python
+import sys, getopt
+import argparse;
+""" 
+Sample API application, 
+Connect to TRex 
+Send UDP packet in specific length 
+Each direction has its own IP range
+Compare  Rx-pkts to TX-pkts assuming ports are loopback
+
+"""
+
+import stl_path
+from trex_control_plane.stl.api import *
+
+H_VER = "trex-x v0.1 "
+
+class t_global(object):
+     args=None;
+
+
+import dpkt
+import time
+import json
+import string
+
+def generate_payload(length):
+    word = ''
+    alphabet_size = len(string.letters)
+    for i in range(length):
+        word += string.letters[(i % alphabet_size)]
+    return word
+
+# simple packet creation
+def create_pkt (frame_size = 9000, direction=0):
+
+    ip_range = {'src': {'start': "10.0.0.1", 'end': "10.0.0.254"},
+                'dst': {'start': "8.0.0.1",  'end': "8.0.0.254"}}
+
+    if (direction == 0):
+        src = ip_range['src']
+        dst = ip_range['dst']
+    else:
+        src = ip_range['dst']
+        dst = ip_range['src']
+
+    vm = [
+        # src
+        STLVmFlowVar(name="src",min_value=src['start'],max_value=src['end'],size=4,op="inc"),
+        STLVmWriteFlowVar(fv_name="src",pkt_offset= "IP.src"),
+
+        # dst
+        STLVmFlowVar(name="dst",min_value=dst['start'],max_value=dst['end'],size=4,op="inc"),
+        STLVmWriteFlowVar(fv_name="dst",pkt_offset= "IP.dst"),
+
+        # checksum
+        STLVmFixIpv4(offset = "IP")
+        ]
+
+    pkt_base  = Ether(src="00:00:00:00:00:01",dst="00:00:00:00:00:02")/IP()/UDP(dport=12,sport=1025)
+    pyld_size = frame_size - len(pkt_base);
+    pkt_pyld   = generate_payload(pyld_size) 
+
+    return STLPktBuilder(pkt = pkt_base/pkt_pyld,
+                         vm  = vm)
+
+
+def simple_burst (duration = 10, frame_size = 9000, speed = '1gbps'):
+   
+    if (frame_size < 60):
+        frame_size = 60
+
+    pkt_dir_0 = create_pkt (frame_size, 0) 
+
+    pkt_dir_1 = create_pkt (frame_size, 1) 
+
+    # create client
+    c = STLClient(server = t_global.args.ip)
+
+    passed = True
+
+    try:
+        # turn this on for some information
+        #c.set_verbose("high")
+
+        # create two streams
+        s1 = STLStream(packet = pkt_dir_0,
+                       mode = STLTXCont(pps = 100))
+
+        # second stream with a phase of 1ms (inter stream gap)
+        s2 = STLStream(packet = pkt_dir_1,
+                       isg = 1000,
+                       mode = STLTXCont(pps = 100))
+
+        if t_global.args.debug:
+            STLStream.dump_to_yaml ("example.yaml", [s1,s2]) # export to YAML so you can run it on simulator ./stl-sim -f example.yaml -o o.pcap 
+
+        # connect to server
+        c.connect()
+
+        # prepare our ports (my machine has 0 <--> 1 with static route)
+        c.reset(ports = [0, 1])
+
+        # add both streams to ports
+        c.add_streams(s1, ports = [0])
+        c.add_streams(s2, ports = [1])
+
+        # clear the stats before injecting
+        c.clear_stats()
+
+        # choose rate and start traffic for 10 seconds on 5 mpps
+        print "Running {0} on ports 0, 1 for 10 seconds, UDP {1}...".format(speed,frame_size+4)
+        c.start(ports = [0, 1], mult = speed, duration = duration)
+
+        # block until done
+        c.wait_on_traffic(ports = [0, 1])
+
+        # read the stats after the test
+        stats = c.get_stats()
+
+        #print stats
+        print json.dumps(stats[0], indent = 4, separators=(',', ': '), sort_keys = True)
+        print json.dumps(stats[1], indent = 4, separators=(',', ': '), sort_keys = True)
+
+        lost_a = stats[0]["opackets"] - stats[1]["ipackets"]
+        lost_b = stats[1]["opackets"] - stats[0]["ipackets"]
+
+        print "\npackets lost from 0 --> 1:   {0} pkts".format(lost_a)
+        print "packets lost from 1 --> 0:   {0} pkts".format(lost_b)
+
+        if (lost_a == 0) and (lost_b == 0):
+            passed = True
+        else:
+            passed = False
+
+    except STLError as e:
+        passed = False
+        print e
+
+    finally:
+        c.disconnect()
+
+    if passed:
+        print "\nPASSED\n"
+    else:
+        print "\nFAILED\n"
+
+def process_options ():
+    parser = argparse.ArgumentParser(usage=""" 
+    connect to TRex and send burst of packets
+
+    examples
+
+     stl_run_udp_simple.py -s 9001  
+
+     stl_run_udp_simple.py -s 9000 -d 2     
+
+     stl_run_udp_simple.py -s 3000 -d 3 -m 10mbps
+
+     stl_run_udp_simple.py -s 3000 -d 3 -m 10mbps --debug
+
+     then run the simulator on the output 
+       ./stl-sim -f example.yaml -o a.pcap  ==> a.pcap include the packet
+
+    """,
+    description="example for TRex api",
+    epilog=" written by hhaim");
+
+    parser.add_argument("-s", "--frame-size", 
+                        dest="frame_size",
+                        help='L2 frame size in bytes without FCS',
+                        default=60,
+                        type = int,
+                        )
+
+    parser.add_argument("--ip", 
+                        dest="ip",
+                        help='remote trex ip default local',
+                        default="127.0.0.1",
+                        type = str
+                        )
+
+
+    parser.add_argument('-d','--duration', 
+                        dest='duration',
+                        help='duration in second ',
+                        default=10,
+                        type = int,
+                        )
+
+
+    parser.add_argument('-m','--multiplier', 
+                        dest='mul',
+                        help='speed in gbps/pps for example 1gbps, 1mbps, 1mpps ',
+                        default="1mbps"
+                        )
+
+    parser.add_argument('--debug', 
+                        action='store_true',
+                        help='see debug into ')
+
+    parser.add_argument('--version', action='version',
+                        version=H_VER )
+
+    t_global.args = parser.parse_args();
+    print t_global.args
+
+
+
+def main():
+    process_options ()
+    simple_burst(duration = t_global.args.duration,  
+                 frame_size = t_global.args.frame_size,
+                 speed = t_global.args.mul
+                 )
+
+if __name__ == "__main__":
+    main()
+
diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_simple_burst.py b/scripts/automation/trex_control_plane/stl/examples/stl_simple_burst.py
new file mode 100644
index 00000000..1e3e7695
--- /dev/null
+++ b/scripts/automation/trex_control_plane/stl/examples/stl_simple_burst.py
@@ -0,0 +1,65 @@
+import stl_path
+from trex_control_plane.stl.api import *
+
+import time
+
+def simple_burst ():
+        
+
+    # create client
+    c = STLClient()
+    passed = True
+    
+    try:
+        pkt = STLPktBuilder(pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)/IP()/'a_payload_example')
+
+        # create two bursts and link them
+        s1 = STLStream(name = 'A',
+                       packet = pkt,
+                       mode = STLTXSingleBurst(total_pkts = 5000),
+                       next = 'B')
+        
+        s2 = STLStream(name = 'B',
+                       self_start = False,
+                       packet = pkt,
+                       mode = STLTXSingleBurst(total_pkts = 3000))
+
+        # connect to server
+        c.connect()
+
+        # prepare our ports
+        c.reset(ports = [0, 3])
+
+        # add both streams to ports
+        stream_ids = c.add_streams([s1, s2], ports = [0, 3])
+
+        # run 5 times
+        for i in xrange(1, 6):
+            c.clear_stats()
+            c.start(ports = [0, 3], mult = "1gbps")
+            c.wait_on_traffic(ports = [0, 3])
+
+            stats = c.get_stats()
+            ipackets  = stats['total']['ipackets']
+
+            print "Test iteration {0} - Packets Received: {1} ".format(i, ipackets)
+            # (5000 + 3000) * 2 ports = 16,000
+            if (ipackets != (16000)):
+                passed = False
+
+    except STLError as e:
+        passed = False
+        print e
+
+    finally:
+        c.disconnect()
+
+    if passed:
+        print "\nTest has passed :-)\n"
+    else:
+        print "\nTest has failed :-(\n"
+
+
+# run the tests
+simple_burst()
+
diff --git a/scripts/automation/trex_control_plane/stl/__init__.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/__init__.py
index e69de29b..e69de29b 100644
--- a/scripts/automation/trex_control_plane/stl/__init__.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/__init__.py
diff --git a/scripts/automation/trex_control_plane/stl/api.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/api.py
index c12628b5..4c0c10fa 100644
--- a/scripts/automation/trex_control_plane/stl/api.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/api.py
@@ -1,6 +1,6 @@
 
 # get external libs
-import trex_control_plane.client_utils.external_packages
+import trex_stl_ext
 
 # client and exceptions
 from trex_stl_exceptions import *
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_async_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_async_client.py
index 9b3b9577..410482b9 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_async_client.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_async_client.py
@@ -10,7 +10,7 @@ import random
 
 from trex_stl_jsonrpc_client import JsonRpcClient, BatchMessage
 
-from common.text_opts import *
+from utils.text_opts import *
 from trex_stl_stats import *
 from trex_stl_types import *
 
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
index 08f640b5..ed11791b 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_client.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
@@ -1,9 +1,8 @@
 #!/router/bin/python
 
 # for API usage the path name must be full
-from trex_control_plane.stl.trex_stl_exceptions import *
-#from trex_control_plane.stl.trex_stl_streams import *
-from trex_stl_streams import *
+from trex_stl_lib.trex_stl_exceptions import *
+from trex_stl_lib.trex_stl_streams import *
 
 from trex_stl_jsonrpc_client import JsonRpcClient, BatchMessage
 import trex_stl_stats
@@ -12,8 +11,8 @@ from trex_stl_port import Port
 from trex_stl_types import *
 from trex_stl_async_client import CTRexAsyncClient
 
-from trex_control_plane.client_utils import parsing_opts, text_tables, general_utils
-from trex_control_plane.common.text_opts import *
+from utils import parsing_opts, text_tables, common
+from utils.text_opts import *
 
 
 from collections import namedtuple
@@ -23,6 +22,7 @@ import datetime
 import re
 import random
 import json
+import traceback
 
 ############################     logger     #############################
 ############################                #############################
@@ -376,7 +376,7 @@ class STLClient(object):
     """docstring for STLClient"""
 
     def __init__(self,
-                 username = general_utils.get_current_user(),
+                 username = common.get_current_user(),
                  server = "localhost",
                  sync_port = 4501,
                  async_port = 4500,
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_exceptions.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_exceptions.py
index d5b3885d..45acc72e 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_exceptions.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_exceptions.py
@@ -1,7 +1,7 @@
 import os
 import sys
 
-from trex_control_plane.common.text_opts import *
+from utils.text_opts import *
 
 # basic error for API
 class STLError(Exception):
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_ext.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_ext.py
new file mode 100644
index 00000000..1092679a
--- /dev/null
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_ext.py
@@ -0,0 +1,74 @@
+import sys
+import os
+import warnings
+
+# if not set - set it to default
+if not 'TREX_STL_EXT_PATH' in globals():
+    CURRENT_PATH        = os.path.dirname(os.path.realpath(__file__))
+    # ../../../../external_libs
+    TREX_STL_EXT_PATH   = os.path.abspath(os.path.join(CURRENT_PATH, os.pardir, os.pardir, os.pardir, os.pardir, 'external_libs'))
+
+# check path exists
+if not os.path.exists(TREX_STL_EXT_PATH):
+    print "Unable to find external packages path: '{0}'".format(TREX_STL_EXT_PATH)
+    print "Please provide the correct path using TREX_STL_EXT_PATH variable"
+    exit(0)
+
+# the modules required
+CLIENT_UTILS_MODULES = ['dpkt-1.8.6',
+                        'yaml-3.11',
+                        'texttable-0.8.4',
+                        'scapy-2.3.1'
+                        ]
+
+def import_client_utils_modules():
+    import_module_list(CLIENT_UTILS_MODULES)
+
+
+def import_module_list(modules_list):
+    assert(isinstance(modules_list, list))
+    for p in modules_list:
+        full_path = os.path.join(TREX_STL_EXT_PATH, p)
+        fix_path = os.path.normcase(full_path)
+        sys.path.insert(1, full_path)
+
+
+    import_platform_dirs()
+  
+
+
+def import_platform_dirs ():
+    # handle platform dirs
+
+    # try fedora 18 first and then cel5.9
+    # we are using the ZMQ module to determine the right platform
+
+    full_path = os.path.join(TREX_STL_EXT_PATH, 'platform/fedora18')
+    fix_path = os.path.normcase(full_path)
+    sys.path.insert(0, full_path)
+    try:
+        # try to import and delete it from the namespace
+        import zmq
+        del zmq
+        return
+    except:
+        sys.path.pop(0)
+        pass
+
+    full_path = os.path.join(TREX_STL_EXT_PATH, 'platform/cel59')
+    fix_path = os.path.normcase(full_path)
+    sys.path.insert(0, full_path)
+    try:
+        # try to import and delete it from the namespace
+        import zmq
+        del zmq
+        return
+
+    except:
+        sys.path.pop(0)
+        sys.modules['zmq'] = None
+        warnings.warn("unable to determine platform type for ZMQ import")
+
+        
+import_client_utils_modules()
+
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_jsonrpc_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_jsonrpc_client.py
index 887681a7..ab3c7282 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_jsonrpc_client.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_jsonrpc_client.py
@@ -2,11 +2,11 @@
 
 import zmq
 import json
-import client_utils.general_utils
 import re
 from time import sleep
 from collections import namedtuple
 from trex_stl_types import *
+from utils.common import random_id_gen
 
 class bcolors:
     BLUE = '\033[94m'
@@ -48,7 +48,8 @@ class JsonRpcClient(object):
         # default values
         self.port   = default_port
         self.server = default_server
-        self.id_gen = client_utils.general_utils.random_id_gen()
+
+        self.id_gen = random_id_gen()
 
 
     def get_connection_details (self):
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_packet_builder_interface.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_interface.py
index b6e7c026..b6e7c026 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_packet_builder_interface.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_interface.py
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_packet_builder_scapy.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py
index 74946c12..0811209a 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_packet_builder_scapy.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py
@@ -585,7 +585,7 @@ class CTRexVmDescTupleGen(CTRexVmDescBase):
 
 ################################################################################################
 
-lass CScapyTRexPktBuilder(CTrexPktBuilderInterface):
+class CScapyTRexPktBuilder(CTrexPktBuilderInterface):
 
     """
     This class defines the TRex API of building a packet using dpkt package.
@@ -794,11 +794,11 @@ lass CScapyTRexPktBuilder(CTrexPktBuilderInterface):
         return p_utl.get_field_offet_by_str(field_name)
 
     def _get_pkt_as_str(self):
-        if self.pkt :
-            str(self.pkt)
+        if self.pkt:
+            return str(self.pkt)
         if self.pkt_raw:
             return self.pkt_raw
-        raise CTRexPacketBuildException(-11,('empty packet') % (var_name) );  
+        raise CTRexPacketBuildException(-11, 'empty packet');  
 
     def _add_tuple_gen(self,tuple_gen):
 
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_port.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py
index b2cf1c90..b2cf1c90 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_port.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_sim.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py
index d61e04bf..1252b752 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_sim.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py
@@ -16,10 +16,13 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+# simulator can be run as a standalone
+import trex_stl_ext
+
 from trex_stl_exceptions import *
 from yaml import YAMLError
 from trex_stl_streams import *
-from client_utils import parsing_opts
+from utils import parsing_opts
 from trex_stl_client import STLClient
 
 import re
@@ -142,6 +145,29 @@ class STLSim(object):
 
         # load streams
         cmds_json = []
+
+        id = 1
+
+        lookup = {}
+        # allocate IDs
+        for stream in stream_list:
+            if stream.get_id() == None:
+                stream.set_id(id)
+                id += 1
+
+            lookup[stream.get_name()] = stream.get_id()
+
+        # resolve names
+        for stream in stream_list:
+            next_id = -1
+            next = stream.get_next()
+            if next:
+                if not next in lookup:
+                    raise STLError("stream dependency error - unable to find '{0}'".format(next))
+                next_id = lookup[next]
+
+            stream.fields['next_stream_id'] = next_id
+
         for stream in stream_list:
             cmd = {"id":1,
                    "jsonrpc": "2.0",
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_stats.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
index f880a914..3f09e47c 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_stats.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
@@ -1,7 +1,7 @@
 #!/router/bin/python
 
-from client_utils import text_tables
-from common.text_opts import format_text, format_threshold, format_num
+from utils import text_tables
+from utils.text_opts import format_text, format_threshold, format_num
 
 from trex_stl_async_client import CTRexAsyncStats
 
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_std.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_std.py
index 72a5ea52..72a5ea52 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_std.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_std.py
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_streams.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py
index 798b2f67..d8e86fef 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_streams.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py
@@ -5,8 +5,6 @@ from trex_stl_packet_builder_interface import CTrexPktBuilderInterface
 from trex_stl_packet_builder_scapy import CScapyTRexPktBuilder, Ether, IP
 from collections import OrderedDict, namedtuple
 
-from trex_control_plane.client_utils.yaml_utils import *
-
 from dpkt import pcap
 import random
 import yaml
@@ -143,6 +141,7 @@ class STLStream(object):
 
         # packet builder
         packet.compile()
+
         # packet and VM
         self.fields['packet'] = packet.dump_pkt()
         self.fields['vm']     = packet.get_vm_data()
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_types.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py
index a7ddacea..1164076b 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_types.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py
@@ -1,6 +1,6 @@
 
 from collections import namedtuple
-from common.text_opts import *
+from utils.text_opts import *
 
 RpcCmdData = namedtuple('RpcCmdData', ['method', 'params'])
 
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/__init__.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/__init__.py
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py
new file mode 100644
index 00000000..117017c3
--- /dev/null
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py
@@ -0,0 +1,47 @@
+import os
+import sys
+import string
+import random
+
+try:
+    import pwd
+except ImportError:
+    import getpass
+    pwd = None
+
+using_python_3 = True if sys.version_info.major == 3 else False
+
+def get_current_user():
+  if pwd:
+      return pwd.getpwuid(os.geteuid()).pw_name
+  else:
+      return getpass.getuser()
+
+
+def user_input():
+    if using_python_3:
+        return input()
+    else:
+        # using python version 2
+        return raw_input()
+
+
+def random_id_gen(length=8):
+    """
+    A generator for creating a random chars id of specific length
+
+    :parameters:
+        length : int
+            the desired length of the generated id
+
+            default: 8
+
+    :return:
+        a random id with each next() request.
+    """
+    id_chars = string.ascii_lowercase + string.digits
+    while True:
+        return_id = ''
+        for i in range(length):
+            return_id += random.choice(id_chars)
+        yield return_id
diff --git a/scripts/automation/trex_control_plane/client_utils/parsing_opts.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py
index 968bbb7e..968bbb7e 100755
--- a/scripts/automation/trex_control_plane/client_utils/parsing_opts.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py
new file mode 100644
index 00000000..78a0ab1f
--- /dev/null
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py
@@ -0,0 +1,192 @@
+import json
+import re
+
+TEXT_CODES = {'bold': {'start': '\x1b[1m',
+                       'end': '\x1b[22m'},
+              'cyan': {'start': '\x1b[36m',
+                       'end': '\x1b[39m'},
+              'blue': {'start': '\x1b[34m',
+                       'end': '\x1b[39m'},
+              'red': {'start': '\x1b[31m',
+                      'end': '\x1b[39m'},
+              'magenta': {'start': '\x1b[35m',
+                          'end': '\x1b[39m'},
+              'green': {'start': '\x1b[32m',
+                        'end': '\x1b[39m'},
+              'yellow': {'start': '\x1b[33m',
+                         'end': '\x1b[39m'},
+              'underline': {'start': '\x1b[4m',
+                            'end': '\x1b[24m'}}
+
+class TextCodesStripper:
+    keys = [re.escape(v['start']) for k,v in TEXT_CODES.iteritems()]
+    keys += [re.escape(v['end']) for k,v in TEXT_CODES.iteritems()]
+    pattern = re.compile("|".join(keys))
+
+    @staticmethod
+    def strip (s):
+        return re.sub(TextCodesStripper.pattern, '', s)
+
+def format_num (size, suffix = "", compact = True, opts = ()):
+    txt = "NaN"
+
+    if type(size) == str:
+        return "N/A"
+
+    u = ''
+
+    if compact:
+        for unit in ['','K','M','G','T','P']:
+            if abs(size) < 1000.0:
+                u = unit
+                break
+            size /= 1000.0
+
+    if isinstance(size, float):
+        txt = "%3.2f" % (size)
+    else:
+        txt = "{:,}".format(size)
+
+    if u or suffix:
+        txt += " {:}{:}".format(u, suffix)
+
+    if isinstance(opts, tuple):
+        return format_text(txt, *opts)
+    else:
+        return format_text(txt, (opts))
+
+
+
+def format_time (t_sec):
+    if t_sec < 0:
+        return "infinite"
+
+    if t_sec < 1:
+        # low numbers
+        for unit in ['ms', 'usec', 'ns']:
+            t_sec *= 1000.0
+            if t_sec >= 1.0:
+                return '{:,.2f} [{:}]'.format(t_sec, unit)
+
+        return "NaN"
+
+    else:
+        # seconds
+        if t_sec < 60.0:
+            return '{:,.2f} [{:}]'.format(t_sec, 'sec')
+
+        # minutes
+        t_sec /= 60.0
+        if t_sec < 60.0:
+            return '{:,.2f} [{:}]'.format(t_sec, 'minutes')
+
+        # hours
+        t_sec /= 60.0
+        if t_sec < 24.0:
+            return '{:,.2f} [{:}]'.format(t_sec, 'hours')
+
+        # days
+        t_sec /= 24.0
+        return '{:,.2f} [{:}]'.format(t_sec, 'days')
+
+
+def format_percentage (size):
+    return "%0.2f %%" % (size)
+
+def bold(text):
+    return text_attribute(text, 'bold')
+
+
+def cyan(text):
+    return text_attribute(text, 'cyan')
+
+
+def blue(text):
+    return text_attribute(text, 'blue')
+
+
+def red(text):
+    return text_attribute(text, 'red')
+
+
+def magenta(text):
+    return text_attribute(text, 'magenta')
+
+
+def green(text):
+    return text_attribute(text, 'green')
+
+def yellow(text):
+    return text_attribute(text, 'yellow')
+
+def underline(text):
+    return text_attribute(text, 'underline')
+
+
+def text_attribute(text, attribute):
+    if isinstance(text, str):
+        return "{start}{txt}{stop}".format(start=TEXT_CODES[attribute]['start'],
+                                           txt=text,
+                                           stop=TEXT_CODES[attribute]['end'])
+    elif isinstance(text, unicode):
+        return u"{start}{txt}{stop}".format(start=TEXT_CODES[attribute]['start'],
+                                            txt=text,
+                                            stop=TEXT_CODES[attribute]['end'])
+    else:
+        raise Exception("not a string")
+
+
+FUNC_DICT = {'blue': blue,
+             'bold': bold,
+             'green': green,
+             'yellow': yellow,
+             'cyan': cyan,
+             'magenta': magenta,
+             'underline': underline,
+             'red': red}
+
+
+def format_text(text, *args):
+    return_string = text
+    for i in args:
+        func = FUNC_DICT.get(i)
+        if func:
+            return_string = func(return_string)
+
+    return return_string
+
+
+def format_threshold (value, red_zone, green_zone):
+    if value >= red_zone[0] and value <= red_zone[1]:
+        return format_text("{0}".format(value), 'red')
+
+    if value >= green_zone[0] and value <= green_zone[1]:
+        return format_text("{0}".format(value), 'green')
+
+    return "{0}".format(value)
+
+# pretty print for JSON
+def pretty_json (json_str, use_colors = True):
+    pretty_str = json.dumps(json.loads(json_str), indent = 4, separators=(',', ': '), sort_keys = True)
+
+    if not use_colors:
+        return pretty_str
+
+    try:
+        # int numbers
+        pretty_str = re.sub(r'([ ]*:[ ]+)(\-?[1-9][0-9]*[^.])',r'\1{0}'.format(blue(r'\2')), pretty_str)
+        # float
+        pretty_str = re.sub(r'([ ]*:[ ]+)(\-?[1-9][0-9]*\.[0-9]+)',r'\1{0}'.format(magenta(r'\2')), pretty_str)
+        # # strings
+        #
+        pretty_str = re.sub(r'([ ]*:[ ]+)("[^"]*")',r'\1{0}'.format(red(r'\2')), pretty_str)
+        pretty_str = re.sub(r"('[^']*')", r'{0}\1{1}'.format(TEXT_CODES['magenta']['start'],
+                                                             TEXT_CODES['red']['start']), pretty_str)
+    except :
+        pass
+
+    return pretty_str
+
+
+if __name__ == "__main__":
+    pass
diff --git a/scripts/automation/trex_control_plane/client_utils/text_tables.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_tables.py
index 6b52a4a9..07753fda 100644
--- a/scripts/automation/trex_control_plane/client_utils/text_tables.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_tables.py
@@ -1,5 +1,5 @@
 from texttable import Texttable
-from trex_control_plane.common.text_opts import format_text
+from text_opts import format_text
 
 class TRexTextTable(Texttable):