Merge GA to Python API

23 files changed, 137 insertions, 52 deletions

diff --git a/scripts/automation/regression/outer_packages.py b/scripts/automation/regression/outer_packages.py
index bec9fe21..61ddc5cd 100755
--- a/scripts/automation/regression/outer_packages.py
+++ b/scripts/automation/regression/outer_packages.py
@@ -9,6 +9,7 @@ if not TREX_PATH or not os.path.isfile('%s/trex_daemon_server' % TREX_PATH):
     TREX_PATH       = os.path.abspath(os.path.join(CURRENT_PATH, os.pardir, os.pardir))
 PATH_TO_PYTHON_LIB  = os.path.abspath(os.path.join(TREX_PATH, 'external_libs'))
 PATH_TO_CTRL_PLANE  = os.path.abspath(os.path.join(TREX_PATH, 'automation', 'trex_control_plane')) 
+PATH_STF_API        = os.path.abspath(os.path.join(PATH_TO_CTRL_PLANE, 'stf')) 
 PATH_STL_API        = os.path.abspath(os.path.join(PATH_TO_CTRL_PLANE, 'stl')) 
 
 
@@ -57,8 +58,9 @@ def import_module_list(modules_list):
 
 def import_nightly_modules ():
     sys.path.append(TREX_PATH)
-    sys.path.append(PATH_TO_CTRL_PLANE)
+    #sys.path.append(PATH_TO_CTRL_PLANE)
     sys.path.append(PATH_STL_API)
+    sys.path.append(PATH_STF_API)
     import_module_list(NIGHTLY_MODULES)
 
 
diff --git a/scripts/automation/regression/trex_unit_test.py b/scripts/automation/regression/trex_unit_test.py
index fb666382..3b1a25f0 100755
--- a/scripts/automation/regression/trex_unit_test.py
+++ b/scripts/automation/regression/trex_unit_test.py
@@ -36,8 +36,8 @@ import misc_methods
 from rednose import RedNose
 import termstyle
 from trex import CTRexScenario
-from stf.trex_client import *
-from stf.trex_exceptions import *
+from trex_stf_lib.trex_client import *
+from trex_stf_lib.trex_exceptions import *
 from trex_stl_lib.api import *
 import trex
 import socket
@@ -67,10 +67,10 @@ STATEFUL_STOP_COMMAND = './trex_daemon_server stop; sleep 1; ./trex_daemon_serve
 STATEFUL_RUN_COMMAND = 'rm /var/log/trex/trex_daemon_server.log; ./trex_daemon_server start; sleep 2; ./trex_daemon_server show'
 TREX_FILES = ('_t-rex-64', '_t-rex-64-o', '_t-rex-64-debug', '_t-rex-64-debug-o')
 
-def trex_remote_command(trex_data, command, background = False, from_scripts = True):
+def trex_remote_command(trex_data, command, background = False, from_scripts = True, timeout = 20):
     if from_scripts:
-        return misc_methods.run_remote_command(trex_data['trex_name'], ('cd %s; ' % CTRexScenario.scripts_path)+ command, background)
-    return misc_methods.run_remote_command(trex_data['trex_name'], command, background)
+        return misc_methods.run_remote_command(trex_data['trex_name'], ('cd %s; ' % CTRexScenario.scripts_path)+ command, background, timeout)
+    return misc_methods.run_remote_command(trex_data['trex_name'], command, background, timeout)
 
 # 1 = running, 0 - not running
 def check_trex_running(trex_data):
@@ -183,7 +183,7 @@ class CTRexTestConfiguringPlugin(Plugin):
             new_path = '/tmp/trex-scripts'
             rsync_template = 'rm -rf /tmp/trex-scripts; mkdir -p %s; rsync -Lc %s /tmp; tar -mxzf /tmp/%s -C %s; mv %s/v*.*/* %s'
             rsync_command = rsync_template % (new_path, self.pkg, os.path.basename(self.pkg), new_path, new_path, new_path)
-            return_code, stdout, stderr = trex_remote_command(self.configuration.trex, rsync_command, from_scripts = False)
+            return_code, stdout, stderr = trex_remote_command(self.configuration.trex, rsync_command, from_scripts = False, timeout = 300)
             if return_code:
                 print('Failed copying')
                 sys.exit(-1)
@@ -339,8 +339,9 @@ if __name__ == "__main__":
     finally:
         save_setup_info()
 
-    if (result == True and not CTRexScenario.is_test_list):
-        print(termstyle.green("""
+    if not CTRexScenario.is_test_list:
+        if result == True:
+            print(termstyle.green("""
                  ..::''''::..
                .;''        ``;.
               ::    ::  ::    ::
@@ -358,8 +359,18 @@ if __name__ == "__main__":
             /_/  /_/ |_/___/___(_)
 
         """))
-        sys.exit(0)
-    sys.exit(-1)
+            sys.exit(0)
+        else:
+            print(termstyle.red("""
+           /\_/\ 
+          ( o.o ) 
+           > ^ < 
+
+This cat is sad, test failed.
+        """))
+            sys.exit(-1)
+
+
 
 
 
diff --git a/scripts/automation/trex_control_plane/doc/conf.py b/scripts/automation/trex_control_plane/doc/conf.py
index a2641ffc..ec133a1c 100755
--- a/scripts/automation/trex_control_plane/doc/conf.py
+++ b/scripts/automation/trex_control_plane/doc/conf.py
@@ -20,7 +20,7 @@ import shlex
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-sys.path.insert(0, os.path.abspath('../stf'))
+sys.path.insert(0, os.path.abspath('../stf/trex_stf_lib'))
 sys.path.insert(0, os.path.abspath('../client_utils'))
 sys.path.insert(0, os.path.abspath('../examples'))
 sys.path.insert(0, os.path.abspath('../common'))
diff --git a/scripts/automation/trex_control_plane/stf/examples/stf_example.py b/scripts/automation/trex_control_plane/stf/examples/stf_example.py
new file mode 100755
index 00000000..f6ebffe7
--- /dev/null
+++ b/scripts/automation/trex_control_plane/stf/examples/stf_example.py
@@ -0,0 +1,53 @@
+import argparse
+import stf_path
+from trex_stf_lib.trex_client import CTRexClient
+
+# sample TRex stateless run
+# assuming server daemon is running.
+
+def minimal_stateful_test(server):
+    print('Connecting to %s' % server)
+    trex_client = CTRexClient(server)
+
+    print('Connected, start TRex')
+    trex_client.start_trex(
+            c = 1,
+            m = 700,
+            f = 'cap2/http_simple.yaml',
+            d = 5,
+            l = 1000,
+            trex_development = True,
+            )
+
+    print('Sample until end')
+    result = trex_client.sample_to_run_finish()
+
+    print('Test results:')
+    print(result)
+
+    print('TX by ports:')
+    tx_ptks_dict = result.get_last_value('trex-global.data', 'opackets-*')
+    print('  |  '.join(['%s: %s' % (k.split('-')[-1], tx_ptks_dict[k]) for k in sorted(tx_ptks_dict.keys())]))
+
+    print('RX by ports:')
+    rx_ptks_dict = result.get_last_value('trex-global.data', 'ipackets-*')
+    print('  |  '.join(['%s: %s' % (k.split('-')[-1], rx_ptks_dict[k]) for k in sorted(rx_ptks_dict.keys())]))
+
+    print('CPU utilization:')
+    print(result.get_value_list('trex-global.data.m_cpu_util'))
+
+    #print('Dump of *latest* result sample, uncomment to see it all')
+    #print(result.get_latest_dump())
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description="Example for TRex Stateful, assuming server daemon is running.")
+    parser.add_argument('-s', '--server',
+                        dest='server',
+                        help='Remote trex address',
+                        default='127.0.0.1',
+                        type = str)
+    args = parser.parse_args()
+
+    minimal_stateful_test(args.server)
+
diff --git a/scripts/automation/trex_control_plane/stf/examples/stf_path.py b/scripts/automation/trex_control_plane/stf/examples/stf_path.py
new file mode 100755
index 00000000..bb401148
--- /dev/null
+++ b/scripts/automation/trex_control_plane/stf/examples/stf_path.py
@@ -0,0 +1,4 @@
+import sys
+
+# FIXME to the write path for trex_stf_lib 
+sys.path.insert(0, "../")
diff --git a/scripts/automation/trex_control_plane/stf/CCustomLogger.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/CCustomLogger.py
index ecf7d519..ecf7d519 100755
--- a/scripts/automation/trex_control_plane/stf/CCustomLogger.py
+++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/CCustomLogger.py
diff --git a/scripts/automation/trex_control_plane/stf/__init__.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/__init__.py
index 5a1da046..5a1da046 100755
--- a/scripts/automation/trex_control_plane/stf/__init__.py
+++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/__init__.py
diff --git a/scripts/automation/trex_control_plane/stf/external_packages.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/external_packages.py
index 7353c397..7353c397 100755
--- a/scripts/automation/trex_control_plane/stf/external_packages.py
+++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/external_packages.py
diff --git a/scripts/automation/trex_control_plane/stf/general_utils.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/general_utils.py
index d2521f02..d2521f02 100755
--- a/scripts/automation/trex_control_plane/stf/general_utils.py
+++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/general_utils.py
diff --git a/scripts/automation/trex_control_plane/stf/outer_packages.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/outer_packages.py
index 5e29f8d6..f8d50ce6 100755
--- a/scripts/automation/trex_control_plane/stf/outer_packages.py
+++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/outer_packages.py
@@ -5,8 +5,8 @@ import os
 
 
 CURRENT_PATH = os.path.dirname(os.path.realpath(__file__))
-PARENT_PATH  = os.path.abspath(os.path.join(CURRENT_PATH, os.pardir, 'external_libs'))
-SCRIPTS_PATH = os.path.abspath(os.path.join(CURRENT_PATH, os.pardir, os.pardir, os.pardir, 'external_libs'))
+PACKAGE_PATH  = os.path.abspath(os.path.join(CURRENT_PATH, os.pardir, os.pardir, 'external_libs'))
+SCRIPTS_PATH = os.path.abspath(os.path.join(CURRENT_PATH, os.pardir, os.pardir, os.pardir, os.pardir, 'external_libs'))
 
 CLIENT_MODULES = ['enum34-1.0.4',
                   'jsonrpclib-pelix-0.2.5',
@@ -22,9 +22,9 @@ def import_module_list(ext_libs_path):
             raise Exception('Library %s is absent in path %s' % (p, ext_libs_path))
         sys.path.insert(1, full_path)
 
-if os.path.exists(PARENT_PATH):
-    import_module_list(PARENT_PATH)
+if os.path.exists(PACKAGE_PATH):
+    import_module_list(PACKAGE_PATH)
 elif os.path.exists(SCRIPTS_PATH):
     import_module_list(SCRIPTS_PATH)
 else:
-    raise Exception('Could not find external libs in path: %s' % [PARENT_PATH, SCRIPTS_PATH])
+    raise Exception('Could not find external libs in path: %s' % [PACKAGE_PATH, SCRIPTS_PATH])
diff --git a/scripts/automation/trex_control_plane/stf/text_opts.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/text_opts.py
index 78a0ab1f..78a0ab1f 100755
--- a/scripts/automation/trex_control_plane/stf/text_opts.py
+++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/text_opts.py
diff --git a/scripts/automation/trex_control_plane/stf/trex_client.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py
index 919253d1..919253d1 100755
--- a/scripts/automation/trex_control_plane/stf/trex_client.py
+++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py
diff --git a/scripts/automation/trex_control_plane/stf/trex_daemon_server.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_daemon_server.py
index 9784d42a..9784d42a 100755
--- a/scripts/automation/trex_control_plane/stf/trex_daemon_server.py
+++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_daemon_server.py
diff --git a/scripts/automation/trex_control_plane/stf/trex_exceptions.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_exceptions.py
index 0de38411..0de38411 100755
--- a/scripts/automation/trex_control_plane/stf/trex_exceptions.py
+++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_exceptions.py
diff --git a/scripts/automation/trex_control_plane/stf/trex_status.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_status.py
index f132720c..f132720c 100644
--- a/scripts/automation/trex_control_plane/stf/trex_status.py
+++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_status.py
diff --git a/scripts/automation/trex_control_plane/stf/trex_status_e.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_status_e.py
index 79a25acc..79a25acc 100755
--- a/scripts/automation/trex_control_plane/stf/trex_status_e.py
+++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_status_e.py
diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_imix.py b/scripts/automation/trex_control_plane/stl/examples/stl_imix.py
index 56fd3cfd..7e43488b 100644
--- a/scripts/automation/trex_control_plane/stl/examples/stl_imix.py
+++ b/scripts/automation/trex_control_plane/stl/examples/stl_imix.py
@@ -12,7 +12,7 @@ import argparse
 # and attach it to both sides and inject
 # at a certain rate for some time
 # finally it checks that all packets arrived
-def imix_test (server):
+def imix_test (server, mult):
     
 
     # create client
@@ -37,7 +37,8 @@ def imix_test (server):
         print("Mapped ports to sides {0} <--> {1}".format(dir_0, dir_1))
 
         # load IMIX profile
-        profile = STLProfile.load_py('../../../../stl/imix.py')
+        profile_file = os.path.join(stl_path.STL_PROFILES_PATH, 'imix.py')
+        profile = STLProfile.load_py(profile_file)
         streams = profile.get_streams()
 
         # add both streams to ports
@@ -47,9 +48,8 @@ def imix_test (server):
         # clear the stats before injecting
         c.clear_stats()
 
-        # choose rate and start traffic for 10 seconds on 5 mpps
+        # choose rate and start traffic for 10 seconds
         duration = 10
-        mult = "30%"
         print("Injecting {0} <--> {1} on total rate of '{2}' for {3} seconds".format(dir_0, dir_1, mult, duration))
 
         c.start(ports = (dir_0 + dir_1), mult = mult, duration = duration, total = True)
@@ -107,8 +107,13 @@ parser.add_argument('-s', '--server',
                     help='Remote trex address',
                     default='127.0.0.1',
                     type = str)
+parser.add_argument('-m', '--mult',
+                    dest='mult',
+                    help='Multiplier of traffic, see Stateless help for more info',
+                    default='30%',
+                    type = str)
 args = parser.parse_args()
 
 # run the tests
-imix_test(args.server)
+imix_test(args.server, args.mult)
 
diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_imix_bidir.py b/scripts/automation/trex_control_plane/stl/examples/stl_imix_bidir.py
index 05a8777b..05615aeb 100644
--- a/scripts/automation/trex_control_plane/stl/examples/stl_imix_bidir.py
+++ b/scripts/automation/trex_control_plane/stl/examples/stl_imix_bidir.py
@@ -29,14 +29,15 @@ def imix_test (server):
         # take all the ports
         c.reset()
 
-    	dir_0 = [0]
-    	dir_1 = [1]
+        dir_0 = [0]
+        dir_1 = [1]
 
         print "Mapped ports to sides {0} <--> {1}".format(dir_0, dir_1)
 
         # load IMIX profile
-        profile1 = STLProfile.load_py('../../../../stl/imix.py', direction=0)
-        profile2 = STLProfile.load_py('../../../../stl/imix.py', direction=1)
+        profile_file = os.path.join(stl_path.STL_PROFILES_PATH, 'imix.py')
+        profile1 = STLProfile.load_py(profile_file, direction=0)
+        profile2 = STLProfile.load_py(profile_file, direction=1)
         stream1 = profile1.get_streams()
         stream2 = profile2.get_streams()
 
diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_path.py b/scripts/automation/trex_control_plane/stl/examples/stl_path.py
index 8f400d23..f1592571 100644
--- a/scripts/automation/trex_control_plane/stl/examples/stl_path.py
+++ b/scripts/automation/trex_control_plane/stl/examples/stl_path.py
@@ -1,4 +1,7 @@
-import sys
+import sys, os
 
 # FIXME to the write path for trex_stl_lib 
 sys.path.insert(0, "../")
+
+STL_PROFILES_PATH = os.path.join(os.pardir, os.pardir, os.pardir, os.pardir, 'stl')
+
diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_profile.py b/scripts/automation/trex_control_plane/stl/examples/stl_profile.py
index 3ae5f855..16d5238e 100644
--- a/scripts/automation/trex_control_plane/stl/examples/stl_profile.py
+++ b/scripts/automation/trex_control_plane/stl/examples/stl_profile.py
@@ -18,8 +18,7 @@ def simple ():
 
         # prepare our ports
         c.reset(ports = my_ports)
-
-        profile_file =   "../../../../stl/udp_1pkt_simple.py"
+        profile_file = os.path.join(stl_path.STL_PROFILES_PATH, 'hlt', 'udp_1pkt_simple.py')
 
         try:
             profile = STLProfile.load(profile_file)
diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_simple_console_like.py b/scripts/automation/trex_control_plane/stl/examples/stl_simple_console_like.py
index 03909e65..1d4ef250 100644
--- a/scripts/automation/trex_control_plane/stl/examples/stl_simple_console_like.py
+++ b/scripts/automation/trex_control_plane/stl/examples/stl_simple_console_like.py
@@ -29,9 +29,10 @@ def simple ():
         print(c.get_port_info(my_ports))
 
         c.ping()
+        profile_file = os.path.join(stl_path.STL_PROFILES_PATH, 'udp_1pkt_simple.py')
 
         print("start")
-        c.start_line (" -f ../../../../stl/udp_1pkt_simple.py -m 10mpps --port 0 1 ")
+        c.start_line (" -f %s -m 10mpps --port 0 1 " % profile_file)
         time.sleep(2);
         c.pause_line("--port 0 1");
         time.sleep(2);
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_ext.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_ext.py
index d6d66ec3..ed0c393d 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_ext.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_ext.py
@@ -9,7 +9,7 @@ TREX_STL_EXT_PATH = os.environ.get('TREX_STL_EXT_PATH')
 # take default
 if not TREX_STL_EXT_PATH or not os.path.exists(TREX_STL_EXT_PATH):
     CURRENT_PATH        = os.path.dirname(os.path.realpath(__file__))
-    TREX_STL_EXT_PATH  = os.path.normpath(os.path.join(CURRENT_PATH, os.pardir, 'external_libs'))
+    TREX_STL_EXT_PATH  = os.path.normpath(os.path.join(CURRENT_PATH, os.pardir, os.pardir, 'external_libs'))
 if not os.path.exists(TREX_STL_EXT_PATH):
     # ../../../../external_libs
     TREX_STL_EXT_PATH   = os.path.normpath(os.path.join(CURRENT_PATH, os.pardir, os.pardir, os.pardir, os.pardir, 'external_libs'))
diff --git a/scripts/run_functional_tests b/scripts/run_functional_tests
index 6212b40b..9ec1bd39 100755
--- a/scripts/run_functional_tests
+++ b/scripts/run_functional_tests
@@ -1,30 +1,36 @@
 #!/bin/bash
 
-source find_python.sh --python2
-cd automation/regression
 
-# Python 2
-echo Python2 test
-$PYTHON trex_unit_test.py --functional $@
-if [ $? -eq 0 ]; then
-    printf "\n$PYTHON test succeeded\n\n"
-else
-    printf "\n*** $PYTHON test failed\n\n"
-    exit -1
+if [[ $@ =~ '--python2' || ! $@ =~ '--python3' ]]; then
+    source find_python.sh --python2
+    cd automation/regression
+
+    # Python 2
+    echo Python2 test
+    $PYTHON trex_unit_test.py --functional $@
+    if [ $? -eq 0 ]; then
+        printf "\n$PYTHON test succeeded\n\n"
+    else
+        printf "\n*** $PYTHON test failed\n\n"
+        exit -1
+    fi
+    cd -
 fi
 
-cd -
-source find_python.sh --python3
-cd -
+if [[ $@ =~ '--python3' || ! $@ =~ '--python2' ]]; then
+    source find_python.sh --python3
+    cd automation/regression
 
-# Python 3
-echo Python3 test
-$PYTHON trex_unit_test.py --functional $@
-if [ $? -eq 0 ]; then
-    printf "\n$PYTHON test succeeded\n\n"
-else
-    printf "\n*** $PYTHON test failed\n\n"
-    exit -1
+    # Python 3
+    echo Python3 test
+    $PYTHON trex_unit_test.py --functional $@
+    if [ $? -eq 0 ]; then
+        printf "\n$PYTHON test succeeded\n\n"
+    else
+        printf "\n*** $PYTHON test failed\n\n"
+        exit -1
+    fi
+    cd -
 fi