regression: stl updates

7 files changed, 80 insertions, 39 deletions

diff --git a/scripts/automation/regression/aggregate_results.py b/scripts/automation/regression/aggregate_results.py
index ea782ccf..31929d50 100755
--- a/scripts/automation/regression/aggregate_results.py
+++ b/scripts/automation/regression/aggregate_results.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 import xml.etree.ElementTree as ET
+import outer_packages
 import argparse
 import glob
 from pprint import pprint
@@ -9,6 +10,13 @@ import copy
 import datetime, time
 import cPickle as pickle
 import subprocess, shlex
+from ansi2html import Ansi2HTMLConverter
+
+converter = Ansi2HTMLConverter(inline = True)
+convert = converter.convert
+
+def ansi2html(text):
+    return convert(text, full = False)
 
 FUNCTIONAL_CATEGORY = 'Functional' # how to display those categories
 ERROR_CATEGORY = 'Error'
@@ -69,7 +77,7 @@ def add_category_of_tests(category, tests, tests_type = None, category_info_dir
     if is_actual_category:
         html_output += '<br><table class="reference">\n'
         
-        if category_info_dir and tests_type != 'stateless':
+        if category_info_dir:
             category_info_file = '%s/report_%s.info' % (category_info_dir, category)
             if os.path.exists(category_info_file):
                 with open(category_info_file) as f:
@@ -91,7 +99,7 @@ def add_category_of_tests(category, tests, tests_type = None, category_info_dir
         html_output += '</table>\n'
 
     if not len(tests):
-        return html_output + pad_tag('<br><font color=red>No tests!</font>', 'b') + '</div>'
+        return html_output + pad_tag('<br><font color=red>No tests!</font>', 'b')
     html_output += '<br>\n<table class="reference" width="100%">\n<tr><th align="left">'
 
     if category == ERROR_CATEGORY:
@@ -137,12 +145,15 @@ def add_category_of_tests(category, tests, tests_type = None, category_info_dir
             start_index_errors = result_text.find('Exception: The test is failed, reasons:')
             if start_index_errors > 0:
                 result_text = result_text[start_index_errors + 10:].strip() # cut traceback
+            result_text = ansi2html(result_text)
             result_text = '<b style="color:000080;">%s:</b><br>%s<br><br>' % (result.capitalize(), result_text.replace('\n', '<br>'))
         stderr = '' if brief and result_text else test.get('stderr', '')
         if stderr:
+            stderr = ansi2html(stderr)
             stderr = '<b style="color:000080;"><text color=000080>Stderr</text>:</b><br>%s<br><br>\n' % stderr.replace('\n', '<br>')
         stdout = '' if brief and result_text else test.get('stdout', '')
         if stdout:
+            stdout = ansi2html(stdout)
             if brief: # cut off server logs
                 stdout = stdout.split('>>>>>>>>>>>>>>>', 1)[0]
             stdout = '<b style="color:000080;">Stdout:</b><br>%s<br><br>\n' % stdout.replace('\n', '<br>')
@@ -453,10 +464,10 @@ if __name__ == '__main__':
     # Setups tests
     for category, tests in setups.items():
         html_output += '<div style="display:none;" id="cat_tglr_%s">' % category
-        if tests.get('stateful'):
+        if 'stateful' in tests:
             html_output += add_category_of_tests(category, tests['stateful'], 'stateful', category_info_dir=args.input_dir)
-        if tests.get('stateless'):
-            html_output += add_category_of_tests(category, tests['stateless'], 'stateless', category_info_dir=args.input_dir)
+        if 'stateless' in tests:
+            html_output += add_category_of_tests(category, tests['stateless'], 'stateless', category_info_dir=(None if 'stateful' in tests else args.input_dir))
         html_output += '</div>'
     # Functional tests
     if len(functional_tests):
diff --git a/scripts/automation/regression/misc_methods.py b/scripts/automation/regression/misc_methods.py
index b5cf79e0..6830be01 100755
--- a/scripts/automation/regression/misc_methods.py
+++ b/scripts/automation/regression/misc_methods.py
@@ -20,22 +20,28 @@ def mix_string (str):
     return str.replace(' ', '_').lower()
 
 # executes given command, returns tuple (return_code, stdout, stderr)
-def run_command(cmd):
-    print 'Running command:', cmd
-    proc = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    (stdout, stderr) = proc.communicate()
-    if stdout:
-        print 'Stdout:\n%s' % stdout
-    if proc.returncode:
-        if stderr:
-            print 'Stderr:\n%s' % stderr
-        print 'Return code: %s' % proc.returncode
-    return (proc.returncode, stdout, stderr)
-
-
-def run_remote_command(host, command_string):
+def run_command(cmd, background = False):
+    if background:
+        print 'Running command in background:', cmd
+        with open(os.devnull, 'w') as tempf:
+            subprocess.Popen(shlex.split(cmd), stdin=tempf, stdout=tempf, stderr=tempf)
+        return (None,)*3
+    else:
+        print 'Running command:', cmd
+        proc = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        (stdout, stderr) = proc.communicate()
+        if stdout:
+            print 'Stdout:\n%s' % stdout
+        if proc.returncode:
+            if stderr:
+                print 'Stderr:\n%s' % stderr
+            print 'Return code: %s' % proc.returncode
+        return (proc.returncode, stdout, stderr)
+
+
+def run_remote_command(host, command_string, background = False):
     cmd = 'ssh -tt %s \'sudo sh -c "%s"\'' % (host, command_string)
-    return run_command(cmd)
+    return run_command(cmd, background)
 
 
 def generate_intf_lists (interfacesList):
diff --git a/scripts/automation/regression/outer_packages.py b/scripts/automation/regression/outer_packages.py
index 6b7c58f9..f55c247d 100755
--- a/scripts/automation/regression/outer_packages.py
+++ b/scripts/automation/regression/outer_packages.py
@@ -11,7 +11,8 @@ PATH_TO_PYTHON_LIB  = os.path.abspath(os.path.join(TREX_PATH, 'external_libs'))
 PATH_TO_CTRL_PLANE  = os.path.abspath(os.path.join(TREX_PATH, 'automation', 'trex_control_plane')) 
 PATH_STL_API        = os.path.abspath(os.path.join(PATH_TO_CTRL_PLANE, 'stl')) 
 
-NIGHTLY_MODULES = ['enum34-1.0.4',
+NIGHTLY_MODULES = ['ansi2html',
+                   'enum34-1.0.4',
                    'nose-1.3.4',
                    'rednose-0.4.1',
                    'progressbar-2.2',
diff --git a/scripts/automation/regression/trex.py b/scripts/automation/regression/trex.py
index 993c5be3..8efa41f6 100644
--- a/scripts/automation/regression/trex.py
+++ b/scripts/automation/regression/trex.py
@@ -21,6 +21,8 @@ class CTRexScenario:
     configuration    = None
     trex             = None
     stl_trex         = None
+    stl_ports_map    = None
+    stl_init_error   = None
     router           = None
     router_cfg       = None
     daemon_log_lines = 0
diff --git a/scripts/automation/regression/trex_unit_test.py b/scripts/automation/regression/trex_unit_test.py
index 753fc733..1bb25077 100755
--- a/scripts/automation/regression/trex_unit_test.py
+++ b/scripts/automation/regression/trex_unit_test.py
@@ -66,8 +66,8 @@ STATEFUL_STOP_COMMAND = './trex_daemon_server stop; sleep 1; ./trex_daemon_serve
 STATEFUL_RUN_COMMAND = 'rm /var/log/trex/trex_daemon_server.log; ./trex_daemon_server start; sleep 2; ./trex_daemon_server show'
 TREX_FILES = ('_t-rex-64', '_t-rex-64-o', '_t-rex-64-debug', '_t-rex-64-debug-o')
 
-def trex_remote_command(trex_data, command):
-    return misc_methods.run_remote_command(trex_data['trex_name'], ('cd %s; ' % CTRexScenario.scripts_path)+ command)
+def trex_remote_command(trex_data, command, background = False):
+    return misc_methods.run_remote_command(trex_data['trex_name'], ('cd %s; ' % CTRexScenario.scripts_path)+ command, background)
 
 # 1 = running, 0 - not running
 def check_trex_running(trex_data):
@@ -89,7 +89,15 @@ def kill_trex_process(trex_data):
                     trex_remote_command(trex_data, 'kill %s' % pid)
             except:
                 continue
-    (return_code, stdout, stderr) = trex_remote_command(trex_data, STATEFUL_STOP_COMMAND)
+
+def address_to_ip(address):
+    for i in range(10):
+        try:
+            return socket.gethostbyname(address)
+        except:
+            continue
+    return socket.gethostbyname(address)
+
 
 class CTRexTestConfiguringPlugin(Plugin):
     def options(self, parser, env = os.environ):
@@ -112,7 +120,7 @@ class CTRexTestConfiguringPlugin(Plugin):
         parser.add_option('--server-logs', '--server_logs', action="store_true", default = False,
                             dest="server_logs",
                             help="Print server side (TRex and trex_daemon) logs per test.")
-        parser.add_option('--kill-running', action="store_true", default = False,
+        parser.add_option('--kill-running', '--kill_running', action="store_true", default = False,
                             dest="kill_running",
                             help="Kills running TRex process on remote server (useful for regression).")
         parser.add_option('--func', '--functional', action="store_true", default = False,
@@ -143,6 +151,7 @@ class CTRexTestConfiguringPlugin(Plugin):
             options.config_path = CTRexScenario.setup_dir
         if options.config_path:
             self.configuration = misc_methods.load_complete_config_file(os.path.join(options.config_path, 'config.yaml'))
+            self.configuration.trex['trex_name'] = address_to_ip(self.configuration.trex['trex_name'])
             self.benchmark = misc_methods.load_benchmark_config_file(os.path.join(options.config_path, 'benchmark.yaml'))
             self.enabled = True
         else:
@@ -177,6 +186,8 @@ class CTRexTestConfiguringPlugin(Plugin):
         # launch TRex daemon on relevant setup
         if not self.no_ssh:
             if self.kill_running:
+                if self.stateful:
+                    trex_remote_command(trex_data, STATEFUL_STOP_COMMAND)
                 kill_trex_process(CTRexScenario.configuration.trex)
                 time.sleep(1)
             elif check_trex_running(CTRexScenario.configuration.trex):
@@ -190,13 +201,10 @@ class CTRexTestConfiguringPlugin(Plugin):
             CTRexScenario.trex = CTRexClient(trex_host = CTRexScenario.configuration.trex['trex_name'], verbose = self.verbose_mode)
         elif self.stateless:
             if not self.no_ssh:
-                trex_remote_command(CTRexScenario.configuration.trex, './t-rex-64 -i&')
+                trex_remote_command(CTRexScenario.configuration.trex, './t-rex-64 -i', background = True)
             CTRexScenario.stl_trex = STLClient(username = 'TRexRegression',
                                                server = CTRexScenario.configuration.trex['trex_name'],
-                                               sync_port = 4501,
-                                               async_port = 4500,
-                                               verbose_level = self.verbose_mode,
-                                               logger = None)
+                                               verbose_level = self.verbose_mode)
         if 'loopback' not in self.modes:
             CTRexScenario.router_cfg = dict(config_dict      = self.configuration.router,
                                             forceImageReload = self.load_image,
@@ -213,6 +221,8 @@ class CTRexTestConfiguringPlugin(Plugin):
             return
         CTRexScenario.is_init = False
         if not self.no_ssh:
+            if self.stateful:
+                trex_remote_command(CTRexScenario.configuration.trex, STATEFUL_STOP_COMMAND)
             kill_trex_process(CTRexScenario.configuration.trex)
 
 
@@ -307,7 +317,7 @@ if __name__ == "__main__":
                 additional_args += ['--with-xunit', xml_arg.replace('.xml', '_stateful.xml')]
             result = result and nose.run(argv = nose_argv + additional_args, addplugins = [red_nose, config_plugin])
         if len(CTRexScenario.test_types['stateless_tests']):
-            additional_args = ['--stl'] + CTRexScenario.test_types['stateless_tests']
+            additional_args = ['--stl', 'stateless_tests/stl_general_test.py:STLBasic_Test.test_connectivity'] + CTRexScenario.test_types['stateless_tests']
             if xml_arg:
                 additional_args += ['--with-xunit', xml_arg.replace('.xml', '_stateless.xml')]
             result = result and nose.run(argv = nose_argv + additional_args, addplugins = [red_nose, config_plugin])
diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_imix.py b/scripts/automation/trex_control_plane/stl/examples/stl_imix.py
index cc7691a3..94165614 100644
--- a/scripts/automation/trex_control_plane/stl/examples/stl_imix.py
+++ b/scripts/automation/trex_control_plane/stl/examples/stl_imix.py
@@ -4,6 +4,7 @@ from trex_stl_lib.api import *
 import time
 import json
 from pprint import pprint
+import argparse
 
 # IMIX test
 # it maps the ports to sides
@@ -11,11 +12,11 @@ from pprint import pprint
 # and attach it to both sides and inject
 # at a certain rate for some time
 # finally it checks that all packets arrived
-def imix_test ():
+def imix_test (server):
     
 
     # create client
-    c = STLClient()
+    c = STLClient(server = server)
     passed = True
 
 
@@ -48,7 +49,7 @@ def imix_test ():
 
         # choose rate and start traffic for 10 seconds on 5 mpps
         duration = 10
-        mult = "5mpps"
+        mult = "30%"
         print "Injecting {0} <--> {1} on total rate of '{2}' for {3} seconds".format(dir_0, dir_1, mult, duration)
 
         c.start(ports = (dir_0 + dir_1), mult = mult, duration = duration, total = True)
@@ -78,9 +79,9 @@ def imix_test ():
         print "Packets injected from {0}: {1:,}".format(dir_1, dir_1_opackets)
 
         print "\npackets lost from {0} --> {1}:   {2:,} pkts".format(dir_0, dir_0, lost_0)
-        print "packets lost from {0} --> {1}:   {2:,} pkts".format(dir_0, dir_0, lost_0)
+        print "packets lost from {0} --> {1}:   {2:,} pkts".format(dir_1, dir_1, lost_1)
 
-        if (lost_0 == 0) and (lost_0 == 0):
+        if (lost_0 <= 0) and (lost_1 <= 0): # less or equal because we might have incoming arps etc.
             passed = True
         else:
             passed = False
@@ -95,10 +96,19 @@ def imix_test ():
 
     if passed:
         print "\nTest has passed :-)\n"
+        sys.exit(0)
     else:
         print "\nTest has failed :-(\n"
+        sys.exit(-1)
 
+parser = argparse.ArgumentParser(description="Example for TRex Stateless, sending IMIX traffic")
+parser.add_argument('-s', '--server',
+                    dest='server',
+                    help='Remote trex address',
+                    default='127.0.0.1',
+                    type = str)
+args = parser.parse_args()
 
 # run the tests
-imix_test()
+imix_test(args.server)
 
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_std.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_std.py
index e0b25b1d..c4cb5d01 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_std.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_std.py
@@ -48,7 +48,6 @@ def stl_map_ports (client, ports = None):
                 tx_port = tx_pkts[pkts]
                 table['map'][port] = tx_port
 
-
     unmapped = list(ports)
     while len(unmapped) > 0:
         port_a = unmapped.pop(0)
@@ -57,7 +56,9 @@ def stl_map_ports (client, ports = None):
         # if unknown - add to the unknown list
         if port_b == None:
             table['unknown'].append(port_a)
-
+        # self-loop, due to bug?
+        elif port_a == port_b:
+            continue
         # bi-directional ports
         elif (table['map'][port_b] == port_a):
             unmapped.remove(port_b)