regression and master daemon updates

5 files changed, 69 insertions, 30 deletions

diff --git a/scripts/automation/regression/stateful_tests/trex_general_test.py b/scripts/automation/regression/stateful_tests/trex_general_test.py
index 38e97597..33b954fc 100755
--- a/scripts/automation/regression/stateful_tests/trex_general_test.py
+++ b/scripts/automation/regression/stateful_tests/trex_general_test.py
@@ -274,9 +274,16 @@ class CTRexGeneral_Test(unittest.TestCase):
         #     raise
 
     def unzip_client_package(self):
+        client_pkg_files = glob('%s/trex_client*.tar.gz' % CTRexScenario.scripts_path)
+        if not len(client_pkg_files):
+            raise Exception('Could not find client package')
+        if len(client_pkg_files) > 1:
+            raise Exception('Found more than one client packages')
         if not os.path.exists('%s/trex_client' % CTRexScenario.scripts_path):
             print('\nUnzipping package')
-            CTRexScenario.trex.master_daemon.unpack_client()
+            return_code, _, stderr = misc_methods.run_command("tar -xzf %s -C %s" % (client_pkg_files[0], CTRexScenario.scripts_path))
+            if return_code:
+                raise Exception('Could not untar the client package: %s' % stderr)
         else:
             print('\nClient package is untarred')
 
@@ -304,7 +311,7 @@ class CTRexGeneral_Test(unittest.TestCase):
             self.trex.force_kill(confirm = False)
         if not self.is_loopback:
             print('')
-            if self.trex: # stateful
+            if not self.stl_trex: # stateful
                 self.router.load_clean_config()
             self.router.clear_counters()
             self.router.clear_packet_drop_stats()
diff --git a/scripts/automation/regression/trex_unit_test.py b/scripts/automation/regression/trex_unit_test.py
index d15762bf..f2cf38cb 100755
--- a/scripts/automation/regression/trex_unit_test.py
+++ b/scripts/automation/regression/trex_unit_test.py
@@ -166,6 +166,10 @@ class CTRexTestConfiguringPlugin(Plugin):
         CTRexScenario.server_logs   = self.server_logs
         CTRexScenario.trex          = CTRexClient(trex_host = self.configuration.trex['trex_name'],
                                                   verbose   = self.json_verbose)
+        if not CTRexScenario.trex.check_master_connectivity():
+            print('Could not connect to master daemon')
+            sys.exit(-1)
+        CTRexScenario.scripts_path = CTRexScenario.trex.get_trex_path()
         if options.ga and CTRexScenario.setup_name:
             CTRexScenario.GAManager  = GAmanager(GoogleID       = 'UA-75220362-4',
                                                  UserID         = CTRexScenario.setup_name,
@@ -179,24 +183,21 @@ class CTRexTestConfiguringPlugin(Plugin):
 
     def begin (self):
         if self.pkg and self.kill_running and not CTRexScenario.is_copied:
-            if not CTRexScenario.trex.check_master_connectivity():
-                print('Could not connect to master daemon')
-                sys.exit(-1)
             print('Updating TRex to %s' % self.pkg)
             if not CTRexScenario.trex.master_daemon.update_trex(self.pkg):
                 print('Failed updating TRex')
                 sys.exit(-1)
             else:
                 print('Updated')
-            CTRexScenario.scripts_path = '/tmp/trex-scripts'
             CTRexScenario.is_copied = True
         if self.functional or self.collect_only:
             return
+        print('Restarting TRex daemon server')
         res = CTRexScenario.trex.restart_trex_daemon()
         if not res:
             print('Could not restart TRex daemon server')
             sys.exit(-1)
-        # launch TRex daemon on relevant setup
+
         trex_cmds = CTRexScenario.trex.get_trex_cmds()
         if trex_cmds:
             if self.kill_running:
diff --git a/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py
index 09ee52ef..0aac043e 100755
--- a/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py
+++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py
@@ -354,6 +354,18 @@ class CTRexClient(object):
             self.prompt_verbose_data()
 
 
+    def get_trex_path(self):
+        '''
+        Returns TRex path on server
+        '''
+        try:
+            return self.master_daemon.get_trex_path()
+        except AppError as err:
+            self._handle_AppError_exception(err.args[0])
+        finally:
+            self.prompt_verbose_data()
+        
+
     def wait_until_kickoff_finish(self, timeout = 40):
         """
         Block the client application until TRex changes state from 'Starting' to either 'Idle' or 'Running'
diff --git a/scripts/master_daemon.py b/scripts/master_daemon.py
index b2e61f02..9214cfc8 100755
--- a/scripts/master_daemon.py
+++ b/scripts/master_daemon.py
@@ -24,6 +24,9 @@ def check_connectivity():
 def add(a, b): # for sanity checks
     return a + b
 
+def get_trex_path():
+    return args.trex_dir
+
 def is_trex_daemon_running():
     ret_code, stdout, stderr = run_command('ps -u root --format comm')
     if ret_code:
@@ -43,7 +46,7 @@ def stop_trex_daemon():
         return False
     return_code, stdout, stderr = run_command('%s stop' % trex_daemon_path)
     if return_code:
-        raise Exception('Could not stop trex_daemon_server, err: %s' % stderr)
+        raise Exception('Could not stop trex_daemon_server, %s' % [return_code, stdout, stderr])
     for i in range(50):
         if not is_trex_daemon_running():
             return True
@@ -134,7 +137,7 @@ def run_command(command, timeout = 15):
     command = 'timeout %s %s' % (timeout, command)
     # pipes might stuck, even with timeout
     with tempfile.TemporaryFile() as stdout_file, tempfile.TemporaryFile() as stderr_file:
-        proc = subprocess.Popen(shlex.split(command), stdout=stdout_file, stderr=stderr_file)
+        proc = subprocess.Popen(shlex.split(command), stdout = stdout_file, stderr = stderr_file, cwd = args.trex_dir)
         proc.wait()
         stdout_file.seek(0)
         stderr_file.seek(0)
@@ -170,6 +173,7 @@ def start_master_daemon_func():
     print('Started master daemon (port %s)' % args.daemon_port)
     server.register_function(add)
     server.register_function(check_connectivity)
+    server.register_function(get_trex_path)
     server.register_function(is_trex_daemon_running)
     server.register_function(restart_trex_daemon)
     server.register_function(start_trex_daemon)
@@ -264,6 +268,7 @@ if os.path.isfile(args.trex_dir):
     raise Exception('Given path is a file')
 if not os.path.exists(args.trex_dir):
     os.makedirs(args.trex_dir)
+os.chmod(args.trex_dir, 0o777)
 if not os.path.exists(tmp_dir):
     os.makedirs(tmp_dir)
 
diff --git a/scripts/trex_daemon_server b/scripts/trex_daemon_server
index 6d08e458..b21a2626 100755
--- a/scripts/trex_daemon_server
+++ b/scripts/trex_daemon_server
@@ -1,6 +1,7 @@
 #!/usr/bin/python
 
 import os, sys, getpass
+import tempfile
 from time import time, sleep
 import subprocess, shlex, multiprocessing
 from argparse import ArgumentParser
@@ -27,25 +28,36 @@ except ImportError:
 
 def run_command(command, timeout = 10):
     commmand = 'timeout %s %s' % (timeout, command)
-    proc = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd = os.getcwd())
-    stdout, stderr = proc.communicate()
-    return (proc.returncode, stdout.decode(errors = 'replace'), stderr.decode(errors = 'replace'))
+    # pipes might stuck, even with timeout
+    with tempfile.TemporaryFile() as stdout_file, tempfile.TemporaryFile() as stderr_file:
+        proc = subprocess.Popen(shlex.split(command), stdout = stdout_file, stderr = stderr_file, cwd = daemon_dir)
+        proc.wait()
+        stdout_file.seek(0)
+        stderr_file.seek(0)
+        return (proc.returncode, stdout_file.read().decode(errors = 'replace'), stderr_file.read().decode(errors = 'replace'))
 
 
 def get_daemon_pid():
-    return_code, stdout, stderr = run_command('netstat -tlnp')
-    if return_code:
-        fail('Failed to determine which program holds port %s, netstat error: %s' % (args.daemon_port, stderr))
-    for line in stdout.splitlines():
-        if '0.0.0.0:%s' % args.daemon_port in line:
-            line_arr = line.split()
-            if '/' not in line_arr[-1]:
-                fail('Expecting pid/program name in netstat line of using port %s, got: %s' % (args.daemon_port, line_arr[-1]))
-            pid, program = line_arr[-1].split('/')
-            if 'python' not in program and 'trex_server' not in program and 'trex_daemon_server' not in program:
-                fail('Some other program holds port %s, not our daemon: %s. Please verify.' % (args.daemon_port, program))
-            return int(pid)
-    return None
+    err = None
+    for i in range(5):
+        try:
+            return_code, stdout, stderr = run_command('netstat -tlnp')
+            if return_code:
+                raise Exception('Failed to run netstat.\nStdout: %s\nStderr: %s' % (stdout, stderr))
+            for line in stdout.splitlines():
+                if '0.0.0.0:%s' % args.daemon_port in line:
+                    line_arr = line.split()
+                    if '/' not in line_arr[-1]:
+                        raise Exception('Expecting pid/program name in netstat line of using port %s, got: %s' % (args.daemon_port, line))
+                    pid, program = line_arr[-1].split('/')
+                    if 'python' not in program and 'trex_server' not in program and 'trex_daemon_server' not in program:
+                        raise Exception('Some other program holds port %s, not our daemon: %s. Please verify.' % (args.daemon_port, program))
+                    return int(pid)
+            return None
+        except Exception as e:
+            err = e
+            sleep(0.1)
+    fail('Could not determine daemon pid, err: %s' % err)
 
 
 def show_daemon_status():
@@ -92,22 +104,23 @@ def kill_daemon():
         print(termstyle.red('TRex server daemon is NOT running'))
         return True
     return_code, stdout, stderr = run_command('kill %s' % pid) # usual kill
-    if return_code:
-        fail('Failed to kill trex_daemon, error: %s' % stderr)
+    #if return_code:
+    #    fail('Failed to kill trex_daemon, error: %s' % stderr)
     for i in range(50):
         if not get_daemon_pid():
             print(termstyle.green('TRex server daemon is killed'))
             return True
         sleep(0.1)
     return_code, stdout, stderr = run_command('kill -9 %s' % pid) # unconditional kill
-    if return_code:
-        fail('Failed to kill trex_daemon, error: %s' % stderr)
+    #if return_code:
+    #    fail('Failed to kill trex_daemon, error: %s' % stderr)
     for i in range(50):
         if not get_daemon_pid():
             print(termstyle.green('TRex server daemon is killed'))
             return True
         sleep(0.1)
-    fail('Failed to kill trex_daemon, even with -9. Please review manually.') # should not happen
+    fail('Failed to kill trex_daemon, even with -9. Please review manually.\n' \
+         'Return code: %s\nStdout: %s\nStderr: %s' % return_code, stdout, stderr) # should not happen
 
 ### Main ###
 
@@ -129,6 +142,7 @@ trex_server.trex_parser.add_argument('action', choices=action_funcs.keys(),
 trex_server.trex_parser.usage = None
 args = trex_server.trex_parser.parse_args()
 
+daemon_dir = os.path.dirname(os.path.realpath(__file__))
 mkpath('/var/log/trex')
 mkpath('/var/run/trex')