CSIT-152: Detect if the T-rex is responding to API after start

Change-Id: I99c8dcbd210175fa5d1af193371249e525dc0102
Signed-off-by: pmikus <pmikus@cisco.com>

2 files changed, 105 insertions, 10 deletions

diff --git a/resources/libraries/python/TrafficGenerator.py b/resources/libraries/python/TrafficGenerator.py
index 127b5b1d9b..6d90fe41b7 100644
--- a/resources/libraries/python/TrafficGenerator.py
+++ b/resources/libraries/python/TrafficGenerator.py
@@ -205,16 +205,35 @@ class TrafficGenerator(object):
                 logger.error('trex-cfg failed: {0}'.format(stdout + stderr))
                 raise RuntimeError('trex-cfg failed')
 
-            (ret, _, _) = ssh.exec_command(
-                "sh -c 'pgrep t-rex && sudo pkill t-rex'")
-
-            (ret, _, _) = ssh.exec_command(
-                "sh -c 'cd {0}/scripts/ && "
-                "sudo nohup ./t-rex-64 -i -c 7 --iom 0 > /dev/null 2>&1 &'"
-                "> /dev/null"\
-                .format(trex_path))
-            if int(ret) != 0:
-                raise RuntimeError('t-rex-64 startup failed')
+            max_startup_retries = 3
+            while max_startup_retries > 0:
+                # kill T-rex only if it is already running
+                (ret, _, _) = ssh.exec_command(
+                    "sh -c 'pgrep t-rex && sudo pkill t-rex'")
+
+                # start T-rex
+                (ret, _, _) = ssh.exec_command(
+                    "sh -c 'cd {0}/scripts/ && "
+                    "sudo nohup ./t-rex-64 -i -c 7 --iom 0 > /dev/null 2>&1 &'"
+                    "> /dev/null"\
+                    .format(trex_path))
+                if int(ret) != 0:
+                    raise RuntimeError('t-rex-64 startup failed')
+
+                # get T-rex server info
+                (ret, _, _) = ssh.exec_command(
+                    "sh -c '{0}/resources/tools/t-rex/t-rex-server-info.py'"\
+                    .format(Constants.REMOTE_FW_DIR),
+                    timeout=120)
+                if int(ret) == 0:
+                    # If we get info T-rex is running
+                    return
+                # try again
+                max_startup_retries -= 1
+            # after max retries T-rex is still not responding to API
+            # critical error occured
+            raise RuntimeError('t-rex-64 startup failed')
+
 
     @staticmethod
     def teardown_traffic_generator(node):
diff --git a/resources/tools/t-rex/t-rex-server-info.py b/resources/tools/t-rex/t-rex-server-info.py
new file mode 100755
index 0000000000..42368df130
--- /dev/null
+++ b/resources/tools/t-rex/t-rex-server-info.py
@@ -0,0 +1,76 @@
+#!/usr/bin/python
+
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This script uses T-REX stateless API to drive t-rex instance.
+
+Requirements:
+- T-REX: https://github.com/cisco-system-traffic-generator/trex-core
+ - compiled and running T-REX process (eg. ./t-rex-64 -i -c 4)
+ - trex_stl_lib.api library
+- Script must be executed on a node with T-REX instance
+
+Functionality:
+1. Verify the API functionality and get server information
+
+"""
+
+import sys
+
+sys.path.insert(0, "/opt/trex-core-2.03/scripts/automation/"+\
+                   "trex_control_plane/stl/")
+from trex_stl_lib.api import *
+
+
+def get_server_system_info():
+    """Check server info and quit.
+
+    :return: nothing
+    """
+
+    # create client
+    client = STLClient()
+
+    try:
+        # connect to server
+        client.connect()
+        # get server info
+        print client.get_server_system_info()
+
+    except STLError as ex_error:
+        print_error(str(ex_error))
+        sys.exit(1)
+
+    finally:
+        client.disconnect()
+
+
+def print_error(msg):
+    """Print error message on stderr.
+
+    :param msg: Error message to print.
+    :type msg: string
+    :return: nothing
+    """
+
+    sys.stderr.write(msg+'\n')
+
+
+def main():
+    """Main function."""
+
+    get_server_system_info()
+
+if __name__ == "__main__":
+    main()