make test: detect hung tests

Run tests in a forked process with a set of pipes to communicate
keep-alives and overall result. This allows us to detect when e.g.
vpp dies mid-API call causing the test to hang waiting for response
(which will never come since vpp died).

Support setting a (per test case) TIMEOUT make test option to set timeout,
with a default timeout of 120 seconds.

Example - fail the test suite if any test-case fails to finish within
300s:

make TIMEOUT=300 test

Change-Id: I0d04f26a7232968f4bf043adf5d5b508f5018717
Signed-off-by: Klement Sekera <ksekera@cisco.com>

1 files changed, 69 insertions, 3 deletions

diff --git a/test/run_tests.py b/test/run_tests.py
index 1b9c677df26..6d477d880f3 100644
--- a/test/run_tests.py
+++ b/test/run_tests.py
@@ -2,10 +2,14 @@
 
 import sys
 import os
+import select
 import unittest
 import argparse
 import importlib
+from multiprocessing import Process, Pipe
 from framework import VppTestRunner
+from debug import spawn_gdb
+from log import global_logger
 
 
 def add_from_dir(suite, directory):
@@ -39,12 +43,38 @@ def add_from_dir(suite, directory):
                 if method.startswith("test_"):
                     suite.addTest(cls(method))
 
+
+def test_runner_wrapper(keep_alive_pipe, result_pipe):
+    result = not VppTestRunner(
+        pipe=keep_alive_pipe,
+        verbosity=verbose,
+        failfast=failfast).run(suite).wasSuccessful()
+    result_pipe.send(result)
+    result_pipe.close()
+    keep_alive_pipe.close()
+
+
+def handle_core(vpp_binary, core_path):
+    try:
+        d = os.getenv("DEBUG")
+    except:
+        d = None
+    if d and d.lower() == "core":
+        spawn_gdb(vpp_binary, core_path, global_logger)
+
+
 if __name__ == '__main__':
     try:
         verbose = int(os.getenv("V", 0))
     except:
         verbose = 0
 
+    default_test_timeout = 120
+    try:
+        test_timeout = int(os.getenv("TIMEOUT", default_test_timeout))
+    except:
+        test_timeout = default_test_timeout
+
     parser = argparse.ArgumentParser(description="VPP unit tests")
     parser.add_argument("-f", "--failfast", action='count',
                         help="fast failure flag")
@@ -56,7 +86,43 @@ if __name__ == '__main__':
 
     suite = unittest.TestSuite()
     for d in args.dir:
-        print("Adding tests from directory tree %s" % d)
+        global_logger.info("Adding tests from directory tree %s" % d)
         add_from_dir(suite, d)
-    sys.exit(not VppTestRunner(verbosity=verbose,
-                               failfast=failfast).run(suite).wasSuccessful())
+    keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False)
+    result_parent_end, result_child_end = Pipe(duplex=False)
+
+    p = Process(target=test_runner_wrapper,
+                args=(keep_alive_child_end,
+                      result_child_end))
+    p.start()
+    last_test_temp_dir = None
+    last_test_vpp_binary = None
+    last_test = None
+    result = None
+    while result is None:
+        readable = select.select([keep_alive_parent_end.fileno(),
+                                  result_parent_end.fileno(),
+                                  ],
+                                 [], [], test_timeout)[0]
+        if result_parent_end.fileno() in readable:
+            result = result_parent_end.recv()
+        elif keep_alive_parent_end.fileno() in readable:
+            while keep_alive_parent_end.poll():
+                last_test, last_test_vpp_binary, last_test_temp_dir =\
+                    keep_alive_parent_end.recv()
+        else:
+            global_logger.critical("Timeout while waiting for child test "
+                                   "runner process (last test running was "
+                                   "`%s' in `%s')!" %
+                                   (last_test, last_test_temp_dir))
+            if last_test_temp_dir and last_test_vpp_binary:
+                core_path = "%s/core" % last_test_temp_dir
+                if os.path.isfile(core_path):
+                    global_logger.error("Core-file exists in test temporary "
+                                        "directory: %s!" % core_path)
+                    handle_core(last_test_vpp_binary, core_path)
+            p.terminate()
+            result = -1
+    keep_alive_parent_end.close()
+    result_parent_end.close()
+    sys.exit(result)