tests: make tests less make dependent

Implement command line argument parsing instead of passing arguments via
environment variables. Add script for running tests without having to
invoke make. Deprecate running tests via make.

Type: improvement
Change-Id: I2e3054a61a2ae25d460e9be00be7d7705fbf943e
Signed-off-by: Klement Sekera <ksekera@cisco.com>
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>

1 files changed, 49 insertions, 100 deletions

diff --git a/test/framework.py b/test/framework.py
index 2c74a03bf37..572207db3f8 100644
--- a/test/framework.py
+++ b/test/framework.py
@@ -9,13 +9,13 @@ import select
 import signal
 import subprocess
 import unittest
-import tempfile
+import re
 import time
 import faulthandler
 import random
 import copy
-import psutil
 import platform
+import shutil
 from collections import deque
 from threading import Thread, Event
 from inspect import getdoc, isclass
@@ -27,6 +27,7 @@ from struct import pack, unpack
 
 import scapy.compat
 from scapy.packet import Raw, Packet
+from config import config, available_cpus, num_cpus, max_vpp_cpus
 import hook as hookmodule
 from vpp_pg_interface import VppPGInterface
 from vpp_sub_interface import VppSubInterface
@@ -45,7 +46,6 @@ from scapy.layers.inet import IPerror, TCPerror, UDPerror, ICMPerror
 from scapy.layers.inet6 import ICMPv6DestUnreach, ICMPv6EchoRequest
 from scapy.layers.inet6 import ICMPv6EchoReply
 
-from cpu_config import available_cpus, num_cpus, max_vpp_cpus
 
 logger = logging.getLogger(__name__)
 
@@ -61,27 +61,7 @@ TEST_RUN = 4
 SKIP_CPU_SHORTAGE = 5
 
 
-class BoolEnvironmentVariable(object):
-
-    def __init__(self, env_var_name, default='n', true_values=None):
-        self.name = env_var_name
-        self.default = default
-        self.true_values = true_values if true_values is not None else \
-            ("y", "yes", "1")
-
-    def __bool__(self):
-        return os.getenv(self.name, self.default).lower() in self.true_values
-
-    if sys.version_info[0] == 2:
-        __nonzero__ = __bool__
-
-    def __repr__(self):
-        return 'BoolEnvironmentVariable(%r, default=%r, true_values=%r)' % \
-               (self.name, self.default, self.true_values)
-
-
-debug_framework = BoolEnvironmentVariable('TEST_DEBUG')
-if debug_framework:
+if config.debug_framework:
     import debug_internal
 
 """
@@ -175,7 +155,7 @@ def pump_output(testclass):
                     limit = -1
                     stdout_fragment = split[-1]
                 testclass.vpp_stdout_deque.extend(split[:limit])
-                if not testclass.cache_vpp_output:
+                if not config.cache_vpp_output:
                     for line in split[:limit]:
                         testclass.logger.info(
                             "VPP STDOUT: %s" % line.rstrip("\n"))
@@ -193,7 +173,7 @@ def pump_output(testclass):
                     stderr_fragment = split[-1]
 
                 testclass.vpp_stderr_deque.extend(split[:limit])
-                if not testclass.cache_vpp_output:
+                if not config.cache_vpp_output:
                     for line in split[:limit]:
                         testclass.logger.error(
                             "VPP STDERR: %s" % line.rstrip("\n"))
@@ -201,13 +181,6 @@ def pump_output(testclass):
                         # flag will take care of properly terminating the loop
 
 
-def _is_skip_aarch64_set():
-    return BoolEnvironmentVariable('SKIP_AARCH64')
-
-
-is_skip_aarch64_set = _is_skip_aarch64_set()
-
-
 def _is_platform_aarch64():
     return platform.machine() == 'aarch64'
 
@@ -215,35 +188,6 @@ def _is_platform_aarch64():
 is_platform_aarch64 = _is_platform_aarch64()
 
 
-def _running_extended_tests():
-    return BoolEnvironmentVariable("EXTENDED_TESTS")
-
-
-running_extended_tests = _running_extended_tests()
-
-
-def _running_gcov_tests():
-    return BoolEnvironmentVariable("GCOV_TESTS")
-
-
-running_gcov_tests = _running_gcov_tests()
-
-
-def get_environ_vpp_worker_count():
-    worker_config = os.getenv("VPP_WORKER_CONFIG", None)
-    if worker_config:
-        elems = worker_config.split(" ")
-        if elems[0] != "workers" or len(elems) != 2:
-            raise ValueError("Wrong VPP_WORKER_CONFIG == '%s' value." %
-                             worker_config)
-        return int(elems[1])
-    else:
-        return 0
-
-
-environ_vpp_worker_count = get_environ_vpp_worker_count()
-
-
 class KeepAliveReporter(object):
     """
     Singleton object which reports test start to parent process
@@ -277,7 +221,7 @@ class KeepAliveReporter(object):
         else:
             desc = test.id()
 
-        self.pipe.send((desc, test.vpp_bin, test.tempdir, test.vpp.pid))
+        self.pipe.send((desc, config.vpp, test.tempdir, test.vpp.pid))
 
 
 class TestCaseTag(Enum):
@@ -411,7 +355,7 @@ class VppTestCase(CPUInterface, unittest.TestCase):
             if cls.has_tag(TestCaseTag.FIXME_VPP_WORKERS):
                 cls.vpp_worker_count = 0
             else:
-                cls.vpp_worker_count = environ_vpp_worker_count
+                cls.vpp_worker_count = config.vpp_worker_count
         return cls.vpp_worker_count
 
     @classmethod
@@ -421,42 +365,38 @@ class VppTestCase(CPUInterface, unittest.TestCase):
     @classmethod
     def setUpConstants(cls):
         """ Set-up the test case class based on environment variables """
-        cls.step = BoolEnvironmentVariable('STEP')
-        # inverted case to handle '' == True
-        c = os.getenv("CACHE_OUTPUT", "1")
-        cls.cache_vpp_output = False if c.lower() in ("n", "no", "0") else True
-        cls.vpp_bin = os.getenv('VPP_BIN', "vpp")
-        extern_plugin_path = os.getenv('EXTERN_PLUGINS')
+        cls.step = config.step
+        cls.plugin_path = ":".join(config.vpp_plugin_dir)
+        cls.test_plugin_path = ":".join(config.vpp_test_plugin_dir)
+        cls.extern_plugin_path = ":".join(config.extern_plugin_dir)
         debug_cli = ""
         if cls.step or cls.debug_gdb or cls.debug_gdbserver:
             debug_cli = "cli-listen localhost:5002"
-        coredump_size = None
-        size = os.getenv("COREDUMP_SIZE")
-        if size is not None:
-            coredump_size = "coredump-size %s" % size
-        if coredump_size is None:
+        size = re.search(r"\d+[gG]", config.coredump_size)
+        if size:
+            coredump_size = f"coredump-size {config.coredump_size}".lower()
+        else:
             coredump_size = "coredump-size unlimited"
-
-        default_variant = os.getenv("VARIANT")
+        default_variant = config.variant
         if default_variant is not None:
             default_variant = "defaults { %s 100 }" % default_variant
         else:
             default_variant = ""
 
-        api_fuzzing = os.getenv("API_FUZZ")
+        api_fuzzing = config.api_fuzz
         if api_fuzzing is None:
             api_fuzzing = 'off'
 
         cls.vpp_cmdline = [
-            cls.vpp_bin,
+            config.vpp,
             "unix", "{", "nodaemon", debug_cli, "full-coredump",
             coredump_size, "runtime-dir", cls.tempdir, "}",
             "api-trace", "{", "on", "}",
             "api-segment", "{", "prefix", cls.get_api_segment_prefix(), "}",
             "cpu", "{", "main-core", str(cls.cpus[0]), ]
-        if extern_plugin_path is not None:
+        if cls.extern_plugin_path not in (None, ""):
             cls.extra_vpp_plugin_config.append(
-                "add-path %s" % extern_plugin_path)
+                "add-path %s" % cls.extern_plugin_path)
         if cls.get_vpp_worker_count():
             cls.vpp_cmdline.extend([
                 "corelist-workers", ",".join([str(x) for x in cls.cpus[1:]])])
@@ -495,15 +435,14 @@ class VppTestCase(CPUInterface, unittest.TestCase):
         print(single_line_delim)
         print("You can debug VPP using:")
         if cls.debug_gdbserver:
-            print("sudo gdb " + cls.vpp_bin +
-                  " -ex 'target remote localhost:{port}'"
-                  .format(port=cls.gdbserver_port))
+            print(f"sudo gdb {config.vpp} "
+                  f"-ex 'target remote localhost:{cls.gdbserver_port}'")
             print("Now is the time to attach gdb by running the above "
                   "command, set up breakpoints etc., then resume VPP from "
                   "within gdb by issuing the 'continue' command")
             cls.gdbserver_port += 1
         elif cls.debug_gdb:
-            print("sudo gdb " + cls.vpp_bin + " -ex 'attach %s'" % cls.vpp.pid)
+            print(f"sudo gdb {config.vpp} -ex 'attach {cls.vpp.pid}'")
             print("Now is the time to attach gdb by running the above "
                   "command and set up breakpoints etc., then resume VPP from"
                   " within gdb by issuing the 'continue' command")
@@ -585,11 +524,23 @@ class VppTestCase(CPUInterface, unittest.TestCase):
 
     @classmethod
     def get_tempdir(cls):
-        if cls.debug_attach:
-            return os.getenv("VPP_IN_GDB_TMP_DIR",
-                             "/tmp/unittest-attach-gdb")
-        else:
-            return tempfile.mkdtemp(prefix='vpp-unittest-%s-' % cls.__name__)
+        tmpdir = f"{config.tmp_dir}/vpp-unittest-{cls.__name__}"
+        if config.wipe_tmp_dir:
+            shutil.rmtree(tmpdir, ignore_errors=True)
+        os.mkdir(tmpdir)
+        return tmpdir
+
+    @classmethod
+    def create_file_handler(cls):
+        if config.log_dir is None:
+            cls.file_handler = FileHandler(f"{cls.tempdir}/log.txt")
+            return
+
+        logdir = f"{config.log_dir}/vpp-unittest-{cls.__name__}"
+        if config.wipe_tmp_dir:
+            shutil.rmtree(logdir, ignore_errors=True)
+        os.mkdir(logdir)
+        cls.file_handler = FileHandler(f"{logdir}/log.txt")
 
     @classmethod
     def setUpClass(cls):
@@ -599,15 +550,13 @@ class VppTestCase(CPUInterface, unittest.TestCase):
         """
         super(VppTestCase, cls).setUpClass()
         cls.logger = get_logger(cls.__name__)
-        seed = os.environ["RND_SEED"]
-        random.seed(seed)
+        random.seed(config.rnd_seed)
         if hasattr(cls, 'parallel_handler'):
             cls.logger.addHandler(cls.parallel_handler)
             cls.logger.propagate = False
-        d = os.getenv("DEBUG", None)
-        cls.set_debug_flags(d)
+        cls.set_debug_flags(config.debug)
         cls.tempdir = cls.get_tempdir()
-        cls.file_handler = FileHandler("%s/log.txt" % cls.tempdir)
+        cls.create_file_handler()
         cls.file_handler.setFormatter(
             Formatter(fmt='%(asctime)s,%(msecs)03d %(message)s',
                       datefmt="%H:%M:%S"))
@@ -617,7 +566,7 @@ class VppTestCase(CPUInterface, unittest.TestCase):
         os.chdir(cls.tempdir)
         cls.logger.info("Temporary dir is %s, api socket is %s",
                         cls.tempdir, cls.get_api_sock_path())
-        cls.logger.debug("Random seed is %s", seed)
+        cls.logger.debug("Random seed is %s", config.rnd_seed)
         cls.setUpConstants()
         cls.reset_packet_infos()
         cls._pcaps = []
@@ -636,7 +585,7 @@ class VppTestCase(CPUInterface, unittest.TestCase):
                 cls.run_vpp()
             cls.reporter.send_keep_alive(cls, 'setUpClass')
             VppTestResult.current_test_case_info = TestCaseInfo(
-                cls.logger, cls.tempdir, cls.vpp.pid, cls.vpp_bin)
+                cls.logger, cls.tempdir, cls.vpp.pid, config.vpp)
             cls.vpp_stdout_deque = deque()
             cls.vpp_stderr_deque = deque()
             if not cls.debug_attach:
@@ -785,7 +734,7 @@ class VppTestCase(CPUInterface, unittest.TestCase):
         cls.quit()
         cls.file_handler.close()
         cls.reset_packet_infos()
-        if debug_framework:
+        if config.debug_framework:
             debug_internal.on_tear_down_class(cls)
 
     def show_commands_at_teardown(self):
@@ -890,7 +839,7 @@ class VppTestCase(CPUInterface, unittest.TestCase):
     def pg_start(cls, trace=True):
         """ Enable the PG, wait till it is done, then clean up """
         for (intf, worker) in cls._old_pcaps:
-            intf.rename_old_pcap_file(intf.get_in_path(worker),
+            intf.handle_old_pcap_file(intf.get_in_path(worker),
                                       intf.in_history_counter)
         cls._old_pcaps = []
         if trace:
@@ -1439,7 +1388,7 @@ class VppTestResult(unittest.TestResult):
     def symlink_failed(self):
         if self.current_test_case_info:
             try:
-                failed_dir = os.getenv('FAILED_DIR')
+                failed_dir = config.failed_dir
                 link_path = os.path.join(
                     failed_dir,
                     '%s-FAILED' %