tests: run tests against a running VPP

Usage:
test/run.py -r  -t {test_filter}
Instead of starting a new instance of VPP, when the -r argument
is provided, test is run against a running VPP instance. Optionally,
one can also set the VPP socket directory using the -d
argument. The default location for socket files is
/var/run/user/${uid}/vpp and /var/run/vpp if VPP is started
as root.

Type: improvement

Change-Id: I05e57a067fcb90fb49973f8159fc17925b741f1a
Signed-off-by: Naveen Joy <najoy@cisco.com>

4 files changed, 291 insertions, 12 deletions

diff --git a/test/config.py b/test/config.py
index b8bbbbcde7c..e73555723ff 100644
--- a/test/config.py
+++ b/test/config.py
@@ -359,6 +359,29 @@ parser.add_argument(
     help=f"if set, keep all pcap files from a test run (default: {default_keep_pcaps})",
 )
 
+parser.add_argument(
+    "-r",
+    "--use-running-vpp",
+    dest="running_vpp",
+    required=False,
+    action="store_true",
+    default=False,
+    help="Runs tests against a running VPP.",
+)
+
+parser.add_argument(
+    "-d",
+    "--socket-dir",
+    dest="socket_dir",
+    required=False,
+    action="store",
+    default="",
+    help="Relative or absolute path to running VPP's socket directory.\n"
+    "The directory must contain VPP's socket files:api.sock & stats.sock.\n"
+    "Default: /var/run/vpp if VPP is started as the root user, else "
+    "/var/run/user/${uid}/vpp.",
+)
+
 config = parser.parse_args()
 
 ws = config.vpp_ws_dir
diff --git a/test/framework.py b/test/framework.py
index 230b2d57c55..c85dec5dbdf 100644
--- a/test/framework.py
+++ b/test/framework.py
@@ -51,6 +51,7 @@ from util import ppp, is_core_present
 from scapy.layers.inet import IPerror, TCPerror, UDPerror, ICMPerror
 from scapy.layers.inet6 import ICMPv6DestUnreach, ICMPv6EchoRequest
 from scapy.layers.inet6 import ICMPv6EchoReply
+from vpp_running import use_running
 
 
 logger = logging.getLogger(__name__)
@@ -302,6 +303,7 @@ class CPUInterface(ABC):
         cls.cpus = cpus
 
 
+@use_running
 class VppTestCase(CPUInterface, unittest.TestCase):
     """This subclass is a base class for VPP test cases that are implemented as
     classes. It provides methods to create and run test case.
@@ -698,7 +700,8 @@ class VppTestCase(CPUInterface, unittest.TestCase):
             )
             cls.vpp_stdout_deque = deque()
             cls.vpp_stderr_deque = deque()
-            if not cls.debug_attach:
+            # Pump thread in a non-debug-attached & not running-vpp
+            if not cls.debug_attach and not hasattr(cls, "running_vpp"):
                 cls.pump_thread_stop_flag = Event()
                 cls.pump_thread_wakeup_pipe = os.pipe()
                 cls.pump_thread = Thread(target=pump_output, args=(cls,))
@@ -775,6 +778,8 @@ class VppTestCase(CPUInterface, unittest.TestCase):
         Disconnect vpp-api, kill vpp and cleanup shared memory files
         """
         cls._debug_quit()
+        if hasattr(cls, "running_vpp"):
+            cls.vpp.quit_vpp()
 
         # first signal that we want to stop the pump thread, then wake it up
         if hasattr(cls, "pump_thread_stop_flag"):
@@ -807,10 +812,16 @@ class VppTestCase(CPUInterface, unittest.TestCase):
                     cls.vpp.kill()
                     outs, errs = cls.vpp.communicate()
             cls.logger.debug("Deleting class vpp attribute on %s", cls.__name__)
-            if not cls.debug_attach:
+            if not cls.debug_attach and not hasattr(cls, "running_vpp"):
                 cls.vpp.stdout.close()
                 cls.vpp.stderr.close()
-            del cls.vpp
+            # If vpp is a dynamic attribute set by the func use_running,
+            # deletion will result in an AttributeError that we can
+            # safetly pass.
+            try:
+                del cls.vpp
+            except AttributeError:
+                pass
 
         if cls.vpp_startup_failed:
             stdout_log = cls.logger.info
diff --git a/test/run.py b/test/run.py
index 07b24d55b82..646354ab060 100755
--- a/test/run.py
+++ b/test/run.py
@@ -21,7 +21,7 @@ import logging
 import os
 from pathlib import Path
 import signal
-from subprocess import Popen, PIPE, STDOUT
+from subprocess import Popen, PIPE, STDOUT, call
 import sys
 import time
 import venv
@@ -31,7 +31,7 @@ import venv
 test_dir = os.path.dirname(os.path.realpath(__file__))
 ws_root = os.path.dirname(test_dir)
 build_root = os.path.join(ws_root, "build-root")
-venv_dir = os.path.join(test_dir, "venv")
+venv_dir = os.path.join(build_root, "test", "venv")
 venv_bin_dir = os.path.join(venv_dir, "bin")
 venv_lib_dir = os.path.join(venv_dir, "lib")
 venv_run_dir = os.path.join(venv_dir, "run")
@@ -215,8 +215,9 @@ def set_environ():
 # Runs a test inside a spawned QEMU VM
 # If a kernel image is not provided, a linux-image-kvm image is
 # downloaded to the test_data_dir
-def vm_test_runner(test_name, kernel_image, test_data_dir, cpu_mask, mem):
+def vm_test_runner(test_name, kernel_image, test_data_dir, cpu_mask, mem, jobs="auto"):
     script = os.path.join(test_dir, "scripts", "run_vpp_in_vm.sh")
+    os.environ["TEST_JOBS"] = str(jobs)
     p = Popen(
         [script, test_name, kernel_image, test_data_dir, cpu_mask, mem],
         stdout=PIPE,
@@ -275,20 +276,53 @@ def set_logging(test_data_dir, test_name):
     logging.basicConfig(filename=filename, level=logging.DEBUG)
 
 
+def run_tests_in_venv(
+    test,
+    jobs,
+    log_dir,
+    socket_dir="",
+    running_vpp=False,
+):
+    """Runs tests in the virtual environment set by venv_dir.
+
+    Arguments:
+    test: Name of the test to run
+    jobs: Maximum concurrent test jobs
+    log_dir: Directory location for storing log files
+    socket_dir: Use running VPP's socket files
+    running_vpp: True if tests are run against a running VPP
+    """
+    script = os.path.join(test_dir, "scripts", "run.sh")
+    args = [
+        f"--venv-dir={venv_dir}",
+        f"--vpp-ws-dir={ws_root}",
+        f"--socket-dir={socket_dir}",
+        f"--filter={test}",
+        f"--jobs={jobs}",
+        f"--log-dir={log_dir}",
+    ]
+    if running_vpp:
+        args = args + [f"--use-running-vpp"]
+    print(f"Running script: {script} " f"{' '.join(args)}")
+    process_args = [script] + args
+    call(process_args)
+
+
 if __name__ == "__main__":
     # Build a Virtual Environment for running tests on host & QEMU
+    # (TODO): Create a single config object by merging the below args with
+    # config.py after gathering dev use-cases.
     parser = argparse.ArgumentParser(
         description="Run VPP Unit Tests", formatter_class=argparse.RawTextHelpFormatter
     )
     parser.add_argument(
         "--vm",
         dest="vm",
-        required=True,
+        required=False,
         action="store_true",
         help="Run Test Inside a QEMU VM",
     )
     parser.add_argument(
-        "-d",
         "--debug",
         dest="debug",
         required=False,
@@ -297,7 +331,6 @@ if __name__ == "__main__":
         help="Run Tests on Debug Build",
     )
     parser.add_argument(
-        "-r",
         "--release",
         dest="release",
         required=False,
@@ -306,12 +339,13 @@ if __name__ == "__main__":
         help="Run Tests on release Build",
     )
     parser.add_argument(
+        "-t",
         "--test",
         dest="test_name",
         required=False,
         action="store",
         default="",
-        help="Tests to Run",
+        help="Test Name or Test filter",
     )
     parser.add_argument(
         "--vm-kernel-image",
@@ -339,7 +373,42 @@ if __name__ == "__main__":
         default="2",
         help="Guest Memory in Gibibytes\n" "E.g. 4 (Default: 2)",
     )
+    parser.add_argument(
+        "--log-dir",
+        action="store",
+        default="/tmp",
+        help="directory where to store directories "
+        "containing log files (default: /tmp)",
+    )
+    parser.add_argument(
+        "--jobs",
+        action="store",
+        default="auto",
+        help="maximum concurrent test jobs",
+    )
+    parser.add_argument(
+        "-r",
+        "--use-running-vpp",
+        dest="running_vpp",
+        required=False,
+        action="store_true",
+        default=False,
+        help="Runs tests against a running VPP.",
+    )
+    parser.add_argument(
+        "-d",
+        "--socket-dir",
+        dest="socket_dir",
+        required=False,
+        action="store",
+        default="",
+        help="Relative or absolute path of running VPP's socket directory "
+        "containing api.sock & stats.sock files.\n"
+        "Default: /var/run/vpp if VPP is started as the root user, else "
+        "/var/run/user/${uid}/vpp.",
+    )
     args = parser.parse_args()
+    vm_tests = False
     # Enable VM tests
     if args.vm and args.test_name:
         test_data_dir = "/tmp/vpp-vm-tests"
@@ -353,7 +422,21 @@ if __name__ == "__main__":
     debug = False if args.release else True
     build_vpp(debug, args.release)
     set_environ()
-    if vm_tests:
+    if args.running_vpp:
+        print("Tests will be run against a running VPP..")
+    elif not vm_tests:
+        print("Tests will be run by spawning a new VPP instance..")
+    # Run tests against a running VPP or a new instance of VPP
+    if not vm_tests:
+        run_tests_in_venv(
+            test=args.test_name,
+            jobs=args.jobs,
+            log_dir=args.log_dir,
+            socket_dir=args.socket_dir,
+            running_vpp=args.running_vpp,
+        )
+    # Run tests against a VPP inside a VM
+    else:
         print("Running VPP unit test(s):{0} inside a QEMU VM".format(args.test_name))
         # Check Available CPUs & Usable Memory
         cpus = expand_mix_string(args.vm_cpu_list)
@@ -366,5 +449,10 @@ if __name__ == "__main__":
             print(f"Error: Mem Size:{args.vm_mem}G > Avail Mem:{avail_mem}G")
             sys.exit(1)
         vm_test_runner(
-            args.test_name, args.kernel_image, test_data_dir, cpus, f"{args.vm_mem}G"
+            args.test_name,
+            args.kernel_image,
+            test_data_dir,
+            cpus,
+            f"{args.vm_mem}G",
+            args.jobs,
         )
diff --git a/test/vpp_running.py b/test/vpp_running.py
new file mode 100644
index 00000000000..e1ffe376546
--- /dev/null
+++ b/test/vpp_running.py
@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+
+# Supporting module for running tests against a running VPP.
+# This module is used by the test framework. Do not invoke this module
+# directly for running tests against a running vpp. Use run.py for
+# running all unit tests.
+
+from glob import glob
+import os
+import sys
+import subprocess
+from config import config
+
+
+def use_running(cls):
+    """Update VPPTestCase to use running VPP's sock files & methods.
+
+    Arguments:
+    cls -- VPPTestCase Class
+    """
+    if config.running_vpp:
+        if os.path.isdir(config.socket_dir):
+            RunningVPP.socket_dir = config.socket_dir
+        else:
+            RunningVPP.socket_dir = RunningVPP.get_default_socket_dir()
+        RunningVPP.get_set_vpp_sock_files()
+        cls.get_stats_sock_path = RunningVPP.get_stats_sock_path
+        cls.get_api_sock_path = RunningVPP.get_api_sock_path
+        cls.run_vpp = RunningVPP.run_vpp
+        cls.quit_vpp = RunningVPP.quit_vpp
+        cls.vpp = RunningVPP
+        cls.running_vpp = True
+    return cls
+
+
+class RunningVPP:
+
+    api_sock = ""  # api_sock file path
+    stats_sock = ""  # stats sock_file path
+    socket_dir = ""  # running VPP's socket directory
+    pid = None  # running VPP's pid
+    returncode = None  # indicates to the framework that VPP is running
+
+    @classmethod
+    def get_stats_sock_path(cls):
+        return cls.stats_sock
+
+    @classmethod
+    def get_api_sock_path(cls):
+        return cls.api_sock
+
+    @classmethod
+    def run_vpp(cls):
+        """VPP is already running -- skip this action."""
+        pass
+
+    @classmethod
+    def quit_vpp(cls):
+        """Indicate quitting to framework by setting returncode=1."""
+        cls.returncode = 1
+
+    @classmethod
+    def terminate(cls):
+        """Indicate termination to framework by setting returncode=1."""
+        cls.returncode = 1
+
+    @classmethod
+    def get_default_socket_dir(cls):
+        """Return running VPP's default socket directory.
+
+        Default socket dir is:
+           /var/run/user/${UID}/vpp  (or)
+           /var/run/vpp, if VPP is started as a root user
+        """
+        if cls.is_running_vpp():
+            vpp_user_id = (
+                subprocess.check_output(["ps", "-o", "uid=", "-p", str(cls.pid)])
+                .decode("utf-8")
+                .strip()
+            )
+            if vpp_user_id == "0":
+                return "/var/run/vpp"
+            else:
+                return f"/var/run/user/{vpp_user_id}/vpp"
+        else:
+            print(
+                "Error: getting default socket dir, as "
+                "a running VPP process could not be found"
+            )
+            sys.exit(1)
+
+    @classmethod
+    def get_set_vpp_sock_files(cls):
+        """Look for *.sock files in the socket_dir and set cls attributes.
+
+        Returns a tuple: (api_sock_file, stats_sock_file)
+        Sets cls.api_sock and cls.stats_sock attributes
+        """
+        # Return if the sock files are already set
+        if cls.api_sock and cls.stats_sock:
+            return (cls.api_sock, cls.stats_sock)
+        # Find running VPP's sock files in the socket dir
+        if os.path.isdir(cls.socket_dir):
+            if not cls.is_running_vpp():
+                print(
+                    "Error: The socket dir for a running VPP directory is, "
+                    "set but a running VPP process could not be found"
+                )
+                sys.exit(1)
+            sock_files = glob(os.path.join(cls.socket_dir + "/" + "*.sock"))
+            for sock_file in sock_files:
+                if "api.sock" in sock_file:
+                    cls.api_sock = os.path.abspath(sock_file)
+                elif "stats.sock" in sock_file:
+                    cls.stats_sock = os.path.abspath(sock_file)
+            if not cls.api_sock:
+                print(
+                    f"Error: Could not find a valid api.sock file "
+                    f"in running VPP's socket directory {cls.socket_dir}"
+                )
+                sys.exit(1)
+            if not cls.stats_sock:
+                print(
+                    f"Error: Could not find a valid stats.sock file "
+                    f"in running VPP's socket directory {cls.socket_dir}"
+                )
+                sys.exit(1)
+            return (cls.api_sock, cls.stats_sock)
+        else:
+            print("Error: The socket dir for a running VPP directory is unset")
+            sys.exit(1)
+
+    @classmethod
+    def is_running_vpp(cls):
+        """Return True if VPP's pid is visible else False."""
+        vpp_pid = subprocess.Popen(
+            ["pgrep", "-d,", "-x", "vpp_main"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
+        )
+        stdout, stderr = vpp_pid.communicate()
+        cls.pid = int(stdout.split(",")[0]) if stdout else None
+        return bool(cls.pid)
+
+    @classmethod
+    def poll(cls):
+        """Return None to indicate that the process hasn't terminated."""
+        return cls.returncode
+
+
+if __name__ == "__main__":
+    RunningVPP.socket_dir = RunningVPP.get_default_socket_dir()
+    RunningVPP.get_set_vpp_sock_files()
+    print(f"Running VPP's sock files")
+    print(f"api_sock_file {RunningVPP.api_sock}")
+    print(f"stats_sock_file {RunningVPP.stats_sock}")