tests: support attaching to existing vpp

Introduce a new option DEBUG=attach to run a test against existing
already running vpp. A new target 'make test-start-gdb' will spawn VPP
in gdb for this purpose. Customization options explained in test-help.

Type: improvement
Change-Id: Ia160a85b33da3b2df292d44bb95729af9dd9da96
Signed-off-by: Klement Sekera <ksekera@cisco.com>

13 files changed, 172 insertions, 73 deletions

diff --git a/Makefile b/Makefile
index 78300e2dfe6..0f0af4af008 100644
--- a/Makefile
+++ b/Makefile
@@ -505,6 +505,14 @@ retest-all-debug:
 	$(eval EXTENDED_TESTS=yes)
 	$(call test,vpp,vpp_debug,retest)
 
+.PHONY: test-start-vpp-in-gdb
+test-start-vpp-in-gdb:
+	$(call test,vpp,vpp,start-gdb)
+
+.PHONY: test-start-vpp-debug-in-gdb
+test-start-vpp-debug-in-gdb:
+	$(call test,vpp,vpp_debug,start-gdb)
+
 ifeq ("$(wildcard $(STARTUP_CONF))","")
 define run
 	@echo "WARNING: STARTUP_CONF not defined or file doesn't exist."
diff --git a/extras/deprecated/vom/test/test_vom.py b/extras/deprecated/vom/test/test_vom.py
index 7dea7697f8c..a77b935263d 100644
--- a/extras/deprecated/vom/test/test_vom.py
+++ b/extras/deprecated/vom/test/test_vom.py
@@ -27,8 +27,8 @@ class VOMTestCase(VppTestCase):
         self.assertIsNotNone(built_root,
                              "Environment variable `%s' not set" % var)
         executable = "%s/vom_test/vom_test" % built_root
-        worker = Worker(
-            [executable, "vpp object model", self.shm_prefix], self.logger)
+        worker = Worker([executable, "vpp object model",
+                         self.get_api_segment_prefix()], self.logger)
         worker.start()
         timeout = 120
         worker.join(timeout)
diff --git a/src/plugins/quic/test/test_quic.py b/src/plugins/quic/test/test_quic.py
index 0e4cb2dacca..1257f4e2b0a 100644
--- a/src/plugins/quic/test/test_quic.py
+++ b/src/plugins/quic/test/test_quic.py
@@ -199,7 +199,7 @@ class QUICEchoExtTestCase(QUICTestCase):
             "uri", self.uri,
             "json",
             self.test_bytes,
-            "socket-name", self.api_sock,
+            "socket-name", self.get_api_sock_path(),
             "quic-setup", self.quic_setup,
             "nthreads", "1",
             "mq-size", f"{self.evt_q_len}"
diff --git a/src/vcl/test/test_vcl.py b/src/vcl/test/test_vcl.py
index 80b9f2f0211..8950470e8fc 100644
--- a/src/vcl/test/test_vcl.py
+++ b/src/vcl/test/test_vcl.py
@@ -87,7 +87,7 @@ class VCLTestCase(VppTestCase):
         self.vapi.session_enable_disable(is_enable=0)
 
     def cut_thru_test(self, server_app, server_args, client_app, client_args):
-        self.env = {'VCL_VPP_API_SOCKET': self.api_sock,
+        self.env = {'VCL_VPP_API_SOCKET': self.get_api_sock_path(),
                     'VCL_APP_SCOPE_LOCAL': "true"}
         worker_server = VCLAppWorker(self.build_dir, server_app, server_args,
                                      self.logger, self.env)
@@ -192,7 +192,7 @@ class VCLTestCase(VppTestCase):
     @unittest.skipUnless(_have_iperf3, "'%s' not found, Skipping.")
     def thru_host_stack_test(self, server_app, server_args,
                              client_app, client_args):
-        self.env = {'VCL_VPP_API_SOCKET': self.api_sock,
+        self.env = {'VCL_VPP_API_SOCKET': self.get_api_sock_path(),
                     'VCL_APP_SCOPE_GLOBAL': "true",
                     'VCL_APP_NAMESPACE_ID': "1",
                     'VCL_APP_NAMESPACE_SECRET': "1234"}
diff --git a/src/vpp-api/test/test_vapi.py b/src/vpp-api/test/test_vapi.py
index 09f9f83e984..d91099210d2 100644
--- a/src/vpp-api/test/test_vapi.py
+++ b/src/vpp-api/test/test_vapi.py
@@ -25,8 +25,8 @@ class VAPITestCase(VppTestCase):
         self.assertIsNotNone(built_root,
                              "Environment variable `%s' not set" % var)
         executable = "%s/vapi_test/vapi_c_test" % built_root
-        worker = Worker(
-            [executable, "vapi client", self.shm_prefix], self.logger)
+        worker = Worker([executable, "vapi client",
+                         self.get_api_segment_prefix()], self.logger)
         worker.start()
         timeout = 60
         worker.join(timeout)
@@ -54,8 +54,8 @@ class VAPITestCase(VppTestCase):
         self.assertIsNotNone(built_root,
                              "Environment variable `%s' not set" % var)
         executable = "%s/vapi_test/vapi_cpp_test" % built_root
-        worker = Worker(
-            [executable, "vapi client", self.shm_prefix], self.logger)
+        worker = Worker([executable, "vapi client",
+                         self.get_api_segment_prefix()], self.logger)
         worker.start()
         timeout = 120
         worker.join(timeout)
diff --git a/test/Makefile b/test/Makefile
index d4d74bb62b4..dc6aa096dd0 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -22,14 +22,7 @@ CORE_TEST_DIRS=$(shell find $(WS_ROOT)/src -not \( -path $(INTERN_PLUGIN_SRC_DIR
 VPP_TEST_DIRS=$(shell ls -d $(TEST_DIR)$(PLUGIN_TEST_DIRS)$(CORE_TEST_DIRS) $(EXTERN_TESTS))
 VPP_TEST_SRC=$(shell for dir in $(VPP_TEST_DIRS) ; do ls $$dir/*.py 2>/dev/null; done)
 
-.PHONY: verify-no-running-vpp
-
-ifdef VPP_ZOMBIE_NOCHECK
-VPP_PIDS=
-else
-VPP_PIDS=$(shell pgrep -d, -x vpp_main)
-endif
-
+FORCE_NO_WIPE=0
 ifeq ($(DEBUG),gdb)
 FORCE_FOREGROUND=1
 else ifeq ($(DEBUG),gdbserver)
@@ -40,6 +33,9 @@ else ifeq ($(DEBUG),gdbserver-all)
 FORCE_FOREGROUND=1
 else ifeq ($(DEBUG),core)
 FORCE_FOREGROUND=1
+else ifeq ($(DEBUG),attach)
+FORCE_FOREGROUND=1
+FORCE_NO_WIPE=1
 else ifeq ($(STEP),yes)
 FORCE_FOREGROUND=1
 else ifeq ($(STEP),y)
@@ -63,16 +59,6 @@ PYTHON_PROFILE_OPTS=-m cProfile $(PROFILE_OUTPUT_OPTS) -s $(PROFILE_SORT_BY)
 FORCE_FOREGROUND=1
 endif
 
-verify-no-running-vpp:
-	@if [ "$(VPP_PIDS)" != "" ]; then \
-		echo; \
-		echo "*** Existing vpp processes detected (PID(s): $(VPP_PIDS)). Running tests under these conditions is not supported. ***"; \
-		echo; \
-		ps -fp $(VPP_PIDS);\
-		echo; \
-		false; \
-	fi
-
 UNITTEST_EXTRA_OPTS=
 UNITTEST_FAILFAST_OPTS=
 
@@ -172,7 +158,7 @@ PLUGIN_SRC_DIR=$(INTERN_PLUGIN_SRC_DIR)
 endif
 
 define retest-func
-@env FORCE_FOREGROUND=$(FORCE_FOREGROUND) FAILED_DIR=$(FAILED_DIR) VENV_PATH=$(VENV_PATH) scripts/setsid_wrapper.sh $(FORCE_FOREGROUND) $(VENV_PATH)/bin/activate $(PYTHON_INTERP) $(PYTHON_PROFILE_OPTS) $(BUILD_TEST_SRC)/run_tests.py -d $(BUILD_TEST_SRC) $(UNITTEST_EXTRA_OPTS) || env FAILED_DIR=$(FAILED_DIR) COMPRESS_FAILED_TEST_LOGS=$(COMPRESS_FAILED_TEST_LOGS) scripts/compress_failed.sh
+@env VPP_IN_GDB=$(VPP_IN_GDB) FORCE_FOREGROUND=$(FORCE_FOREGROUND) FAILED_DIR=$(FAILED_DIR) VENV_PATH=$(VENV_PATH) scripts/setsid_wrapper.sh $(FORCE_FOREGROUND) $(VENV_PATH)/bin/activate $(PYTHON_INTERP) $(PYTHON_PROFILE_OPTS) $(BUILD_TEST_SRC)/run_tests.py -d $(BUILD_TEST_SRC) $(UNITTEST_EXTRA_OPTS) || env FAILED_DIR=$(FAILED_DIR) COMPRESS_FAILED_TEST_LOGS=$(COMPRESS_FAILED_TEST_LOGS) scripts/compress_failed.sh
 endef
 
 .PHONY: sanity
@@ -197,7 +183,7 @@ else
 PARALLEL_ILLEGAL=1
 endif
 
-sanity: test-dep verify-no-running-vpp
+sanity: test-dep
 	@bash -c "test $(PARALLEL_ILLEGAL) -eq 0 ||\
 	    (echo \"*******************************************************************\" &&\
 		 echo \"* Sanity check failed, TEST_JOBS is not 1 or 'auto' and DEBUG, STEP or PROFILE is set\" &&\
@@ -264,7 +250,7 @@ shell: test-dep
 .PHONY: reset
 reset:
 	@rm -f /dev/shm/vpp-unittest-*
-	@rm -rf /tmp/vpp-unittest-*
+	@if [ $(FORCE_NO_WIPE) -eq "0" ] ; then rm -rf /tmp/vpp-unittest-*;  fi
 	@rm -f /tmp/api_post_mortem.*
 	@rm -rf $(FAILED_DIR)
 
@@ -333,6 +319,12 @@ checkstyle-diff: $(BUILD_TEST_SRC) $(PIP_INSTALL_DONE)
 	@echo "* Test framework PEP8 compliance check passed (checked changed files)"
 	@echo "*********************************************************************"
 
+.PHONY: start-gdb
+start-gdb: sanity
+	$(eval VPP_IN_GDB=1)
+	$(eval FORCE_FOREGROUND=1)
+	$(call retest-func)
+
 .PHONY: checkstyle
 checkstyle: $(BUILD_TEST_SRC) $(PIP_INSTALL_DONE)
 	@bash -c "source $(VENV_PATH)/bin/activate &&\
@@ -361,7 +353,6 @@ help:
 	@echo " retest-all-debug       - run functional and extended tests (debug build)"
 	@echo " test-cov               - generate code coverage report for test framework"
 	@echo " test-gcov                      - build and run functional tests (gcov build)"
-
 	@echo " test-wipe              - wipe (temporary) files generated by unit tests"
 	@echo " test-wipe-cov          - wipe code coverage report for test framework"
 	@echo " test-wipe-doc          - wipe documentation for test framework"
@@ -369,8 +360,8 @@ help:
 	@echo " test-wipe-all          - wipe (temporary) files generated by unit tests, docs, and coverage"
 	@echo " test-shell             - enter shell with test environment"
 	@echo " test-shell-debug       - enter shell with test environment (debug build)"
-	@echo " test-checkstyle      - check PEP8 compliance for test framework"
-	@echo " test-refresh-deps    - refresh the Python dependencies for the tests"
+	@echo " test-checkstyle        - check PEP8 compliance for test framework"
+	@echo " test-refresh-deps      - refresh the Python dependencies for the tests"
 	@echo ""
 	@echo "Arguments controlling test runs:"
 	@echo " V=[0|1|2]              - set test verbosity level"
@@ -387,6 +378,8 @@ help:
 	@echo "                          and tearing down a testcase"
 	@echo "    DEBUG=gdbserver     - run gdb inside a gdb server, otherwise"
 	@echo "                          same as above"
+	@echo "    DEBUG=attach        - attach test case to already running vpp in gdb (see test-start-vpp-in-gdb)"
+	@echo ""
 	@echo " STEP=[yes|no]          - ease debugging by stepping through a testcase"
 	@echo " SANITY=[yes|no]        - perform sanity import of vpp-api/sanity vpp run before running tests (default: yes)"
 	@echo " EXTENDED_TESTS=[1|y]   - used by '[re]test-all' & '[re]test-all-debug' to run extended tests"
@@ -404,7 +397,6 @@ help:
 	@echo "                          e.g. VARIANT=skx test the skx march variants"
 	@echo "                          e.g. VARIANT=icl test the icl march variants"
 	@echo ""
-	@echo " VPP_ZOMBIE_NOCHECK=1   - skip checking for vpp (zombie) processes (CAUTION)"
 	@echo " COREDUMP_SIZE=<size>   - pass <size> as unix { coredump-size <size> } argument to vpp"
 	@echo "                          e.g. COREDUMP_SIZE=4g"
 	@echo "                               COREDUMP_SIZE=unlimited"
@@ -423,6 +415,17 @@ help:
 	@echo ""
 	@echo " RND_SEED=seed          - Seed RND with given seed"
 	@echo ""
+	@echo "Starting VPP in GDB for use with DEBUG=attach:"
+	@echo ""
+	@echo " test-start-vpp-in-gdb       - start VPP in gdb (release)"
+	@echo " test-start-vpp-debug-in-gdb - start VPP in gdb (debug)"
+	@echo ""
+	@echo "Arguments controlling VPP in GDB runs:"
+	@echo " "
+	@echo " VPP_IN_GDB_TMP_DIR     - specify directory to run VPP IN (default: /tmp/unittest-attach-gdb)"
+	@echo " VPP_IN_GDB_NO_RMDIR=0  - don't remove existing tmp dir but fail instead"
+	@echo " VPP_IN_GDB_CMDLINE=1   - add 'interactive' to VPP arguments to run with command line"
+	@echo ""
 	@echo "Creating test documentation"
 	@echo " test-doc               - generate documentation for test framework"
 	@echo " test-wipe-doc          - wipe documentation for test framework"
diff --git a/test/debug.py b/test/debug.py
index d1c89c66f3c..e79f082615d 100644
--- a/test/debug.py
+++ b/test/debug.py
@@ -4,6 +4,9 @@ import os
 import pexpect
 import sys
 
+from sanity_run_vpp import SanityTestCase
+from shutil import rmtree
+
 gdb_path = '/usr/bin/gdb'
 
 
@@ -22,3 +25,40 @@ def spawn_gdb(binary_path, core_path):
     else:
         sys.stderr.write("Debugger '%s' does not exist or is not "
                          "an executable..\n" % gdb_path)
+
+
+def start_vpp_in_gdb():
+    # here we use SanityTestCase as a dummy to inherit functionality,
+    # but any test case class could be used ...
+    SanityTestCase.set_debug_flags("attach")
+    SanityTestCase.tempdir = SanityTestCase.get_tempdir()
+    if os.path.exists(SanityTestCase.tempdir):
+        if os.getenv("VPP_IN_GDB_NO_RMDIR", "0") in ["1", "y", "yes"]:
+            raise FileExistsError(
+                "Temporary directory exists and removal denied.")
+        print("Removing existing temp dir '%s'." % SanityTestCase.tempdir)
+        rmtree(SanityTestCase.tempdir)
+    print("Creating temp dir '%s'." % SanityTestCase.tempdir)
+    os.mkdir(SanityTestCase.tempdir)
+    SanityTestCase.setUpConstants()
+    vpp_cmdline = SanityTestCase.vpp_cmdline
+    if os.getenv("VPP_IN_GDB_CMDLINE", "y").lower() in ["1", "y", "yes"]:
+        print("Hacking cmdline to make VPP interactive.")
+        vpp_cmdline.insert(vpp_cmdline.index("nodaemon"), "interactive")
+    print("VPP cmdline is %s" % " ".join(vpp_cmdline))
+    print("Running GDB.")
+
+    if os.path.isfile(gdb_path) and os.access(gdb_path, os.X_OK):
+        gdb_cmdline = "%s --args %s " % (gdb_path, " ".join(vpp_cmdline))
+        print("GDB cmdline is %s" % gdb_cmdline)
+        gdb = pexpect.spawn(gdb_cmdline)
+        gdb.interact()
+        try:
+            gdb.terminate(True)
+        except:
+            pass
+        if gdb.isalive():
+            raise Exception("GDB refused to die...")
+    else:
+        sys.stderr.write("Debugger '%s' does not exist or is not "
+                         "an executable..\n" % gdb_path)
diff --git a/test/framework.py b/test/framework.py
index 8e50a467834..67ac495547c 100644
--- a/test/framework.py
+++ b/test/framework.py
@@ -281,6 +281,17 @@ tag_run_solo = create_tag_decorator(TestCaseTag.RUN_SOLO)
 tag_fixme_vpp_workers = create_tag_decorator(TestCaseTag.FIXME_VPP_WORKERS)
 
 
+class DummyVpp:
+    returncode = None
+    pid = 0xcafebafe
+
+    def poll(self):
+        pass
+
+    def terminate(self):
+        pass
+
+
 class VppTestCase(unittest.TestCase):
     """This subclass is a base class for VPP test cases that are implemented as
     classes. It provides methods to create and run test case.
@@ -330,6 +341,7 @@ class VppTestCase(unittest.TestCase):
         cls.debug_gdb = False
         cls.debug_gdbserver = False
         cls.debug_all = False
+        cls.debug_attach = False
         if d is None:
             return
         dl = d.lower()
@@ -339,6 +351,8 @@ class VppTestCase(unittest.TestCase):
             cls.debug_gdb = True
         elif dl == "gdbserver" or dl == "gdbserver-all":
             cls.debug_gdbserver = True
+        elif dl == "attach":
+            cls.debug_attach = True
         else:
             raise Exception("Unrecognized DEBUG option: '%s'" % d)
         if dl == "gdb-all" or dl == "gdbserver-all":
@@ -377,11 +391,9 @@ class VppTestCase(unittest.TestCase):
     def setUpConstants(cls):
         """ Set-up the test case class based on environment variables """
         cls.step = BoolEnvironmentVariable('STEP')
-        d = os.getenv("DEBUG", None)
         # inverted case to handle '' == True
         c = os.getenv("CACHE_OUTPUT", "1")
         cls.cache_vpp_output = False if c.lower() in ("n", "no", "0") else True
-        cls.set_debug_flags(d)
         cls.vpp_bin = os.getenv('VPP_BIN', "vpp")
         cls.plugin_path = os.getenv('VPP_PLUGIN_PATH')
         cls.test_plugin_path = os.getenv('VPP_TEST_PLUGIN_PATH')
@@ -434,15 +446,15 @@ class VppTestCase(unittest.TestCase):
             "unix", "{", "nodaemon", debug_cli, "full-coredump",
             coredump_size, "runtime-dir", cls.tempdir, "}",
             "api-trace", "{", "on", "}",
-            "api-segment", "{", "prefix", cls.shm_prefix, "}",
+            "api-segment", "{", "prefix", cls.get_api_segment_prefix(), "}",
             "cpu", "{", "main-core", str(cpu_core_number), ]
         if cls.vpp_worker_count:
             cls.vpp_cmdline.extend(["workers", str(cls.vpp_worker_count)])
         cls.vpp_cmdline.extend([
             "}",
             "physmem", "{", "max-size", "32m", "}",
-            "statseg", "{", "socket-name", cls.stats_sock, "}",
-            "socksvr", "{", "socket-name", cls.api_sock, "}",
+            "statseg", "{", "socket-name", cls.get_stats_sock_path(), "}",
+            "socksvr", "{", "socket-name", cls.get_api_sock_path(), "}",
             "node { ", default_variant, "}",
             "api-fuzz {", api_fuzzing, "}",
             "plugins", "{", "plugin", "dpdk_plugin.so", "{", "disable", "}",
@@ -458,8 +470,9 @@ class VppTestCase(unittest.TestCase):
         if cls.test_plugin_path is not None:
             cls.vpp_cmdline.extend(["test_plugin_path", cls.test_plugin_path])
 
-        cls.logger.info("vpp_cmdline args: %s" % cls.vpp_cmdline)
-        cls.logger.info("vpp_cmdline: %s" % " ".join(cls.vpp_cmdline))
+        if not cls.debug_attach:
+            cls.logger.info("vpp_cmdline args: %s" % cls.vpp_cmdline)
+            cls.logger.info("vpp_cmdline: %s" % " ".join(cls.vpp_cmdline))
 
     @classmethod
     def wait_for_enter(cls):
@@ -491,6 +504,10 @@ class VppTestCase(unittest.TestCase):
         input("Press ENTER to continue running the testcase...")
 
     @classmethod
+    def attach_vpp(cls):
+        cls.vpp = DummyVpp()
+
+    @classmethod
     def run_vpp(cls):
         cmdline = cls.vpp_cmdline
 
@@ -547,37 +564,52 @@ class VppTestCase(unittest.TestCase):
                                  corefile, curr_size)
 
     @classmethod
+    def get_stats_sock_path(cls):
+        return "%s/stats.sock" % cls.tempdir
+
+    @classmethod
+    def get_api_sock_path(cls):
+        return "%s/api.sock" % cls.tempdir
+
+    @classmethod
+    def get_api_segment_prefix(cls):
+        return os.path.basename(cls.tempdir)  # Only used for VAPI
+
+    @classmethod
+    def get_tempdir(cls):
+        if cls.debug_attach:
+            return os.getenv("VPP_IN_GDB_TMP_DIR",
+                             "/tmp/unittest-attach-gdb")
+        else:
+            return tempfile.mkdtemp(prefix='vpp-unittest-%s-' % cls.__name__)
+
+    @classmethod
     def setUpClass(cls):
         """
         Perform class setup before running the testcase
         Remove shared memory files, start vpp and connect the vpp-api
         """
         super(VppTestCase, cls).setUpClass()
-        gc.collect()  # run garbage collection first
         cls.logger = get_logger(cls.__name__)
         seed = os.environ["RND_SEED"]
         random.seed(seed)
         if hasattr(cls, 'parallel_handler'):
             cls.logger.addHandler(cls.parallel_handler)
             cls.logger.propagate = False
-
-        cls.tempdir = tempfile.mkdtemp(
-            prefix='vpp-unittest-%s-' % cls.__name__)
-        cls.stats_sock = "%s/stats.sock" % cls.tempdir
-        cls.api_sock = "%s/api.sock" % cls.tempdir
+        d = os.getenv("DEBUG", None)
+        cls.set_debug_flags(d)
+        cls.tempdir = cls.get_tempdir()
         cls.file_handler = FileHandler("%s/log.txt" % cls.tempdir)
         cls.file_handler.setFormatter(
             Formatter(fmt='%(asctime)s,%(msecs)03d %(message)s',
                       datefmt="%H:%M:%S"))
         cls.file_handler.setLevel(DEBUG)
         cls.logger.addHandler(cls.file_handler)
-        cls.logger.debug("--- setUpClass() for %s called ---" %
-                         cls.__name__)
-        cls.shm_prefix = os.path.basename(cls.tempdir)  # Only used for VAPI
+        cls.logger.debug("--- setUpClass() for %s called ---" % cls.__name__)
         os.chdir(cls.tempdir)
         cls.logger.info("Temporary dir is %s, api socket is %s",
-                        cls.tempdir, cls.api_sock)
-        cls.logger.debug("Random seed is %s" % seed)
+                        cls.tempdir, cls.get_api_sock_path())
+        cls.logger.debug("Random seed is %s", seed)
         cls.setUpConstants()
         cls.reset_packet_infos()
         cls._pcaps = []
@@ -590,18 +622,22 @@ class VppTestCase(unittest.TestCase):
         # need to catch exceptions here because if we raise, then the cleanup
         # doesn't get called and we might end with a zombie vpp
         try:
-            cls.run_vpp()
+            if cls.debug_attach:
+                cls.attach_vpp()
+            else:
+                cls.run_vpp()
             cls.reporter.send_keep_alive(cls, 'setUpClass')
             VppTestResult.current_test_case_info = TestCaseInfo(
                 cls.logger, cls.tempdir, cls.vpp.pid, cls.vpp_bin)
             cls.vpp_stdout_deque = deque()
             cls.vpp_stderr_deque = deque()
-            cls.pump_thread_stop_flag = Event()
-            cls.pump_thread_wakeup_pipe = os.pipe()
-            cls.pump_thread = Thread(target=pump_output, args=(cls,))
-            cls.pump_thread.daemon = True
-            cls.pump_thread.start()
-            if cls.debug_gdb or cls.debug_gdbserver:
+            if not cls.debug_attach:
+                cls.pump_thread_stop_flag = Event()
+                cls.pump_thread_wakeup_pipe = os.pipe()
+                cls.pump_thread = Thread(target=pump_output, args=(cls,))
+                cls.pump_thread.daemon = True
+                cls.pump_thread.start()
+            if cls.debug_gdb or cls.debug_gdbserver or cls.debug_attach:
                 cls.vapi_response_timeout = 0
             cls.vapi = VppPapiProvider(cls.__name__, cls,
                                        cls.vapi_response_timeout)
@@ -610,7 +646,7 @@ class VppTestCase(unittest.TestCase):
             else:
                 hook = hookmodule.PollHook(cls)
             cls.vapi.register_hook(hook)
-            cls.statistics = VPPStats(socketname=cls.stats_sock)
+            cls.statistics = VPPStats(socketname=cls.get_stats_sock_path())
             try:
                 hook.poll_vpp()
             except VppDiedError:
@@ -630,6 +666,10 @@ class VppTestCase(unittest.TestCase):
                                    "VPP-API connection failed, did you forget "
                                    "to 'continue' VPP from within gdb?", RED))
                 raise e
+            if cls.debug_attach:
+                last_line = cls.vapi.cli("show thread").split("\n")[-2]
+                cls.vpp_worker_count = int(last_line.split(" ")[0])
+                print("Detected VPP with %s workers." % cls.vpp_worker_count)
         except vpp_papi.VPPRuntimeError as e:
             cls.logger.debug("%s" % e)
             cls.quit()
@@ -684,7 +724,7 @@ class VppTestCase(unittest.TestCase):
                                  cls.__name__)
                 del cls.vapi
             cls.vpp.poll()
-            if cls.vpp.returncode is None:
+            if not cls.debug_attach and cls.vpp.returncode is None:
                 cls.wait_for_coredump()
                 cls.logger.debug("Sending TERM to vpp")
                 cls.vpp.terminate()
@@ -696,8 +736,9 @@ class VppTestCase(unittest.TestCase):
                     outs, errs = cls.vpp.communicate()
             cls.logger.debug("Deleting class vpp attribute on %s",
                              cls.__name__)
-            cls.vpp.stdout.close()
-            cls.vpp.stderr.close()
+            if not cls.debug_attach:
+                cls.vpp.stdout.close()
+                cls.vpp.stderr.close()
             del cls.vpp
 
         if cls.vpp_startup_failed:
diff --git a/test/run_tests.py b/test/run_tests.py
index 59c9d83d512..008828c1e86 100644
--- a/test/run_tests.py
+++ b/test/run_tests.py
@@ -19,7 +19,7 @@ import framework
 from framework import VppTestRunner, running_extended_tests, VppTestCase, \
     get_testcase_doc_name, get_test_description, PASS, FAIL, ERROR, SKIP, \
     TEST_RUN
-from debug import spawn_gdb
+from debug import spawn_gdb, start_vpp_in_gdb
 from log import get_parallel_logger, double_line_delim, RED, YELLOW, GREEN, \
     colorize, single_line_delim
 from discover_tests import discover_tests
@@ -779,11 +779,15 @@ if __name__ == '__main__':
 
     retries = parse_digit_env("RETRIES", 0)
 
-    debug = os.getenv("DEBUG", "n").lower() in ["gdb", "gdbserver"]
+    debug = os.getenv("DEBUG", "n").lower() in ["gdb", "gdbserver", "attach"]
 
     debug_core = os.getenv("DEBUG", "").lower() == "core"
     compress_core = framework.BoolEnvironmentVariable("CORE_COMPRESS")
 
+    if os.getenv("VPP_IN_GDB", "n").lower() in ["1", "y", "yes"]:
+        start_vpp_in_gdb()
+        exit()
+
     step = framework.BoolEnvironmentVariable("STEP")
     force_foreground = framework.BoolEnvironmentVariable("FORCE_FOREGROUND")
 
@@ -815,8 +819,9 @@ if __name__ == '__main__':
 
     if run_interactive and concurrent_tests > 1:
         raise NotImplementedError(
-            'Running tests interactively (DEBUG is gdb or gdbserver or STEP '
-            'is set) in parallel (TEST_JOBS is more than 1) is not supported')
+            'Running tests interactively (DEBUG is gdb[server] or ATTACH or '
+            'STEP is set) in parallel (TEST_JOBS is more than 1) is not '
+            'supported')
 
     parser = argparse.ArgumentParser(description="VPP unit tests")
     parser.add_argument("-f", "--failfast", action='store_true',
diff --git a/test/sanity_run_vpp.py b/test/sanity_run_vpp.py
index 5eb68853b1f..f91dc833eeb 100644
--- a/test/sanity_run_vpp.py
+++ b/test/sanity_run_vpp.py
@@ -43,8 +43,8 @@ if __name__ == '__main__':
     y.close()
 
     if rc == 0:
-        print('Sanity test case passed.\n')
+        print('Sanity test case passed.')
     else:
-        print('Sanity test case failed.\n')
+        print('Sanity test case failed.')
 
     sys.exit(rc)
diff --git a/test/test_stats_client.py b/test/test_stats_client.py
index 0fa87a370c1..bdc98118aeb 100644
--- a/test/test_stats_client.py
+++ b/test/test_stats_client.py
@@ -34,7 +34,7 @@ class StatsClientTestCase(VppTestCase):
         initial_fds = p.num_fds()
 
         for _ in range(100):
-            stats = VPPStats(socketname=cls.stats_sock)
+            stats = VPPStats(socketname=cls.get_stats_sock_path())
             stats.disconnect()
 
         ending_fds = p.num_fds()
diff --git a/test/test_vlib.py b/test/test_vlib.py
index 64218eabbb3..a9a5f6aeb89 100644
--- a/test/test_vlib.py
+++ b/test/test_vlib.py
@@ -192,7 +192,8 @@ class TestVlib(VppTestCase):
         """ Private Binary API Segment Test (takes 70 seconds) """
 
         vat_path = self.vpp_bin + '_api_test'
-        vat = pexpect.spawn(vat_path, ['socket-name', self.api_sock])
+        vat = pexpect.spawn(vat_path, ['socket-name',
+                                       self.get_api_sock_path()])
         vat.expect("vat# ", timeout=10)
         vat.sendline('sock_init_shm')
         vat.expect("vat# ", timeout=10)
@@ -203,5 +204,6 @@ class TestVlib(VppTestCase):
         time.sleep(70)
         self.logger.info("Reaper should be complete...")
 
+
 if __name__ == '__main__':
     unittest.main(testRunner=VppTestRunner)
diff --git a/test/vpp_papi_provider.py b/test/vpp_papi_provider.py
index d677ab316b2..4b58d6c8bde 100644
--- a/test/vpp_papi_provider.py
+++ b/test/vpp_papi_provider.py
@@ -143,7 +143,7 @@ class VppPapiProvider(object):
         self.vpp = VPPApiClient(logger=test_class.logger,
                                 read_timeout=read_timeout,
                                 use_socket=True,
-                                server_address=test_class.api_sock)
+                                server_address=test_class.get_api_sock_path())
         self._events = queue.Queue()
 
     def __enter__(self):
@@ -252,7 +252,7 @@ class VppPapiProvider(object):
         """Connect the API to VPP"""
         # This might be called before VPP is prepared to listen to the socket
         retries = 0
-        while not os.path.exists(self.test_class.api_sock):
+        while not os.path.exists(self.test_class.get_api_sock_path()):
             time.sleep(0.5)
             retries += 1
             if retries > 120: