tests: vpp_qemu_utils with concurrency handling

Type: test

Enhance vpp_qemu_utils functions with mutex locking, handle namespace
and
host interface existence, unique namespace/interface name, error
handling and retries, check error code explicitly.

Change-Id: I1ea66eeefbc1fee9b58e8b9886f4dd6fd8d33444
Signed-off-by: Ivan Ivanets <iivanets@cisco.com>

4 files changed, 367 insertions, 228 deletions

diff --git a/test/asf/test_http_static.py b/test/asf/test_http_static.py
index 701bfe783ea..368826f75d4 100644
--- a/test/asf/test_http_static.py
+++ b/test/asf/test_http_static.py
@@ -1,13 +1,14 @@
 from config import config
-from asfframework import VppAsfTestCase, VppTestRunner
+from asfframework import VppAsfTestCase, VppTestRunner, get_testcase_dirname
 import unittest
 import subprocess
 import tempfile
+import os
 from vpp_qemu_utils import (
     create_host_interface,
-    delete_host_interfaces,
+    delete_all_host_interfaces,
     create_namespace,
-    delete_namespace,
+    delete_all_namespaces,
 )
 
 
@@ -28,23 +29,36 @@ class TestHttpStaticVapi(VppAsfTestCase):
         cls.temp2 = tempfile.NamedTemporaryFile()
         cls.temp2.write(b"Hello world2")
 
+        cls.ns_history_name = (
+            f"{config.tmp_dir}/{get_testcase_dirname(cls.__name__)}/history_ns.txt"
+        )
+        cls.if_history_name = (
+            f"{config.tmp_dir}/{get_testcase_dirname(cls.__name__)}/history_if.txt"
+        )
+
         try:
-            create_namespace("HttpStatic")
-        except Exception:
-            cls.logger.warning("Unable to create a namespace, retrying.")
-            delete_namespace("HttpStatic")
-            create_namespace("HttpStatic")
+            # CleanUp
+            delete_all_namespaces(cls.ns_history_name)
+            delete_all_host_interfaces(cls.if_history_name)
 
-        create_host_interface("vppHost", "vppOut", "HttpStatic", "10.10.1.1/24")
+            cls.ns_name = create_namespace(cls.ns_history_name)
+            cls.host_if_name, cls.vpp_if_name = create_host_interface(
+                cls.if_history_name, cls.ns_name, "10.10.1.1/24"
+            )
 
-        cls.vapi.cli("create host-interface name vppOut")
-        cls.vapi.cli("set int state host-vppOut up")
-        cls.vapi.cli("set int ip address host-vppOut 10.10.1.2/24")
+        except Exception as e:
+            cls.logger.warning(f"Unable to complete setup: {e}")
+            raise unittest.SkipTest("Skipping tests due to setup failure.")
+
+        cls.vapi.cli(f"create host-interface name {cls.vpp_if_name}")
+        cls.vapi.cli(f"set int state host-{cls.vpp_if_name} up")
+        cls.vapi.cli(f"set int ip address host-{cls.vpp_if_name} 10.10.1.2/24")
 
     @classmethod
     def tearDownClass(cls):
-        delete_namespace("HttpStatic")
-        delete_host_interfaces("vppHost")
+        delete_all_namespaces(cls.ns_history_name)
+        delete_all_host_interfaces(cls.if_history_name)
+
         cls.temp.close()
         cls.temp2.close()
         super(TestHttpStaticVapi, cls).tearDownClass()
@@ -61,7 +75,7 @@ class TestHttpStaticVapi(VppAsfTestCase):
                 "ip",
                 "netns",
                 "exec",
-                "HttpStatic",
+                self.ns_name,
                 "curl",
                 "-v",
                 f"10.10.1.2/{self.temp.name[5:]}",
@@ -77,7 +91,7 @@ class TestHttpStaticVapi(VppAsfTestCase):
                 "ip",
                 "netns",
                 "exec",
-                "HttpStatic",
+                self.ns_name,
                 "curl",
                 f"10.10.1.2/{self.temp2.name[5:]}",
             ],
@@ -103,23 +117,35 @@ class TestHttpStaticCli(VppAsfTestCase):
         cls.temp2 = tempfile.NamedTemporaryFile()
         cls.temp2.write(b"Hello world2")
 
+        cls.ns_history_name = (
+            f"{config.tmp_dir}/{get_testcase_dirname(cls.__name__)}/history_ns.txt"
+        )
+        cls.if_history_name = (
+            f"{config.tmp_dir}/{get_testcase_dirname(cls.__name__)}/history_if.txt"
+        )
+
         try:
-            create_namespace("HttpStatic2")
-        except Exception:
-            cls.logger.warning("Unable to create namespace, retrying.")
-            delete_namespace("HttpStatic2")
-            create_namespace("HttpStatic2")
+            delete_all_namespaces(cls.ns_history_name)
+            delete_all_host_interfaces(cls.if_history_name)
 
-        create_host_interface("vppHost2", "vppOut2", "HttpStatic2", "10.10.1.1/24")
+            cls.ns_name = create_namespace(cls.ns_history_name)
+            cls.host_if_name, cls.vpp_if_name = create_host_interface(
+                cls.if_history_name, cls.ns_name, "10.10.1.1/24"
+            )
 
-        cls.vapi.cli("create host-interface name vppOut2")
-        cls.vapi.cli("set int state host-vppOut2 up")
-        cls.vapi.cli("set int ip address host-vppOut2 10.10.1.2/24")
+        except Exception as e:
+            cls.logger.warning(f"Unable to complete setup: {e}")
+            raise unittest.SkipTest("Skipping tests due to setup failure.")
+
+        cls.vapi.cli(f"create host-interface name {cls.vpp_if_name}")
+        cls.vapi.cli(f"set int state host-{cls.vpp_if_name} up")
+        cls.vapi.cli(f"set int ip address host-{cls.vpp_if_name} 10.10.1.2/24")
 
     @classmethod
     def tearDownClass(cls):
-        delete_namespace("HttpStatic2")
-        delete_host_interfaces("vppHost2")
+        delete_all_namespaces(cls.ns_history_name)
+        delete_all_host_interfaces(cls.if_history_name)
+
         cls.temp.close()
         cls.temp2.close()
         super(TestHttpStaticCli, cls).tearDownClass()
@@ -135,7 +161,7 @@ class TestHttpStaticCli(VppAsfTestCase):
                 "ip",
                 "netns",
                 "exec",
-                "HttpStatic2",
+                self.ns_name,
                 "curl",
                 f"10.10.1.2/{self.temp.name[5:]}",
             ],
@@ -149,7 +175,7 @@ class TestHttpStaticCli(VppAsfTestCase):
                 "ip",
                 "netns",
                 "exec",
-                "HttpStatic2",
+                self.ns_name,
                 "curl",
                 f"10.10.1.2/{self.temp2.name[5:]}",
             ],
diff --git a/test/asf/test_prom.py b/test/asf/test_prom.py
index f536fd19d34..ffb86a1d8d3 100644
--- a/test/asf/test_prom.py
+++ b/test/asf/test_prom.py
@@ -1,12 +1,13 @@
 from config import config
-from asfframework import VppAsfTestCase, VppTestRunner
+from asfframework import VppAsfTestCase, VppTestRunner, get_testcase_dirname
 import unittest
 import subprocess
+import os
 from vpp_qemu_utils import (
     create_host_interface,
-    delete_host_interfaces,
+    delete_all_host_interfaces,
     create_namespace,
-    delete_namespace,
+    delete_all_namespaces,
 )
 
 
@@ -22,17 +23,36 @@ class TestProm(VppAsfTestCase):
     def setUpClass(cls):
         super(TestProm, cls).setUpClass()
 
-        create_namespace("HttpStaticProm")
-        create_host_interface("vppHost", "vppOut", "HttpStaticProm", "10.10.1.1/24")
+        cls.ns_history_name = (
+            f"{config.tmp_dir}/{get_testcase_dirname(cls.__name__)}/history_ns.txt"
+        )
+        cls.if_history_name = (
+            f"{config.tmp_dir}/{get_testcase_dirname(cls.__name__)}/history_if.txt"
+        )
+
+        try:
+            # CleanUp
+            delete_all_namespaces(cls.ns_history_name)
+            delete_all_host_interfaces(cls.if_history_name)
 
-        cls.vapi.cli("create host-interface name vppOut")
-        cls.vapi.cli("set int state host-vppOut up")
-        cls.vapi.cli("set int ip address host-vppOut 10.10.1.2/24")
+            cls.ns_name = create_namespace(cls.ns_history_name)
+
+            cls.host_if_name, cls.vpp_if_name = create_host_interface(
+                cls.if_history_name, cls.ns_name, "10.10.1.1/24"
+            )
+        except Exception as e:
+            cls.logger.warning(f"Unable to complete setup: {e}")
+            raise unittest.SkipTest("Skipping tests due to setup failure.")
+
+        cls.vapi.cli(f"create host-interface name {cls.vpp_if_name}")
+        cls.vapi.cli(f"set int state host-{cls.vpp_if_name} up")
+        cls.vapi.cli(f"set int ip address host-{cls.vpp_if_name} 10.10.1.2/24")
 
     @classmethod
     def tearDownClass(cls):
-        delete_namespace(["HttpStaticProm"])
-        delete_host_interfaces("vppHost")
+        delete_all_namespaces(cls.ns_history_name)
+        delete_all_host_interfaces(cls.if_history_name)
+
         super(TestProm, cls).tearDownClass()
 
     def test_prom(self):
@@ -46,7 +66,7 @@ class TestProm(VppAsfTestCase):
                 "ip",
                 "netns",
                 "exec",
-                "HttpStaticProm",
+                self.ns_name,
                 "curl",
                 f"10.10.1.2/stats.prom",
             ],
diff --git a/test/vm_vpp_interfaces.py b/test/vm_vpp_interfaces.py
index 0f1e33d679b..66f88969ccd 100644
--- a/test/vm_vpp_interfaces.py
+++ b/test/vm_vpp_interfaces.py
@@ -3,9 +3,9 @@ import unittest
 from ipaddress import ip_address, ip_interface
 from vpp_qemu_utils import (
     create_namespace,
-    delete_namespace,
+    delete_all_namespaces,
     create_host_interface,
-    delete_host_interfaces,
+    delete_all_host_interfaces,
     set_interface_mtu,
     disable_interface_gso,
     add_namespace_route,
@@ -18,6 +18,7 @@ from config import config
 from vpp_papi import VppEnum
 import time
 import sys
+import os
 from vm_test_config import test_config
 
 #
@@ -226,7 +227,16 @@ class TestVPPInterfacesQemu:
         # prevent conflicts when TEST_JOBS > 1
         self.client_namespace = test_config["client_namespace"] + str(test["id"])
         self.server_namespace = test_config["server_namespace"] + str(test["id"])
-        create_namespace([self.client_namespace, self.server_namespace])
+        self.ns_history_file = (
+            f"{config.tmp_dir}/vpp-unittest-{self.__class__.__name__}/history_ns.txt"
+        )
+        self.if_history_name = (
+            f"{config.tmp_dir}/vpp-unittest-{self.__class__.__name__}/history_if.txt"
+        )
+        delete_all_namespaces(self.ns_history_file)
+        create_namespace(
+            self.ns_history_file, ns=[self.client_namespace, self.server_namespace]
+        )
         # Set a unique iPerf port for parallel server and client runs
         self.iperf_port = 5000 + test["id"]
         # IPerf client & server ingress/egress interface indexes in VPP
@@ -258,11 +268,11 @@ class TestVPPInterfacesQemu:
             "iprf_server_interface_on_vpp"
         ] + str(test["id"])
         # Handle client interface types
+        delete_all_host_interfaces(self.if_history_name)
         for client_if_type in client_if_types:
             if client_if_type == "af_packet":
                 create_host_interface(
-                    self.iprf_client_host_interface_on_linux,
-                    self.iprf_client_host_interface_on_vpp,
+                    self.if_history_name,
                     self.client_namespace,
                     (
                         layer2["client_ip4_prefix"]
@@ -274,6 +284,8 @@ class TestVPPInterfacesQemu:
                         if x_connect_mode == "L2"
                         else layer3["client_ip6_prefix"]
                     ),
+                    vpp_if_name=self.iprf_client_host_interface_on_vpp,
+                    host_if_name=self.iprf_client_host_interface_on_linux,
                 )
                 self.ingress_if_idx = self.create_af_packet(
                     version=client_if_version,
@@ -352,11 +364,12 @@ class TestVPPInterfacesQemu:
         for server_if_type in server_if_types:
             if server_if_type == "af_packet":
                 create_host_interface(
-                    self.iprf_server_host_interface_on_linux,
-                    self.iprf_server_host_interface_on_vpp,
+                    self.if_history_name,
                     self.server_namespace,
                     server_ip4_prefix,
                     server_ip6_prefix,
+                    vpp_if_name=self.iprf_server_host_interface_on_vpp,
+                    host_if_name=self.iprf_server_host_interface_on_linux,
                 )
                 self.egress_if_idx = self.create_af_packet(
                     version=server_if_version,
@@ -480,12 +493,7 @@ class TestVPPInterfacesQemu:
         except Exception:
             pass
         try:
-            delete_host_interfaces(
-                self.iprf_client_host_interface_on_linux,
-                self.iprf_server_host_interface_on_linux,
-                self.iprf_client_host_interface_on_vpp,
-                self.iprf_server_host_interface_on_vpp,
-            )
+            delete_all_host_interfaces(self.if_history_name)
         except Exception:
             pass
         try:
@@ -506,12 +514,7 @@ class TestVPPInterfacesQemu:
         except Exception:
             pass
         try:
-            delete_namespace(
-                [
-                    self.client_namespace,
-                    self.server_namespace,
-                ]
-            )
+            delete_all_namespaces(self.ns_history_file)
         except Exception:
             pass
         try:
diff --git a/test/vpp_qemu_utils.py b/test/vpp_qemu_utils.py
index 03b8632b15f..3831d84afe9 100644
--- a/test/vpp_qemu_utils.py
+++ b/test/vpp_qemu_utils.py
@@ -5,123 +5,213 @@
 import subprocess
 import sys
 import os
-import multiprocessing as mp
+import time
+import random
+import string
+from multiprocessing import Lock, Process
+
+lock = Lock()
 
 
 def can_create_namespaces(namespace="vpp_chk_4212"):
     """Check if the environment allows creating the namespaces"""
-
-    try:
-        result = subprocess.run(["ip", "netns", "add", namespace], capture_output=True)
-        if result.returncode != 0:
-            return False
-        result = subprocess.run(["ip", "netns", "del", namespace], capture_output=True)
-        if result.returncode != 0:
+    with lock:
+        try:
+            result = subprocess.run(
+                ["ip", "netns", "add", namespace], capture_output=True
+            )
+            if result.returncode != 0:
+                return False
+            subprocess.run(["ip", "netns", "del", namespace], capture_output=True)
+            return True
+        except Exception:
             return False
-        return True
-    except Exception:
-        return False
 
 
-def create_namespace(ns):
-    """create one or more namespaces.
+def create_namespace(history_file, ns=None):
+    """Create one or more namespaces."""
+
+    with lock:
+        namespaces = []
+        retries = 5
+
+        if ns is None:
+            result = None
+
+            for retry in range(retries):
+                suffix = "".join(
+                    random.choices(string.ascii_lowercase + string.digits, k=8)
+                )
+                new_namespace_name = f"vpp_ns{suffix}"
+                # Check if the namespace already exists
+                result = subprocess.run(
+                    ["ip", "netns", "add", new_namespace_name],
+                    capture_output=True,
+                    text=True,
+                )
+                if result.returncode == 0:
+                    with open(history_file, "a") as ns_file:
+                        ns_file.write(f"{new_namespace_name}\n")
+                    return new_namespace_name
+            error_message = result.stderr if result else "Unknown error"
+            raise Exception(
+                f"Failed to generate a unique namespace name after {retries} attempts."
+                f"Error from last attempt: {error_message}"
+            )
+        elif isinstance(ns, str):
+            namespaces = [ns]
+        else:
+            namespaces = ns
 
-    arguments:
-    ns -- a string value or an iterable of namespace names
-    """
-    if isinstance(ns, str):
-        namespaces = [ns]
-    else:
-        namespaces = ns
-    try:
         for namespace in namespaces:
-            result = subprocess.run(["ip", "netns", "add", namespace])
-            if result.returncode != 0:
-                raise Exception(f"Error while creating namespace {namespace}")
-    except subprocess.CalledProcessError as e:
-        raise Exception("Error creating namespace:", e.output)
+            for attempt in range(retries):
+                result = subprocess.run(
+                    ["ip", "netns", "add", namespace],
+                    capture_output=True,
+                    text=True,
+                )
+
+                if result.returncode == 0:
+                    with open(history_file, "a") as ns_file:
+                        ns_file.write(f"{namespace}\n")
+                    break
+                if attempt >= retries - 1:
+                    raise Exception(
+                        f"Failed to create namespace {namespace} after {retries} attempts. Error: {result.stderr.decode()}"
+                    )
+        return ns
 
 
 def add_namespace_route(ns, prefix, gw_ip):
     """Add a route to a namespace.
-
     arguments:
     ns -- namespace string value
     prefix -- NETWORK/MASK or "default"
     gw_ip -- Gateway IP
     """
-    try:
-        subprocess.run(
-            ["ip", "netns", "exec", ns, "ip", "route", "add", prefix, "via", gw_ip],
-            capture_output=True,
-        )
-    except subprocess.CalledProcessError as e:
-        raise Exception("Error adding route to namespace:", e.output)
+    with lock:
+        try:
+            subprocess.run(
+                ["ip", "netns", "exec", ns, "ip", "route", "add", prefix, "via", gw_ip],
+                capture_output=True,
+            )
+        except subprocess.CalledProcessError as e:
+            raise Exception("Error adding route to namespace:", e.output)
 
 
-def delete_host_interfaces(*host_interface_names):
-    """Delete host interfaces.
+def delete_all_host_interfaces(history_file):
+    """Delete all host interfaces whose names have been added to the history file."""
+
+    with lock:
+        if os.path.exists(history_file):
+            with open(history_file, "r") as if_file:
+                for line in if_file:
+                    if_name = line.strip()
+                    if if_name:
+                        _delete_host_interfaces(if_name)
+                os.remove(history_file)
 
+
+def _delete_host_interfaces(*host_interface_names):
+    """Delete host interfaces.
     arguments:
     host_interface_names - sequence of host interface names to be deleted
     """
     for host_interface_name in host_interface_names:
-        try:
-            subprocess.run(
-                ["ip", "link", "del", host_interface_name], capture_output=True
+        retries = 3
+        for attempt in range(retries):
+            check_result = subprocess.run(
+                ["ip", "link", "show", host_interface_name],
+                capture_output=True,
+                text=True,
             )
-        except subprocess.CalledProcessError as e:
-            raise Exception("Error deleting host interface:", e.output)
+            if check_result.returncode != 0:
+                break
+
+            result = subprocess.run(
+                ["ip", "link", "del", host_interface_name],
+                capture_output=True,
+                text=True,
+            )
+
+            if result.returncode == 0:
+                break
+            if attempt < retries - 1:
+                time.sleep(1)
+            else:
+                raise Exception(
+                    f"Failed to delete host interface {host_interface_name} after {retries} attempts"
+                )
 
 
 def create_host_interface(
-    host_interface_name, vpp_interface_name, host_namespace, *host_ip_prefixes
+    history_file, host_namespace, *host_ip_prefixes, vpp_if_name=None, host_if_name=None
 ):
     """Create a host interface of type veth.
-
     arguments:
-    host_interface_name -- name of the veth interface on the host side
-    vpp_interface_name -- name of the veth interface on the VPP side
     host_namespace -- host namespace into which the host_interface needs to be set
     host_ip_prefixes -- a sequence of ip/prefix-lengths to be set
                         on the host_interface
+    vpp_if_name -- name of the veth interface on the VPP side
+    host_if_name -- name of the veth interface on the host side
     """
-    try:
-        process = subprocess.run(
-            [
-                "ip",
-                "link",
-                "add",
-                "name",
-                vpp_interface_name,
-                "type",
-                "veth",
-                "peer",
-                "name",
-                host_interface_name,
-            ],
-            capture_output=True,
-        )
-        if process.returncode != 0:
-            print(f"Error creating host interface: {process.stderr}")
-            sys.exit(1)
+    with lock:
+        retries = 5
+
+        for attempt in range(retries):
+            if_name = (
+                host_if_name
+                or f"hostif{''.join(random.choices(string.ascii_lowercase + string.digits, k=8))}"
+            )
+            new_vpp_if_name = (
+                vpp_if_name
+                or f"vppout{''.join(random.choices(string.ascii_lowercase + string.digits, k=8))}"
+            )
+
+            result = subprocess.run(
+                [
+                    "ip",
+                    "link",
+                    "add",
+                    "name",
+                    new_vpp_if_name,
+                    "type",
+                    "veth",
+                    "peer",
+                    "name",
+                    if_name,
+                ],
+                capture_output=True,
+            )
+            if result.returncode == 0:
+                host_if_name = if_name
+                vpp_if_name = new_vpp_if_name
+                with open(history_file, "a") as if_file:
+                    if_file.write(f"{host_if_name}\n{vpp_if_name}\n")
+                break
+            if attempt >= retries - 1:
+                raise Exception(
+                    f"Failed to create host interface {if_name} and vpp {new_vpp_if_name} after {retries} attempts. Error: {result.stderr.decode()}"
+                )
 
-        process = subprocess.run(
-            ["ip", "link", "set", host_interface_name, "netns", host_namespace],
+        result = subprocess.run(
+            ["ip", "link", "set", host_if_name, "netns", host_namespace],
             capture_output=True,
         )
-        if process.returncode != 0:
-            print(f"Error setting host interface namespace: {process.stderr}")
-            sys.exit(1)
+        if result.returncode != 0:
+            raise Exception(
+                f"Error setting host interface namespace: {result.stderr.decode()}"
+            )
 
-        process = subprocess.run(
-            ["ip", "link", "set", "dev", vpp_interface_name, "up"], capture_output=True
+        result = subprocess.run(
+            ["ip", "link", "set", "dev", vpp_if_name, "up"], capture_output=True
         )
-        if process.returncode != 0:
-            print(f"Error bringing up the host interface: {process.stderr}")
-            sys.exit(1)
+        if result.returncode != 0:
+            raise Exception(
+                f"Error bringing up the host interface: {result.stderr.decode()}"
+            )
 
-        process = subprocess.run(
+        result = subprocess.run(
             [
                 "ip",
                 "netns",
@@ -131,20 +221,18 @@ def create_host_interface(
                 "link",
                 "set",
                 "dev",
-                host_interface_name,
+                host_if_name,
                 "up",
             ],
             capture_output=True,
         )
-        if process.returncode != 0:
-            print(
-                f"Error bringing up the host interface in namespace: "
-                f"{process.stderr}"
+        if result.returncode != 0:
+            raise Exception(
+                f"Error bringing up the host interface in namespace: {result.stderr.decode()}"
             )
-            sys.exit(1)
 
         for host_ip_prefix in host_ip_prefixes:
-            process = subprocess.run(
+            result = subprocess.run(
                 [
                     "ip",
                     "netns",
@@ -155,96 +243,120 @@ def create_host_interface(
                     "add",
                     host_ip_prefix,
                     "dev",
-                    host_interface_name,
+                    host_if_name,
                 ],
                 capture_output=True,
             )
-            if process.returncode != 0:
-                print(
-                    f"Error setting ip prefix on the host interface: "
-                    f"{process.stderr}"
+            if result.returncode != 0:
+                raise Exception(
+                    f"Error setting ip prefix on the host interface: {result.stderr.decode()}"
                 )
-                sys.exit(1)
-    except subprocess.CalledProcessError as e:
-        raise Exception("Error adding route to namespace:", e.output)
+
+        return host_if_name, vpp_if_name
 
 
 def set_interface_mtu(namespace, interface, mtu, logger):
-    """set an mtu number on a linux device interface."""
+    """Set an MTU number on a linux device interface."""
     args = ["ip", "link", "set", "mtu", str(mtu), "dev", interface]
     if namespace:
         args = ["ip", "netns", "exec", namespace] + args
-    try:
-        logger.debug(
-            f"Setting mtu:{mtu} on linux interface:{interface} "
-            f"in namespace:{namespace}"
-        )
-        subprocess.run(args)
-    except subprocess.CalledProcessError as e:
-        raise Exception("Error updating mtu:", e.output)
+    with lock:
+        retries = 3
+        for attempt in range(retries):
+            result = subprocess.run(args, capture_output=True)
+            if result.returncode == 0:
+                break
+            if attempt < retries - 1:
+                time.sleep(1)
+            else:
+                raise Exception(
+                    f"Failed to set MTU on interface {interface} in namespace {namespace} after {retries} attempts"
+                )
 
 
 def enable_interface_gso(namespace, interface):
-    """enable gso offload on a linux device interface."""
+    """Enable GSO offload on a linux device interface."""
     args = ["ethtool", "-K", interface, "rx", "on", "tx", "on"]
     if namespace:
         args = ["ip", "netns", "exec", namespace] + args
-    try:
-        process = subprocess.run(args, capture_output=True)
-        if process.returncode != 0:
-            print(
-                f"Error enabling GSO offload on linux device interface: "
-                f"{process.stderr}"
+    with lock:
+        result = subprocess.run(args, capture_output=True)
+        if result.returncode != 0:
+            raise Exception(
+                f"Error enabling GSO offload on interface {interface} in namespace {namespace}: {result.stderr.decode()}"
             )
-            sys.exit(1)
-    except subprocess.CalledProcessError as e:
-        raise Exception("Error enabling gso:", e.output)
 
 
 def disable_interface_gso(namespace, interface):
-    """disable gso offload on a linux device interface."""
+    """Disable GSO offload on a linux device interface."""
     args = ["ethtool", "-K", interface, "rx", "off", "tx", "off"]
     if namespace:
         args = ["ip", "netns", "exec", namespace] + args
-    try:
-        process = subprocess.run(args, capture_output=True)
-        if process.returncode != 0:
-            print(
-                f"Error disabling GSO offload on linux device interface: "
-                f"{process.stderr}"
+    with lock:
+        result = subprocess.run(args, capture_output=True)
+        if result.returncode != 0:
+            raise Exception(
+                f"Error disabling GSO offload on interface {interface} in namespace {namespace}: {result.stderr.decode()}"
             )
-            sys.exit(1)
-    except subprocess.CalledProcessError as e:
-        raise Exception("Error disabling gso:", e.output)
 
 
-def delete_namespace(ns):
-    """delete one or more namespaces.
+def delete_all_namespaces(history_file):
+    """Delete all namespaces whose names have been added to the history file."""
+    with lock:
+        if os.path.exists(history_file):
+            with open(history_file, "r") as ns_file:
+                for line in ns_file:
+                    ns_name = line.strip()
+                    if ns_name:
+                        _delete_namespace(ns_name)
+                os.remove(history_file)
+
+
+def _delete_namespace(ns):
+    """Delete one or more namespaces.
 
     arguments:
-    namespaces -- a list of namespace names
+    ns -- a list of namespace names or namespace
     """
     if isinstance(ns, str):
         namespaces = [ns]
     else:
         namespaces = ns
-    try:
-        for namespace in namespaces:
+
+    existing_namespaces = subprocess.run(
+        ["ip", "netns", "list"], capture_output=True, text=True
+    ).stdout.splitlines()
+    existing_namespaces = {line.split()[0] for line in existing_namespaces}
+
+    for namespace in namespaces:
+        if namespace not in existing_namespaces:
+            continue
+
+        retries = 3
+        for attempt in range(retries):
             result = subprocess.run(
                 ["ip", "netns", "del", namespace], capture_output=True
             )
-            if result.returncode != 0:
-                raise Exception(f"Error while deleting namespace {namespace}")
-    except subprocess.CalledProcessError as e:
-        raise Exception("Error deleting namespace:", e.output)
+            if result.returncode == 0:
+                break
+            if attempt < retries - 1:
+                time.sleep(1)
+            else:
+                raise Exception(
+                    f"Failed to delete namespace {namespace} after {retries} attempts"
+                )
 
 
 def list_namespace(ns):
-    """List the IP address of a namespace"""
-    try:
-        subprocess.run(["ip", "netns", "exec", ns, "ip", "addr"])
-    except subprocess.CalledProcessError as e:
-        raise Exception("Error listing namespace IP:", e.output)
+    """List the IP address of a namespace."""
+    with lock:
+        result = subprocess.run(
+            ["ip", "netns", "exec", ns, "ip", "addr"], capture_output=True
+        )
+        if result.returncode != 0:
+            raise Exception(
+                f"Error listing IP addresses in namespace {ns}: {result.stderr.decode()}"
+            )
 
 
 def libmemif_test_app(memif_sock_path, logger):
@@ -257,52 +369,30 @@ def libmemif_test_app(memif_sock_path, logger):
 
     def build_libmemif_app():
         if not os.path.exists(libmemif_app):
-            print(f"Building app:{libmemif_app} for memif interface testing")
+            logger.info(f"Building app:{libmemif_app} for memif interface testing")
             libmemif_app_dir = os.path.join(ws_root, "extras", "libmemif", "build")
-            if not os.path.exists(libmemif_app_dir):
-                os.makedirs(libmemif_app_dir)
+            os.makedirs(libmemif_app_dir, exist_ok=True)
             os.chdir(libmemif_app_dir)
-            try:
-                p = subprocess.run(["cmake", ".."], capture_output=True)
-                logger.debug(p.stdout)
-                if p.returncode != 0:
-                    print(f"libmemif app:{libmemif_app} cmake error:{p.stderr}")
-                    sys.exit(1)
-                p = subprocess.run(["make"], capture_output=True)
-                logger.debug(p.stdout)
-                if p.returncode != 0:
-                    print(f"Error building libmemif app:{p.stderr}")
-                    sys.exit(1)
-            except subprocess.CalledProcessError as e:
-                raise Exception("Error building libmemif_test_app:", e.output)
+            subprocess.run(["cmake", ".."], check=True)
+            subprocess.run(["make"], check=True)
 
     def start_libmemif_app():
         """Restart once if the initial run fails."""
         max_tries = 2
         run = 0
-        if not os.path.exists(libmemif_app):
-            raise Exception(
-                f"Error could not locate the libmemif test app:{libmemif_app}"
-            )
-        args = [libmemif_app, "-b", "9216", "-s", memif_sock_path]
         while run < max_tries:
-            try:
-                process = subprocess.run(args, capture_output=True)
-                logger.debug(process.stdout)
-                if process.returncode != 0:
-                    msg = f"Error starting libmemif app:{libmemif_app}"
-                    logger.error(msg)
-                    raise Exception(msg)
-            except Exception:
-                msg = f"re-starting libmemif app:{libmemif_app}"
-                logger.error(msg)
-                continue
-            else:
+            result = subprocess.run(
+                [libmemif_app, "-b", "9216", "-s", memif_sock_path], capture_output=True
+            )
+            if result.returncode == 0:
                 break
-            finally:
-                run += 1
+            logger.error(
+                f"Restarting libmemif app due to error: {result.stderr.decode()}"
+            )
+            run += 1
+            time.sleep(1)
 
     build_libmemif_app()
-    process = mp.Process(target=start_libmemif_app)
+    process = Process(target=start_libmemif_app)
     process.start()
     return process