Add memif with DMA test suites

Change-Id: I8c27b5956c0c26bb2de48321675e285ed961412f
Signed-off-by: xinfengx <xinfengx.zhao@intel.com>

7 files changed, 322 insertions, 4 deletions

diff --git a/resources/libraries/python/ContainerUtils.py b/resources/libraries/python/ContainerUtils.py
index 158eb4bc48..fc32248f6b 100644
--- a/resources/libraries/python/ContainerUtils.py
+++ b/resources/libraries/python/ContainerUtils.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2023 Cisco and/or its affiliates.
+# Copyright (c) 2024 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -256,6 +256,11 @@ class ContainerManager:
                 self._configure_vpp_chain_ipsec(
                     mid1=mid1, mid2=mid2, sid1=sid1, sid2=sid2,
                     guest_dir=guest_dir, nf_instance=idx, **kwargs)
+            elif chain_topology == u"chain_dma":
+                self._configure_vpp_chain_dma(
+                    mid1=mid1, mid2=mid2, sid1=sid1, sid2=sid2,
+                    guest_dir=guest_dir, **kwargs
+                )
             else:
                 raise RuntimeError(
                     f"Container topology {chain_topology} not implemented"
@@ -278,6 +283,25 @@ class ContainerManager:
             f"{self.engine.container.name}-{kwargs[u'sid2']}"
         )
 
+    def _configure_vpp_chain_dma(self, **kwargs):
+        """Configure VPP in chain topology with l2xc (dma).
+
+        :param kwargs: Named parameters.
+        :type kwargs: dict
+        """
+        dma_wqs = kwargs[f"dma_wqs"]
+        self.engine.create_vpp_startup_config_dma(dma_wqs)
+
+        self.engine.create_vpp_exec_config(
+            u"memif_create_chain_dma.exec",
+            mid1=kwargs[u"mid1"], mid2=kwargs[u"mid2"],
+            sid1=kwargs[u"sid1"], sid2=kwargs[u"sid2"],
+            socket1=f"{kwargs[u'guest_dir']}/memif-"
+            f"{self.engine.container.name}-{kwargs[u'sid1']}",
+            socket2=f"{kwargs[u'guest_dir']}/memif-"
+            f"{self.engine.container.name}-{kwargs[u'sid2']}"
+        )
+
     def _configure_vpp_cross_horiz(self, **kwargs):
         """Configure VPP in cross horizontal topology (single memif).
 
@@ -768,6 +792,22 @@ class ContainerEngine:
             f'echo "{vpp_config.get_config_str()}" | tee /etc/vpp/startup.conf'
         )
 
+    def create_vpp_startup_config_dma(self, dma_devices):
+        """Create startup configuration of VPP DMA.
+
+        :param dma_devices: DMA devices list.
+        :type dma_devices: list
+        """
+        vpp_config = self.create_base_vpp_startup_config()
+        vpp_config.add_plugin(u"enable", u"dma_intel_plugin.so")
+        vpp_config.add_dma_dev(dma_devices)
+
+        # Apply configuration
+        self.execute(u"mkdir -p /etc/vpp/")
+        self.execute(
+            f'echo "{vpp_config.get_config_str()}" | tee /etc/vpp/startup.conf'
+        )
+
     def create_vpp_exec_config(self, template_file, **kwargs):
         """Create VPP exec configuration on container.
 
diff --git a/resources/libraries/python/DMAUtil.py b/resources/libraries/python/DMAUtil.py
new file mode 100644
index 0000000000..f904ea4e3d
--- /dev/null
+++ b/resources/libraries/python/DMAUtil.py
@@ -0,0 +1,213 @@
+# Copyright (c) 2024 Intel and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""DMA util library."""
+
+from re import search
+from resources.libraries.python.topology import NodeType, Topology
+from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
+
+
+class DMAUtil:
+    """Common DMA utilities"""
+
+    @staticmethod
+    def get_dma_resource(node, dma_device):
+        """Get DMA resource from DMA device.
+
+        :param node: Topology node.
+        :param dma_device: DMA device.
+        :type node: dict
+        :type dma_device: str
+        :returns: DMA resource.
+        :rtype: dict
+        """
+
+        cmd = f"grep -H . /sys/bus/pci/devices/{dma_device}/dsa*/*"
+        _, stdout, stderr = exec_cmd(node, cmd, sudo=True)
+
+        dma_info = dict()
+        dma_info["dma_device"] = dma_device
+        dma_info["engine"] = list()
+        dma_info["wq"] = list()
+        dma_info["group"] = list()
+
+        for line in stdout.split():
+            g1 = search(r"/(dsa\d+)/(.+):(.+)", line)
+            if g1 is not None:
+                dma_info["dma_name"] = g1.group(1)
+                dma_info[f"{g1.group(2)}"] = g1.group(3)
+
+        for line in stderr.split():
+            g2 = search(r"/(dsa\d+)/((engine|group|wq)\d+\.\d+)", line)
+            if g2 is not None:
+                dev_type = g2.group(3)
+                dev = g2.group(2)
+                dma_info[dev_type].append(dev)
+
+        return dma_info
+
+    @staticmethod
+    def disable_dma_device(node, dma_name):
+        """Disable DMA device.
+
+        :param node: Topology node.
+        :param dma_name: DMA name.
+        :type node: dict
+        :type dma_name: str
+        """
+        cmd = f"cat /sys/bus/dsa/devices/{dma_name}/state"
+        stdout, _ = exec_cmd_no_error(
+                node, cmd, sudo=True,
+                message="Failed to get dma state.")
+        if stdout.strip() == "disabled":
+            return
+
+        cmd = f"accel-config disable-device -f {dma_name}"
+        exec_cmd_no_error(
+            node, cmd, sudo=True,
+            message="Failed to disable DMA on DUT.")
+
+    @staticmethod
+    def enable_dma_device(node, dma_name, groups, engines, wqs, wq_size,
+            max_batch_size, max_transfer_size):
+        """Enable DMA device.
+
+        :param node: Topology node.
+        :param dma_name: DMA name.
+        :param groups: DMA groups.
+        :param engines: DMA engines.
+        :param wqs: DMA work queues.
+        :param wq_size: DMA work queue size.
+        :param max_batch_size: Wq max batch size.
+        :param max_transfer_size: Wq max transfer size.
+        :type node: dict
+        :type dma_name: str
+        :type groups: list
+        :type engines: list
+        :type wqs: list
+        :type wq_size: int
+        :type max_batch_size: int
+        :type max_transfer_size: int
+        """
+
+        # Configure Device
+        cmd = f"accel-config config-device {dma_name}"
+
+        exec_cmd_no_error(
+            node, cmd, sudo=True,
+            message="Failed to configure DMA device on DUT.")
+
+        # Configure DMA group
+        for i, group in enumerate(groups):
+            cmd = f"accel-config config-group " \
+                    f"{dma_name}/{group} --read-buffers-reserved=0"
+
+            exec_cmd_no_error(
+                node, cmd, sudo=True,
+                message="Failed to configure DMA group on DUT.")
+
+        # Configure DMA engine
+        for i, engine in enumerate(engines):
+            cmd = f"accel-config config-engine " \
+                    f"{dma_name}/{engine} --group-id={i}"
+
+            exec_cmd_no_error(
+                node, cmd, sudo=True,
+                message="Failed to configure DMA engine on DUT.")
+
+        # Configure DMA work queue
+        for i, wq in enumerate(wqs):
+            cmd = f"accel-config config-wq {dma_name}/{wq} " \
+                f" --group-id={i%len(engines)} --type=user " \
+                f" --priority=10 --block-on-fault=1 " \
+                f" --wq-size={wq_size} --mode=dedicated " \
+                f" --name={dma_name}_{i} " \
+                f" --max-batch-size={max_batch_size} " \
+                f" --max-transfer-size={max_transfer_size} "
+
+            exec_cmd_no_error(
+                node, cmd, sudo=True,
+                message="Failed to configure DMA work queue on DUT.")
+
+        # Enable DMA and work queues
+        cmd = f"accel-config enable-device {dma_name}"
+        exec_cmd_no_error(
+            node, cmd, sudo=True,
+            message="Failed to enable DMA device on DUT.")
+
+        dma_wqs = [f"{dma_name}/{wq}" for wq in wqs]
+        cmd = f"accel-config enable-wq {' '.join(dma_wqs)}"
+        exec_cmd_no_error(
+            node, cmd, sudo=True,
+            message="Failed to enable DMA work queue on DUT.")
+
+    @staticmethod
+    def enable_dmas_and_wqs_on_dut(node, wq_num):
+        """Enable DMAs and work queues on DUT.
+
+        :param node: Topology node.
+        :param wq_num: Number of work queues.
+        :type node: dict
+        :type wq_num: int
+        :returns: DMA work queues enabled.
+        :rtype: list
+        """
+        if node["type"] == NodeType.DUT:
+            dma_devs = Topology.get_bus(node)
+
+        enabled_wqs = list()
+
+        for dev in dma_devs.values():
+            if "Intel-DSA" not in dev["model"]:
+                continue
+
+            dev_pci = dev["pci_address"]
+            dma_info = DMAUtil.get_dma_resource(node, dev_pci)
+
+            dma_name = dma_info["dma_name"]
+            groups = dma_info["group"]
+            engines = dma_info["engine"]
+            wqs = dma_info["wq"]
+            wq_num_per_dma = wq_num//len(dma_devs) if wq_num > 1 else 1
+            max_transfer_size = \
+                    int(dma_info["max_transfer_size"])//wq_num_per_dma
+            wq_size = int(dma_info["max_work_queues_size"])//wq_num_per_dma
+            max_batch_size = int(dma_info["max_batch_size"])
+
+            DMAUtil.disable_dma_device(node, dma_name)
+
+            DMAUtil.enable_dma_device(node,
+                    dma_name,
+                    groups[:wq_num_per_dma],
+                    engines[:wq_num_per_dma],
+                    wqs[:wq_num_per_dma],
+                    wq_size,
+                    max_batch_size,
+                    max_transfer_size
+                    )
+            enabled_wqs += wqs[:wq_num_per_dma]
+
+            cmd = f"lspci -vvv -s {dev_pci}"
+            exec_cmd_no_error(
+                node, cmd, sudo=True, message="Failed")
+
+        cmd = "accel-config list"
+        exec_cmd_no_error(
+            node, cmd, sudo=True, message="Failed")
+
+        cmd = "cat /proc/cmdline"
+        exec_cmd_no_error(
+            node, cmd, sudo=True, message="Failed")
+
+        return enabled_wqs
diff --git a/resources/libraries/python/VppConfigGenerator.py b/resources/libraries/python/VppConfigGenerator.py
index 0628e8ee47..25b5802fb2 100644
--- a/resources/libraries/python/VppConfigGenerator.py
+++ b/resources/libraries/python/VppConfigGenerator.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2023 Cisco and/or its affiliates.
+# Copyright (c) 2024 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -670,6 +670,16 @@ class VppConfigGenerator:
         path = [u"session", u"local-endpoints-table-memory"]
         self.add_config_item(self._nodeconfig, value, path)
 
+    def add_dma_dev(self, devices):
+        """Add DMA devices configuration.
+
+        :param devices: DMA devices or work queues.
+        :type devices: list
+        """
+        for device in devices:
+            path = [u"dsa", f"dev {device}"]
+            self.add_config_item(self._nodeconfig, u"", path)
+
     def write_config(self, filename=None):
         """Generate and write VPP startup configuration to file.
 
diff --git a/resources/libraries/python/topology.py b/resources/libraries/python/topology.py
index 7e53a4ba60..22ed3666c3 100644
--- a/resources/libraries/python/topology.py
+++ b/resources/libraries/python/topology.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2022 Cisco and/or its affiliates.
+# Copyright (c) 2024 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -1087,6 +1087,19 @@ class Topology:
         except KeyError:
             return None
 
+    def get_bus(node):
+        """Return bus configuration of the node.
+
+        :param node: Node created from topology.
+        :type node: dict
+        :returns: bus configuration string.
+        :rtype: str
+        """
+        try:
+            return node[u"bus"]
+        except KeyError:
+            return None
+
     @staticmethod
     def get_uio_driver(node):
         """Return uio-driver configuration of the node.
diff --git a/resources/libraries/robot/features/dma.robot b/resources/libraries/robot/features/dma.robot
new file mode 100644
index 0000000000..677f39c9df
--- /dev/null
+++ b/resources/libraries/robot/features/dma.robot
@@ -0,0 +1,28 @@
+# Copyright (c) 2024 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+| Documentation | DMA Keywords
+|
+| Library | resources.libraries.python.DMAUtil
+| Resource | resources/libraries/robot/nginx/default.robot
+
+*** Keywords ***
+| Enable DMA WQs on all DUTs
+| | [Documentation] | Enable DMA WQs on all DUTs.
+| |
+| | FOR | ${dut} | IN | @{duts}
+| | | ${dma_wqs}= | Enable DMAs And WQs On DUT
+| | | ... | ${nodes['${dut}']} | ${cpu_count_int}
+| | | Set Test Variable | ${${dut}_dma_wqs} | ${dma_wqs}
+| | END
diff --git a/resources/libraries/robot/shared/container.robot b/resources/libraries/robot/shared/container.robot
index 253f1f063f..9f08e6b1d4 100644
--- a/resources/libraries/robot/shared/container.robot
+++ b/resources/libraries/robot/shared/container.robot
@@ -1,4 +1,4 @@
-# Copyright (c) 2023 Cisco and/or its affiliates.
+# Copyright (c) 2024 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -240,6 +240,10 @@
 | | ... | ${container_chain_topology}
 | | ... | dut1_if=${DUT1_${int}2}[0]
 | | ... | dut2_if=${DUT2_${int}2}[0]
+| | ... | ELSE IF | '${container_chain_topology}' == 'chain_dma'
+| | ... | ${group}.Configure VPP In All Containers
+| | ... | ${container_chain_topology}
+| | ... | dma_wqs=${DUT1_dma_wqs}
 | | ... | ELSE
 | | ... | ${group}.Configure VPP In All Containers
 | | ... | ${container_chain_topology}
diff --git a/resources/templates/container/memif_create_chain_dma.exec b/resources/templates/container/memif_create_chain_dma.exec
new file mode 100644
index 0000000000..e4dac71943
--- /dev/null
+++ b/resources/templates/container/memif_create_chain_dma.exec
@@ -0,0 +1,10 @@
+create memif socket id $sid1 filename $socket1
+create interface memif id $mid1 socket-id $sid1 master use-dma
+set int state memif$sid1/$mid1 up
+
+create memif socket id $sid2 filename $socket2
+create interface memif id $mid2 socket-id $sid2 master use-dma
+set int state memif$sid2/$mid2 up
+
+set interface l2 xconnect memif$sid2/$mid2 memif$sid1/$mid1
+set interface l2 xconnect memif$sid1/$mid1 memif$sid2/$mid2
\ No newline at end of file