diff options
Diffstat (limited to 'resources/libraries')
| -rw-r--r-- | resources/libraries/python/ContainerUtils.py | 42 | ||||
| -rw-r--r-- | resources/libraries/python/DMAUtil.py | 213 | ||||
| -rw-r--r-- | resources/libraries/python/VppConfigGenerator.py | 12 | ||||
| -rw-r--r-- | resources/libraries/python/topology.py | 15 | ||||
| -rw-r--r-- | resources/libraries/robot/features/dma.robot | 28 | ||||
| -rw-r--r-- | resources/libraries/robot/shared/container.robot | 6 |
6 files changed, 312 insertions, 4 deletions
diff --git a/resources/libraries/python/ContainerUtils.py b/resources/libraries/python/ContainerUtils.py index 158eb4bc48..fc32248f6b 100644 --- a/resources/libraries/python/ContainerUtils.py +++ b/resources/libraries/python/ContainerUtils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Cisco and/or its affiliates. +# Copyright (c) 2024 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -256,6 +256,11 @@ class ContainerManager: self._configure_vpp_chain_ipsec( mid1=mid1, mid2=mid2, sid1=sid1, sid2=sid2, guest_dir=guest_dir, nf_instance=idx, **kwargs) + elif chain_topology == u"chain_dma": + self._configure_vpp_chain_dma( + mid1=mid1, mid2=mid2, sid1=sid1, sid2=sid2, + guest_dir=guest_dir, **kwargs + ) else: raise RuntimeError( f"Container topology {chain_topology} not implemented" @@ -278,6 +283,25 @@ class ContainerManager: f"{self.engine.container.name}-{kwargs[u'sid2']}" ) + def _configure_vpp_chain_dma(self, **kwargs): + """Configure VPP in chain topology with l2xc (dma). + + :param kwargs: Named parameters. + :type kwargs: dict + """ + dma_wqs = kwargs[f"dma_wqs"] + self.engine.create_vpp_startup_config_dma(dma_wqs) + + self.engine.create_vpp_exec_config( + u"memif_create_chain_dma.exec", + mid1=kwargs[u"mid1"], mid2=kwargs[u"mid2"], + sid1=kwargs[u"sid1"], sid2=kwargs[u"sid2"], + socket1=f"{kwargs[u'guest_dir']}/memif-" + f"{self.engine.container.name}-{kwargs[u'sid1']}", + socket2=f"{kwargs[u'guest_dir']}/memif-" + f"{self.engine.container.name}-{kwargs[u'sid2']}" + ) + def _configure_vpp_cross_horiz(self, **kwargs): """Configure VPP in cross horizontal topology (single memif). @@ -768,6 +792,22 @@ class ContainerEngine: f'echo "{vpp_config.get_config_str()}" | tee /etc/vpp/startup.conf' ) + def create_vpp_startup_config_dma(self, dma_devices): + """Create startup configuration of VPP DMA. + + :param dma_devices: DMA devices list. + :type dma_devices: list + """ + vpp_config = self.create_base_vpp_startup_config() + vpp_config.add_plugin(u"enable", u"dma_intel_plugin.so") + vpp_config.add_dma_dev(dma_devices) + + # Apply configuration + self.execute(u"mkdir -p /etc/vpp/") + self.execute( + f'echo "{vpp_config.get_config_str()}" | tee /etc/vpp/startup.conf' + ) + def create_vpp_exec_config(self, template_file, **kwargs): """Create VPP exec configuration on container. diff --git a/resources/libraries/python/DMAUtil.py b/resources/libraries/python/DMAUtil.py new file mode 100644 index 0000000000..f904ea4e3d --- /dev/null +++ b/resources/libraries/python/DMAUtil.py @@ -0,0 +1,213 @@ +# Copyright (c) 2024 Intel and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DMA util library.""" + +from re import search +from resources.libraries.python.topology import NodeType, Topology +from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error + + +class DMAUtil: + """Common DMA utilities""" + + @staticmethod + def get_dma_resource(node, dma_device): + """Get DMA resource from DMA device. + + :param node: Topology node. + :param dma_device: DMA device. + :type node: dict + :type dma_device: str + :returns: DMA resource. + :rtype: dict + """ + + cmd = f"grep -H . /sys/bus/pci/devices/{dma_device}/dsa*/*" + _, stdout, stderr = exec_cmd(node, cmd, sudo=True) + + dma_info = dict() + dma_info["dma_device"] = dma_device + dma_info["engine"] = list() + dma_info["wq"] = list() + dma_info["group"] = list() + + for line in stdout.split(): + g1 = search(r"/(dsa\d+)/(.+):(.+)", line) + if g1 is not None: + dma_info["dma_name"] = g1.group(1) + dma_info[f"{g1.group(2)}"] = g1.group(3) + + for line in stderr.split(): + g2 = search(r"/(dsa\d+)/((engine|group|wq)\d+\.\d+)", line) + if g2 is not None: + dev_type = g2.group(3) + dev = g2.group(2) + dma_info[dev_type].append(dev) + + return dma_info + + @staticmethod + def disable_dma_device(node, dma_name): + """Disable DMA device. + + :param node: Topology node. + :param dma_name: DMA name. + :type node: dict + :type dma_name: str + """ + cmd = f"cat /sys/bus/dsa/devices/{dma_name}/state" + stdout, _ = exec_cmd_no_error( + node, cmd, sudo=True, + message="Failed to get dma state.") + if stdout.strip() == "disabled": + return + + cmd = f"accel-config disable-device -f {dma_name}" + exec_cmd_no_error( + node, cmd, sudo=True, + message="Failed to disable DMA on DUT.") + + @staticmethod + def enable_dma_device(node, dma_name, groups, engines, wqs, wq_size, + max_batch_size, max_transfer_size): + """Enable DMA device. + + :param node: Topology node. + :param dma_name: DMA name. + :param groups: DMA groups. + :param engines: DMA engines. + :param wqs: DMA work queues. + :param wq_size: DMA work queue size. + :param max_batch_size: Wq max batch size. + :param max_transfer_size: Wq max transfer size. + :type node: dict + :type dma_name: str + :type groups: list + :type engines: list + :type wqs: list + :type wq_size: int + :type max_batch_size: int + :type max_transfer_size: int + """ + + # Configure Device + cmd = f"accel-config config-device {dma_name}" + + exec_cmd_no_error( + node, cmd, sudo=True, + message="Failed to configure DMA device on DUT.") + + # Configure DMA group + for i, group in enumerate(groups): + cmd = f"accel-config config-group " \ + f"{dma_name}/{group} --read-buffers-reserved=0" + + exec_cmd_no_error( + node, cmd, sudo=True, + message="Failed to configure DMA group on DUT.") + + # Configure DMA engine + for i, engine in enumerate(engines): + cmd = f"accel-config config-engine " \ + f"{dma_name}/{engine} --group-id={i}" + + exec_cmd_no_error( + node, cmd, sudo=True, + message="Failed to configure DMA engine on DUT.") + + # Configure DMA work queue + for i, wq in enumerate(wqs): + cmd = f"accel-config config-wq {dma_name}/{wq} " \ + f" --group-id={i%len(engines)} --type=user " \ + f" --priority=10 --block-on-fault=1 " \ + f" --wq-size={wq_size} --mode=dedicated " \ + f" --name={dma_name}_{i} " \ + f" --max-batch-size={max_batch_size} " \ + f" --max-transfer-size={max_transfer_size} " + + exec_cmd_no_error( + node, cmd, sudo=True, + message="Failed to configure DMA work queue on DUT.") + + # Enable DMA and work queues + cmd = f"accel-config enable-device {dma_name}" + exec_cmd_no_error( + node, cmd, sudo=True, + message="Failed to enable DMA device on DUT.") + + dma_wqs = [f"{dma_name}/{wq}" for wq in wqs] + cmd = f"accel-config enable-wq {' '.join(dma_wqs)}" + exec_cmd_no_error( + node, cmd, sudo=True, + message="Failed to enable DMA work queue on DUT.") + + @staticmethod + def enable_dmas_and_wqs_on_dut(node, wq_num): + """Enable DMAs and work queues on DUT. + + :param node: Topology node. + :param wq_num: Number of work queues. + :type node: dict + :type wq_num: int + :returns: DMA work queues enabled. + :rtype: list + """ + if node["type"] == NodeType.DUT: + dma_devs = Topology.get_bus(node) + + enabled_wqs = list() + + for dev in dma_devs.values(): + if "Intel-DSA" not in dev["model"]: + continue + + dev_pci = dev["pci_address"] + dma_info = DMAUtil.get_dma_resource(node, dev_pci) + + dma_name = dma_info["dma_name"] + groups = dma_info["group"] + engines = dma_info["engine"] + wqs = dma_info["wq"] + wq_num_per_dma = wq_num//len(dma_devs) if wq_num > 1 else 1 + max_transfer_size = \ + int(dma_info["max_transfer_size"])//wq_num_per_dma + wq_size = int(dma_info["max_work_queues_size"])//wq_num_per_dma + max_batch_size = int(dma_info["max_batch_size"]) + + DMAUtil.disable_dma_device(node, dma_name) + + DMAUtil.enable_dma_device(node, + dma_name, + groups[:wq_num_per_dma], + engines[:wq_num_per_dma], + wqs[:wq_num_per_dma], + wq_size, + max_batch_size, + max_transfer_size + ) + enabled_wqs += wqs[:wq_num_per_dma] + + cmd = f"lspci -vvv -s {dev_pci}" + exec_cmd_no_error( + node, cmd, sudo=True, message="Failed") + + cmd = "accel-config list" + exec_cmd_no_error( + node, cmd, sudo=True, message="Failed") + + cmd = "cat /proc/cmdline" + exec_cmd_no_error( + node, cmd, sudo=True, message="Failed") + + return enabled_wqs diff --git a/resources/libraries/python/VppConfigGenerator.py b/resources/libraries/python/VppConfigGenerator.py index 0628e8ee47..25b5802fb2 100644 --- a/resources/libraries/python/VppConfigGenerator.py +++ b/resources/libraries/python/VppConfigGenerator.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Cisco and/or its affiliates. +# Copyright (c) 2024 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -670,6 +670,16 @@ class VppConfigGenerator: path = [u"session", u"local-endpoints-table-memory"] self.add_config_item(self._nodeconfig, value, path) + def add_dma_dev(self, devices): + """Add DMA devices configuration. + + :param devices: DMA devices or work queues. + :type devices: list + """ + for device in devices: + path = [u"dsa", f"dev {device}"] + self.add_config_item(self._nodeconfig, u"", path) + def write_config(self, filename=None): """Generate and write VPP startup configuration to file. diff --git a/resources/libraries/python/topology.py b/resources/libraries/python/topology.py index 7e53a4ba60..22ed3666c3 100644 --- a/resources/libraries/python/topology.py +++ b/resources/libraries/python/topology.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 Cisco and/or its affiliates. +# Copyright (c) 2024 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -1087,6 +1087,19 @@ class Topology: except KeyError: return None + def get_bus(node): + """Return bus configuration of the node. + + :param node: Node created from topology. + :type node: dict + :returns: bus configuration string. + :rtype: str + """ + try: + return node[u"bus"] + except KeyError: + return None + @staticmethod def get_uio_driver(node): """Return uio-driver configuration of the node. diff --git a/resources/libraries/robot/features/dma.robot b/resources/libraries/robot/features/dma.robot new file mode 100644 index 0000000000..677f39c9df --- /dev/null +++ b/resources/libraries/robot/features/dma.robot @@ -0,0 +1,28 @@ +# Copyright (c) 2024 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +| Documentation | DMA Keywords +| +| Library | resources.libraries.python.DMAUtil +| Resource | resources/libraries/robot/nginx/default.robot + +*** Keywords *** +| Enable DMA WQs on all DUTs +| | [Documentation] | Enable DMA WQs on all DUTs. +| | +| | FOR | ${dut} | IN | @{duts} +| | | ${dma_wqs}= | Enable DMAs And WQs On DUT +| | | ... | ${nodes['${dut}']} | ${cpu_count_int} +| | | Set Test Variable | ${${dut}_dma_wqs} | ${dma_wqs} +| | END diff --git a/resources/libraries/robot/shared/container.robot b/resources/libraries/robot/shared/container.robot index 253f1f063f..9f08e6b1d4 100644 --- a/resources/libraries/robot/shared/container.robot +++ b/resources/libraries/robot/shared/container.robot @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Cisco and/or its affiliates. +# Copyright (c) 2024 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -240,6 +240,10 @@ | | ... | ${container_chain_topology} | | ... | dut1_if=${DUT1_${int}2}[0] | | ... | dut2_if=${DUT2_${int}2}[0] +| | ... | ELSE IF | '${container_chain_topology}' == 'chain_dma' +| | ... | ${group}.Configure VPP In All Containers +| | ... | ${container_chain_topology} +| | ... | dma_wqs=${DUT1_dma_wqs} | | ... | ELSE | | ... | ${group}.Configure VPP In All Containers | | ... | ${container_chain_topology} |