aboutsummaryrefslogtreecommitdiffstats
path: root/resources/libraries/python
diff options
context:
space:
mode:
authorpmikus <pmikus@cisco.com>2021-05-26 16:06:02 +0000
committerPeter Mikus <pmikus@cisco.com>2021-06-04 06:45:23 +0000
commit4a0b0f2b633bb9fd15a7dc8357650a9ac7846edd (patch)
tree71151a129bfc83d9da837283c78972f0f72125ee /resources/libraries/python
parent5468f3b23b892541026ab29aa37b5131e7e30a82 (diff)
Performance: Add AF_XDP tests
- enabling for fortville, columbiaville - enabling experimental for mlx Signed-off-by: pmikus <pmikus@cisco.com> Change-Id: I1b7ceb54769f4a0089ac7309350499e60c5cca0a
Diffstat (limited to 'resources/libraries/python')
-rw-r--r--resources/libraries/python/Constants.py14
-rw-r--r--resources/libraries/python/CpuUtils.py30
-rw-r--r--resources/libraries/python/InterfaceUtil.py178
-rw-r--r--resources/libraries/python/IrqUtil.py99
-rw-r--r--resources/libraries/python/topology.py4
5 files changed, 295 insertions, 30 deletions
diff --git a/resources/libraries/python/Constants.py b/resources/libraries/python/Constants.py
index 4afcc0dd6a..992a3140be 100644
--- a/resources/libraries/python/Constants.py
+++ b/resources/libraries/python/Constants.py
@@ -333,12 +333,12 @@ class Constants:
NIC_NAME_TO_DRIVER = {
u"Intel-X520-DA2": [u"vfio-pci"],
u"Intel-X553": [u"vfio-pci"],
- u"Intel-X710": [u"vfio-pci", u"avf"],
- u"Intel-XL710": [u"vfio-pci", u"avf"],
- u"Intel-XXV710": [u"vfio-pci", u"avf"],
- u"Intel-E810CQ": [u"vfio-pci", u"avf"],
+ u"Intel-X710": [u"vfio-pci", u"avf", u"af_xdp"],
+ u"Intel-XL710": [u"vfio-pci", u"avf", u"af_xdp"],
+ u"Intel-XXV710": [u"vfio-pci", u"avf", u"af_xdp"],
+ u"Intel-E810CQ": [u"vfio-pci", u"avf", u"af_xdp"],
u"Amazon-Nitro-50G": [u"vfio-pci"],
- u"Mellanox-CX556A": [u"rdma-core"],
+ u"Mellanox-CX556A": [u"rdma-core", u"af_xdp"],
}
# Each driver needs different prugin to work.
@@ -346,6 +346,7 @@ class Constants:
u"vfio-pci": u"dpdk_plugin.so",
u"avf": u"avf_plugin.so",
u"rdma-core": u"rdma_plugin.so",
+ u"af_xdp": u"af_xdp_plugin.so",
}
# Tags to differentiate tests for different NIC driver.
@@ -353,6 +354,7 @@ class Constants:
u"vfio-pci": u"DRV_VFIO_PCI",
u"avf": u"DRV_AVF",
u"rdma-core": u"DRV_RDMA_CORE",
+ u"af_xdp": u"DRV_AF_XDP",
}
# Suite names have to be different, add prefix.
@@ -360,6 +362,7 @@ class Constants:
u"vfio-pci": u"",
u"avf": u"avf-",
u"rdma-core": u"rdma-",
+ u"af_xdp": u"af-xdp-",
}
# Number of virtual functions of physical nic.
@@ -367,6 +370,7 @@ class Constants:
u"vfio-pci": u"nic_vfs}= | 0",
u"avf": u"nic_vfs}= | 1",
u"rdma-core": u"nic_vfs}= | 0",
+ u"af_xdp": u"nic_vfs}= | 0",
}
# Not each driver is supported by each NIC.
diff --git a/resources/libraries/python/CpuUtils.py b/resources/libraries/python/CpuUtils.py
index 293d6b6913..e23404b1dd 100644
--- a/resources/libraries/python/CpuUtils.py
+++ b/resources/libraries/python/CpuUtils.py
@@ -334,6 +334,36 @@ class CpuUtils:
return result
@staticmethod
+ def get_affinity_af_xdp(
+ node, pf_key, cpu_skip_cnt=0, cpu_cnt=1):
+ """Get affinity for AF_XDP interface. Result will be used to pin IRQs.
+
+ :param node: Topology node.
+ :param pf_key: Topology interface.
+ :param cpu_skip_cnt: Amount of CPU cores to skip.
+ :param cpu_cnt: CPU threads count.
+ :type node: dict
+ :type pf_key: str
+ :type cpu_skip_cnt: int
+ :type cpu_cnt: int
+ :returns: List of CPUs allocated to AF_XDP interface.
+ :rtype: list
+ """
+ if pf_key:
+ cpu_node = Topology.get_interface_numa_node(node, pf_key)
+ else:
+ cpu_node = 0
+
+ smt_used = CpuUtils.is_smt_enabled(node[u"cpuinfo"])
+ if smt_used:
+ cpu_cnt = cpu_cnt // CpuUtils.NR_OF_THREADS
+
+ return CpuUtils.cpu_slice_of_list_per_node(
+ node, cpu_node, skip_cnt=cpu_skip_cnt, cpu_cnt=cpu_cnt,
+ smt_used=smt_used
+ )
+
+ @staticmethod
def get_affinity_nf(
nodes, node, nf_chains=1, nf_nodes=1, nf_chain=1, nf_node=1,
vs_dtc=1, nf_dtc=1, nf_mtcr=2, nf_dtcr=1):
diff --git a/resources/libraries/python/InterfaceUtil.py b/resources/libraries/python/InterfaceUtil.py
index bf36635915..939a34b8d1 100644
--- a/resources/libraries/python/InterfaceUtil.py
+++ b/resources/libraries/python/InterfaceUtil.py
@@ -111,6 +111,13 @@ class RdmaMode(IntEnum):
RDMA_API_MODE_DV = 2
+class AfXdpMode(IntEnum):
+ """AF_XDP interface mode."""
+ AF_XDP_API_MODE_AUTO = 0
+ AF_XDP_API_MODE_COPY = 1
+ AF_XDP_API_MODE_ZERO_COPY = 2
+
+
class InterfaceUtil:
"""General utilities for managing interfaces"""
@@ -229,6 +236,26 @@ class InterfaceUtil:
)
@staticmethod
+ def set_interface_state_pci(
+ node, pf_pcis, namespace=None, state=u"up"):
+ """Set operational state for interface specified by PCI address.
+
+ :param node: Topology node.
+ :param pf_pcis: List of node's interfaces PCI addresses.
+ :param namespace: Exec command in namespace. (Optional, Default: none)
+ :param state: Up/Down. (Optional, default: up)
+ :type nodes: dict
+ :type pf_pcis: list
+ :type namespace: str
+ :type state: str
+ """
+ for pf_pci in pf_pcis:
+ pf_eth = InterfaceUtil.pci_to_eth(node, pf_pci)
+ InterfaceUtil.set_linux_interface_state(
+ node, pf_eth, namespace=namespace, state=state
+ )
+
+ @staticmethod
def set_interface_mtu(node, pf_pcis, mtu=9200):
"""Set Ethernet MTU for specified interfaces.
@@ -246,25 +273,43 @@ class InterfaceUtil:
exec_cmd_no_error(node, cmd, sudo=True)
@staticmethod
- def set_interface_flow_control(node, pf_pcis, rx=u"off", tx=u"off"):
+ def set_interface_channels(
+ node, pf_pcis, num_queues=1, channel=u"combined"):
+ """Set interface channels for specified interfaces.
+
+ :param node: Topology node.
+ :param pf_pcis: List of node's interfaces PCI addresses.
+ :param num_queues: Number of channels. (Optional, Default: 1)
+ :param channel: Channel type. (Optional, Default: combined)
+ :type nodes: dict
+ :type pf_pcis: list
+ :type num_queues: int
+ :type channel: str
+ """
+ for pf_pci in pf_pcis:
+ pf_eth = InterfaceUtil.pci_to_eth(node, pf_pci)
+ cmd = f"ethtool --set-channels {pf_eth} {channel} {num_queues}"
+ exec_cmd_no_error(node, cmd, sudo=True)
+
+ @staticmethod
+ def set_interface_flow_control(node, pf_pcis, rxf=u"off", txf=u"off"):
"""Set Ethernet flow control for specified interfaces.
:param node: Topology node.
:param pf_pcis: List of node's interfaces PCI addresses.
- :param rx: RX flow. Default: off.
- :param tx: TX flow. Default: off.
+ :param rxf: RX flow. (Optional, Default: off).
+ :param txf: TX flow. (Optional, Default: off).
:type nodes: dict
:type pf_pcis: list
- :type rx: str
- :type tx: str
+ :type rxf: str
+ :type txf: str
"""
for pf_pci in pf_pcis:
pf_eth = InterfaceUtil.pci_to_eth(node, pf_pci)
- cmd = f"ethtool -A {pf_eth} rx off tx off"
+ cmd = f"ethtool -A {pf_eth} rx {rxf} tx {txf}"
ret_code, _, _ = exec_cmd(node, cmd, sudo=True)
if int(ret_code) not in (0, 78):
- raise RuntimeError("Failed to set MTU on {pf_eth}!")
-
+ raise RuntimeError("Failed to set flow control on {pf_eth}!")
@staticmethod
def set_pci_parameter(node, pf_pcis, key, value):
@@ -309,8 +354,7 @@ class InterfaceUtil:
with PapiSocketExecutor(node) as papi_exec:
papi_exec.add(cmd, **args).get_reply(err_msg)
except AssertionError as err:
- # TODO: Make failure tolerance optional.
- logger.debug(f"Setting MTU failed. Expected?\n{err}")
+ logger.debug(f"Setting MTU failed.\n{err}")
@staticmethod
def vpp_set_interfaces_mtu_on_node(node, mtu=9200):
@@ -1183,6 +1227,58 @@ class InterfaceUtil:
return Topology.get_interface_by_sw_index(node, sw_if_index)
@staticmethod
+ def vpp_create_af_xdp_interface(
+ node, if_key, num_rx_queues=None, rxq_size=0, txq_size=0,
+ mode=u"auto"):
+ """Create AF_XDP interface on VPP node.
+
+ :param node: DUT node from topology.
+ :param if_key: Physical interface key from topology file of interface
+ to be bound to compatible driver.
+ :param num_rx_queues: Number of RX queues. (Optional, Default: none)
+ :param rxq_size: Size of RXQ (0 = Default API; 512 = Default VPP).
+ :param txq_size: Size of TXQ (0 = Default API; 512 = Default VPP).
+ :param mode: AF_XDP interface mode. (Optional, Default: auto).
+ :type node: dict
+ :type if_key: str
+ :type num_rx_queues: int
+ :type rxq_size: int
+ :type txq_size: int
+ :type mode: str
+ :returns: Interface key (name) in topology file.
+ :rtype: str
+ :raises RuntimeError: If it is not possible to create AF_XDP interface
+ on the node.
+ """
+ PapiSocketExecutor.run_cli_cmd(
+ node, u"set logging class af_xdp level debug"
+ )
+
+ cmd = u"af_xdp_create"
+ pci_addr = Topology.get_interface_pci_addr(node, if_key)
+ args = dict(
+ name=InterfaceUtil.pci_to_eth(node, pci_addr),
+ host_if=InterfaceUtil.pci_to_eth(node, pci_addr),
+ rxq_num=int(num_rx_queues) if num_rx_queues else 0,
+ rxq_size=rxq_size,
+ txq_size=txq_size,
+ mode=getattr(AfXdpMode, f"AF_XDP_API_MODE_{mode.upper()}").value
+ )
+ err_msg = f"Failed to create AF_XDP interface on host {node[u'host']}"
+ with PapiSocketExecutor(node) as papi_exec:
+ sw_if_index = papi_exec.add(cmd, **args).get_sw_if_index(err_msg)
+
+ InterfaceUtil.vpp_set_interface_mac(
+ node, sw_if_index, Topology.get_interface_mac(node, if_key)
+ )
+ InterfaceUtil.add_eth_interface(
+ node, sw_if_index=sw_if_index, ifc_pfx=u"eth_af_xdp",
+ host_if_key=if_key
+ )
+
+ return Topology.get_interface_by_sw_index(node, sw_if_index)
+
+ @staticmethod
def vpp_create_rdma_interface(
node, if_key, num_rx_queues=None, rxq_size=0, txq_size=0,
mode=u"auto"):
@@ -1219,7 +1315,7 @@ class InterfaceUtil:
rxq_size=rxq_size,
txq_size=txq_size,
mode=getattr(RdmaMode, f"RDMA_API_MODE_{mode.upper()}").value,
- # TODO: Set True for non-jumbo packets.
+ # Note: Set True for non-jumbo packets.
no_multi_seg=False,
max_pktlen=0,
)
@@ -1577,9 +1673,52 @@ class InterfaceUtil:
exec_cmd_no_error(node, cmd, sudo=True)
@staticmethod
- def init_avf_interface(node, ifc_key, numvfs=1, osi_layer=u"L2"):
- """Init PCI device by creating VIFs and bind them to vfio-pci for AVF
- driver testing on DUT.
+ def init_interface(node, ifc_key, driver, numvfs=0, osi_layer=u"L2"):
+ """Init PCI device. Check driver compatibility and bind to proper
+ drivers. Optionally create NIC VFs.
+
+ :param node: DUT node.
+ :param ifc_key: Interface key from topology file.
+ :param driver: Base driver to use.
+ :param numvfs: Number of VIFs to initialize, 0 - disable the VIFs.
+ :param osi_layer: OSI Layer type to initialize TG with.
+ Default value "L2" sets linux interface spoof off.
+ :type node: dict
+ :type ifc_key: str
+ :type driver: str
+ :type numvfs: int
+ :type osi_layer: str
+ :returns: Virtual Function topology interface keys.
+ :rtype: list
+ :raises RuntimeError: If a reason preventing initialization is found.
+ """
+ kernel_driver = Topology.get_interface_driver(node, ifc_key)
+ vf_keys = []
+ if driver == u"avf":
+ if kernel_driver not in (
+ u"ice", u"iavf", u"i40e", u"i40evf"):
+ raise RuntimeError(
+ f"AVF needs ice or i40e compatible driver, not "
+ f"{kernel_driver} at node {node[u'host']} ifc {ifc_key}"
+ )
+ vf_keys = InterfaceUtil.init_generic_interface(
+ node, ifc_key, numvfs=numvfs, osi_layer=osi_layer
+ )
+ elif driver == u"af_xdp":
+ if kernel_driver not in (
+ u"ice", u"iavf", u"i40e", u"i40evf", u"mlx5_core"):
+ raise RuntimeError(
+ f"AF_XDP needs ice or i40e or rdma compatible driver, not "
+ f"{kernel_driver} at node {node[u'host']} ifc {ifc_key}"
+ )
+ vf_keys = InterfaceUtil.init_generic_interface(
+ node, ifc_key, numvfs=numvfs, osi_layer=osi_layer
+ )
+ return vf_keys
+
+ @staticmethod
+ def init_generic_interface(node, ifc_key, numvfs=0, osi_layer=u"L2"):
+ """Init PCI device. Bind to proper drivers. Optionally create NIC VFs.
:param node: DUT node.
:param ifc_key: Interface key from topology file.
@@ -1599,11 +1738,6 @@ class InterfaceUtil:
pf_mac_addr = Topology.get_interface_mac(node, ifc_key).split(":")
uio_driver = Topology.get_uio_driver(node)
kernel_driver = Topology.get_interface_driver(node, ifc_key)
- if kernel_driver not in (u"ice", u"iavf", u"i40e", u"i40evf"):
- raise RuntimeError(
- f"AVF needs ice or i40e compatible driver, not {kernel_driver}"
- f"at node {node[u'host']} ifc {ifc_key}"
- )
current_driver = DUTSetup.get_pci_dev_driver(
node, pf_pci_addr.replace(u":", r"\:"))
@@ -1726,8 +1860,7 @@ class InterfaceUtil:
@staticmethod
def vpp_round_robin_rx_placement(
- node, prefix, dp_worker_limit=None
- ):
+ node, prefix, dp_worker_limit=None):
"""Set Round Robin interface RX placement on all worker threads
on node.
@@ -1762,8 +1895,7 @@ class InterfaceUtil:
@staticmethod
def vpp_round_robin_rx_placement_on_all_duts(
- nodes, prefix, dp_core_limit=None
- ):
+ nodes, prefix, dp_core_limit=None):
"""Set Round Robin interface RX placement on all worker threads
on all DUTs.
diff --git a/resources/libraries/python/IrqUtil.py b/resources/libraries/python/IrqUtil.py
new file mode 100644
index 0000000000..1ef228eeac
--- /dev/null
+++ b/resources/libraries/python/IrqUtil.py
@@ -0,0 +1,99 @@
+# Copyright (c) 2021 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""IRQ handling library."""
+
+from resources.libraries.python.CpuUtils import CpuUtils
+from resources.libraries.python.InterfaceUtil import InterfaceUtil
+from resources.libraries.python.ssh import exec_cmd_no_error
+from resources.libraries.python.topology import Topology
+
+
+class IrqUtil:
+ """Contains methods for managing IRQs."""
+
+ @staticmethod
+ def get_pci_interface_irqs(node, pci_addr):
+ """Get IRQs for interface in linux specified by PCI address.
+
+ :param node: Topology node.
+ :param pci_addr: Linux interface PCI address.
+ :type node: dict
+ :type pci_addr: str
+ :returns: List of IRQs attached to specified interface.
+ :rtype: list
+ """
+ interface = InterfaceUtil.pci_to_eth(node, pci_addr)
+ return IrqUtil.get_interface_irqs(node, interface)
+
+ @staticmethod
+ def get_interface_irqs(node, interface):
+ """Get IRQs for interface in linux.
+
+ :param node: Topology node.
+ :param interface: Linux interface name.
+ :type node: dict
+ :type interface: str
+ :returns: List of IRQs attached to specified interface.
+ :rtype: list
+ """
+ irqs = []
+
+ command = f"grep '{interface}-.*TxRx' /proc/interrupts | cut -f1 -d:"
+ message = f"Failed to get IRQs for {interface} on {node['host']}!"
+ stdout, _ = exec_cmd_no_error(
+ node, command, timeout=30, sudo=True, message=message
+ )
+
+ for line in stdout.splitlines():
+ irqs.append(int(line.strip()))
+
+ return irqs
+
+ @staticmethod
+ def set_interface_irqs_affinity(node, interface, cpu_skip_cnt=0, cpu_cnt=1):
+ """Set IRQs affinity for interface in linux.
+
+ :param node: Topology node.
+ :param interface: Topology interface.
+ :param cpu_skip_cnt: Amount of CPU cores to skip.
+ :param cpu_cnt: CPU threads count. (Optional, Default: 0)
+ :param cpu_list: List of CPUs. (Optional, Default: 1)
+ :type node: dict
+ :type interface: str
+ :type cpu_skip_cnt: int
+ :type cpu_cnt: int
+ """
+ cpu_list = CpuUtils.get_affinity_af_xdp(
+ node, interface, cpu_skip_cnt=cpu_skip_cnt, cpu_cnt=cpu_cnt
+ )
+ interface = Topology.get_interface_name(node, interface)
+ irq_list = IrqUtil.get_interface_irqs(node, interface)
+
+ for irq, cpu in zip(irq_list, cpu_list):
+ if cpu < 32:
+ mask = 1 << cpu
+ mask = f"{mask:x}"
+ else:
+ groups = int(cpu/32)
+ mask_fill = u""
+ for _ in range(groups):
+ mask_fill = f"{mask_fill},00000000"
+ mask = 1 << (cpu - (32 * groups))
+ mask = f"{mask:x}{mask_fill}"
+
+ command = f"sh -c 'echo {mask} > /proc/irq/{irq}/smp_affinity'"
+ message = f"Failed to set IRQ affinity for {irq} on {node['host']}!"
+ exec_cmd_no_error(
+ node, command, timeout=30, sudo=True, message=message
+ )
diff --git a/resources/libraries/python/topology.py b/resources/libraries/python/topology.py
index c39e5afabb..e829700b22 100644
--- a/resources/libraries/python/topology.py
+++ b/resources/libraries/python/topology.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2020 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
@@ -174,7 +174,7 @@ class Topology:
port_types = (
u"subinterface", u"vlan_subif", u"memif", u"tap", u"vhost",
u"loopback", u"gre_tunnel", u"vxlan_tunnel", u"eth_bond",
- u"eth_avf", u"eth_rdma", u"geneve_tunnel"
+ u"eth_avf", u"eth_rdma", u"geneve_tunnel", u"eth_af_xdp"
)
for node_data in nodes.values():