diff options
| -rw-r--r-- | resources/libraries/python/CpuUtils.py | 47 | ||||
| -rw-r--r-- | resources/libraries/python/InterfaceUtil.py | 92 | ||||
| -rw-r--r-- | resources/libraries/python/TrafficGenerator.py | 31 | ||||
| -rw-r--r-- | resources/tools/testbed-setup/ansible/roles/cleanup/tasks/tg.yaml | 2 | ||||
| -rwxr-xr-x | resources/tools/testbed-setup/ansible/roles/tg/files/csit-initialize-docker-tg.sh | 10 | ||||
| -rw-r--r-- | tests/dpdk/perf/__init__.robot | 12 | ||||
| -rw-r--r-- | tests/vpp/device/__init__.robot | 3 | ||||
| -rw-r--r-- | tests/vpp/perf/__init__.robot | 4 |
8 files changed, 90 insertions, 111 deletions
diff --git a/resources/libraries/python/CpuUtils.py b/resources/libraries/python/CpuUtils.py index 842c16d7ef..e4fff010f1 100644 --- a/resources/libraries/python/CpuUtils.py +++ b/resources/libraries/python/CpuUtils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2020 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -141,7 +141,7 @@ class CpuUtils: @staticmethod def cpu_slice_of_list_per_node( node, cpu_node, skip_cnt=0, cpu_cnt=0, smt_used=False): - """Return string of node related list of CPU numbers. + """Return node related subset of list of CPU numbers. :param node: Node dictionary with cpuinfo. :param cpu_node: Numa node number. @@ -283,12 +283,6 @@ class CpuUtils: dtc_is_integer = isinstance(nf_dtc, int) if not smt_used and not dtc_is_integer: raise RuntimeError(u"Cannot allocate if SMT is not enabled!") - # TODO: Please reword the following todo if it is still relevant - # TODO: Workaround as we are using physical core as main unit, we must - # adjust number of physical dataplane cores in case of float for further - # array referencing. As rounding method in Py2.7 and Py3.x differs, we - # are using static mapping. This can be rewritten using flat arrays and - # different logic (from Physical core unit to Logical core unit). if not dtc_is_integer: nf_dtc = 1 @@ -361,3 +355,40 @@ class CpuUtils: nf_nodes=nf_nodes, nf_chain=nf_chain, nf_node=nf_node, nf_mtcr=nf_mtcr, nf_dtcr=nf_dtcr, nf_dtc=nf_dtc, skip_cnt=skip_cnt ) + + @staticmethod + def get_affinity_trex( + node, if1_pci, if2_pci, tg_mtc=1, tg_dtc=1, tg_ltc=1): + """Get affinity for T-Rex. Result will be used to pin T-Rex threads. + + :param node: TG node. + :param if1_pci: TG first interface. + :param if2_pci: TG second interface. + :param tg_mtc: TG main thread count. + :param tg_dtc: TG dataplane thread count. + :param tg_ltc: TG latency thread count. + :type node: dict + :type if1_pci: str + :type if2_pci: str + :type tg_mtc: int + :type tg_dtc: int + :type tg_ltc: int + :returns: List of CPUs allocated to T-Rex including numa node. + :rtype: int, int, int, list + """ + interface_list = [if1_pci, if2_pci] + cpu_node = Topology.get_interfaces_numa_node(node, *interface_list) + + master_thread_id = CpuUtils.cpu_slice_of_list_per_node( + node, cpu_node, skip_cnt=0, cpu_cnt=tg_mtc, + smt_used=False) + + threads = CpuUtils.cpu_slice_of_list_per_node( + node, cpu_node, skip_cnt=tg_mtc, cpu_cnt=tg_dtc, + smt_used=False) + + latency_thread_id = CpuUtils.cpu_slice_of_list_per_node( + node, cpu_node, skip_cnt=tg_mtc + tg_dtc, cpu_cnt=tg_ltc, + smt_used=False) + + return master_thread_id[0], latency_thread_id[0], cpu_node, threads diff --git a/resources/libraries/python/InterfaceUtil.py b/resources/libraries/python/InterfaceUtil.py index bd3652b22f..5724af01c0 100644 --- a/resources/libraries/python/InterfaceUtil.py +++ b/resources/libraries/python/InterfaceUtil.py @@ -113,8 +113,6 @@ class RdmaMode(IntEnum): class InterfaceUtil: """General utilities for managing interfaces""" - __UDEV_IF_RULES_FILE = u"/etc/udev/rules.d/10-network.rules" - @staticmethod def pci_to_int(pci_str): """Convert PCI address from string format (0000:18:0a.0) to @@ -542,45 +540,6 @@ class InterfaceUtil: return DUTSetup.get_pci_dev_driver(node, pci_addr) @staticmethod - def tg_set_interfaces_udev_rules(node): - """Set udev rules for interfaces. - - Create udev rules file in /etc/udev/rules.d where are rules for each - interface used by TG node, based on MAC interface has specific name. - So after unbind and bind again to kernel driver interface has same - name as before. This must be called after TG has set name for each - port in topology dictionary. - udev rule example - SUBSYSTEM=="net", ACTION=="add", ATTR{address}=="52:54:00:e1:8a:0f", - NAME="eth1" - - :param node: Node to set udev rules on (must be TG node). - :type node: dict - :raises RuntimeError: If setting of udev rules fails. - """ - ssh = SSH() - ssh.connect(node) - - cmd = f"rm -f {InterfaceUtil.__UDEV_IF_RULES_FILE}" - ret_code, _, _ = ssh.exec_command_sudo(cmd) - if int(ret_code) != 0: - raise RuntimeError(f"'{cmd}' failed on '{node[u'host']}'") - - for interface in node[u"interfaces"].values(): - rule = u'SUBSYSTEM==\\"net\\", ACTION==\\"add\\", ATTR{address}' + \ - u'==\\"' + interface[u"mac_address"] + u'\\", NAME=\\"' + \ - interface[u"name"] + u'\\"' - cmd = f"sh -c \"echo '{rule}'\" >> " \ - f"{InterfaceUtil.__UDEV_IF_RULES_FILE}'" - - ret_code, _, _ = ssh.exec_command_sudo(cmd) - if int(ret_code) != 0: - raise RuntimeError(f"'{cmd}' failed on '{node[u'host']}'") - - cmd = u"/etc/init.d/udev restart" - ssh.exec_command_sudo(cmd) - - @staticmethod def tg_set_interfaces_default_driver(node): """Set interfaces default driver specified in topology yaml file. @@ -667,7 +626,7 @@ class InterfaceUtil: InterfaceUtil.update_nic_interface_names(node) @staticmethod - def update_tg_interface_data_on_node(node, skip_tg_udev=False): + def update_tg_interface_data_on_node(node): """Update interface name for TG/linux node in DICT__nodes. .. note:: @@ -679,9 +638,7 @@ class InterfaceUtil: "00:00:00:00:00:00": "lo" :param node: Node selected from DICT__nodes. - :param skip_tg_udev: Skip udev rename on TG node. :type node: dict - :type skip_tg_udev: bool :raises RuntimeError: If getting of interface name and MAC fails. """ # First setup interface driver specified in yaml file @@ -706,10 +663,6 @@ class InterfaceUtil: continue interface[u"name"] = name - # Set udev rules for interfaces - if not skip_tg_udev: - InterfaceUtil.tg_set_interfaces_udev_rules(node) - @staticmethod def iface_update_numa_node(node): """For all interfaces from topology file update numa node based on @@ -752,25 +705,8 @@ class InterfaceUtil: raise RuntimeError(f"Update numa node failed for: {if_pci}") @staticmethod - def update_all_numa_nodes(nodes, skip_tg=False): - """For all nodes and all their interfaces from topology file update numa - node information based on information from the node. - - :param nodes: Nodes in the topology. - :param skip_tg: Skip TG node - :type nodes: dict - :type skip_tg: bool - :returns: Nothing. - """ - for node in nodes.values(): - if node[u"type"] == NodeType.DUT: - InterfaceUtil.iface_update_numa_node(node) - elif node[u"type"] == NodeType.TG and not skip_tg: - InterfaceUtil.iface_update_numa_node(node) - - @staticmethod def update_all_interface_data_on_all_nodes( - nodes, skip_tg=False, skip_tg_udev=False, numa_node=False): + nodes, skip_tg=False, skip_vpp=False): """Update interface names on all nodes in DICT__nodes. This method updates the topology dictionary by querying interface lists @@ -778,25 +714,17 @@ class InterfaceUtil: :param nodes: Nodes in the topology. :param skip_tg: Skip TG node. - :param skip_tg_udev: Skip udev rename on TG node. - :param numa_node: Retrieve numa_node location. + :param skip_vpp: Skip VPP node. :type nodes: dict :type skip_tg: bool - :type skip_tg_udev: bool - :type numa_node: bool + :type skip_vpp: bool """ - for node_data in nodes.values(): - if node_data[u"type"] == NodeType.DUT: - InterfaceUtil.update_vpp_interface_data_on_node(node_data) - elif node_data[u"type"] == NodeType.TG and not skip_tg: - InterfaceUtil.update_tg_interface_data_on_node( - node_data, skip_tg_udev) - - if numa_node: - if node_data[u"type"] == NodeType.DUT: - InterfaceUtil.iface_update_numa_node(node_data) - elif node_data[u"type"] == NodeType.TG and not skip_tg: - InterfaceUtil.iface_update_numa_node(node_data) + for node in nodes.values(): + if node[u"type"] == NodeType.DUT and not skip_vpp: + InterfaceUtil.update_vpp_interface_data_on_node(node) + elif node[u"type"] == NodeType.TG and not skip_tg: + InterfaceUtil.update_tg_interface_data_on_node(node) + InterfaceUtil.iface_update_numa_node(node) @staticmethod def create_vlan_subinterface(node, interface, vlan): diff --git a/resources/libraries/python/TrafficGenerator.py b/resources/libraries/python/TrafficGenerator.py index 8976dff348..1b519d5713 100644 --- a/resources/libraries/python/TrafficGenerator.py +++ b/resources/libraries/python/TrafficGenerator.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2020 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -19,6 +19,7 @@ from robot.api import logger from robot.libraries.BuiltIn import BuiltIn from .Constants import Constants +from .CpuUtils import CpuUtils from .DropRateSearch import DropRateSearch from .MLRsearch.AbstractMeasurer import AbstractMeasurer from .MLRsearch.MultipleLossRatioSearch import MultipleLossRatioSearch @@ -278,6 +279,11 @@ class TrafficGenerator(AbstractMeasurer): if1_adj_addr, if2_adj_addr = if2_adj_addr, if1_adj_addr self._ifaces_reordered = True + master_thread_id, latency_thread_id, socket, threads = \ + CpuUtils.get_affinity_trex( + self._node, tg_if1, tg_if2, + tg_dtc=Constants.TREX_CORE_COUNT) + if osi_layer in (u"L2", u"L3"): dst_mac0 = f"0x{if1_adj_addr.replace(u':', u',0x')}" src_mac0 = f"0x{if1_addr.replace(u':', u',0x')}" @@ -287,6 +293,7 @@ class TrafficGenerator(AbstractMeasurer): self._node, f"sh -c 'cat << EOF > /etc/trex_cfg.yaml\n" f"- version: 2\n" + f" c: {len(threads)}\n" f" limit_memory: {Constants.TREX_LIMIT_MEMORY}\n" f" interfaces: [\"{if1_pci}\",\"{if2_pci}\"]\n" f" port_info:\n" @@ -294,14 +301,21 @@ class TrafficGenerator(AbstractMeasurer): f" src_mac: [{src_mac0}]\n" f" - dest_mac: [{dst_mac1}]\n" f" src_mac: [{src_mac1}]\n" + f" platform :\n" + f" master_thread_id: {master_thread_id}\n" + f" latency_thread_id: {latency_thread_id}\n" + f" dual_if:\n" + f" - socket: {socket}\n" + f" threads: {threads}\n" f"EOF'", - sudo=True, message=u"TRex config generation error" + sudo=True, message=u"TRex config generation!" ) elif osi_layer == u"L7": exec_cmd_no_error( self._node, f"sh -c 'cat << EOF > /etc/trex_cfg.yaml\n" f"- version: 2\n" + f" c: {len(threads)}\n" f" limit_memory: {Constants.TREX_LIMIT_MEMORY}\n" f" interfaces: [\"{if1_pci}\",\"{if2_pci}\"]\n" f" port_info:\n" @@ -309,11 +323,17 @@ class TrafficGenerator(AbstractMeasurer): f" default_gw: [{if1_adj_addr}]\n" f" - ip: [{if2_addr}]\n" f" default_gw: [{if2_adj_addr}]\n" + f" platform :\n" + f" master_thread_id: {master_thread_id}\n" + f" latency_thread_id: {latency_thread_id}\n" + f" dual_if:\n" + f" - socket: {socket}\n" + f" threads: {threads}\n" f"EOF'", - sudo=True, message=u"TRex config generation error" + sudo=True, message=u"TRex config generation!" ) else: - raise ValueError(u"Unknown Test Type") + raise ValueError(u"Unknown Test Type!") self._startup_trex(osi_layer) @@ -348,7 +368,6 @@ class TrafficGenerator(AbstractMeasurer): cd_cmd = f"cd '{Constants.TREX_INSTALL_DIR}/scripts/'" trex_cmd = OptionString([u"nohup", u"./t-rex-64"]) trex_cmd.add(u"-i") - trex_cmd.add(f"-c {Constants.TREX_CORE_COUNT}") trex_cmd.add(u"--prefix $(hostname)") trex_cmd.add(u"--hdrh") trex_cmd.add(u"--no-scapy-server") @@ -392,8 +411,6 @@ class TrafficGenerator(AbstractMeasurer): :rtype: bool :raises RuntimeError: If node type is not a TG. """ - # No need to check subtype, we know it is TREX. - ret, _, _ = exec_cmd(node, u"pidof t-rex", sudo=True) return bool(int(ret) == 0) diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/tg.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/tg.yaml index f58cb59a1a..a026ec2acd 100644 --- a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/tg.yaml +++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/tg.yaml @@ -5,6 +5,7 @@ import_tasks: kill_process.yaml vars: process: "_t-rex" + when: docker_tg is undefined tags: kill-process - name: Kill processes - WRK @@ -12,3 +13,4 @@ vars: process: "wrk" tags: kill-process + when: docker_tg is undefined
\ No newline at end of file diff --git a/resources/tools/testbed-setup/ansible/roles/tg/files/csit-initialize-docker-tg.sh b/resources/tools/testbed-setup/ansible/roles/tg/files/csit-initialize-docker-tg.sh index 0c6dbee492..2d307b1c38 100755 --- a/resources/tools/testbed-setup/ansible/roles/tg/files/csit-initialize-docker-tg.sh +++ b/resources/tools/testbed-setup/ansible/roles/tg/files/csit-initialize-docker-tg.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2020 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -21,6 +21,7 @@ case "${1:-start}" in "start" ) # Run TG for cnt in $(seq 1 ${2:-1}); do + docker network create --driver bridge csit-nw-tg${cnt} # If the IMAGE is not already loaded then docker run will pull the # IMAGE, and all image dependencies, before it starts the container. dcr_image="snergster/csit-sut:latest" @@ -47,10 +48,11 @@ case "${1:-start}" in dcr_stc_params+="--volume /dev/hugepages:/dev/hugepages " params=(${dcr_stc_params} --name csit-tg-"${cnt}" "${dcr_image}") - docker run "${params[@]}" + docker run --network=csit-nw-tg${cnt} "${params[@]}" done ;; "stop" ) - docker rm --force $(docker ps --all --quiet --filter name=csit-tg) + docker rm --force $(docker ps --all --quiet --filter name=csit) + docker network rm $(docker network ls --filter name=csit --quiet) ;; -esac +esac
\ No newline at end of file diff --git a/tests/dpdk/perf/__init__.robot b/tests/dpdk/perf/__init__.robot index 10878c2d19..096bc8865b 100644 --- a/tests/dpdk/perf/__init__.robot +++ b/tests/dpdk/perf/__init__.robot @@ -1,4 +1,4 @@ -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2020 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -20,11 +20,11 @@ | Library | resources.libraries.python.DPDK.DPDKTools | | Suite Setup | Run Keywords | Setup performance global Variables -| ... | AND | Setup Framework | ${nodes} -| ... | AND | Install DPDK test on all DUTs | ${nodes} -| ... | AND | Get CPU Info from All Nodes | ${nodes} -| ... | AND | Update All Numa Nodes -| ... | ${nodes} | skip_tg=${True} +| ... | AND | Setup Framework | ${nodes} +| ... | AND | Install DPDK test on all DUTs | ${nodes} +| ... | AND | Get CPU Info from All Nodes | ${nodes} +| ... | AND | Update All Interface Data on All Nodes | ${nodes} +| ... | skip_tg=${True} | skip_vpp=${True} | | Suite Teardown | Cleanup Framework | ${nodes} diff --git a/tests/vpp/device/__init__.robot b/tests/vpp/device/__init__.robot index f166c124c4..0d31fc1876 100644 --- a/tests/vpp/device/__init__.robot +++ b/tests/vpp/device/__init__.robot @@ -1,4 +1,4 @@ -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2020 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -26,7 +26,6 @@ | ... | AND | Verify Vpp on All Duts | ${nodes} | ... | AND | Get CPU Info from All Nodes | ${nodes} | ... | AND | Update All Interface Data on All Nodes | ${nodes} -| ... | skip_tg_udev=${True} | numa_node=${True} | | Suite Teardown | Cleanup Framework | ${nodes} diff --git a/tests/vpp/perf/__init__.robot b/tests/vpp/perf/__init__.robot index 90afe801cd..e5a2c751a0 100644 --- a/tests/vpp/perf/__init__.robot +++ b/tests/vpp/perf/__init__.robot @@ -1,4 +1,4 @@ -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2020 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -28,7 +28,7 @@ | ... | AND | Show Vpp Version on All Duts | ${nodes} | ... | AND | Get CPU Info from All Nodes | ${nodes} | ... | AND | Update All Interface Data on All Nodes | ${nodes} -| ... | skip_tg=${True} | numa_node=${True} +| ... | skip_tg=${True} | | Suite Teardown | Cleanup Framework | ${nodes} |