diff options
| -rwxr-xr-x | scripts/dpdk_nic_bind.py | 41 | ||||
| -rwxr-xr-x | scripts/dpdk_setup_ports.py | 236 | ||||
| -rw-r--r-- | src/main_dpdk.cpp | 7 |
3 files changed, 200 insertions, 84 deletions
diff --git a/scripts/dpdk_nic_bind.py b/scripts/dpdk_nic_bind.py index b1f38c09..bd4ec739 100755 --- a/scripts/dpdk_nic_bind.py +++ b/scripts/dpdk_nic_bind.py @@ -40,6 +40,7 @@ sys.path.append(text_tables_path) import texttable sys.path.remove(text_tables_path) import re +import termios # The PCI device class for ETHERNET devices ETHERNET_CLASS = "0200" @@ -341,27 +342,34 @@ def get_pid_using_pci(pci_list): f_cont = f.read() for pci in pci_list: if '/%s/' % pci in f_cont: - with open('/proc/%s/status' % pid) as f: - for line in f.readlines(): - key, val = line.split(':', 1) - if key.strip() == 'Tgid' and val.strip() == pid: - return int(pid) + return int(pid) def read_pid_cmdline(pid): cmdline_file = '/proc/%s/cmdline' % pid if not os.path.exists(cmdline_file): return None with open(cmdline_file, 'rb') as f: - return f.read().replace(b'\0', b' ').decode() + return f.read().replace(b'\0', b' ').decode(errors = 'replace') def confirm(msg, default = False): try: if not os.isatty(1): return default + termios.tcflush(sys.stdin, termios.TCIOFLUSH) return strtobool(raw_input(msg)) except: return default +def read_line(msg = '', default = ''): + try: + if not os.isatty(1): + return default + termios.tcflush(sys.stdin, termios.TCIOFLUSH) + return raw_input(msg).strip() + except KeyboardInterrupt: + print('') + sys.exit(1) + def unbind_one(dev_id, force): '''Unbind the device identified by "dev_id" from its current driver''' dev = devices[dev_id] @@ -550,10 +558,10 @@ def show_status(): display_devices("Other network devices", no_drv,\ "unused=%(Module_str)s") -def get_macs_from_trex(pci_addr_list): +def get_info_from_trex(pci_addr_list): if not pci_addr_list: return {} - pci_mac_dict = {} + pci_info_dict = {} run_command = 'sudo ./t-rex-64 --dump-interfaces %s' % ' '.join(pci_addr_list) proc = subprocess.Popen(shlex.split(run_command), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines = True) @@ -564,18 +572,19 @@ def get_macs_from_trex(pci_addr_list): else: print('Error upon running TRex to get MAC info:\n%s' % stdout) sys.exit(1) - pci_mac_str = 'PCI: (\S+).+?MAC: (\S+)' + pci_mac_str = 'PCI: (\S+).+?MAC: (\S+).+?Driver: (\S+)' pci_mac_re = re.compile(pci_mac_str) for line in stdout.splitlines(): match = pci_mac_re.match(line) if match: pci = match.group(1) - mac = match.group(2) if pci not in pci_addr_list: # sanity check, should not happen - print('Internal error while getting MACs of DPDK bound interfaces, unknown PCI: %s' % pci) - return {} - pci_mac_dict[pci] = mac - return pci_mac_dict + print('Internal error while getting info of DPDK bound interfaces, unknown PCI: %s' % pci) + sys.exit(1) + pci_info_dict[pci] = {} + pci_info_dict[pci]['MAC'] = match.group(2) + pci_info_dict[pci]['TRex_Driver'] = match.group(3) + return pci_info_dict def show_table(): '''Function called when the script is passed the "--table" option. @@ -586,11 +595,11 @@ def show_table(): if devices[d].get("Driver_str") in dpdk_drivers: dpdk_drv.append(d) - for pci, mac in get_macs_from_trex(dpdk_drv).items(): + for pci, info in get_info_from_trex(dpdk_drv).items(): if pci not in dpdk_drv: # sanity check, should not happen print('Internal error while getting MACs of DPDK bound interfaces, unknown PCI: %s' % pci) return - devices[pci]['MAC'] = mac + devices[pci].update(info) table = texttable.Texttable(max_width=-1) table.header(['ID', 'NUMA', 'PCI', 'MAC', 'Name', 'Driver', 'Linux IF', 'Active']) diff --git a/scripts/dpdk_setup_ports.py b/scripts/dpdk_setup_ports.py index b32163da..ab85d97c 100755 --- a/scripts/dpdk_setup_ports.py +++ b/scripts/dpdk_setup_ports.py @@ -57,15 +57,11 @@ class ConfigCreator(object): if self.has_zero_lcore: del self.cpu_topology[zero_lcore_numa][zero_lcore_core] self.cpu_topology[zero_lcore_numa][zero_lcore_core] = zero_lcore_siblings - Device_str = None for interface in self.interfaces: for mandatory_interface_field in ConfigCreator.mandatory_interface_fields: if mandatory_interface_field not in interface: raise DpdkSetup("Expected '%s' field in interface dictionary, got: %s" % (mandatory_interface_field, interface)) - if Device_str is None: - Device_str = interface['Device_str'] - elif Device_str != interface['Device_str']: - raise DpdkSetup('Interfaces should be of same type, got:\n\t* %s\n\t* %s' % (Device_str, interface['Device_str'])) + Device_str = self._verify_devices_same_type(self.interfaces) if '40Gb' in Device_str: self.speed = 40 else: @@ -95,11 +91,20 @@ class ConfigCreator(object): self.zmq_rpc_port = zmq_rpc_port self.ignore_numa = ignore_numa - def _convert_mac(self, mac_string): + @staticmethod + def _convert_mac(mac_string): if not ConfigCreator.mac_re.match(mac_string): raise DpdkSetup('MAC address should be in format of 12:34:56:78:9a:bc, got: %s' % mac_string) return ', '.join([('0x%s' % elem).lower() for elem in mac_string.split(':')]) + @staticmethod + def _verify_devices_same_type(interfaces_list): + Device_str = interfaces_list[0]['Device_str'] + for interface in interfaces_list: + if Device_str != interface['Device_str']: + raise DpdkSetup('Interfaces should be of same type, got:\n\t* %s\n\t* %s' % (Device_str, interface['Device_str'])) + return Device_str + def create_config(self, filename = None, print_config = False): config_str = '### Config file generated by dpdk_setup_ports.py ###\n\n' config_str += '- port_limit: %s\n' % len(self.interfaces) @@ -167,22 +172,16 @@ class ConfigCreator(object): except Exception as e: raise DpdkSetup('Could not create correct yaml config.\nGenerated YAML:\n%s\nEncountered error:\n%s' % (config_str, e)) + if print_config: + print(config_str) if filename: if os.path.exists(filename): - result = None - try: - result = strtobool(raw_input('File %s already exist, overwrite? (y/N)' % filename)) - except: - pass - finally: - if not result: - print('Skipping.') - return config_str + if not dpdk_nic_bind.confirm('File %s already exist, overwrite? (y/N)' % filename): + print('Skipping.') + return config_str with open(filename, 'w') as f: f.write(config_str) print('Saved.') - if print_config: - print(config_str) return config_str @@ -354,7 +353,84 @@ Other network devices print('DPDK interfaces are in use. Unbinding them might cause following process to hang:\n%s' % cmdline) if not dpdk_nic_bind.confirm('Confirm (y/N):'): return - print('TODO: unbind %s' % dpdk_interfaces) + drivers_table = { + 'rte_ixgbe_pmd': 'ixgbe', + 'rte_igb_pmd': 'igb', + 'rte_i40e_pmd': 'i40e', + 'rte_em_pmd': 'e1000', + 'rte_vmxnet3_pmd': 'vmxnet3', + 'rte_virtio_pmd': 'virtio', + 'rte_enic_pmd': 'enic', + } + for pci, info in dpdk_nic_bind.get_info_from_trex(dpdk_interfaces).items(): + if pci not in self.m_devices: + raise DpdkSetup('Internal error: PCI %s is not found among devices' % pci) + dev = self.m_devices[pci] + if info['TRex_Driver'] not in drivers_table: + print('Got unknown driver %s, description: %s' % (info['TRex_Driver'], dev['Device_str'])) + else: + print('Returning to Linux %s' % pci) + dpdk_nic_bind.bind_one(pci, drivers_table[info['TRex_Driver']], False) + + def _get_cpu_topology(self): + cpu_topology_file = '/proc/cpuinfo' + # physical processor -> physical core -> logical processing units (threads) + cpu_topology = OrderedDict() + if not os.path.exists(cpu_topology_file): + raise DpdkSetup('File with CPU topology (%s) does not exist.' % cpu_topology_file) + with open(cpu_topology_file) as f: + for lcore in f.read().split('\n\n'): + if not lcore: + continue + lcore_dict = OrderedDict() + for line in lcore.split('\n'): + key, val = line.split(':', 1) + lcore_dict[key.strip()] = val.strip() + numa = int(lcore_dict['physical id']) + if numa not in cpu_topology: + cpu_topology[numa] = OrderedDict() + core = int(lcore_dict['core id']) + if core not in cpu_topology[numa]: + cpu_topology[numa][core] = [] + cpu_topology[numa][core].append(int(lcore_dict['processor'])) + return cpu_topology + + # input: list of different descriptions of interfaces: index, pci, name etc. + # binds to dpdk not bound to any driver wanted interfaces + # output: list of maps of devices in dpdk_* format (self.m_devices.values()) + def _get_wanted_interfaces(self, input_interfaces): + wanted_interfaces = [] + sorted_pci = sorted(self.m_devices.keys()) + for interface in input_interfaces: + dev = None + try: + interface = int(interface) + if interface < 0 or interface >= len(sorted_pci): + raise DpdkSetup('Index of an interface should be in range 0:%s' % (len(sorted_pci) - 1)) + dev = self.m_devices[sorted_pci[interface]] + except ValueError: + for d in self.m_devices.values(): + if interface in (d['Interface'], d['Slot'], d['Slot_str']): + dev = d + break + if not dev: + raise DpdkSetup('Could not find information about this interface: %s' % interface) + if dev in wanted_interfaces: + raise DpdkSetup('Interface %s is specified twice' % interface) + dev['Interface_argv'] = interface + wanted_interfaces.append(dev) + + unbound = [] + for interface in wanted_interfaces: + if 'Driver_str' not in interface: + unbound.append(interface['Slot']) + if unbound: + for pci, info in dpdk_nic_bind.get_info_from_trex(unbound).items(): + if pci not in self.m_devices: + raise DpdkSetup('Internal error: PCI %s is not found among devices' % pci) + self.m_devices[pci].update(info) + + return wanted_interfaces def do_create(self): create_interfaces = map_driver.args.create_interfaces @@ -367,35 +443,12 @@ Other network devices # gather info about NICS from dpdk_nic_bind.py if not self.m_devices: self.run_dpdk_lspci() - wanted_interfaces = [] - for interface in copy.copy(create_interfaces): - for m_device in self.m_devices.values(): - if interface in (m_device['Interface'], m_device['Slot'], m_device['Slot_str']): - if m_device in wanted_interfaces: - raise DpdkSetup('Interface %s is specified twice' % interface) - m_device['Interface_argv'] = interface - wanted_interfaces.append(m_device) - create_interfaces.remove(interface) - break - # verify all interfaces identified - if len(create_interfaces): - raise DpdkSetup('Could not find information about those interfaces: %s' % create_interfaces) - - dpdk_bound = [] - for interface in wanted_interfaces: - if 'Driver_str' not in interface: - self.do_bind_one(interface['Slot']) - dpdk_bound.append(interface['Slot']) - elif interface['Driver_str'] in dpdk_nic_bind.dpdk_drivers: - dpdk_bound.append(interface['Slot']) - if dpdk_bound: - for pci, mac in dpdk_nic_bind.get_macs_from_trex(dpdk_bound).items(): - self.m_devices[pci]['MAC'] = mac + wanted_interfaces = self._get_wanted_interfaces(create_interfaces) dest_macs = map_driver.args.dest_macs for i, interface in enumerate(wanted_interfaces): if 'MAC' not in interface: - raise DpdkSetup('Cound not determine MAC of interface: %s. Please verify with -t flag.' % wanted_interfaces[i]['Interface_argv']) + raise DpdkSetup('Cound not determine MAC of interface: %s. Please verify with -t flag.' % interface['Interface_argv']) interface['src_mac'] = interface['MAC'] if isinstance(dest_macs, list) and len(dest_macs) > i: interface['dest_mac'] = dest_macs[i] @@ -404,32 +457,76 @@ Other network devices wanted_interfaces[dual_index]['dest_mac'] = interface['MAC'] # loopback wanted_interfaces[dual_index]['loopback_dest'] = True - cpu_topology_file = '/proc/cpuinfo' - # physical processor -> physical core -> logical processing units (threads) - cpu_topology = OrderedDict() - if not os.path.exists(cpu_topology_file): - raise DpdkSetup('File with CPU topology (%s) does not exist.' % cpu_topology_file) - with open(cpu_topology_file) as f: - for lcore in f.read().split('\n\n'): - if not lcore: - continue - lcore_dict = OrderedDict() - for line in lcore.split('\n'): - key, val = line.split(':', 1) - lcore_dict[key.strip()] = val.strip() - numa = int(lcore_dict['physical id']) - if numa not in cpu_topology: - cpu_topology[numa] = OrderedDict() - core = int(lcore_dict['core id']) - if core not in cpu_topology[numa]: - cpu_topology[numa][core] = [] - cpu_topology[numa][core].append(int(lcore_dict['processor'])) - - config = ConfigCreator(cpu_topology, wanted_interfaces, include_lcores = map_driver.args.create_include, exclude_lcores = map_driver.args.create_exclude, + config = ConfigCreator(self._get_cpu_topology(), wanted_interfaces, include_lcores = map_driver.args.create_include, exclude_lcores = map_driver.args.create_exclude, only_first_thread = map_driver.args.no_ht, ignore_numa = map_driver.args.ignore_numa, prefix = map_driver.args.prefix, zmq_rpc_port = map_driver.args.zmq_rpc_port, zmq_pub_port = map_driver.args.zmq_pub_port) config.create_config(filename = map_driver.args.o, print_config = map_driver.args.dump) + def do_interactive_create(self): + if not self.m_devices: + self.run_dpdk_lspci() + dpdk_nic_bind.show_table() + print('Please choose interfaces either by ID, PCI or Linux IF.') + print('Stateful will use order of interfaces: Client1 Server1 Client2 Server2 etc. for flows.') + print('Stateless can be in any order.') + print('Try to set each pair of interfaces on same NUMA for performance.') + input = dpdk_nic_bind.read_line('List of interfaces: ') + create_interfaces = input.replace(',', ' ').replace(';', ' ').split() + if not len(create_interfaces): + raise DpdkSetup('Please specify interfaces to use in the config') + if len(create_interfaces) % 2: + raise DpdkSetup('Please specify even number of interfaces') + wanted_interfaces = self._get_wanted_interfaces(create_interfaces) + ConfigCreator._verify_devices_same_type(wanted_interfaces) + print('') + + dest_macs = map_driver.args.dest_macs + ignore_numa = False + for i, interface in enumerate(wanted_interfaces): + if 'MAC' not in interface: + raise DpdkSetup('Cound not determine MAC of interface: %s. Please verify with -t flag.' % interface['Interface_argv']) + interface['src_mac'] = interface['MAC'] + dual_index = i + 1 - (i % 2) * 2 + dual_int = wanted_interfaces[dual_index] + if not ignore_numa and interface['NUMA'] != dual_int['NUMA']: + print('NUMA is different at pair of interfaces: %s and %s. It will reduce performance.' % (interface['Interface_argv'], dual_int['Interface_argv'])) + if dpdk_nic_bind.confirm('Ignore and continue? (y/N): '): + ignore_numa = True + else: + return + if isinstance(dest_macs, list) and len(dest_macs): + dest_mac = dest_macs.pop(0) + loopback_dest = False + print('For interface %s, take destination MAC from --dest-mac argument (%s)' % (interface['Interface_argv'], dest_mac)) + else: + dest_mac = dual_int['MAC'] + loopback_dest = True + print("For interface %s, take destination MAC of it's dual interface %s (%s). Change it in case of router DUT." % (interface['Interface_argv'], dual_int['Interface_argv'], dest_mac)) + try: + input_mac = dpdk_nic_bind.read_line('Press ENTER to confirm or enter new destination MAC: ') + if input_mac: + ConfigCreator._convert_mac(input_mac) # verify format + dest_mac = input_mac + loopback_dest = False + except KeyboardInterrupt: + print('') + return + wanted_interfaces[i]['dest_mac'] = dest_mac + wanted_interfaces[i]['loopback_dest'] = loopback_dest + + config = ConfigCreator(self._get_cpu_topology(), wanted_interfaces, include_lcores = map_driver.args.create_include, exclude_lcores = map_driver.args.create_exclude, + only_first_thread = map_driver.args.no_ht, ignore_numa = map_driver.args.ignore_numa or ignore_numa, + prefix = map_driver.args.prefix, zmq_rpc_port = map_driver.args.zmq_rpc_port, zmq_pub_port = map_driver.args.zmq_pub_port) + if dpdk_nic_bind.confirm('Print preview of generated config? (Y/n)', default = True): + config.create_config(print_config = True) + if dpdk_nic_bind.confirm('Save the config to file? (Y/n)', default = True): + print('Default filename is /etc/trex_cfg.yaml') + filename = dpdk_nic_bind.read_line('Press ENTER to confirm or enter new file: ') + if not filename: + filename = '/etc/trex_cfg.yaml' + config.create_config(filename = filename) + + def parse_parent_cfg (parent_cfg): parent_parser = argparse.ArgumentParser() parent_parser.add_argument('--cfg', default='') @@ -446,6 +543,9 @@ Examples: To return to Linux the DPDK bound interfaces (for ifconfig etc.) sudo ./dpdk_set_ports.py -l +To create TRex config file using interactive mode + sudo ./dpdk_set_ports.py -l + To create a default config file (example1) sudo ./dpdk_setup_ports.py -c 02:00.0 02:00.1 -o /etc/trex_cfg.yaml @@ -474,6 +574,10 @@ To see more detailed info on interfaces (table): help=argparse.SUPPRESS ) + parser.add_argument("-i", "--interactive", action='store_true', + help=""" Create TRex config in interactive mode """, + ) + parser.add_argument("-c", "--create", nargs='*', default=None, dest='create_interfaces', metavar='<interface>', help="""Try to create a configuration file by specifying needed interfaces by PCI address or Linux names: eth1 etc.""", ) @@ -553,6 +657,8 @@ def main (): if map_driver.args.create_interfaces is not None: obj.do_create(); + elif map_driver.args.interactive: + obj.do_interactive_create(); elif map_driver.args.linux: obj.do_return_to_linux(); else: diff --git a/src/main_dpdk.cpp b/src/main_dpdk.cpp index f0320921..4b5cedbc 100644 --- a/src/main_dpdk.cpp +++ b/src/main_dpdk.cpp @@ -4968,12 +4968,13 @@ void dump_interfaces_info() { struct rte_pci_addr pci_addr; for (uint8_t port_id=0; port_id<m_max_ports; port_id++) { - // PCI and MAC + // PCI, MAC and Driver pci_addr = rte_eth_devices[port_id].pci_dev->addr; rte_eth_macaddr_get(port_id, &mac_addr); ether_format_addr(mac_str, sizeof mac_str, &mac_addr); - printf("PCI: %04x:%02x:%02x.%d - MAC: %s\n", - pci_addr.domain, pci_addr.bus, pci_addr.devid, pci_addr.function, mac_str); + printf("PCI: %04x:%02x:%02x.%d - MAC: %s - Driver: %s\n", + pci_addr.domain, pci_addr.bus, pci_addr.devid, pci_addr.function, mac_str, + rte_eth_devices[port_id].pci_dev->driver->name); } } |