From c118763130d9901bf5271d2a8fc0bfac4be88b95 Mon Sep 17 00:00:00 2001 From: Yaroslav Brustinov Date: Sun, 5 Mar 2017 23:30:13 +0200 Subject: Deal with loaded Kernel modules in Python: Try loading igb_uio if it's avaiable, next try vfio-pci if it's available. Last effort is compiling igb_uio. Change-Id: I99d1d6c969f795d58a403587d6d0c395548ba3f7 Signed-off-by: Yaroslav Brustinov --- scripts/dpdk_nic_bind.py | 73 ++++++++++++------ scripts/dpdk_setup_ports.py | 184 ++++++++++++++++++++++++++++++++------------ scripts/trex-cfg | 48 ------------ 3 files changed, 185 insertions(+), 120 deletions(-) diff --git a/scripts/dpdk_nic_bind.py b/scripts/dpdk_nic_bind.py index ce82c9f4..08954aa0 100755 --- a/scripts/dpdk_nic_bind.py +++ b/scripts/dpdk_nic_bind.py @@ -75,6 +75,7 @@ status_flag = False table_flag = False force_flag = False args = [] +loaded_modules = [] try: raw_input @@ -152,6 +153,9 @@ def check_output(args, stderr=None, **kwargs): return subprocess.Popen(args, stdout=subprocess.PIPE, stderr=stderr, **kwargs).communicate()[0] +kernel_ver = check_output(['uname', '-r'], universal_newlines = True).strip() + + def find_module(mod): '''find the .ko file for kernel module named mod. Searches the $RTE_SDK/$RTE_TARGET directory, the kernel @@ -167,8 +171,8 @@ def find_module(mod): # check using depmod try: depmod_out = check_output(["modinfo", "-n", mod], \ - stderr=subprocess.STDOUT, universal_newlines = True).lower() - if "error" not in depmod_out: + stderr=subprocess.STDOUT, universal_newlines = True) + if "error" not in depmod_out.lower(): path = depmod_out.strip() if exists(path): return path @@ -176,27 +180,30 @@ def find_module(mod): pass # check for a copy based off current path - tools_dir = dirname(abspath(sys.argv[0])) - if (tools_dir.endswith("tools")): - base_dir = dirname(tools_dir) - find_out = check_output(["find", base_dir, "-name", mod + ".ko"], universal_newlines = True) - if len(find_out) > 0: #something matched - path = find_out.splitlines()[0] - if exists(path): - return path + drivers_dir = '/lib/modules/%s/kernel/drivers' % kernel_ver + find_out = check_output(["find", drivers_dir, "-name", mod + ".ko\*"], universal_newlines = True) + if find_out: #something matched + path = find_out.splitlines()[0] + if exists(path): + return path + +def get_loaded_modules(): + if not loaded_modules: + with open("/proc/modules") as fd: + loaded_mods = fd.readlines() + for line in loaded_mods: + loaded_modules.append(line.split()[0]) + return loaded_modules def check_modules(): '''Checks that igb_uio is loaded''' global dpdk_drivers - with open("/proc/modules") as fd: - loaded_mods = fd.readlines() - # list of supported modules mods = [{"Name" : driver, "Found" : False} for driver in dpdk_drivers] # first check if module is loaded - for line in loaded_mods: + for line in get_loaded_modules(): for mod in mods: if line.startswith(mod["Name"]): mod["Found"] = True @@ -354,6 +361,16 @@ def get_tcp_port_usage(port): return None return res[0][7] +def is_module_used(module): + refcnt_file = '/sys/module/%s/refcnt' % module + if not os.path.exists(refcnt_file): + return False + with open(refcnt_file) as f: + ref_cnt = int(f.read().strip()) + if ref_cnt: + return True + return False + def get_pid_using_pci(pci_list): if not isinstance(pci_list, list): pci_list = [pci_list] @@ -406,21 +423,28 @@ def unbind_one(dev_id, force): (dev[b"Slot"], dev[b"Device_str"], dev[b"Interface"])) return - if not force and dev.get('Driver_str') in dpdk_drivers and get_igb_uio_usage(): + # Mellanox NICs do not need unbind + if not force and dev['Driver_str'] in dpdk_and_kernel: + print('Mellanox NICs do not need unbinding.') + if not confirm('Confirm unbind (y/N)'): + print('Not unbinding.') + sys.exit(-1) + + if not force and dev['Driver_str'] in dpdk_drivers and is_module_used(dev['Driver_str']): pid = get_pid_using_pci(dev_id) if pid: cmdline = read_pid_cmdline(pid) print('Interface %s is in use by process:\n%s' % (dev_id, cmdline)) if not confirm('Unbinding might hang the process. Confirm unbind (y/N)'): print('Not unbinding.') - return + sys.exit(-1) # prevent us disconnecting ourselves if dev["Active"] and not force: print('netstat indicates that interface %s is active.' % dev_id) if not confirm('Confirm unbind (y/N)'): print('Not unbinding.') - return + sys.exit(-1) # write to /sys to unbind filename = "/sys/bus/pci/drivers/%s/unbind" % dev["Driver_str"] @@ -443,23 +467,23 @@ def bind_one(dev_id, driver, force): print("netstat indicates that interface %s is active" % dev_id) if not confirm("Confirm bind (y/N)"): print('Not binding.') - return + sys.exit(-1) # unbind any existing drivers we don't want if has_driver(dev_id): if dev["Driver_str"] == driver: print("%s already bound to driver %s, skipping\n" % (dev_id, driver)) - return + return True else: saved_driver = dev["Driver_str"] - if not force and get_igb_uio_usage(): + if not force and is_module_used(saved_driver): pid = get_pid_using_pci(dev_id) if pid: cmdline = read_pid_cmdline(pid) print('Interface %s is in use by process:\n%s' % (dev_id, cmdline)) if not confirm('Binding to other driver might hang the process. Confirm unbind (y/N)'): print('Not binding.') - return + sys.exit(-1) unbind_one(dev_id, force) dev["Driver_str"] = "" # clear driver string @@ -492,13 +516,14 @@ def bind_one(dev_id, driver, force): try: f.write(dev_id) f.close() + return True except: # for some reason, closing dev_id after adding a new PCI ID to new_id # results in IOError. however, if the device was successfully bound, # we don't care for any errors and can safely ignore IOError tmp = get_pci_device_details(dev_id) if "Driver_str" in tmp and tmp["Driver_str"] == driver: - return + return True print("Error: bind failed for %s - Cannot bind to driver %s" % (dev_id, driver)) if saved_driver is not None: # restore any previous driver bind_one(dev_id, saved_driver, force) @@ -518,7 +543,9 @@ def bind_all(dev_list, driver, force=False): dev_list = list(map(dev_id_from_dev_name, dev_list)) for d in dev_list: - bind_one(d, driver, force) + res = bind_one(d, driver, force) + if not res: + sys.exit(-1) # when binding devices to a generic driver (i.e. one that doesn't have a # PCI ID table), some devices that are not bound to any other driver could diff --git a/scripts/dpdk_setup_ports.py b/scripts/dpdk_setup_ports.py index 6eb7481d..8e5a23be 100755 --- a/scripts/dpdk_setup_ports.py +++ b/scripts/dpdk_setup_ports.py @@ -24,8 +24,7 @@ import platform # 32 : no errors - mlx share object should be loaded MLX_EXIT_CODE = 32 -out = subprocess.check_output(['lsmod']) -DPDK_DRIVER = 'vfio-pci' if out.find('vfio_pci') != -1 else 'igb_uio' +class VFIOBindErr(Exception): pass class ConfigCreator(object): mandatory_interface_fields = ['Slot_str', 'Device_str', 'NUMA'] @@ -248,6 +247,52 @@ class ConfigCreator(object): print('Saved to %s.' % filename) return config_str +# only load igb_uio if it's available +def load_igb_uio(): + if 'igb_uio' in dpdk_nic_bind.get_loaded_modules(): + return True + km = './ko/%s/igb_uio.ko' % dpdk_nic_bind.kernel_ver + if os.path.exists(km): + return os.system('insmod %s' % km) == 0 + +# try to compile igb_uio if it's missing +def compile_and_load_igb_uio(): + loaded_mods = dpdk_nic_bind.get_loaded_modules() + if 'igb_uio' in loaded_mods: + return + if 'uio' not in loaded_mods: + res = os.system('modprobe uio') + if res: + print('Failed inserting uio module, please check if it is installed') + sys.exit(-1) + km = './ko/%s/igb_uio.ko' % dpdk_nic_bind.kernel_ver + if not os.path.exists(km): + print("ERROR: We don't have precompiled igb_uio.ko module for your kernel version") + print('Will try compiling automatically.') + try: + subprocess.check_output('make', cwd = './ko/src', stderr = subprocess.STDOUT) + subprocess.check_output(['make', 'install'], cwd = './ko/src', stderr = subprocess.STDOUT) + print('\nSuccess.') + except Exception as e: + print('\nAutomatic compilation failed: (%s)' % e) + print('You can try compiling yourself, using the following commands:') + print(' $cd ko/src') + print(' $make') + print(' $make install') + print(' $cd -') + print('Then, try to run TRex again.') + print('Note: you might need additional Linux packages for that:') + print(' * yum based (Fedora, CentOS, RedHat):') + print(' sudo yum install kernel-devel-`uname -r`') + print(' sudo yum group install "Development tools"') + print(' * apt based (Ubuntu):') + print(' sudo apt install linux-headers-`uname -r`') + print(' sudo apt install build-essential') + sys.exit(-1) + res = os.system('insmod %s' % km) + if res: + print('Failed inserting igb_uio module') + sys.exit(-1) class map_driver(object): args=None; @@ -442,17 +487,33 @@ Other network devices raise DpdkSetup('Error: port_limit should not be higher than number of interfaces in config file: %s\n' % fcfg) - def do_bind_one (self,key,mellanox): - if mellanox: - drv="mlx5_core" - else: - drv=DPDK_DRIVER - - cmd='%s dpdk_nic_bind.py --bind=%s %s ' % (sys.executable, drv,key) + def do_bind_all(self, drv, pci, force = False): + assert type(pci) is list + cmd = '{ptn} dpdk_nic_bind.py --bind={drv} {pci} {frc}'.format( + ptn = sys.executable, + drv = drv, + pci = ' '.join(pci), + frc = '--force' if force else '') print(cmd) - res=os.system(cmd); - if res!=0: - raise DpdkSetup('') + return os.system(cmd) + + # pros: no need to compile .ko per Kernel version + # cons: need special config/hw (not always works) + def try_bind_to_vfio_pci(self, to_bind_list): + krnl_params_file = '/proc/cmdline' + if not os.path.exists(krnl_params_file): + raise VFIOBindErr('Could not find file with Kernel boot parameters: %s' % krnl_params_file) + with open(krnl_params_file) as f: + krnl_params = f.read() + if 'iommu=' in krnl_params: + if 'vfio_pci' not in dpdk_nic_bind.get_loaded_modules(): + ret = os.system('modprobe vfio_pci') + if ret: + raise VFIOBindErr('Could not load vfio_pci') + ret = self.do_bind_all('vfio-pci', to_bind_list) + if ret: + raise VFIOBindErr('Binding to vfio_pci failed') + raise VFIOBindErr('vfio-pci is not an option here') def pci_name_to_full_name (self,pci_name): @@ -507,18 +568,15 @@ Other network devices err=" %s does not have Vendor_str " %key; raise DpdkSetup(err) - if self.m_devices[key]['Vendor_str'].find("Mellanox")>-1 : - Mellanox_cnt=Mellanox_cnt+1 + if 'Mellanox' in self.m_devices[key]['Vendor_str']: + Mellanox_cnt += 1 if not map_driver.parent_args.dump_interfaces: - if ((Mellanox_cnt>0) and (Mellanox_cnt!= len(if_list))): + if (Mellanox_cnt > 0) and (Mellanox_cnt != len(if_list)): err=" All driver should be from one vendor. you have at least one driver from Mellanox but not all "; raise DpdkSetup(err) - - - if not map_driver.parent_args.dump_interfaces: - if Mellanox_cnt>0 : + if Mellanox_cnt > 0: self.set_only_mellanox_nics() if self.get_only_mellanox_nics(): @@ -537,24 +595,11 @@ Other network devices if only_check_all_mlx: - if Mellanox_cnt >0: + if Mellanox_cnt > 0: exit(MLX_EXIT_CODE); else: exit(0); - for key in if_list: - if key not in self.m_devices: - err=" %s does not exist " %key; - raise DpdkSetup(err) - - if 'Driver_str' in self.m_devices[key]: - if self.m_devices[key]['Driver_str'] not in (dpdk_nic_bind.dpdk_drivers+dpdk_nic_bind.dpdk_and_kernel) : - self.do_bind_one (key,(Mellanox_cnt>0)) - pass; - else: - self.do_bind_one (key,(Mellanox_cnt>0)) - pass; - if if_list and map_driver.args.parent and self.m_cfg_dict[0].get('enable_zmq_pub', True): publisher_port = self.m_cfg_dict[0].get('zmq_pub_port', 4500) pid = dpdk_nic_bind.get_tcp_port_usage(publisher_port) @@ -574,37 +619,78 @@ Other network devices print("Could not start scapy_daemon_server, which is needed by GUI to create packets.\nIf you don't need it, use --no-scapy-server flag.") sys.exit(-1) - if Mellanox_cnt: - return MLX_EXIT_CODE - else: - return 0 + to_bind_list = [] + for key in if_list: + if key not in self.m_devices: + err=" %s does not exist " %key; + raise DpdkSetup(err) + + if self.m_devices[key].get('Driver_str') not in (dpdk_nic_bind.dpdk_drivers + dpdk_nic_bind.dpdk_and_kernel): + to_bind_list.append(key) + + if to_bind_list: + if Mellanox_cnt > 0: + ret = self.do_bind_all('mlx5_core', to_bind_list) + if ret: + raise DpdkSetup('Unable to bind interfaces to driver mlx5_core.') + return MLX_EXIT_CODE + else: + # if igb_uio is ready, use it as safer choice, afterwards try vfio-pci + if load_igb_uio(): + print('Trying to bind to igb_uio ...') + ret = self.do_bind_all('igb_uio', to_bind_list) + if ret: + raise DpdkSetup('Unable to bind interfaces to driver igb_uio.') # module present, loaded, but unable to bind + return + + try: + print('Trying to bind to vfio-pci ...') + self.try_bind_to_vfio_pci(to_bind_list) + return + except VFIOBindErr as e: + pass + #print(e) + + print('Trying to compile and bind to igb_uio ...') + compile_and_load_igb_uio() + ret = self.do_bind_all('igb_uio', to_bind_list) + if ret: + raise DpdkSetup('Unable to bind interfaces to driver igb_uio.') def do_return_to_linux(self): if not self.m_devices: self.run_dpdk_lspci() dpdk_interfaces = [] + check_drivers = set() for device in self.m_devices.values(): if device.get('Driver_str') in dpdk_nic_bind.dpdk_drivers: dpdk_interfaces.append(device['Slot']) + check_drivers.add(device['Driver_str']) if not dpdk_interfaces: print('No DPDK bound interfaces.') return - if dpdk_nic_bind.get_igb_uio_usage(): + any_driver_used = False + for driver in check_drivers: + if dpdk_nic_bind.is_module_used(driver): + any_driver_used = True + if any_driver_used: pid = dpdk_nic_bind.get_pid_using_pci(dpdk_interfaces) if pid: cmdline = dpdk_nic_bind.read_pid_cmdline(pid) print('DPDK interfaces are in use. Unbinding them might cause following process to hang:\npid: %s, cmd: %s' % (pid, cmdline)) if not dpdk_nic_bind.confirm('Confirm (y/N):'): - return + sys.exit(-1) + + # DPDK => Linux drivers_table = { 'net_ixgbe': 'ixgbe', - 'net_ixgbe_vf': 'ixgbe_vf', - 'net_e1000_igb': 'e1000_igb', + 'net_ixgbe_vf': 'ixgbevf', + 'net_e1000_igb': 'igb', 'net_i40e': 'i40e', - 'net_i40e_vf': 'i40e_vf', - 'net_e1000_em': 'e1000_em', + 'net_i40e_vf': 'i40evf', + 'net_e1000_em': 'e1000', 'net_vmxnet3': 'vmxnet3', 'net_virtio': 'virtio-pci', 'net_enic': 'enic', @@ -617,10 +703,13 @@ Other network devices raise DpdkSetup('Internal error: PCI %s is not found among devices' % pci) dev = self.m_devices[pci] if info['TRex_Driver'] not in drivers_table: - print("Got unknown driver '%s', description: %s" % (info['TRex_Driver'], dev['Device_str'])) + raise DpdkSetup("Got unknown driver '%s', description: %s" % (info['TRex_Driver'], dev['Device_str'])) + linux_driver = drivers_table[info['TRex_Driver']] + if linux_driver not in dpdk_nic_bind.get_loaded_modules(): + print("No Linux driver installed, or wrong module name: %s" % linux_driver) else: print('Returning to Linux %s' % pci) - dpdk_nic_bind.bind_one(pci, drivers_table[info['TRex_Driver']], False) + dpdk_nic_bind.bind_one(pci, linux_driver, False) def _get_cpu_topology(self): cpu_topology_file = '/proc/cpuinfo' @@ -915,12 +1004,9 @@ To return to Linux the DPDK bound interfaces (for ifconfig etc.) To create TRex config file using interactive mode sudo ./dpdk_set_ports.py -i -To create a default config file (example1) +To create a default config file (example) sudo ./dpdk_setup_ports.py -c 02:00.0 02:00.1 -o /etc/trex_cfg.yaml -To create a default config file (example2) - sudo ./dpdk_setup_ports.py -c eth1 eth2 --dest-macs 11:11:11:11:11:11 22:22:22:22:22:22 --dump - To show interfaces status sudo ./dpdk_set_ports.py -s diff --git a/scripts/trex-cfg b/scripts/trex-cfg index 5945a074..5d87b562 100755 --- a/scripts/trex-cfg +++ b/scripts/trex-cfg @@ -43,54 +43,6 @@ for file in /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepag fi done -PATH=$PATH:/sbin:/usr/sbin - -VFIO_PCI_KO=`find /lib/modules/$(uname -r) -type f -name vfio-pci.ko` -if [ $VFIO_PCI_KO ] && grep "iommu=pt" /proc/cmdline | grep -q "intel_iommu=on" ; then - modprobe vfio-pci -elif ! lsmod | grep -q igb_uio ; then - echo "Loading kernel drivers for the first time" - modprobe uio - if [ $? -ne 0 ]; then - echo "Failed inserting uio module, please check if it is installed" - exit -1 - fi - km=ko/$SYS/igb_uio.ko - if [ ! -e $km ]; then - echo "ERROR: We don't have precompiled igb_uio.ko module for your kernel version" - echo Will try compiling automatically. - { - cd ko/src && - make && - make install && - cd - - } &> /dev/null || { - echo -e "Automatic compilation failed.\n" - echo "You can try compiling yourself, using the following commands:" - echo " \$cd ko/src " - echo " \$make " - echo " \$make install " - echo -e " \$cd - \n" - echo -e "Then, try to run TRex again.\n" - echo 'Note: you might need additional Linux packages for that:' - echo ' * yum based (Fedora, CentOS, RedHat):' - echo ' sudo yum install kernel-devel-`uname -r`' - echo ' sudo yum group install "Development tools"' - echo ' * apt based (Ubuntu):' - echo ' sudo apt install linux-headers-`uname -r`' - echo ' sudo apt install build-essential' - exit -1 - } - echo Success. - fi - if [ -e $km ]; then - insmod $km - if [ $? -ne 0 ]; then - echo "Failed inserting igb_uio module" - exit -1 - fi - fi -fi # try to bind the ports from the configuration file (new DPDK) PARENT_ARGS="$0 $@" -- cgit 1.2.3-korg