Deal with loaded Kernel modules in Python:

Try loading igb_uio if it's avaiable, next try vfio-pci if it's available.
Last effort is compiling igb_uio.

Change-Id: I99d1d6c969f795d58a403587d6d0c395548ba3f7
Signed-off-by: Yaroslav Brustinov <ybrustin@cisco.com>

3 files changed, 185 insertions, 120 deletions

diff --git a/scripts/dpdk_nic_bind.py b/scripts/dpdk_nic_bind.py
index ce82c9f4..08954aa0 100755
--- a/scripts/dpdk_nic_bind.py
+++ b/scripts/dpdk_nic_bind.py
@@ -75,6 +75,7 @@ status_flag = False
 table_flag = False
 force_flag = False
 args = []
+loaded_modules = []
 
 try:
     raw_input
@@ -152,6 +153,9 @@ def check_output(args, stderr=None, **kwargs):
     return subprocess.Popen(args, stdout=subprocess.PIPE,
                             stderr=stderr, **kwargs).communicate()[0]
 
+kernel_ver = check_output(['uname', '-r'], universal_newlines = True).strip()
+
+
 def find_module(mod):
     '''find the .ko file for kernel module named mod.
     Searches the $RTE_SDK/$RTE_TARGET directory, the kernel
@@ -167,8 +171,8 @@ def find_module(mod):
     # check using depmod
     try:
         depmod_out = check_output(["modinfo", "-n", mod], \
-                                  stderr=subprocess.STDOUT, universal_newlines = True).lower()
-        if "error" not in depmod_out:
+                                  stderr=subprocess.STDOUT, universal_newlines = True)
+        if "error" not in depmod_out.lower():
             path = depmod_out.strip()
             if exists(path):
                 return path
@@ -176,27 +180,30 @@ def find_module(mod):
         pass
 
     # check for a copy based off current path
-    tools_dir = dirname(abspath(sys.argv[0]))
-    if (tools_dir.endswith("tools")):
-        base_dir = dirname(tools_dir)
-        find_out = check_output(["find", base_dir, "-name", mod + ".ko"], universal_newlines = True)
-        if len(find_out) > 0: #something matched
-            path = find_out.splitlines()[0]
-            if exists(path):
-                return path
+    drivers_dir = '/lib/modules/%s/kernel/drivers' % kernel_ver
+    find_out = check_output(["find", drivers_dir, "-name", mod + ".ko\*"], universal_newlines = True)
+    if find_out: #something matched
+        path = find_out.splitlines()[0]
+        if exists(path):
+            return path
+
+def get_loaded_modules():
+    if not loaded_modules:
+        with open("/proc/modules") as fd:
+            loaded_mods = fd.readlines()
+        for line in loaded_mods:
+            loaded_modules.append(line.split()[0])
+    return loaded_modules
 
 def check_modules():
     '''Checks that igb_uio is loaded'''
     global dpdk_drivers
 
-    with open("/proc/modules") as fd:
-        loaded_mods = fd.readlines()
-
     # list of supported modules
     mods =  [{"Name" : driver, "Found" : False} for driver in dpdk_drivers]
 
     # first check if module is loaded
-    for line in loaded_mods:
+    for line in get_loaded_modules():
         for mod in mods:
             if line.startswith(mod["Name"]):
                 mod["Found"] = True
@@ -354,6 +361,16 @@ def get_tcp_port_usage(port):
         return None
     return res[0][7]
 
+def is_module_used(module):
+    refcnt_file = '/sys/module/%s/refcnt' % module
+    if not os.path.exists(refcnt_file):
+        return False
+    with open(refcnt_file) as f:
+        ref_cnt = int(f.read().strip())
+    if ref_cnt:
+        return True
+    return False
+
 def get_pid_using_pci(pci_list):
     if not isinstance(pci_list, list):
         pci_list = [pci_list]
@@ -406,21 +423,28 @@ def unbind_one(dev_id, force):
             (dev[b"Slot"], dev[b"Device_str"], dev[b"Interface"]))
         return
 
-    if not force and dev.get('Driver_str') in dpdk_drivers and get_igb_uio_usage():
+    # Mellanox NICs do not need unbind
+    if not force and dev['Driver_str'] in dpdk_and_kernel:
+        print('Mellanox NICs do not need unbinding.')
+        if not confirm('Confirm unbind (y/N)'):
+            print('Not unbinding.')
+            sys.exit(-1)
+
+    if not force and dev['Driver_str'] in dpdk_drivers and is_module_used(dev['Driver_str']):
         pid = get_pid_using_pci(dev_id)
         if pid:
             cmdline = read_pid_cmdline(pid)
             print('Interface %s is in use by process:\n%s' % (dev_id, cmdline))
             if not confirm('Unbinding might hang the process. Confirm unbind (y/N)'):
                 print('Not unbinding.')
-                return
+                sys.exit(-1)
 
     # prevent us disconnecting ourselves
     if dev["Active"] and not force:
         print('netstat indicates that interface %s is active.' % dev_id)
         if not confirm('Confirm unbind (y/N)'):
             print('Not unbinding.')
-            return
+            sys.exit(-1)
 
     # write to /sys to unbind
     filename = "/sys/bus/pci/drivers/%s/unbind" % dev["Driver_str"]
@@ -443,23 +467,23 @@ def bind_one(dev_id, driver, force):
         print("netstat indicates that interface %s is active" % dev_id)
         if not confirm("Confirm bind (y/N)"):
             print('Not binding.')
-            return
+            sys.exit(-1)
 
     # unbind any existing drivers we don't want
     if has_driver(dev_id):
         if dev["Driver_str"] == driver:
             print("%s already bound to driver %s, skipping\n" % (dev_id, driver))
-            return
+            return True
         else:
             saved_driver = dev["Driver_str"]
-            if not force and get_igb_uio_usage():
+            if not force and is_module_used(saved_driver):
                 pid = get_pid_using_pci(dev_id)
                 if pid:
                     cmdline = read_pid_cmdline(pid)
                     print('Interface %s is in use by process:\n%s' % (dev_id, cmdline))
                     if not confirm('Binding to other driver might hang the process. Confirm unbind (y/N)'):
                         print('Not binding.')
-                        return
+                        sys.exit(-1)
 
             unbind_one(dev_id, force)
             dev["Driver_str"] = "" # clear driver string
@@ -492,13 +516,14 @@ def bind_one(dev_id, driver, force):
     try:
         f.write(dev_id)
         f.close()
+        return True
     except:
         # for some reason, closing dev_id after adding a new PCI ID to new_id
         # results in IOError. however, if the device was successfully bound,
         # we don't care for any errors and can safely ignore IOError
         tmp = get_pci_device_details(dev_id)
         if "Driver_str" in tmp and tmp["Driver_str"] == driver:
-            return
+            return True
         print("Error: bind failed for %s - Cannot bind to driver %s" % (dev_id, driver))
         if saved_driver is not None: # restore any previous driver
             bind_one(dev_id, saved_driver, force)
@@ -518,7 +543,9 @@ def bind_all(dev_list, driver, force=False):
     dev_list = list(map(dev_id_from_dev_name, dev_list))
 
     for d in dev_list:
-        bind_one(d, driver, force)
+        res = bind_one(d, driver, force)
+        if not res:
+            sys.exit(-1)
 
     # when binding devices to a generic driver (i.e. one that doesn't have a
     # PCI ID table), some devices that are not bound to any other driver could
diff --git a/scripts/dpdk_setup_ports.py b/scripts/dpdk_setup_ports.py
index 6eb7481d..8e5a23be 100755
--- a/scripts/dpdk_setup_ports.py
+++ b/scripts/dpdk_setup_ports.py
@@ -24,8 +24,7 @@ import platform
 # 32  : no errors - mlx share object should be loaded 
 MLX_EXIT_CODE = 32
 
-out = subprocess.check_output(['lsmod'])
-DPDK_DRIVER = 'vfio-pci' if out.find('vfio_pci') != -1 else 'igb_uio'
+class VFIOBindErr(Exception): pass
 
 class ConfigCreator(object):
     mandatory_interface_fields = ['Slot_str', 'Device_str', 'NUMA']
@@ -248,6 +247,52 @@ class ConfigCreator(object):
             print('Saved to %s.' % filename)
         return config_str
 
+# only load igb_uio if it's available
+def load_igb_uio():
+    if 'igb_uio' in dpdk_nic_bind.get_loaded_modules():
+        return True
+    km = './ko/%s/igb_uio.ko' % dpdk_nic_bind.kernel_ver
+    if os.path.exists(km):
+        return os.system('insmod %s' % km) == 0
+
+# try to compile igb_uio if it's missing
+def compile_and_load_igb_uio():
+    loaded_mods = dpdk_nic_bind.get_loaded_modules()
+    if 'igb_uio' in loaded_mods:
+        return
+    if 'uio' not in loaded_mods:
+        res = os.system('modprobe uio')
+        if res:
+            print('Failed inserting uio module, please check if it is installed')
+            sys.exit(-1)
+    km = './ko/%s/igb_uio.ko' % dpdk_nic_bind.kernel_ver
+    if not os.path.exists(km):
+        print("ERROR: We don't have precompiled igb_uio.ko module for your kernel version")
+        print('Will try compiling automatically.')
+        try:
+            subprocess.check_output('make', cwd = './ko/src', stderr = subprocess.STDOUT)
+            subprocess.check_output(['make', 'install'], cwd = './ko/src', stderr = subprocess.STDOUT)
+            print('\nSuccess.')
+        except Exception as e:
+            print('\nAutomatic compilation failed: (%s)' % e)
+            print('You can try compiling yourself, using the following commands:')
+            print('  $cd ko/src')
+            print('  $make')
+            print('  $make install')
+            print('  $cd -')
+            print('Then, try to run TRex again.')
+            print('Note: you might need additional Linux packages for that:')
+            print('  * yum based (Fedora, CentOS, RedHat):')
+            print('        sudo yum install kernel-devel-`uname -r`')
+            print('        sudo yum group install "Development tools"')
+            print('  * apt based (Ubuntu):')
+            print('        sudo apt install linux-headers-`uname -r`')
+            print('        sudo apt install build-essential')
+            sys.exit(-1)
+    res = os.system('insmod %s' % km)
+    if res:
+        print('Failed inserting igb_uio module')
+        sys.exit(-1)
 
 class map_driver(object):
     args=None;
@@ -442,17 +487,33 @@ Other network devices
             raise DpdkSetup('Error: port_limit should not be higher than number of interfaces in config file: %s\n' % fcfg)
 
 
-    def do_bind_one (self,key,mellanox):
-        if mellanox:
-            drv="mlx5_core"
-        else:
-            drv=DPDK_DRIVER
-
-        cmd='%s dpdk_nic_bind.py --bind=%s %s ' % (sys.executable, drv,key)
+    def do_bind_all(self, drv, pci, force = False):
+        assert type(pci) is list
+        cmd = '{ptn} dpdk_nic_bind.py --bind={drv} {pci} {frc}'.format(
+            ptn = sys.executable,
+            drv = drv,
+            pci = ' '.join(pci),
+            frc = '--force' if force else '')
         print(cmd)
-        res=os.system(cmd);
-        if res!=0:
-            raise DpdkSetup('')
+        return os.system(cmd)
+
+    # pros: no need to compile .ko per Kernel version
+    # cons: need special config/hw (not always works)
+    def try_bind_to_vfio_pci(self, to_bind_list):
+        krnl_params_file = '/proc/cmdline'
+        if not os.path.exists(krnl_params_file):
+            raise VFIOBindErr('Could not find file with Kernel boot parameters: %s' % krnl_params_file)
+        with open(krnl_params_file) as f:
+            krnl_params = f.read()
+        if 'iommu=' in krnl_params:
+            if 'vfio_pci' not in dpdk_nic_bind.get_loaded_modules():
+                ret = os.system('modprobe vfio_pci')
+                if ret:
+                    raise VFIOBindErr('Could not load vfio_pci')
+            ret = self.do_bind_all('vfio-pci', to_bind_list)
+            if ret:
+                raise VFIOBindErr('Binding to vfio_pci failed')
+        raise VFIOBindErr('vfio-pci is not an option here')
 
 
     def pci_name_to_full_name (self,pci_name):
@@ -507,18 +568,15 @@ Other network devices
                 err=" %s does not have Vendor_str " %key;
                 raise DpdkSetup(err)
 
-            if self.m_devices[key]['Vendor_str'].find("Mellanox")>-1 :
-                Mellanox_cnt=Mellanox_cnt+1
+            if 'Mellanox' in self.m_devices[key]['Vendor_str']:
+                Mellanox_cnt += 1
 
 
         if not map_driver.parent_args.dump_interfaces:
-            if  ((Mellanox_cnt>0) and (Mellanox_cnt!= len(if_list))):
+            if (Mellanox_cnt > 0) and (Mellanox_cnt != len(if_list)):
                err=" All driver should be from one vendor. you have at least one driver from Mellanox but not all "; 
                raise DpdkSetup(err)
-
-
-        if not map_driver.parent_args.dump_interfaces:
-            if  Mellanox_cnt>0 :
+            if Mellanox_cnt > 0:
                 self.set_only_mellanox_nics()
 
         if self.get_only_mellanox_nics():
@@ -537,24 +595,11 @@ Other network devices
 
 
         if only_check_all_mlx:
-            if Mellanox_cnt >0:
+            if Mellanox_cnt > 0:
                 exit(MLX_EXIT_CODE);
             else:
                 exit(0);
 
-        for key in if_list:
-            if key not in self.m_devices:
-                err=" %s does not exist " %key;
-                raise DpdkSetup(err)
-
-            if 'Driver_str' in self.m_devices[key]:
-                if self.m_devices[key]['Driver_str'] not in (dpdk_nic_bind.dpdk_drivers+dpdk_nic_bind.dpdk_and_kernel) :
-                    self.do_bind_one (key,(Mellanox_cnt>0))
-                    pass;
-            else:
-                self.do_bind_one (key,(Mellanox_cnt>0))
-                pass;
-
         if if_list and map_driver.args.parent and self.m_cfg_dict[0].get('enable_zmq_pub', True):
             publisher_port = self.m_cfg_dict[0].get('zmq_pub_port', 4500)
             pid = dpdk_nic_bind.get_tcp_port_usage(publisher_port)
@@ -574,37 +619,78 @@ Other network devices
                 print("Could not start scapy_daemon_server, which is needed by GUI to create packets.\nIf you don't need it, use --no-scapy-server flag.")
                 sys.exit(-1)
 
-        if Mellanox_cnt:
-            return MLX_EXIT_CODE
-        else:
-            return 0
 
+        to_bind_list = []
+        for key in if_list:
+            if key not in self.m_devices:
+                err=" %s does not exist " %key;
+                raise DpdkSetup(err)
+
+            if self.m_devices[key].get('Driver_str') not in (dpdk_nic_bind.dpdk_drivers + dpdk_nic_bind.dpdk_and_kernel):
+                to_bind_list.append(key)
+
+        if to_bind_list:
+            if Mellanox_cnt > 0:
+                ret = self.do_bind_all('mlx5_core', to_bind_list)
+                if ret:
+                    raise DpdkSetup('Unable to bind interfaces to driver mlx5_core.')
+                return MLX_EXIT_CODE
+            else:
+                # if igb_uio is ready, use it as safer choice, afterwards try vfio-pci
+                if load_igb_uio():
+                    print('Trying to bind to igb_uio ...')
+                    ret = self.do_bind_all('igb_uio', to_bind_list)
+                    if ret:
+                        raise DpdkSetup('Unable to bind interfaces to driver igb_uio.') # module present, loaded, but unable to bind
+                    return
+
+                try:
+                    print('Trying to bind to vfio-pci ...')
+                    self.try_bind_to_vfio_pci(to_bind_list)
+                    return
+                except VFIOBindErr as e:
+                    pass
+                    #print(e)
+
+                print('Trying to compile and bind to igb_uio ...')
+                compile_and_load_igb_uio()
+                ret = self.do_bind_all('igb_uio', to_bind_list)
+                if ret:
+                    raise DpdkSetup('Unable to bind interfaces to driver igb_uio.')
 
 
     def do_return_to_linux(self):
         if not self.m_devices:
             self.run_dpdk_lspci()
         dpdk_interfaces = []
+        check_drivers = set()
         for device in self.m_devices.values():
             if device.get('Driver_str') in dpdk_nic_bind.dpdk_drivers:
                 dpdk_interfaces.append(device['Slot'])
+                check_drivers.add(device['Driver_str'])
         if not dpdk_interfaces:
             print('No DPDK bound interfaces.')
             return
-        if dpdk_nic_bind.get_igb_uio_usage():
+        any_driver_used = False
+        for driver in check_drivers:
+            if dpdk_nic_bind.is_module_used(driver):
+                any_driver_used = True
+        if any_driver_used:
             pid = dpdk_nic_bind.get_pid_using_pci(dpdk_interfaces)
             if pid:
                 cmdline = dpdk_nic_bind.read_pid_cmdline(pid)
                 print('DPDK interfaces are in use. Unbinding them might cause following process to hang:\npid: %s, cmd: %s' % (pid, cmdline))
                 if not dpdk_nic_bind.confirm('Confirm (y/N):'):
-                    return
+                    sys.exit(-1)
+
+        # DPDK => Linux
         drivers_table = {
             'net_ixgbe': 'ixgbe',
-            'net_ixgbe_vf': 'ixgbe_vf',
-            'net_e1000_igb': 'e1000_igb',
+            'net_ixgbe_vf': 'ixgbevf',
+            'net_e1000_igb': 'igb',
             'net_i40e': 'i40e',
-            'net_i40e_vf': 'i40e_vf',
-            'net_e1000_em': 'e1000_em',
+            'net_i40e_vf': 'i40evf',
+            'net_e1000_em': 'e1000',
             'net_vmxnet3': 'vmxnet3',
             'net_virtio': 'virtio-pci',
             'net_enic': 'enic',
@@ -617,10 +703,13 @@ Other network devices
                 raise DpdkSetup('Internal error: PCI %s is not found among devices' % pci)
             dev = self.m_devices[pci]
             if info['TRex_Driver'] not in drivers_table:
-                print("Got unknown driver '%s', description: %s" % (info['TRex_Driver'], dev['Device_str']))
+                raise DpdkSetup("Got unknown driver '%s', description: %s" % (info['TRex_Driver'], dev['Device_str']))
+            linux_driver = drivers_table[info['TRex_Driver']]
+            if linux_driver not in dpdk_nic_bind.get_loaded_modules():
+                print("No Linux driver installed, or wrong module name: %s" % linux_driver)
             else:
                 print('Returning to Linux %s' % pci)
-                dpdk_nic_bind.bind_one(pci, drivers_table[info['TRex_Driver']], False)
+                dpdk_nic_bind.bind_one(pci, linux_driver, False)
 
     def _get_cpu_topology(self):
         cpu_topology_file = '/proc/cpuinfo'
@@ -915,12 +1004,9 @@ To return to Linux the DPDK bound interfaces (for ifconfig etc.)
 To create TRex config file using interactive mode
   sudo ./dpdk_set_ports.py -i
 
-To create a default config file (example1)
+To create a default config file (example)
   sudo ./dpdk_setup_ports.py -c 02:00.0 02:00.1 -o /etc/trex_cfg.yaml
 
-To create a default config file (example2)
-  sudo ./dpdk_setup_ports.py -c eth1 eth2 --dest-macs 11:11:11:11:11:11 22:22:22:22:22:22 --dump
-
 To show interfaces status
   sudo ./dpdk_set_ports.py -s
 
diff --git a/scripts/trex-cfg b/scripts/trex-cfg
index 5945a074..5d87b562 100755
--- a/scripts/trex-cfg
+++ b/scripts/trex-cfg
@@ -43,54 +43,6 @@ for file in /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepag
     fi
 done
 
-PATH=$PATH:/sbin:/usr/sbin
-
-VFIO_PCI_KO=`find /lib/modules/$(uname -r) -type f -name vfio-pci.ko`
-if [ $VFIO_PCI_KO ] && grep "iommu=pt" /proc/cmdline | grep -q "intel_iommu=on" ; then
-    modprobe vfio-pci
-elif ! lsmod | grep -q igb_uio ; then
-    echo "Loading kernel drivers for the first time"
-    modprobe uio
-    if [ $? -ne 0 ]; then
-        echo "Failed inserting uio module, please check if it is installed"
-        exit -1
-    fi
-    km=ko/$SYS/igb_uio.ko
-    if [ ! -e $km ]; then
-        echo "ERROR: We don't have precompiled igb_uio.ko module for your kernel version"
-        echo Will try compiling automatically.
-        {
-            cd ko/src &&
-            make &&
-            make install &&
-            cd -
-        } &> /dev/null || {
-            echo -e "Automatic compilation failed.\n"
-            echo "You can try compiling yourself, using the following commands:"
-            echo "  \$cd ko/src  "
-            echo "  \$make  "
-            echo "  \$make install  "
-            echo -e "  \$cd -  \n"
-            echo -e "Then, try to run TRex again.\n"
-            echo 'Note: you might need additional Linux packages for that:'
-            echo '  * yum based (Fedora, CentOS, RedHat):'
-            echo '        sudo yum install kernel-devel-`uname -r`'
-            echo '        sudo yum group install "Development tools"'
-            echo '  * apt based (Ubuntu):'
-            echo '        sudo apt install linux-headers-`uname -r`'
-            echo '        sudo apt install build-essential'
-            exit -1
-        }
-        echo Success.
-    fi
-    if [ -e $km ]; then
-        insmod $km
-        if [ $? -ne 0 ]; then
-            echo "Failed inserting igb_uio module"
-            exit -1
-        fi
-    fi
-fi
 
 # try to bind the ports from the configuration file (new DPDK)
 PARENT_ARGS="$0 $@"