vmxnet3: auto bind support

For creating the vmxnet3 interface, add the bind option to automatically bind
the pci to vfio-pci module which removes the need for manual bind. Manual bind
still works, should people prefer to go that route.

Change-Id: Ife75926f8755d754a08dd0ecff0f1de326ad5ba1
Signed-off-by: Steven Luong <sluong@cisco.com>

7 files changed, 63 insertions, 23 deletions

diff --git a/src/plugins/vmxnet3/README.md b/src/plugins/vmxnet3/README.md
index b5829939506..722954d51a3 100644
--- a/src/plugins/vmxnet3/README.md
+++ b/src/plugins/vmxnet3/README.md
@@ -7,11 +7,11 @@ This plugin provides native PCI driver support for VMWare vmxnet3.
  * This code is tested with vfio-pci driver installed with Ubuntu 18.04 which
 has kernel version 4.15.0-33-generic.
 
- * This code is tested with ESXi vSwitch version 6.0, release build 3620759.
+ * This driver is tested with ESXi vSwitch version 6.5/6.7 for LRO/TSO support, VMware Workstation 15 Pro (no LRO/TSO), and VMware Fusion 11 Pro (no LRO/TSO)
 
  * Driver requires MSI-X interrupt support, which is not supported by
-uio_pci_generic driver, so vfio-pci needs to be used. On systems without IOMMU
-vfio driver can still be used with recent kernels which support no-iommu mode.
+uio_pci_generic driver. So vfio-pci must be used. On systems without IOMMU,
+vfio driver can still be used with 4.15.0-33-generic kernel (Ubuntu 18.04) which supports no-iommu mode.
 
 ##Known issues
 
@@ -25,20 +25,27 @@ vfio driver can still be used with recent kernels which support no-iommu mode.
 sudo modprobe vfio-pci
 ```
 
-2. (systems without IOMMU only) enable unsafe NOIOMMU mode
+2. Make sure the interface is down
+```
+sudo ifconfig <if-name> down
+```
+
+Steps 3 and 4 are optional. They can be accomplished by specifying the optional keyword "bind" when creating the vmxnet3 interface.
+
+3. (systems without IOMMU only) enable unsafe NOIOMMU mode
 ```
 echo Y | sudo tee /sys/module/vfio/parameters/enable_unsafe_noiommu_mode
 ```
 
-3. Bind interface to vfio-pci
+4. Bind interface to vfio-pci
 ```
 sudo dpdk-devbind.py --bind vfio-pci 0b:00.0
 ```
 
 ### Interface Creation
-Interface can be dynamically created with following CLI:
+Interface can be dynamically created with following CLI, with or without the bind option. If step 3 and 4 were executed, bind can be omitted.
 ```
-create interface vmxnet3 0000:0b:00.0
+create interface vmxnet3 0000:0b:00.0 bind
 set int state vmxnet3-0/b/0/0 up
 ```
 
diff --git a/src/plugins/vmxnet3/cli.c b/src/plugins/vmxnet3/cli.c
index 0f0cf60fa40..776901e8ac2 100644
--- a/src/plugins/vmxnet3/cli.c
+++ b/src/plugins/vmxnet3/cli.c
@@ -44,6 +44,8 @@ vmxnet3_create_command_fn (vlib_main_t * vm, unformat_input_t * input,
 	;
       else if (unformat (line_input, "elog"))
 	args.enable_elog = 1;
+      else if (unformat (line_input, "bind"))
+	args.bind = 1;
       else if (unformat (line_input, "rx-queue-size %u", &args.rxq_size))
 	;
       else if (unformat (line_input, "tx-queue-size %u", &args.txq_size))
@@ -69,7 +71,7 @@ VLIB_CLI_COMMAND (vmxnet3_create_command, static) = {
   .path = "create interface vmxnet3",
   .short_help = "create interface vmxnet3 <pci-address>"
                 "[rx-queue-size <size>] [tx-queue-size <size>]"
-                "[num-tx-queues <number>] [num-rx-queues <number>]",
+                "[num-tx-queues <number>] [num-rx-queues <number>] [bind]",
   .function = vmxnet3_create_command_fn,
 };
 /* *INDENT-ON* */
diff --git a/src/plugins/vmxnet3/vmxnet3.api b/src/plugins/vmxnet3/vmxnet3.api
index 804ba538be5..7408836af7b 100644
--- a/src/plugins/vmxnet3/vmxnet3.api
+++ b/src/plugins/vmxnet3/vmxnet3.api
@@ -28,6 +28,8 @@ option version = "1.0.0";
     @param rxq_num - number of receive queues (optional - default is 1)
     @param txq_size - transmit queue size (optional - default is 1024)
     @param txq_num - number of transmit queues (optional - default is 1)
+    @param bind - automatically bind PCI to vfio-pci module
+		      (optional - default is 0)
 */
 
 define vmxnet3_create
@@ -41,6 +43,7 @@ define vmxnet3_create
   u16 rxq_num;
   u16 txq_size;
   u16 txq_num;
+  u8 bind;
 };
 
 /** \brief
diff --git a/src/plugins/vmxnet3/vmxnet3.c b/src/plugins/vmxnet3/vmxnet3.c
index 43f9cbed793..de1002381b6 100644
--- a/src/plugins/vmxnet3/vmxnet3.c
+++ b/src/plugins/vmxnet3/vmxnet3.c
@@ -658,9 +658,10 @@ vmxnet3_create_if (vlib_main_t * vm, vmxnet3_create_if_args_t * args)
   pool_foreach (vd, vmxm->devices, ({
     if (vd->pci_addr.as_u32 == args->addr.as_u32)
       {
-	args->rv = VNET_API_ERROR_INVALID_VALUE;
+	args->rv = VNET_API_ERROR_ADDRESS_IN_USE;
 	args->error =
-	  clib_error_return (error, "PCI address in use");
+	  clib_error_return (error, "%U: %s", format_vlib_pci_addr,
+			     &args->addr, "pci address in use");
 	vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
 		  format_vlib_pci_addr, &args->addr, "pci address in use");
 	return;
@@ -668,24 +669,31 @@ vmxnet3_create_if (vlib_main_t * vm, vmxnet3_create_if_args_t * args)
   }));
   /* *INDENT-ON* */
 
-  pool_get (vmxm->devices, vd);
-  vd->num_tx_queues = args->txq_num;
-  vd->num_rx_queues = args->rxq_num;
-  vd->dev_instance = vd - vmxm->devices;
-  vd->per_interface_next_index = ~0;
-  vd->pci_addr = args->addr;
-
-  if (args->enable_elog)
-    vd->flags |= VMXNET3_DEVICE_F_ELOG;
+  if (args->bind)
+    {
+      error = vlib_pci_bind_to_uio (vm, &args->addr, (char *) "auto");
+      if (error)
+	{
+	  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
+	  args->error =
+	    clib_error_return (error, "%U: %s", format_vlib_pci_addr,
+			       &args->addr,
+			       "error encountered on binding pci device");
+	  vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
+		    format_vlib_pci_addr, &args->addr,
+		    "error encountered on binding pci devicee");
+	  return;
+	}
+    }
 
   if ((error =
        vlib_pci_device_open (vm, &args->addr, vmxnet3_pci_device_ids, &h)))
     {
-      pool_put (vmxm->devices, vd);
       args->rv = VNET_API_ERROR_INVALID_INTERFACE;
       args->error =
-	clib_error_return (error, "pci-addr %U", format_vlib_pci_addr,
-			   &args->addr);
+	clib_error_return (error, "%U: %s", format_vlib_pci_addr,
+			   &args->addr,
+			   "error encountered on pci device open");
       vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
 		format_vlib_pci_addr, &args->addr,
 		"error encountered on pci device open");
@@ -696,6 +704,16 @@ vmxnet3_create_if (vlib_main_t * vm, vmxnet3_create_if_args_t * args)
    * Do not use vmxnet3_log_error prior to this line since the macro
    * references vd->pci_dev_handle
    */
+  pool_get (vmxm->devices, vd);
+  vd->num_tx_queues = args->txq_num;
+  vd->num_rx_queues = args->rxq_num;
+  vd->dev_instance = vd - vmxm->devices;
+  vd->per_interface_next_index = ~0;
+  vd->pci_addr = args->addr;
+
+  if (args->enable_elog)
+    vd->flags |= VMXNET3_DEVICE_F_ELOG;
+
   vd->pci_dev_handle = h;
   vd->numa_node = vlib_pci_get_numa_node (vm, h);
   vd->num_intrs = vd->num_rx_queues + 1;	// +1 for the event interrupt
diff --git a/src/plugins/vmxnet3/vmxnet3.h b/src/plugins/vmxnet3/vmxnet3.h
index 7bc05ec4267..65aa929ddcd 100644
--- a/src/plugins/vmxnet3/vmxnet3.h
+++ b/src/plugins/vmxnet3/vmxnet3.h
@@ -605,6 +605,7 @@ typedef struct
   u16 rxq_num;
   u16 txq_size;
   u16 txq_num;
+  u8 bind;
   /* return */
   i32 rv;
   u32 sw_if_index;
diff --git a/src/plugins/vmxnet3/vmxnet3_api.c b/src/plugins/vmxnet3/vmxnet3_api.c
index 0a66ffe0778..5e1bb9e2f18 100644
--- a/src/plugins/vmxnet3/vmxnet3_api.c
+++ b/src/plugins/vmxnet3/vmxnet3_api.c
@@ -72,6 +72,8 @@ vl_api_vmxnet3_create_t_handler (vl_api_vmxnet3_create_t * mp)
   args.rxq_size = ntohs (mp->rxq_size);
   args.txq_size = ntohs (mp->txq_size);
   args.txq_num = ntohs (mp->txq_num);
+  args.rxq_num = ntohs (mp->rxq_num);
+  args.bind = mp->bind;
 
   vmxnet3_create_if (vm, &args);
   rv = args.rv;
diff --git a/src/plugins/vmxnet3/vmxnet3_test.c b/src/plugins/vmxnet3/vmxnet3_test.c
index bb9f93e95cc..fe491b2a41d 100644
--- a/src/plugins/vmxnet3/vmxnet3_test.c
+++ b/src/plugins/vmxnet3/vmxnet3_test.c
@@ -117,12 +117,16 @@ api_vmxnet3_create (vat_main_t * vam)
 	}
       else if (unformat (i, "elog"))
 	args.enable_elog = 1;
+      else if (unformat (i, "bind"))
+	args.bind = 1;
       else if (unformat (i, "rx-queue-size %u", &args.rxq_size))
 	;
       else if (unformat (i, "tx-queue-size %u", &args.txq_size))
 	;
       else if (unformat (i, "num-tx-queues %u", &args.txq_num))
 	;
+      else if (unformat (i, "num-rx-queues %u", &args.rxq_num))
+	;
       else
 	{
 	  clib_warning ("unknown input '%U'", format_unformat_error, i);
@@ -137,6 +141,8 @@ api_vmxnet3_create (vat_main_t * vam)
   mp->rxq_size = clib_host_to_net_u16 (args.rxq_size);
   mp->txq_size = clib_host_to_net_u16 (args.txq_size);
   mp->txq_num = clib_host_to_net_u16 (args.txq_num);
+  mp->rxq_num = clib_host_to_net_u16 (args.rxq_num);
+  mp->bind = args.bind;
 
   S (mp);
   W (ret);
@@ -289,7 +295,8 @@ vl_api_vmxnet3_details_t_handler (vl_api_vmxnet3_details_t * mp)
  */
 #define foreach_vpe_api_msg					\
 _(vmxnet3_create, "<pci-address> [rx-queue-size <size>] "	\
-              "[tx-queue-size <size>] [num-tx-queues <num>]")	\
+  "[tx-queue-size <size>] [num-tx-queues <num>]"		\
+  "[num-rx-queues <num>] [bind]")				\
 _(vmxnet3_delete, "sw_if_index <sw_if_index>")                  \
 _(vmxnet3_dump, "")