vlib: pci improvements

 - logging
 - pass vlib_main_t to all APIs
 - open vfio container only when needed

Change-Id: I897e53e0af3f91c3a99f0c827401d1c0ec2e478a
Signed-off-by: Damjan Marion <damarion@cisco.com>

6 files changed, 50 insertions, 46 deletions

diff --git a/src/plugins/avf/device.c b/src/plugins/avf/device.c
index c23746be1b2..e71b74ec8fc 100644
--- a/src/plugins/avf/device.c
+++ b/src/plugins/avf/device.c
@@ -1036,11 +1036,10 @@ VLIB_REGISTER_NODE (avf_process_node, static)  = {
 /* *INDENT-ON* */
 
 static void
-avf_irq_0_handler (vlib_pci_dev_handle_t h, u16 line)
+avf_irq_0_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h, u16 line)
 {
-  vlib_main_t *vm = vlib_get_main ();
   avf_main_t *am = &avf_main;
-  uword pd = vlib_pci_get_private_data (h);
+  uword pd = vlib_pci_get_private_data (vm, h);
   avf_device_t *ad = pool_elt_at_index (am->devices, pd);
   u32 icr0;
 
@@ -1075,12 +1074,11 @@ avf_irq_0_handler (vlib_pci_dev_handle_t h, u16 line)
 }
 
 static void
-avf_irq_n_handler (vlib_pci_dev_handle_t h, u16 line)
+avf_irq_n_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h, u16 line)
 {
   vnet_main_t *vnm = vnet_get_main ();
-  vlib_main_t *vm = vlib_get_main ();
   avf_main_t *am = &avf_main;
-  uword pd = vlib_pci_get_private_data (h);
+  uword pd = vlib_pci_get_private_data (vm, h);
   avf_device_t *ad = pool_elt_at_index (am->devices, pd);
   u16 qid;
   int i;
@@ -1126,7 +1124,7 @@ avf_delete_if (vlib_main_t * vm, avf_device_t * ad)
       ethernet_delete_interface (vnm, ad->hw_if_index);
     }
 
-  vlib_pci_device_close (ad->pci_dev_handle);
+  vlib_pci_device_close (vm, ad->pci_dev_handle);
 
   vlib_physmem_free (vm, am->physmem_region, ad->atq);
   vlib_physmem_free (vm, am->physmem_region, ad->arq);
@@ -1197,7 +1195,8 @@ avf_create_if (vlib_main_t * vm, avf_create_if_args_t * args)
   if (args->enable_elog)
     ad->flags |= AVF_DEVICE_F_ELOG;
 
-  if ((error = vlib_pci_device_open (&args->addr, avf_pci_device_ids, &h)))
+  if ((error = vlib_pci_device_open (vm, &args->addr, avf_pci_device_ids,
+				     &h)))
     {
       pool_put (am->devices, ad);
       args->rv = VNET_API_ERROR_INVALID_INTERFACE;
@@ -1208,21 +1207,23 @@ avf_create_if (vlib_main_t * vm, avf_create_if_args_t * args)
     }
   ad->pci_dev_handle = h;
 
-  vlib_pci_set_private_data (h, ad->dev_instance);
+  vlib_pci_set_private_data (vm, h, ad->dev_instance);
 
-  if ((error = vlib_pci_bus_master_enable (h)))
+  if ((error = vlib_pci_bus_master_enable (vm, h)))
     goto error;
 
-  if ((error = vlib_pci_map_region (h, 0, &ad->bar0)))
+  if ((error = vlib_pci_map_region (vm, h, 0, &ad->bar0)))
     goto error;
 
-  if ((error = vlib_pci_register_msix_handler (h, 0, 1, &avf_irq_0_handler)))
+  if ((error = vlib_pci_register_msix_handler (vm, h, 0, 1,
+					       &avf_irq_0_handler)))
     goto error;
 
-  if ((error = vlib_pci_register_msix_handler (h, 1, 1, &avf_irq_n_handler)))
+  if ((error = vlib_pci_register_msix_handler (vm, h, 1, 1,
+					       &avf_irq_n_handler)))
     goto error;
 
-  if ((error = vlib_pci_enable_msix_irq (h, 0, 2)))
+  if ((error = vlib_pci_enable_msix_irq (vm, h, 0, 2)))
     goto error;
 
   if (am->physmem_region_alloc == 0)
@@ -1258,7 +1259,7 @@ avf_create_if (vlib_main_t * vm, avf_create_if_args_t * args)
   if (error)
     goto error;
 
-  if ((error = vlib_pci_intr_enable (h)))
+  if ((error = vlib_pci_intr_enable (vm, h)))
     goto error;
 
   /* FIXME detect */
diff --git a/src/plugins/avf/format.c b/src/plugins/avf/format.c
index 4eab3198766..3a1e1ff7f44 100644
--- a/src/plugins/avf/format.c
+++ b/src/plugins/avf/format.c
@@ -25,10 +25,11 @@
 u8 *
 format_avf_device_name (u8 * s, va_list * args)
 {
+  vlib_main_t *vm = vlib_get_main ();
   u32 i = va_arg (*args, u32);
   avf_main_t *am = &avf_main;
   avf_device_t *ad = vec_elt_at_index (am->devices, i);
-  vlib_pci_addr_t *addr = vlib_pci_get_addr (ad->pci_dev_handle);
+  vlib_pci_addr_t *addr = vlib_pci_get_addr (vm, ad->pci_dev_handle);
 
   s = format (s, "avf-%x/%x/%x/%x",
 	      addr->domain, addr->bus, addr->slot, addr->function);
diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c
index 58ba6c003c7..7d9ec7e5c80 100644
--- a/src/plugins/dpdk/device/init.c
+++ b/src/plugins/dpdk/device/init.c
@@ -842,6 +842,7 @@ dpdk_lib_init (dpdk_main_t * dm)
 static void
 dpdk_bind_devices_to_uio (dpdk_config_main_t * conf)
 {
+  vlib_main_t *vm = vlib_get_main ();
   clib_error_t *error;
   u8 *pci_addr = 0;
   int num_whitelisted = vec_len (conf->dev_confs);
@@ -860,7 +861,7 @@ dpdk_bind_devices_to_uio (dpdk_config_main_t * conf)
       vlib_pci_free_device_info (d);
       d = 0;
       }
-    d = vlib_pci_get_device_info (addr, &error);
+    d = vlib_pci_get_device_info (vm, addr, &error);
     if (error)
     {
       clib_error_report (error);
@@ -928,7 +929,7 @@ dpdk_bind_devices_to_uio (dpdk_config_main_t * conf)
         continue;
       }
 
-    error = vlib_pci_bind_to_uio (addr, (char *) conf->uio_driver_name);
+    error = vlib_pci_bind_to_uio (vm, addr, (char *) conf->uio_driver_name);
 
     if (error)
       {
diff --git a/src/plugins/ixge/ixge.c b/src/plugins/ixge/ixge.c
index 5cf83cfb670..09394ca4fc0 100644
--- a/src/plugins/ixge/ixge.c
+++ b/src/plugins/ixge/ixge.c
@@ -2245,10 +2245,11 @@ CLIB_MULTIARCH_SELECT_FN (ixge_input)
 static u8 *
 format_ixge_device_name (u8 * s, va_list * args)
 {
+  vlib_main_t *vm = vlib_get_main ();
   u32 i = va_arg (*args, u32);
   ixge_main_t *xm = &ixge_main;
   ixge_device_t *xd = vec_elt_at_index (xm->devices, i);
-  vlib_pci_addr_t *addr = vlib_pci_get_addr (xd->pci_dev_handle);
+  vlib_pci_addr_t *addr = vlib_pci_get_addr (vm, xd->pci_dev_handle);
   return format (s, "TenGigabitEthernet%x/%x/%x/%x",
 		 addr->domain, addr->bus, addr->slot, addr->function);
 }
@@ -2337,6 +2338,7 @@ format_ixge_device (u8 * s, va_list * args)
 {
   u32 dev_instance = va_arg (*args, u32);
   CLIB_UNUSED (int verbose) = va_arg (*args, int);
+  vlib_main_t *vm = vlib_get_main ();
   ixge_main_t *xm = &ixge_main;
   ixge_device_t *xd = vec_elt_at_index (xm->devices, dev_instance);
   ixge_phy_t *phy = xd->phys + xd->phy_index;
@@ -2351,8 +2353,8 @@ format_ixge_device (u8 * s, va_list * args)
 
   {
 
-    vlib_pci_addr_t *addr = vlib_pci_get_addr (xd->pci_dev_handle);
-    vlib_pci_device_info_t *d = vlib_pci_get_device_info (addr, 0);
+    vlib_pci_addr_t *addr = vlib_pci_get_addr (vm, xd->pci_dev_handle);
+    vlib_pci_device_info_t *d = vlib_pci_get_device_info (vm, addr, 0);
 
     if (d)
       s = format (s, "\n%UPCIe %U", format_white_space, indent + 2,
@@ -2822,11 +2824,9 @@ VLIB_INIT_FUNCTION (ixge_init);
 
 
 static void
-ixge_pci_intr_handler (vlib_pci_dev_handle_t h)
+ixge_pci_intr_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h)
 {
-  ixge_main_t *xm = &ixge_main;
-  vlib_main_t *vm = xm->vlib_main;
-  uword private_data = vlib_pci_get_private_data (h);
+  uword private_data = vlib_pci_get_private_data (vm, h);
 
   vlib_node_set_interrupt_pending (vm, ixge_input_node.index);
 
@@ -2845,8 +2845,8 @@ ixge_pci_init (vlib_main_t * vm, vlib_pci_dev_handle_t h)
   clib_error_t *error = 0;
   void *r;
   ixge_device_t *xd;
-  vlib_pci_addr_t *addr = vlib_pci_get_addr (h);
-  vlib_pci_device_info_t *d = vlib_pci_get_device_info (addr, 0);
+  vlib_pci_addr_t *addr = vlib_pci_get_addr (vm, h);
+  vlib_pci_device_info_t *d = vlib_pci_get_device_info (vm, addr, 0);
 
   /* Allocate physmem region for DMA buffers */
   if (xm->physmem_region_allocated == 0)
@@ -2859,7 +2859,7 @@ ixge_pci_init (vlib_main_t * vm, vlib_pci_dev_handle_t h)
   if (error)
     return error;
 
-  error = vlib_pci_map_region (h, 0, &r);
+  error = vlib_pci_map_region (vm, h, 0, &r);
   if (error)
     return error;
 
@@ -2877,7 +2877,7 @@ ixge_pci_init (vlib_main_t * vm, vlib_pci_dev_handle_t h)
   xd->pci_function = addr->function;
   xd->per_interface_next_index = ~0;
 
-  vlib_pci_set_private_data (h, xd->device_index);
+  vlib_pci_set_private_data (vm, h, xd->device_index);
 
   /* Chip found so enable node. */
   {
@@ -2895,12 +2895,12 @@ ixge_pci_init (vlib_main_t * vm, vlib_pci_dev_handle_t h)
       xm->process_node_index = ixge_process_node.index;
     }
 
-  error = vlib_pci_bus_master_enable (h);
+  error = vlib_pci_bus_master_enable (vm, h);
 
   if (error)
     return error;
 
-  return vlib_pci_intr_enable (h);
+  return vlib_pci_intr_enable (vm, h);
 }
 
 /* *INDENT-OFF* */
diff --git a/src/plugins/vmxnet3/format.c b/src/plugins/vmxnet3/format.c
index 56904652a79..c1a74367436 100644
--- a/src/plugins/vmxnet3/format.c
+++ b/src/plugins/vmxnet3/format.c
@@ -25,10 +25,11 @@
 u8 *
 format_vmxnet3_device_name (u8 * s, va_list * args)
 {
+  vlib_main_t *vm = vlib_get_main ();
   u32 i = va_arg (*args, u32);
   vmxnet3_main_t *vmxm = &vmxnet3_main;
   vmxnet3_device_t *vd = vec_elt_at_index (vmxm->devices, i);
-  vlib_pci_addr_t *addr = vlib_pci_get_addr (vd->pci_dev_handle);
+  vlib_pci_addr_t *addr = vlib_pci_get_addr (vm, vd->pci_dev_handle);
 
   s = format (s, "vmxnet3-%x/%x/%x/%x",
 	      addr->domain, addr->bus, addr->slot, addr->function);
diff --git a/src/plugins/vmxnet3/vmxnet3.c b/src/plugins/vmxnet3/vmxnet3.c
index ac99411a4e2..375224142f9 100644
--- a/src/plugins/vmxnet3/vmxnet3.c
+++ b/src/plugins/vmxnet3/vmxnet3.c
@@ -428,11 +428,11 @@ vmxnet3_device_init (vlib_main_t * vm, vmxnet3_device_t * vd,
 }
 
 static void
-vmxnet3_irq_0_handler (vlib_pci_dev_handle_t h, u16 line)
+vmxnet3_irq_0_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h, u16 line)
 {
   vnet_main_t *vnm = vnet_get_main ();
   vmxnet3_main_t *vmxm = &vmxnet3_main;
-  uword pd = vlib_pci_get_private_data (h);
+  uword pd = vlib_pci_get_private_data (vm, h);
   vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, pd);
   u16 qid = line;
 
@@ -441,11 +441,11 @@ vmxnet3_irq_0_handler (vlib_pci_dev_handle_t h, u16 line)
 }
 
 static void
-vmxnet3_irq_1_handler (vlib_pci_dev_handle_t h, u16 line)
+vmxnet3_irq_1_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h, u16 line)
 {
   vnet_main_t *vnm = vnet_get_main ();
   vmxnet3_main_t *vmxm = &vmxnet3_main;
-  uword pd = vlib_pci_get_private_data (h);
+  uword pd = vlib_pci_get_private_data (vm, h);
   vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, pd);
   u32 ret;
 
@@ -521,7 +521,7 @@ vmxnet3_create_if (vlib_main_t * vm, vmxnet3_create_if_args_t * args)
     vd->flags |= VMXNET3_DEVICE_F_ELOG;
 
   if ((error =
-       vlib_pci_device_open (&args->addr, vmxnet3_pci_device_ids, &h)))
+       vlib_pci_device_open (vm, &args->addr, vmxnet3_pci_device_ids, &h)))
     {
       pool_put (vmxm->devices, vd);
       args->rv = VNET_API_ERROR_INVALID_INTERFACE;
@@ -532,29 +532,29 @@ vmxnet3_create_if (vlib_main_t * vm, vmxnet3_create_if_args_t * args)
     }
   vd->pci_dev_handle = h;
 
-  vlib_pci_set_private_data (h, vd->dev_instance);
+  vlib_pci_set_private_data (vm, h, vd->dev_instance);
 
-  if ((error = vlib_pci_bus_master_enable (h)))
+  if ((error = vlib_pci_bus_master_enable (vm, h)))
     goto error;
 
-  if ((error = vlib_pci_map_region (h, 0, (void **) &vd->bar[0])))
+  if ((error = vlib_pci_map_region (vm, h, 0, (void **) &vd->bar[0])))
     goto error;
 
-  if ((error = vlib_pci_map_region (h, 1, (void **) &vd->bar[1])))
+  if ((error = vlib_pci_map_region (vm, h, 1, (void **) &vd->bar[1])))
     goto error;
 
-  if ((error = vlib_pci_register_msix_handler (h, 0, 1,
+  if ((error = vlib_pci_register_msix_handler (vm, h, 0, 1,
 					       &vmxnet3_irq_0_handler)))
     goto error;
 
-  if ((error = vlib_pci_register_msix_handler (h, 1, 1,
+  if ((error = vlib_pci_register_msix_handler (vm, h, 1, 1,
 					       &vmxnet3_irq_1_handler)))
     goto error;
 
-  if ((error = vlib_pci_enable_msix_irq (h, 0, 2)))
+  if ((error = vlib_pci_enable_msix_irq (vm, h, 0, 2)))
     goto error;
 
-  if ((error = vlib_pci_intr_enable (h)))
+  if ((error = vlib_pci_intr_enable (vm, h)))
     goto error;
 
   if ((error = vmxnet3_device_init (vm, vd, args)))
@@ -611,7 +611,7 @@ vmxnet3_delete_if (vlib_main_t * vm, vmxnet3_device_t * vd)
       ethernet_delete_interface (vnm, vd->hw_if_index);
     }
 
-  vlib_pci_device_close (vd->pci_dev_handle);
+  vlib_pci_device_close (vm, vd->pci_dev_handle);
 
   /* *INDENT-OFF* */
   vec_foreach_index (i, vd->rxqs)