dpdk: Decoupling the meaning of xd->device_index in dpdk_plugin

Prior to the change, dpdk plugin assumes xd->device_index is
used both as index for internal dpdk_main->devices array
and DPDK port index to call into DPDK APIs.

However, when running on top of Failsafe PMDs,
DPDK port index range may no longer be contiguous (as noted:
http://dpdk.org/ml/archives/dev/2018-March/092375.html
for related changes in DPDK). Because this, dpdk plugin can
no longer iterate through all available DPDK ports
with a for 0->rte_eth_dev_count() loop and the assumption of
device_index no longer holds.

This is part of initial effort to enable vpp running over
dpdk on failsafe PMD in Microsoft Azure(3/4).

Change-Id: I416fd80f2d40e12e139f8f3492814da98343eae7
Signed-off-by: Rui Cai <rucai@microsoft.com>

1 files changed, 24 insertions, 13 deletions

diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c
index d7c369a9c80..b5a8e3bcd4b 100644
--- a/src/plugins/dpdk/device/init.c
+++ b/src/plugins/dpdk/device/init.c
@@ -107,9 +107,9 @@ dpdk_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi, u32 flags)
       if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP)
 	{
 	  if (xd->flags & DPDK_DEVICE_FLAG_PROMISC)
-	    rte_eth_promiscuous_enable (xd->device_index);
+	    rte_eth_promiscuous_enable (xd->port_id);
 	  else
-	    rte_eth_promiscuous_disable (xd->device_index);
+	    rte_eth_promiscuous_disable (xd->port_id);
 	}
     }
   else if (ETHERNET_INTERFACE_FLAG_CONFIG_MTU (flags))
@@ -254,7 +254,8 @@ dpdk_lib_init (dpdk_main_t * dm)
       vnet_buffer (&ptd->buffer_template)->sw_if_index[VLIB_TX] = (u32) ~ 0;
     }
 
-  for (i = 0; i < nports; i++)
+  /* *INDENT-OFF* */
+  RTE_ETH_FOREACH_DEV(i)
     {
       u8 addr[6];
       u8 vlan_strip = 0;
@@ -507,8 +508,8 @@ dpdk_lib_init (dpdk_main_t * dm)
       if (xd->tx_q_used < tm->n_vlib_mains)
 	dpdk_device_lock_init (xd);
 
+      xd->port_id = i;
       xd->device_index = xd - dm->devices;
-      ASSERT (i == xd->device_index);
       xd->per_interface_next_index = ~0;
 
       /* assign interface to input thread */
@@ -621,12 +622,10 @@ dpdk_lib_init (dpdk_main_t * dm)
 	{
 	  int i;
 	  q = 0;
-	  /* *INDENT-OFF* */
 	  clib_bitmap_foreach (i, devconf->workers, ({
 	    vnet_hw_interface_assign_rx_thread (dm->vnet_main, xd->hw_if_index, q++,
 					     vdm->first_worker_thread_index + i);
 	  }));
-	  /* *INDENT-ON* */
 	}
       else
 	for (q = 0; q < xd->rx_q_used; q++)
@@ -684,10 +683,10 @@ dpdk_lib_init (dpdk_main_t * dm)
       if (vlan_strip)
 	{
 	  int vlan_off;
-	  vlan_off = rte_eth_dev_get_vlan_offload (xd->device_index);
+	  vlan_off = rte_eth_dev_get_vlan_offload (xd->port_id);
 	  vlan_off |= ETH_VLAN_STRIP_OFFLOAD;
 	  xd->port_conf.rxmode.hw_vlan_strip = vlan_off;
-	  if (rte_eth_dev_set_vlan_offload (xd->device_index, vlan_off) == 0)
+	  if (rte_eth_dev_set_vlan_offload (xd->port_id, vlan_off) == 0)
 	    dpdk_log_info ("VLAN strip enabled for interface\n");
 	  else
 	    dpdk_log_warn ("VLAN strip cannot be supported by interface\n");
@@ -702,8 +701,9 @@ dpdk_lib_init (dpdk_main_t * dm)
       if (dm->conf->no_multi_seg)
 	mtu = mtu > ETHER_MAX_LEN ? ETHER_MAX_LEN : mtu;
 
-      rte_eth_dev_set_mtu (xd->device_index, mtu);
+      rte_eth_dev_set_mtu (xd->port_id, mtu);
     }
+  /* *INDENT-ON* */
 
   if (nb_desc > dm->conf->num_mbufs)
     dpdk_log_err ("%d mbufs allocated but total rx/tx ring size is %d\n",
@@ -1377,7 +1377,7 @@ dpdk_update_link_state (dpdk_device_t * xd, f64 now)
 
   xd->time_last_link_update = now ? now : xd->time_last_link_update;
   memset (&xd->link, 0, sizeof (xd->link));
-  rte_eth_link_get_nowait (xd->device_index, &xd->link);
+  rte_eth_link_get_nowait (xd->port_id, &xd->link);
 
   if (LINK_STATE_ELOGS)
     {
@@ -1509,6 +1509,7 @@ dpdk_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f)
   dpdk_device_t *xd;
   vlib_thread_main_t *tm = vlib_get_thread_main ();
   int i;
+  int j;
 
   error = dpdk_lib_init (dm);
 
@@ -1535,10 +1536,19 @@ dpdk_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f)
     int nports = rte_eth_dev_count ();
     if (nports > 0)
       {
-	for (i = 0; i < nports; i++)
+	/* *INDENT-OFF* */
+	RTE_ETH_FOREACH_DEV(i)
 	  {
-	    xd = &dm->devices[i];
-	    ASSERT (i == xd->device_index);
+	    xd = NULL;
+	    for (j = 0; j < nports; j++)
+	      {
+		if (dm->devices[j].port_id == i)
+		  {
+		    xd = &dm->devices[j];
+		  }
+	      }
+	    ASSERT (xd != NULL);
+
 	    if (xd->pmd == VNET_DPDK_PMD_BOND)
 	      {
 		u8 addr[6];
@@ -1625,6 +1635,7 @@ dpdk_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f)
 		  }
 	      }
 	  }
+	/* *INDENT-ON* */
       }
   }