dpdk: improve error handling during device initialization

Change-Id: Ib390164abb07ca0d38fd49e7e2e6b4e9ea856405
Signed-off-by: Damjan Marion <damarion@cisco.com>

6 files changed, 98 insertions, 79 deletions

diff --git a/src/plugins/dpdk/device/cli.c b/src/plugins/dpdk/device/cli.c
index c7e5090dd81..aeeb772d4fb 100644
--- a/src/plugins/dpdk/device/cli.c
+++ b/src/plugins/dpdk/device/cli.c
@@ -537,7 +537,10 @@ set_dpdk_if_desc (vlib_main_t * vm, unformat_input_t * input,
   if (nb_tx_desc != (u32) ~ 0)
     xd->nb_tx_desc = nb_tx_desc;
 
-  error = dpdk_device_setup (xd);
+  dpdk_device_setup (xd);
+
+  if (vec_len (xd->errors))
+    return clib_error_return (0, "%U", format_dpdk_device_errors, xd);
 
 done:
   unformat_free (line_input);
diff --git a/src/plugins/dpdk/device/common.c b/src/plugins/dpdk/device/common.c
index 79c5888d9ad..1a9688e75e5 100644
--- a/src/plugins/dpdk/device/common.c
+++ b/src/plugins/dpdk/device/common.c
@@ -24,24 +24,28 @@
 #include <dpdk/device/dpdk_priv.h>
 #include <vppinfra/error.h>
 
-clib_error_t *
-dpdk_error_return (clib_error_t * error, char *str, dpdk_device_t * xd,
-		   int rv)
+void
+dpdk_device_error (dpdk_device_t * xd, char *str, int rv)
 {
-  return clib_error_return (error, "%s[%d]: %s(%d)", str, xd->device_index,
-			    rte_strerror (rv), rv);
+  xd->errors = clib_error_return (xd->errors, "%s[port:%d, errno:%d]: %s",
+				  str, xd->device_index, rv,
+				  rte_strerror (rv));
 }
 
-clib_error_t *
+void
 dpdk_device_setup (dpdk_device_t * xd)
 {
   dpdk_main_t *dm = &dpdk_main;
-  clib_error_t *err = 0;
+  vnet_main_t *vnm = vnet_get_main ();
+  vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, xd->vlib_sw_if_index);
   int rv;
   int j;
 
   ASSERT (vlib_get_thread_index () == 0);
 
+  clib_error_free (xd->errors);
+  sw->flags &= ~VNET_SW_INTERFACE_FLAG_ERROR;
+
   if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP)
     {
       vnet_hw_interface_set_flags (dm->vnet_main, xd->hw_if_index, 0);
@@ -52,7 +56,10 @@ dpdk_device_setup (dpdk_device_t * xd)
 			      xd->tx_q_used, &xd->port_conf);
 
   if (rv < 0)
-    return dpdk_error_return (err, "rte_eth_dev_configure", xd, rv);
+    {
+      dpdk_device_error (xd, "rte_eth_dev_configure", rv);
+      goto error;
+    }
 
   /* Set up one TX-queue per worker thread */
   for (j = 0; j < xd->tx_q_used; j++)
@@ -65,7 +72,7 @@ dpdk_device_setup (dpdk_device_t * xd)
 	rv = rte_eth_tx_queue_setup (xd->device_index, j, xd->nb_tx_desc,
 				     SOCKET_ID_ANY, &xd->tx_conf);
       if (rv < 0)
-	err = dpdk_error_return (err, "rte_eth_tx_queue_setup", xd, rv);
+	dpdk_device_error (xd, "rte_eth_tx_queue_setup", rv);
     }
 
   for (j = 0; j < xd->rx_q_used; j++)
@@ -86,28 +93,40 @@ dpdk_device_setup (dpdk_device_t * xd)
 				     dm->pktmbuf_pools[socket_id]);
 
       if (rv < 0)
-	err = dpdk_error_return (err, "rte_eth_rx_queue_setup", xd, rv);
+	dpdk_device_error (xd, "rte_eth_rx_queue_setup", rv);
     }
 
-  if (err)
-    return err;
+  if (vec_len (xd->errors))
+    goto error;
 
   if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP)
-    err = dpdk_device_start (xd);
+    dpdk_device_start (xd);
+
+  if (vec_len (xd->errors))
+    goto error;
+
+  return;
 
-  return err;
+error:
+  xd->flags |= DPDK_DEVICE_FLAG_PMD_INIT_FAIL;
+  sw->flags |= VNET_SW_INTERFACE_FLAG_ERROR;
 }
 
-clib_error_t *
+void
 dpdk_device_start (dpdk_device_t * xd)
 {
   int rv;
-  clib_error_t *err = 0;
+
+  if (xd->flags & DPDK_DEVICE_FLAG_PMD_INIT_FAIL)
+    return;
 
   rv = rte_eth_dev_start (xd->device_index);
 
   if (rv)
-    return dpdk_error_return (err, "rte_eth_dev_start", xd, rv);
+    {
+      dpdk_device_error (xd, "rte_eth_dev_start", rv);
+      return;
+    }
 
   if (xd->default_mac_address)
     rv =
@@ -116,7 +135,7 @@ dpdk_device_start (dpdk_device_t * xd)
 					xd->default_mac_address);
 
   if (rv)
-    err = dpdk_error_return (err, "rte_eth_dev_default_mac_addr_set", xd, rv);
+    dpdk_device_error (xd, "rte_eth_dev_default_mac_addr_set", rv);
 
   if (xd->flags & DPDK_DEVICE_FLAG_PROMISC)
     rte_eth_promiscuous_enable (xd->device_index);
@@ -135,16 +154,28 @@ dpdk_device_start (dpdk_device_t * xd)
 	  rte_eth_allmulticast_enable (dpdk_port);
 	}
     }
-
-  return err;
 }
 
-clib_error_t *
+void
 dpdk_device_stop (dpdk_device_t * xd)
 {
+  if (xd->flags & DPDK_DEVICE_FLAG_PMD_INIT_FAIL)
+    return;
+
+  rte_eth_allmulticast_disable (xd->device_index);
   rte_eth_dev_stop (xd->device_index);
 
-  return 0;
+  /* For bonded interface, stop slave links */
+  if (xd->pmd == VNET_DPDK_PMD_BOND)
+    {
+      u8 slink[16];
+      int nlink = rte_eth_bond_slaves_get (xd->device_index, slink, 16);
+      while (nlink >= 1)
+	{
+	  u8 dpdk_port = slink[--nlink];
+	  rte_eth_dev_stop (dpdk_port);
+	}
+    }
 }
 
 /*
diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c
index 51d6eacbeb4..6a0b1acf6c1 100644
--- a/src/plugins/dpdk/device/device.c
+++ b/src/plugins/dpdk/device/device.c
@@ -612,38 +612,16 @@ dpdk_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
   uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
   dpdk_main_t *dm = &dpdk_main;
   dpdk_device_t *xd = vec_elt_at_index (dm->devices, hif->dev_instance);
-  int rv = 0;
+
+  if (xd->flags & DPDK_DEVICE_FLAG_PMD_INIT_FAIL)
+    return clib_error_return (0, "Interface not initialized");
 
   if (is_up)
     {
       f64 now = vlib_time_now (dm->vlib_main);
 
       if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) == 0)
-	{
-	  rv = rte_eth_dev_start (xd->device_index);
-	  if (!rv && xd->default_mac_address)
-	    rv = rte_eth_dev_default_mac_addr_set (xd->device_index,
-						   (struct ether_addr *)
-						   xd->default_mac_address);
-	}
-
-      if (xd->flags & DPDK_DEVICE_FLAG_PROMISC)
-	rte_eth_promiscuous_enable (xd->device_index);
-      else
-	rte_eth_promiscuous_disable (xd->device_index);
-
-      rte_eth_allmulticast_enable (xd->device_index);
-
-      if (xd->pmd == VNET_DPDK_PMD_BOND)
-	{
-	  u8 slink[16];
-	  int nlink = rte_eth_bond_slaves_get (xd->device_index, slink, 16);
-	  while (nlink >= 1)
-	    {
-	      u8 dpdk_port = slink[--nlink];
-	      rte_eth_allmulticast_enable (dpdk_port);
-	    }
-	}
+	dpdk_device_start (xd);
 
       xd->flags |= DPDK_DEVICE_FLAG_ADMIN_UP;
       dpdk_update_counters (xd, now);
@@ -652,27 +630,12 @@ dpdk_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
   else
     {
       xd->flags &= ~DPDK_DEVICE_FLAG_ADMIN_UP;
-
-      rte_eth_allmulticast_disable (xd->device_index);
       vnet_hw_interface_set_flags (vnm, xd->hw_if_index, 0);
-      rte_eth_dev_stop (xd->device_index);
 
-      /* For bonded interface, stop slave links */
-      if (xd->pmd == VNET_DPDK_PMD_BOND)
-	{
-	  u8 slink[16];
-	  int nlink = rte_eth_bond_slaves_get (xd->device_index, slink, 16);
-	  while (nlink >= 1)
-	    {
-	      u8 dpdk_port = slink[--nlink];
-	      rte_eth_dev_stop (dpdk_port);
-	    }
-	}
+      if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) != 0)
+	dpdk_device_stop (xd);
     }
 
-  if (rv < 0)
-    clib_warning ("rte_eth_dev_%s error: %d", is_up ? "start" : "stop", rv);
-
   return /* no error */ 0;
 }
 
diff --git a/src/plugins/dpdk/device/dpdk.h b/src/plugins/dpdk/device/dpdk.h
index 7e974491efe..6364d163632 100644
--- a/src/plugins/dpdk/device/dpdk.h
+++ b/src/plugins/dpdk/device/dpdk.h
@@ -166,7 +166,7 @@ typedef struct
 #define DPDK_DEVICE_FLAG_ADMIN_UP           (1 << 0)
 #define DPDK_DEVICE_FLAG_PROMISC            (1 << 1)
 #define DPDK_DEVICE_FLAG_PMD                (1 << 2)
-
+#define DPDK_DEVICE_FLAG_PMD_INIT_FAIL      (1 << 3)
 #define DPDK_DEVICE_FLAG_MAYBE_MULTISEG     (1 << 4)
 #define DPDK_DEVICE_FLAG_HAVE_SUBIF         (1 << 5)
 #define DPDK_DEVICE_FLAG_HQOS               (1 << 6)
@@ -207,6 +207,9 @@ typedef struct
 
   /* mac address */
   u8 *default_mac_address;
+
+  /* error string */
+  clib_error_t *errors;
 } dpdk_device_t;
 
 #define DPDK_STATS_POLL_INTERVAL      (10.0)
@@ -399,9 +402,9 @@ typedef struct
   u8 data[256];			/* First 256 data bytes, used for hexdump */
 } dpdk_rx_dma_trace_t;
 
-clib_error_t *dpdk_device_setup (dpdk_device_t * xd);
-clib_error_t *dpdk_device_start (dpdk_device_t * xd);
-clib_error_t *dpdk_device_stop (dpdk_device_t * xd);
+void dpdk_device_setup (dpdk_device_t * xd);
+void dpdk_device_start (dpdk_device_t * xd);
+void dpdk_device_stop (dpdk_device_t * xd);
 
 #define foreach_dpdk_error						\
   _(NONE, "no error")							\
@@ -424,6 +427,7 @@ void dpdk_update_link_state (dpdk_device_t * xd, f64 now);
 
 format_function_t format_dpdk_device_name;
 format_function_t format_dpdk_device;
+format_function_t format_dpdk_device_errors;
 format_function_t format_dpdk_tx_dma_trace;
 format_function_t format_dpdk_rx_dma_trace;
 format_function_t format_dpdk_rte_mbuf;
diff --git a/src/plugins/dpdk/device/format.c b/src/plugins/dpdk/device/format.c
index 3b75563d0b5..0b67eae3c45 100644
--- a/src/plugins/dpdk/device/format.c
+++ b/src/plugins/dpdk/device/format.c
@@ -377,6 +377,20 @@ format_dpdk_tx_offload_caps (u8 * s, va_list * args)
 #undef _
 
 u8 *
+format_dpdk_device_errors (u8 * s, va_list * args)
+{
+  dpdk_device_t *xd = va_arg (*args, dpdk_device_t *);
+  clib_error_t *e;
+  uword indent = format_get_indent (s);
+
+  vec_foreach (e, xd->errors)
+  {
+    s = format (s, "%U%v\n", format_white_space, indent, e->what);
+  }
+  return s;
+}
+
+u8 *
 format_dpdk_device (u8 * s, va_list * args)
 {
   u32 dev_instance = va_arg (*args, u32);
@@ -511,6 +525,12 @@ format_dpdk_device (u8 * s, va_list * args)
       vec_free (xs);
     }
 
+  if (vec_len (xd->errors))
+    {
+      s = format (s, "%UErrors:\n  %U", format_white_space, indent,
+		  format_dpdk_device_errors, xd);
+    }
+
   return s;
 }
 
diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c
index 0ee28db54a4..bcb787c82cf 100755
--- a/src/plugins/dpdk/device/init.c
+++ b/src/plugins/dpdk/device/init.c
@@ -118,11 +118,7 @@ dpdk_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi, u32 flags)
       rte_eth_dev_set_mtu (xd->device_index, hi->max_packet_bytes);
 
       if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP)
-	{
-	  clib_error_t *error;
-	  error = dpdk_device_start (xd);
-	  clib_error_report (error);
-	}
+	dpdk_device_start (xd);
 
     }
   return old;
@@ -223,7 +219,6 @@ dpdk_lib_init (dpdk_main_t * dm)
       u8 vlan_strip = 0;
       int j;
       struct rte_eth_dev_info dev_info;
-      clib_error_t *rv;
       struct rte_eth_link l;
       dpdk_device_config_t *devconf = 0;
       vlib_pci_addr_t pci_addr;
@@ -570,13 +565,16 @@ dpdk_lib_init (dpdk_main_t * dm)
 
       hi = vnet_get_hw_interface (dm->vnet_main, xd->hw_if_index);
 
-      rv = dpdk_device_setup (xd);
+      dpdk_device_setup (xd);
 
-      if (rv)
-	return rv;
+      if (vec_len (xd->errors))
+	clib_warning ("setup failed for device %U. Errors:\n  %U",
+		      format_dpdk_device_name, i,
+		      format_dpdk_device_errors, xd);
 
       if (devconf->hqos_enabled)
 	{
+	  clib_error_t *rv;
 	  rv = dpdk_port_setup_hqos (xd, &devconf->hqos);
 	  if (rv)
 	    return rv;