More optimizations for Tx and Rx loops

  1) Parse multiple output packets in loop.
  2) Remove checking for error flags, they are never set because
     ODP parser is disabled.
  3) Do not call vlib_buffer_advance if not necessary.
  4) Remove lock in TX path. Queues are synchronized by ODP.
  5) Optimize detection of packet type.
  6) Create a wrapper for call to odp_packet_user_area.

Change-Id: Ib50c9be9a62d67824b58ef2cd443b7fc59471655
Signed-off-by: Michal Mazur <mkm@semihalf.com>

1 files changed, 74 insertions, 70 deletions

diff --git a/src/plugins/odp/node.c b/src/plugins/odp/node.c
index 1a8b04c9..8eb1b0cb 100755
--- a/src/plugins/odp/node.c
+++ b/src/plugins/odp/node.c
@@ -51,7 +51,7 @@ format_odp_packet_input_trace (u8 * s, va_list * args)
 static_always_inline void
 odp_prefetch_buffer (odp_packet_t pkt)
 {
-  vlib_buffer_t *b = (vlib_buffer_t *) odp_packet_user_area (pkt);
+  vlib_buffer_t *b = vlib_buffer_from_odp_packet (pkt);
   CLIB_PREFETCH (pkt, CLIB_CACHE_LINE_BYTES, LOAD);
   CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, STORE);
 }
@@ -59,7 +59,7 @@ odp_prefetch_buffer (odp_packet_t pkt)
 static_always_inline void
 odp_prefetch_ethertype (odp_packet_t pkt)
 {
-  vlib_buffer_t *b = (vlib_buffer_t *) odp_packet_user_area (pkt);
+  vlib_buffer_t *b = vlib_buffer_from_odp_packet (pkt);
   CLIB_PREFETCH (vlib_buffer_get_current (b) +
 		 STRUCT_OFFSET_OF (ethernet_header_t, type),
 		 CLIB_CACHE_LINE_BYTES, LOAD);
@@ -74,25 +74,28 @@ odp_packet_queue_mode (odp_packet_if_t * oif, odp_packet_t pkt_tbl[],
   odp_pktio_t pktio = oif->pktio;
   odp_event_t evt_tbl[req_pkts];
   u64 sched_wait;
+  odp_queue_t rxq = ODP_QUEUE_INVALID;
 
   if (pktio == ODP_PKTIO_INVALID)
     {
       clib_warning ("invalid oif->pktio value");
       return 0;
     }
-  if ((oif->m.rx_mode == APPL_MODE_PKT_QUEUE) &&
-      (oif->rxq[queue_id] == ODP_QUEUE_INVALID))
+  if (oif->m.rx_mode == APPL_MODE_PKT_QUEUE)
     {
-      clib_warning ("invalid rxq[%d] queue", queue_id);
-      return 0;
+      rxq = oif->rxq[queue_id];
+      if (rxq == ODP_QUEUE_INVALID)
+	{
+	  clib_warning ("invalid rxq[%d] queue", queue_id);
+	  return 0;
+	}
     }
 
   while (req_pkts)
     {
       if (oif->m.rx_mode == APPL_MODE_PKT_QUEUE)
 	{
-	  i = odp_queue_deq_multi (oif->rxq[queue_id],
-				   &evt_tbl[num_evts], req_pkts);
+	  i = odp_queue_deq_multi (rxq, &evt_tbl[num_evts], req_pkts);
 	}
       else
 	{
@@ -139,38 +142,20 @@ odp_packet_burst_mode (odp_packet_if_t * oif, odp_packet_t pkt_tbl[],
   return num_pkts;
 }
 
-always_inline int
-vlib_buffer_is_ip4 (vlib_buffer_t * b)
-{
-  ethernet_header_t *h = (ethernet_header_t *) vlib_buffer_get_current (b);
-  return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP4));
-}
-
-always_inline int
-vlib_buffer_is_ip6 (vlib_buffer_t * b)
-{
-  ethernet_header_t *h = (ethernet_header_t *) vlib_buffer_get_current (b);
-  return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6));
-}
-
-always_inline int
-vlib_buffer_is_mpls (vlib_buffer_t * b)
-{
-  ethernet_header_t *h = (ethernet_header_t *) vlib_buffer_get_current (b);
-  return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS));
-}
-
 always_inline u32
 odp_rx_next_from_etype (void *mb, vlib_buffer_t * b0)
 {
-  if (PREDICT_TRUE (vlib_buffer_is_ip4 (b0)))
+  ethernet_header_t *h = (ethernet_header_t *) vlib_buffer_get_current (b0);
+  u16 type = h->type;
+
+  if (type == clib_host_to_net_u16 (ETHERNET_TYPE_IP4))
     return VNET_DEVICE_INPUT_NEXT_IP4_INPUT;
-  else if (PREDICT_TRUE (vlib_buffer_is_ip6 (b0)))
+  if (type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6))
     return VNET_DEVICE_INPUT_NEXT_IP6_INPUT;
-  else if (PREDICT_TRUE (vlib_buffer_is_mpls (b0)))
+  if (type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS))
     return VNET_DEVICE_INPUT_NEXT_MPLS_INPUT;
-  else
-    return VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
+
+  return VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
 }
 
 static_always_inline void
@@ -197,7 +182,7 @@ odp_adjust_buffer (vlib_buffer_t * buf, odp_packet_t pkt,
               tr->hw_if_index = (oif)->hw_if_index;             \
             }
 
-void
+static void
 odp_trace_buffer_x4 (uword * n_trace, vlib_main_t * vm,
 		     vlib_node_runtime_t * node, odp_packet_if_t * oif,
 		     vlib_buffer_t * b0, vlib_buffer_t * b1,
@@ -221,54 +206,62 @@ odp_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
   u32 n_rx_bytes = 0;
   u32 *to_next = 0;
   odp_packet_t pkt_tbl[VLIB_FRAME_SIZE];
-  int pkts = 0, i;
+  int i;
   int n_left = 0, n_left_to_next = VLIB_FRAME_SIZE;
-  u32 next0 = next_index;
-  u32 next1 = next_index;
-  u32 next2 = next_index;
-  u32 next3 = next_index;
+  u32 next0;
+  u32 next1;
+  u32 next2;
+  u32 next3;
 
   do
     {
       if ((oif->m.rx_mode == (APPL_MODE_PKT_QUEUE)) ||
 	  (oif->m.rx_mode == (APPL_MODE_PKT_SCHED)))
 	{
-	  pkts =
+	  n_left =
 	    odp_packet_queue_mode (oif, pkt_tbl, queue_id, n_left_to_next);
 	}
       else
 	{
-	  pkts =
+	  n_left =
 	    odp_packet_burst_mode (oif, pkt_tbl, queue_id, n_left_to_next);
 	}
 
-      n_left = drop_err_pkts (pkt_tbl, pkts);
       if (n_left == 0)
 	break;
 
       i = 0;
 
-      if (n_rx_packets == 0)
-	vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 
       while ((n_left >= 4) && (n_left_to_next >= 4))
 	{
 	  u32 bi0 = 0, bi1 = 0, bi2 = 0, bi3 = 0;
 	  vlib_buffer_t *b0, *b1, *b2, *b3;
+	  odp_packet_t pkt0, pkt1, pkt2, pkt3;
+
+	  pkt0 = pkt_tbl[i++];
+	  pkt1 = pkt_tbl[i++];
+	  pkt2 = pkt_tbl[i++];
+	  pkt3 = pkt_tbl[i++];
 
-	  b0 = (vlib_buffer_t *) odp_packet_user_area (pkt_tbl[i]);
-	  b1 = (vlib_buffer_t *) odp_packet_user_area (pkt_tbl[i + 1]);
-	  b2 = (vlib_buffer_t *) odp_packet_user_area (pkt_tbl[i + 2]);
-	  b3 = (vlib_buffer_t *) odp_packet_user_area (pkt_tbl[i + 3]);
+	  b0 = vlib_buffer_from_odp_packet (pkt0);
+	  b1 = vlib_buffer_from_odp_packet (pkt1);
+	  b2 = vlib_buffer_from_odp_packet (pkt2);
+	  b3 = vlib_buffer_from_odp_packet (pkt3);
 	  bi0 = vlib_get_buffer_index (vm, b0);
 	  bi1 = vlib_get_buffer_index (vm, b1);
 	  bi2 = vlib_get_buffer_index (vm, b2);
 	  bi3 = vlib_get_buffer_index (vm, b3);
+	  b0->l2_priv_data = pkt0;
+	  b1->l2_priv_data = pkt1;
+	  b2->l2_priv_data = pkt2;
+	  b3->l2_priv_data = pkt3;
 
-	  odp_adjust_buffer (b0, pkt_tbl[i], oif);
-	  odp_adjust_buffer (b1, pkt_tbl[i + 1], oif);
-	  odp_adjust_buffer (b2, pkt_tbl[i + 2], oif);
-	  odp_adjust_buffer (b3, pkt_tbl[i + 3], oif);
+	  odp_adjust_buffer (b0, pkt0, oif);
+	  odp_adjust_buffer (b1, pkt1, oif);
+	  odp_adjust_buffer (b2, pkt2, oif);
+	  odp_adjust_buffer (b3, pkt3, oif);
 
 	  if (PREDICT_FALSE (oif->per_interface_next_index != ~0))
 	    {
@@ -276,20 +269,31 @@ odp_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 	      next1 = oif->per_interface_next_index;
 	      next2 = oif->per_interface_next_index;
 	      next3 = oif->per_interface_next_index;
+	      if (oif->per_interface_next_index !=
+		  VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT)
+		{
+		  vlib_buffer_advance (b0, sizeof (ethernet_header_t));
+		  vlib_buffer_advance (b1, sizeof (ethernet_header_t));
+		  vlib_buffer_advance (b2, sizeof (ethernet_header_t));
+		  vlib_buffer_advance (b3, sizeof (ethernet_header_t));
+		}
 	    }
 	  else
 	    {
-	      next0 = odp_rx_next_from_etype (pkt_tbl[i], b0);
-	      next1 = odp_rx_next_from_etype (pkt_tbl[i + 1], b1);
-	      next2 = odp_rx_next_from_etype (pkt_tbl[i + 2], b2);
-	      next3 = odp_rx_next_from_etype (pkt_tbl[i + 3], b3);
+	      next0 = odp_rx_next_from_etype (pkt0, b0);
+	      next1 = odp_rx_next_from_etype (pkt1, b1);
+	      next2 = odp_rx_next_from_etype (pkt2, b2);
+	      next3 = odp_rx_next_from_etype (pkt3, b3);
+	      if (next0 != VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT)
+		vlib_buffer_advance (b0, sizeof (ethernet_header_t));
+	      if (next1 != VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT)
+		vlib_buffer_advance (b1, sizeof (ethernet_header_t));
+	      if (next2 != VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT)
+		vlib_buffer_advance (b2, sizeof (ethernet_header_t));
+	      if (next3 != VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT)
+		vlib_buffer_advance (b3, sizeof (ethernet_header_t));
 	    }
 
-	  vlib_buffer_advance (b0, device_input_next_node_advance[next0]);
-	  vlib_buffer_advance (b1, device_input_next_node_advance[next1]);
-	  vlib_buffer_advance (b2, device_input_next_node_advance[next2]);
-	  vlib_buffer_advance (b3, device_input_next_node_advance[next3]);
-
 	  /* trace */
 	  if (PREDICT_FALSE ((n_trace) > 0))
 	    odp_trace_buffer_x4 (&n_trace, vm, node, oif, b0, b1, b2, b3,
@@ -312,7 +316,6 @@ odp_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  n_rx_bytes += b1->current_length;
 	  n_rx_bytes += b2->current_length;
 	  n_rx_bytes += b3->current_length;
-	  i += 4;
 	  n_left -= 4;
 	  n_rx_packets += 4;
 	}
@@ -321,12 +324,13 @@ odp_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 	{
 	  u32 bi0 = 0;
 	  vlib_buffer_t *b0;
+	  odp_packet_t pkt = pkt_tbl[i++];
 
-	  b0 = (vlib_buffer_t *) odp_packet_user_area (pkt_tbl[i]);
+	  b0 = vlib_buffer_from_odp_packet (pkt);
 	  bi0 = vlib_get_buffer_index (vm, b0);
-	  b0->l2_priv_data = pkt_tbl[i];
+	  b0->l2_priv_data = pkt;
 
-	  b0->current_length = odp_packet_len (pkt_tbl[i]);
+	  b0->current_length = odp_packet_len (pkt);
 	  b0->current_data = 0;
 	  b0->total_length_not_including_first_buffer = 0;
 	  b0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
@@ -336,9 +340,10 @@ odp_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  if (PREDICT_FALSE (oif->per_interface_next_index != ~0))
 	    next0 = oif->per_interface_next_index;
 	  else
-	    next0 = odp_rx_next_from_etype (pkt_tbl[i], b0);
+	    next0 = odp_rx_next_from_etype (pkt, b0);
 
-	  vlib_buffer_advance (b0, device_input_next_node_advance[next0]);
+	  if (next0 != VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT)
+	    vlib_buffer_advance (b0, sizeof (ethernet_header_t));
 
 	  /* trace */
 	  ODP_TRACE_BUFFER (n_trace, b0, next0, vm, node, oif);
@@ -352,8 +357,7 @@ odp_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 					   n_left_to_next, bi0, next0);
 
 	  /* next packet */
-	  n_rx_bytes += odp_packet_len (pkt_tbl[i]);
-	  i++;
+	  n_rx_bytes += b0->current_length;
 	  n_left--;
 	  n_rx_packets++;
 	}