dpdk: add l2_hdr_offset and l3_hdr_offset in vlib_buffer_t

Change-Id: I0a6d1257e391c3b6f7da6498bd5f7d4c545d17e9
Signed-off-by: Damjan Marion <damarion@cisco.com>

1 files changed, 57 insertions, 56 deletions

diff --git a/src/plugins/dpdk/device/node.c b/src/plugins/dpdk/device/node.c
index 1240b8d01a7..4863390ec2e 100644
--- a/src/plugins/dpdk/device/node.c
+++ b/src/plugins/dpdk/device/node.c
@@ -36,40 +36,21 @@ static char *dpdk_error_strings[] = {
 };
 #endif
 
-always_inline int
-vlib_buffer_is_ip4 (vlib_buffer_t * b)
-{
-  ethernet_header_t *h = (ethernet_header_t *) vlib_buffer_get_current (b);
-  return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP4));
-}
-
-always_inline int
-vlib_buffer_is_ip6 (vlib_buffer_t * b)
-{
-  ethernet_header_t *h = (ethernet_header_t *) vlib_buffer_get_current (b);
-  return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6));
-}
-
-always_inline int
-vlib_buffer_is_mpls (vlib_buffer_t * b)
-{
-  ethernet_header_t *h = (ethernet_header_t *) vlib_buffer_get_current (b);
-  return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS));
-}
-
 always_inline u32
-dpdk_rx_next_from_etype (struct rte_mbuf * mb, vlib_buffer_t * b0)
+dpdk_rx_next_from_etype (struct rte_mbuf *mb)
 {
-  if (PREDICT_TRUE (vlib_buffer_is_ip4 (b0)))
+  ethernet_header_t *h = rte_pktmbuf_mtod (mb, ethernet_header_t *);
+  if (PREDICT_TRUE (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP4)))
     {
       if (PREDICT_TRUE ((mb->ol_flags & PKT_RX_IP_CKSUM_GOOD) != 0))
 	return VNET_DEVICE_INPUT_NEXT_IP4_NCS_INPUT;
       else
 	return VNET_DEVICE_INPUT_NEXT_IP4_INPUT;
     }
-  else if (PREDICT_TRUE (vlib_buffer_is_ip6 (b0)))
+  else if (PREDICT_TRUE (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6)))
     return VNET_DEVICE_INPUT_NEXT_IP6_INPUT;
-  else if (PREDICT_TRUE (vlib_buffer_is_mpls (b0)))
+  else
+    if (PREDICT_TRUE (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)))
     return VNET_DEVICE_INPUT_NEXT_MPLS_INPUT;
   else
     return VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
@@ -78,6 +59,7 @@ dpdk_rx_next_from_etype (struct rte_mbuf * mb, vlib_buffer_t * b0)
 always_inline u32
 dpdk_rx_next_from_packet_start (struct rte_mbuf * mb, vlib_buffer_t * b0)
 {
+  ethernet_header_t *h = rte_pktmbuf_mtod (mb, ethernet_header_t *);
   word start_delta;
   int rv;
 
@@ -86,16 +68,17 @@ dpdk_rx_next_from_packet_start (struct rte_mbuf * mb, vlib_buffer_t * b0)
 
   vlib_buffer_advance (b0, -start_delta);
 
-  if (PREDICT_TRUE (vlib_buffer_is_ip4 (b0)))
+  if (PREDICT_TRUE (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP4)))
     {
       if (PREDICT_TRUE ((mb->ol_flags & PKT_RX_IP_CKSUM_GOOD) != 0))
 	rv = VNET_DEVICE_INPUT_NEXT_IP4_NCS_INPUT;
       else
 	rv = VNET_DEVICE_INPUT_NEXT_IP4_INPUT;
     }
-  else if (PREDICT_TRUE (vlib_buffer_is_ip6 (b0)))
+  else if (PREDICT_TRUE (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6)))
     rv = VNET_DEVICE_INPUT_NEXT_IP6_INPUT;
-  else if (PREDICT_TRUE (vlib_buffer_is_mpls (b0)))
+  else
+    if (PREDICT_TRUE (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)))
     rv = VNET_DEVICE_INPUT_NEXT_MPLS_INPUT;
   else
     rv = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
@@ -326,6 +309,7 @@ dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd,
       u32 bi2, next2;
       u32 bi3, next3;
       u8 error0, error1, error2, error3;
+      i16 offset0, offset1, offset2, offset3;
       u64 or_ol_flags;
 
       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
@@ -366,21 +350,10 @@ dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd,
 	  b2 = vlib_buffer_from_rte_mbuf (mb2);
 	  b3 = vlib_buffer_from_rte_mbuf (mb3);
 
-	  clib_memcpy64_x4 (b0, b1, b2, b3, bt);
-
 	  dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 9]);
 	  dpdk_prefetch_ethertype (xd->rx_vectors[queue_id][mb_index + 5]);
 
-	  /* current_data must be set to -RTE_PKTMBUF_HEADROOM in template */
-	  b0->current_data += mb0->data_off;
-	  b1->current_data += mb1->data_off;
-	  b2->current_data += mb2->data_off;
-	  b3->current_data += mb3->data_off;
-
-	  b0->current_length = mb0->data_len;
-	  b1->current_length = mb1->data_len;
-	  b2->current_length = mb2->data_len;
-	  b3->current_length = mb3->data_len;
+	  clib_memcpy64_x4 (b0, b1, b2, b3, bt);
 
 	  dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 10]);
 	  dpdk_prefetch_ethertype (xd->rx_vectors[queue_id][mb_index + 7]);
@@ -403,10 +376,10 @@ dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd,
 	    }
 	  else
 	    {
-	      next0 = dpdk_rx_next_from_etype (mb0, b0);
-	      next1 = dpdk_rx_next_from_etype (mb1, b1);
-	      next2 = dpdk_rx_next_from_etype (mb2, b2);
-	      next3 = dpdk_rx_next_from_etype (mb3, b3);
+	      next0 = dpdk_rx_next_from_etype (mb0);
+	      next1 = dpdk_rx_next_from_etype (mb1);
+	      next2 = dpdk_rx_next_from_etype (mb2);
+	      next3 = dpdk_rx_next_from_etype (mb3);
 	    }
 
 	  dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 11]);
@@ -426,16 +399,43 @@ dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd,
 	      b3->error = node->errors[error3];
 	    }
 
-	  vlib_buffer_advance (b0, device_input_next_node_advance[next0]);
-	  vlib_buffer_advance (b1, device_input_next_node_advance[next1]);
-	  vlib_buffer_advance (b2, device_input_next_node_advance[next2]);
-	  vlib_buffer_advance (b3, device_input_next_node_advance[next3]);
-
+	  offset0 = device_input_next_node_advance[next0];
+	  b0->current_data = mb0->data_off + offset0 - RTE_PKTMBUF_HEADROOM;
+	  b0->flags |= device_input_next_node_flags[next0];
+	  vnet_buffer (b0)->l3_hdr_offset = b0->current_data;
+	  vnet_buffer (b0)->l2_hdr_offset =
+	    mb0->data_off - RTE_PKTMBUF_HEADROOM;
+	  b0->current_length = mb0->data_len - offset0;
 	  n_rx_bytes += mb0->pkt_len;
+
+	  offset1 = device_input_next_node_advance[next1];
+	  b1->current_data = mb1->data_off + offset1 - RTE_PKTMBUF_HEADROOM;
+	  b1->flags |= device_input_next_node_flags[next1];
+	  vnet_buffer (b1)->l3_hdr_offset = b1->current_data;
+	  vnet_buffer (b1)->l2_hdr_offset =
+	    mb1->data_off - RTE_PKTMBUF_HEADROOM;
+	  b1->current_length = mb1->data_len - offset1;
 	  n_rx_bytes += mb1->pkt_len;
+
+	  offset2 = device_input_next_node_advance[next2];
+	  b2->current_data = mb2->data_off + offset2 - RTE_PKTMBUF_HEADROOM;
+	  b2->flags |= device_input_next_node_flags[next2];
+	  vnet_buffer (b2)->l3_hdr_offset = b2->current_data;
+	  vnet_buffer (b2)->l2_hdr_offset =
+	    mb2->data_off - RTE_PKTMBUF_HEADROOM;
+	  b2->current_length = mb2->data_len - offset2;
 	  n_rx_bytes += mb2->pkt_len;
+
+	  offset3 = device_input_next_node_advance[next3];
+	  b3->current_data = mb3->data_off + offset3 - RTE_PKTMBUF_HEADROOM;
+	  b3->flags |= device_input_next_node_flags[next3];
+	  vnet_buffer (b3)->l3_hdr_offset = b3->current_data;
+	  vnet_buffer (b3)->l2_hdr_offset =
+	    mb3->data_off - RTE_PKTMBUF_HEADROOM;
+	  b3->current_length = mb3->data_len - offset3;
 	  n_rx_bytes += mb3->pkt_len;
 
+
 	  /* Process subsequent segments of multi-segment packets */
 	  if (maybe_multiseg)
 	    {
@@ -488,10 +488,6 @@ dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd,
 
 	  clib_memcpy (b0, bt, CLIB_CACHE_LINE_BYTES);
 
-	  ASSERT (b0->current_data == -RTE_PKTMBUF_HEADROOM);
-	  b0->current_data += mb0->data_off;
-	  b0->current_length = mb0->data_len;
-
 	  bi0 = vlib_get_buffer_index (vm, b0);
 
 	  to_next[0] = bi0;
@@ -501,13 +497,18 @@ dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd,
 	  if (PREDICT_FALSE (xd->per_interface_next_index != ~0))
 	    next0 = xd->per_interface_next_index;
 	  else
-	    next0 = dpdk_rx_next_from_etype (mb0, b0);
+	    next0 = dpdk_rx_next_from_etype (mb0);
 
 	  dpdk_rx_error_from_mb (mb0, &next0, &error0);
 	  b0->error = node->errors[error0];
 
-	  vlib_buffer_advance (b0, device_input_next_node_advance[next0]);
-
+	  offset0 = device_input_next_node_advance[next0];
+	  b0->current_data = mb0->data_off + offset0 - RTE_PKTMBUF_HEADROOM;
+	  b0->flags |= device_input_next_node_flags[next0];
+	  vnet_buffer (b0)->l3_hdr_offset = b0->current_data;
+	  vnet_buffer (b0)->l2_hdr_offset =
+	    mb0->data_off - RTE_PKTMBUF_HEADROOM;
+	  b0->current_length = mb0->data_len - offset0;
 	  n_rx_bytes += mb0->pkt_len;
 
 	  /* Process subsequent segments of multi-segment packets */