1 files changed, 43 insertions, 432 deletions

diff --git a/vnet/vnet/devices/dpdk/node.c b/vnet/vnet/devices/dpdk/node.c
index 9a0f9c8f..27670f37 100644
--- a/vnet/vnet/devices/dpdk/node.c
+++ b/vnet/vnet/devices/dpdk/node.c
@@ -22,6 +22,7 @@
 #include <vnet/devices/dpdk/dpdk.h>
 #include <vnet/classify/vnet_classify.h>
 #include <vnet/mpls-gre/packet.h>
+#include <vnet/handoff.h>
 
 #include "dpdk_priv.h"
 
@@ -48,240 +49,45 @@
  */
 #define VMWARE_LENGTH_BUG_WORKAROUND 0
 
-typedef struct {
-  u32 cached_next_index;
-
-  /* convenience variables */
-  vlib_main_t * vlib_main;
-  vnet_main_t * vnet_main;
-} handoff_dispatch_main_t;
-
-typedef struct {
-  u32 buffer_index;
-  u32 next_index;
-  u32 sw_if_index;
-} handoff_dispatch_trace_t;
-
-/* packet trace format function */
-static u8 * format_handoff_dispatch_trace (u8 * s, va_list * args)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  handoff_dispatch_trace_t * t = va_arg (*args, handoff_dispatch_trace_t *);
-
-  s = format (s, "HANDOFF_DISPATCH: sw_if_index %d next_index %d buffer 0x%x",
-      t->sw_if_index,
-      t->next_index,
-      t->buffer_index);
-  return s;
-}
-
-handoff_dispatch_main_t handoff_dispatch_main;
-
-vlib_node_registration_t handoff_dispatch_node;
-
-#define foreach_handoff_dispatch_error \
-_(EXAMPLE, "example packets")
-
-typedef enum {
-#define _(sym,str) HANDOFF_DISPATCH_ERROR_##sym,
-  foreach_handoff_dispatch_error
-#undef _
-  HANDOFF_DISPATCH_N_ERROR,
-} handoff_dispatch_error_t;
-
-static char * handoff_dispatch_error_strings[] = {
-#define _(sym,string) string,
-  foreach_handoff_dispatch_error
+static char * dpdk_error_strings[] = {
+#define _(n,s) s,
+    foreach_dpdk_error
 #undef _
 };
 
-static inline
-void vlib_put_handoff_queue_elt (vlib_frame_queue_elt_t * hf)
-{
-  CLIB_MEMORY_BARRIER();
-  hf->valid = 1;
-}
-
-static uword
-handoff_dispatch_node_fn (vlib_main_t * vm,
-		  vlib_node_runtime_t * node,
-		  vlib_frame_t * frame)
+always_inline int
+dpdk_mbuf_is_ip4(struct rte_mbuf *mb)
 {
-  u32 n_left_from, * from, * to_next;
-  dpdk_rx_next_t next_index;
-
-  from = vlib_frame_vector_args (frame);
-  n_left_from = frame->n_vectors;
-  next_index = node->cached_next_index;
-
-  while (n_left_from > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index,
-			   to_next, n_left_to_next);
-
-      while (n_left_from >= 4 && n_left_to_next >= 2)
-	{
-          u32 bi0, bi1;
-	  vlib_buffer_t * b0, * b1;
-          u32 next0, next1;
-          u32 sw_if_index0, sw_if_index1;
-          
-	  /* Prefetch next iteration. */
-	  {
-	    vlib_buffer_t * p2, * p3;
-            
-	    p2 = vlib_get_buffer (vm, from[2]);
-	    p3 = vlib_get_buffer (vm, from[3]);
-            
-	    vlib_prefetch_buffer_header (p2, LOAD);
-	    vlib_prefetch_buffer_header (p3, LOAD);
-	  }
-
-          /* speculatively enqueue b0 and b1 to the current next frame */
-	  to_next[0] = bi0 = from[0];
-	  to_next[1] = bi1 = from[1];
-	  from += 2;
-	  to_next += 2;
-	  n_left_from -= 2;
-	  n_left_to_next -= 2;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-	  b1 = vlib_get_buffer (vm, bi1);
-
-          next0 = vnet_buffer(b0)->io_handoff.next_index;
-          next1 = vnet_buffer(b1)->io_handoff.next_index;
-
-          if (PREDICT_FALSE(vm->trace_main.trace_active_hint))
-            {
-            if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
-              {
-                vlib_trace_buffer (vm, node, next0, b0, /* follow_chain */ 0);
-                handoff_dispatch_trace_t *t =
-                  vlib_add_trace (vm, node, b0, sizeof (*t));
-                sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
-                t->sw_if_index = sw_if_index0;
-                t->next_index = next0;
-                t->buffer_index = bi0;
-              }
-            if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
-              {
-                vlib_trace_buffer (vm, node, next1, b1, /* follow_chain */ 0);
-                handoff_dispatch_trace_t *t =
-                  vlib_add_trace (vm, node, b1, sizeof (*t));
-                sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
-                t->sw_if_index = sw_if_index1;
-                t->next_index = next1;
-                t->buffer_index = bi1;
-              }
-            }
-            
-          /* verify speculative enqueues, maybe switch current next frame */
-          vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
-                                           to_next, n_left_to_next,
-                                           bi0, bi1, next0, next1);
-        }
-      
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-          u32 bi0;
-	  vlib_buffer_t * b0;
-          u32 next0;
-          u32 sw_if_index0;
-
-          /* speculatively enqueue b0 to the current next frame */
-	  bi0 = from[0];
-	  to_next[0] = bi0;
-	  from += 1;
-	  to_next += 1;
-	  n_left_from -= 1;
-	  n_left_to_next -= 1;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-
-          next0 = vnet_buffer(b0)->io_handoff.next_index;
-
-          if (PREDICT_FALSE(vm->trace_main.trace_active_hint))
-            {
-            if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
-              {
-                vlib_trace_buffer (vm, node, next0, b0, /* follow_chain */ 0);
-                handoff_dispatch_trace_t *t =
-                  vlib_add_trace (vm, node, b0, sizeof (*t));
-                sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
-                t->sw_if_index = sw_if_index0;
-                t->next_index = next0;
-                t->buffer_index = bi0;
-              }
-            }
-
-          /* verify speculative enqueue, maybe switch current next frame */
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, next0);
-	}
-
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-
-  return frame->n_vectors;
+#if RTE_VERSION >= RTE_VERSION_NUM(2, 1, 0, 0)
+  return RTE_ETH_IS_IPV4_HDR(mb->packet_type) != 0;
+#else
+  return (mb_flags & (PKT_RX_IPV4_HDR | PKT_RX_IPV4_HDR_EXT)) != 0;
+#endif
 }
 
-VLIB_REGISTER_NODE (handoff_dispatch_node) = {
-  .function = handoff_dispatch_node_fn,
-  .name = "handoff-dispatch",
-  .vector_size = sizeof (u32),
-  .format_trace = format_handoff_dispatch_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-  .flags = VLIB_NODE_FLAG_IS_HANDOFF,
-  
-  .n_errors = ARRAY_LEN(handoff_dispatch_error_strings),
-  .error_strings = handoff_dispatch_error_strings,
-
-  .n_next_nodes = DPDK_RX_N_NEXT,
-
-  .next_nodes = {
-        [DPDK_RX_NEXT_DROP] = "error-drop",
-        [DPDK_RX_NEXT_ETHERNET_INPUT] = "ethernet-input",
-        [DPDK_RX_NEXT_IP4_INPUT] = "ip4-input",
-        [DPDK_RX_NEXT_IP6_INPUT] = "ip6-input",
-        [DPDK_RX_NEXT_MPLS_INPUT] = "mpls-gre-input",
-  },
-};
-
-VLIB_NODE_FUNCTION_MULTIARCH (handoff_dispatch_node, handoff_dispatch_node_fn)
-
-clib_error_t *handoff_dispatch_init (vlib_main_t *vm)
+always_inline int
+dpdk_mbuf_is_ip6(struct rte_mbuf *mb)
 {
-  handoff_dispatch_main_t * mp = &handoff_dispatch_main;
-    
-  mp->vlib_main = vm;
-  mp->vnet_main = &vnet_main;
-
-  return 0;
+#if RTE_VERSION >= RTE_VERSION_NUM(2, 1, 0, 0)
+  return RTE_ETH_IS_IPV6_HDR(mb->packet_type) != 0;
+#else
+  return (mb_flags & (PKT_RX_IPV6_HDR | PKT_RX_IPV6_HDR_EXT)) != 0;
+#endif
 }
 
-VLIB_INIT_FUNCTION (handoff_dispatch_init);
-
-u32 dpdk_get_handoff_node_index (void)
+always_inline int
+vlib_buffer_is_mpls(vlib_buffer_t * b)
 {
-  return handoff_dispatch_node.index;
+  ethernet_header_t *h = (ethernet_header_t *) b->data;
+  return (h->type == clib_host_to_net_u16(ETHERNET_TYPE_MPLS_UNICAST));
 }
 
-static char * dpdk_error_strings[] = {
-#define _(n,s) s,
-    foreach_dpdk_error
-#undef _
-};
-
 always_inline void
 dpdk_rx_next_and_error_from_mb_flags_x1 (dpdk_device_t *xd, struct rte_mbuf *mb,
                                          vlib_buffer_t *b0,
 					 u8 * next0, u8 * error0)
 {
-  u8 is0_ip4, is0_ip6, is0_mpls, n0;
+  u8 n0;
   uint16_t mb_flags = mb->ol_flags;
 
   if (PREDICT_FALSE(mb_flags & (
@@ -306,37 +112,30 @@ dpdk_rx_next_and_error_from_mb_flags_x1 (dpdk_device_t *xd, struct rte_mbuf *mb,
     {
       *error0 = DPDK_ERROR_NONE;
       if (PREDICT_FALSE(xd->per_interface_next_index != ~0))
-	n0 = xd->per_interface_next_index;
+	{
+	  n0 = xd->per_interface_next_index;
+	  b0->flags |= BUFFER_HANDOFF_NEXT_VALID;
+	  if (PREDICT_TRUE (dpdk_mbuf_is_ip4(mb)))
+	    vnet_buffer(b0)->handoff.next_index = HANDOFF_DISPATCH_NEXT_IP4_INPUT;
+	  else if (PREDICT_TRUE(dpdk_mbuf_is_ip6(mb)))
+	    vnet_buffer(b0)->handoff.next_index = HANDOFF_DISPATCH_NEXT_IP6_INPUT;
+          else if (PREDICT_TRUE(vlib_buffer_is_mpls(b0)))
+	    vnet_buffer(b0)->handoff.next_index = HANDOFF_DISPATCH_NEXT_MPLS_INPUT;
+	  else
+	    vnet_buffer(b0)->handoff.next_index = HANDOFF_DISPATCH_NEXT_ETHERNET_INPUT;
+	}
       else if (PREDICT_FALSE(xd->vlan_subifs || (mb_flags & PKT_RX_VLAN_PKT)))
 	n0 = DPDK_RX_NEXT_ETHERNET_INPUT;
       else
 	{
-	  n0 = DPDK_RX_NEXT_ETHERNET_INPUT;
-#if RTE_VERSION >= RTE_VERSION_NUM(2, 1, 0, 0)
-	  is0_ip4 = RTE_ETH_IS_IPV4_HDR(mb->packet_type) != 0;
-#else
-	  is0_ip4 = (mb_flags & (PKT_RX_IPV4_HDR | PKT_RX_IPV4_HDR_EXT)) != 0;
-#endif
-
-	  if (PREDICT_TRUE(is0_ip4))
+	  if (PREDICT_TRUE (dpdk_mbuf_is_ip4(mb)))
 	    n0 = DPDK_RX_NEXT_IP4_INPUT;
+	  else if (PREDICT_TRUE(dpdk_mbuf_is_ip6(mb)))
+	    n0 = DPDK_RX_NEXT_IP6_INPUT;
+          else if (PREDICT_TRUE(vlib_buffer_is_mpls(b0)))
+	    n0 = DPDK_RX_NEXT_MPLS_INPUT;
 	  else
-	    {
-#if RTE_VERSION >= RTE_VERSION_NUM(2, 1, 0, 0)
-	      is0_ip6 = RTE_ETH_IS_IPV6_HDR(mb->packet_type) != 0;
-#else
-	      is0_ip6 = 
-		      (mb_flags & (PKT_RX_IPV6_HDR | PKT_RX_IPV6_HDR_EXT)) != 0;
-#endif
-              if (PREDICT_TRUE(is0_ip6))
-	        n0 = DPDK_RX_NEXT_IP6_INPUT;
-              else
-                {
-                  ethernet_header_t *h0 = (ethernet_header_t *) b0->data;
-                  is0_mpls = (h0->type == clib_host_to_net_u16(ETHERNET_TYPE_MPLS_UNICAST));
-	          n0 = is0_mpls ? DPDK_RX_NEXT_MPLS_INPUT : n0;
-                }
-	    }
+	    n0 = DPDK_RX_NEXT_ETHERNET_INPUT;
 	}
     }
   *next0 = n0;
@@ -908,194 +707,6 @@ void dpdk_set_next_node (dpdk_rx_next_t next, char *name)
     }
 }
 
-inline vlib_frame_queue_elt_t * 
-vlib_get_handoff_queue_elt (u32 vlib_worker_index) 
-{
-  vlib_frame_queue_t *fq;
-  vlib_frame_queue_elt_t *elt;
-  u64 new_tail;
-  
-  fq = vlib_frame_queues[vlib_worker_index];
-  ASSERT (fq);
-
-  new_tail = __sync_add_and_fetch (&fq->tail, 1);
-
-  /* Wait until a ring slot is available */
-  while (new_tail >= fq->head_hint + fq->nelts)
-      vlib_worker_thread_barrier_check ();
-
-  elt = fq->elts + (new_tail & (fq->nelts-1));
-
-  /* this would be very bad... */
-  while (elt->valid) 
-    ;
-
-  elt->msg_type = VLIB_FRAME_QUEUE_ELT_DISPATCH_FRAME;
-  elt->last_n_vectors = elt->n_vectors = 0;
-
-  return elt;
-}
-
-static inline vlib_frame_queue_elt_t *
-dpdk_get_handoff_queue_elt ( 
-    u32 vlib_worker_index, 
-    vlib_frame_queue_elt_t ** handoff_queue_elt_by_worker_index)
-{
-  vlib_frame_queue_elt_t *elt;
-
-  if (handoff_queue_elt_by_worker_index [vlib_worker_index])
-      return handoff_queue_elt_by_worker_index [vlib_worker_index];
-
-  elt = vlib_get_handoff_queue_elt (vlib_worker_index);
-
-  handoff_queue_elt_by_worker_index [vlib_worker_index] = elt;
-
-  return elt;
-}
-
-static inline vlib_frame_queue_t *
-is_vlib_handoff_queue_congested (
-    u32 vlib_worker_index,
-    u32 queue_hi_thresh,
-    vlib_frame_queue_t ** handoff_queue_by_worker_index)
-{
-  vlib_frame_queue_t *fq;
-
-  fq = handoff_queue_by_worker_index [vlib_worker_index];
-  if (fq != (vlib_frame_queue_t *)(~0)) 
-      return fq;
-  
-  fq = vlib_frame_queues[vlib_worker_index];
-  ASSERT (fq);
-
-  if (PREDICT_FALSE(fq->tail >= (fq->head_hint + queue_hi_thresh))) {
-    /* a valid entry in the array will indicate the queue has reached
-     * the specified threshold and is congested
-     */
-    handoff_queue_by_worker_index [vlib_worker_index] = fq;
-    fq->enqueue_full_events++;
-    return fq;
-  }
-
-  return NULL;
-}
-
-static inline u64 ipv4_get_key (ip4_header_t *ip)
-{
-   u64  hash_key;
-
-   hash_key = *((u64*)(&ip->address_pair)) ^ ip->protocol;
-
-   return hash_key;
-}
-
-static inline u64 ipv6_get_key (ip6_header_t *ip)
-{
-   u64  hash_key;
-
-   hash_key = ip->src_address.as_u64[0] ^
-              rotate_left(ip->src_address.as_u64[1],13) ^
-              rotate_left(ip->dst_address.as_u64[0],26) ^
-              rotate_left(ip->dst_address.as_u64[1],39) ^
-              ip->protocol;
-
-   return hash_key;
-}
-
-
-#define MPLS_BOTTOM_OF_STACK_BIT_MASK   0x00000100U
-#define MPLS_LABEL_MASK                 0xFFFFF000U
-
-static inline u64 mpls_get_key (mpls_unicast_header_t *m)
-{
-   u64                     hash_key;
-   u8                      ip_ver;
-
-
-   /* find the bottom of the MPLS label stack. */
-   if (PREDICT_TRUE(m->label_exp_s_ttl & 
-                    clib_net_to_host_u32(MPLS_BOTTOM_OF_STACK_BIT_MASK))) {
-       goto bottom_lbl_found;
-   }
-   m++;
-
-   if (PREDICT_TRUE(m->label_exp_s_ttl & 
-                    clib_net_to_host_u32(MPLS_BOTTOM_OF_STACK_BIT_MASK))) {
-       goto bottom_lbl_found;
-   }
-   m++;
-
-   if (m->label_exp_s_ttl & clib_net_to_host_u32(MPLS_BOTTOM_OF_STACK_BIT_MASK)) {
-       goto bottom_lbl_found;
-   }
-   m++;
-
-   if (m->label_exp_s_ttl & clib_net_to_host_u32(MPLS_BOTTOM_OF_STACK_BIT_MASK)) {
-       goto bottom_lbl_found;
-   }
-   m++;
-
-   if (m->label_exp_s_ttl & clib_net_to_host_u32(MPLS_BOTTOM_OF_STACK_BIT_MASK)) {
-       goto bottom_lbl_found;
-   }
-   
-   /* the bottom label was not found - use the last label */
-   hash_key = m->label_exp_s_ttl & clib_net_to_host_u32(MPLS_LABEL_MASK);
-
-   return hash_key;
-   
-
-bottom_lbl_found:
-   m++;
-   ip_ver = (*((u8 *)m) >> 4);
-
-   /* find out if it is IPV4 or IPV6 header */
-   if (PREDICT_TRUE(ip_ver == 4)) {
-       hash_key = ipv4_get_key((ip4_header_t *)m);
-   } else if (PREDICT_TRUE(ip_ver == 6)) {
-       hash_key = ipv6_get_key((ip6_header_t *)m);
-   } else {
-       /* use the bottom label */
-       hash_key = (m-1)->label_exp_s_ttl & clib_net_to_host_u32(MPLS_LABEL_MASK);
-   }
-
-   return hash_key;
-
-}
-
-static inline u64 eth_get_key (ethernet_header_t *h0)
-{
-   u64 hash_key;
-
-
-   if (PREDICT_TRUE(h0->type) == clib_host_to_net_u16(ETHERNET_TYPE_IP4)) {
-       hash_key = ipv4_get_key((ip4_header_t *)(h0+1));
-   } else if (h0->type == clib_host_to_net_u16(ETHERNET_TYPE_IP6)) {
-       hash_key = ipv6_get_key((ip6_header_t *)(h0+1));
-   } else if (h0->type == clib_host_to_net_u16(ETHERNET_TYPE_MPLS_UNICAST)) {
-       hash_key = mpls_get_key((mpls_unicast_header_t *)(h0+1));
-   } else if ((h0->type == clib_host_to_net_u16(ETHERNET_TYPE_VLAN)) || 
-              (h0->type == clib_host_to_net_u16(ETHERNET_TYPE_DOT1AD))) {
-       ethernet_vlan_header_t * outer = (ethernet_vlan_header_t *)(h0 + 1);
-       
-       outer = (outer->type == clib_host_to_net_u16(ETHERNET_TYPE_VLAN)) ? 
-                                  outer+1 : outer;
-       if (PREDICT_TRUE(outer->type) == clib_host_to_net_u16(ETHERNET_TYPE_IP4)) {
-           hash_key = ipv4_get_key((ip4_header_t *)(outer+1));
-       } else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6)) {
-           hash_key = ipv6_get_key((ip6_header_t *)(outer+1));
-       } else if (outer->type == clib_host_to_net_u16(ETHERNET_TYPE_MPLS_UNICAST)) {
-           hash_key = mpls_get_key((mpls_unicast_header_t *)(outer+1));
-       }  else {
-           hash_key = outer->type; 
-       }
-   } else {
-       hash_key  = 0;
-   }
-
-   return hash_key;
-}
-
 /*
  * This function is used when dedicated IO threads feed the worker threads.
  *
@@ -1395,7 +1006,7 @@ void dpdk_io_thread (vlib_worker_thread_t * w,
                 
               vnet_buffer(b0)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
               vnet_buffer(b0)->sw_if_index[VLIB_TX] = (u32)~0;
-              vnet_buffer(b0)->io_handoff.next_index = next0;
+              vnet_buffer(b0)->handoff.next_index = next0;
               n_rx_bytes += mb->pkt_len;
 
               /* Process subsequent segments of multi-segment packets */
@@ -1796,7 +1407,7 @@ dpdk_io_input (vlib_main_t * vm,
 
           vnet_buffer(b0)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
           vnet_buffer(b0)->sw_if_index[VLIB_TX] = (u32)~0;
-          vnet_buffer(b0)->io_handoff.next_index = next0;
+          vnet_buffer(b0)->handoff.next_index = next0;
           n_rx_bytes += mb->pkt_len;
 
           /* Process subsequent segments of multi-segment packets */