1 files changed, 90 insertions, 116 deletions

diff --git a/src/plugins/odp/node.c b/src/plugins/odp/node.c
index d03480c1..a1ebce0c 100755
--- a/src/plugins/odp/node.c
+++ b/src/plugins/odp/node.c
@@ -10,7 +10,6 @@
 #include <vnet/ip/ip.h>
 #include <vnet/ethernet/ethernet.h>
 #include <vnet/devices/devices.h>
-#include <vnet/feature/feature.h>
 #include <odp/odp_packet.h>
 
 #define foreach_odp_packet_input_error
@@ -49,30 +48,11 @@ format_odp_packet_input_trace (u8 * s, va_list * args)
   return s;
 }
 
-always_inline void
-buffer_add_to_chain (vlib_main_t * vm, u32 bi, u32 first_bi, u32 prev_bi)
-{
-  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
-  vlib_buffer_t *first_b = vlib_get_buffer (vm, first_bi);
-  vlib_buffer_t *prev_b = vlib_get_buffer (vm, prev_bi);
-
-  /* update first buffer */
-  first_b->total_length_not_including_first_buffer += b->current_length;
-
-  /* update previous buffer */
-  prev_b->next_buffer = bi;
-  prev_b->flags |= VLIB_BUFFER_NEXT_PRESENT;
-
-  /* update current buffer */
-  b->next_buffer = 0;
-
-
-}
-
-always_inline int
+int
 odp_packet_queue_mode (odp_pktio_t pktio, u32 mode, odp_packet_t pkt_tbl[])
 {
-  u32 num_evts = 0, num_pkts = 0, i = 0;
+  u32 num_evts = 0, num_pkts = 0;
+  int i;
   odp_queue_t inq;
   odp_event_t evt_tbl[VLIB_FRAME_SIZE];
   u64 sched_wait = odp_schedule_wait_time (ODP_TIME_MSEC_IN_NS * 100);
@@ -91,11 +71,18 @@ odp_packet_queue_mode (odp_pktio_t pktio, u32 mode, odp_packet_t pkt_tbl[])
       return -1;
     }
 
-  if (inq != ODP_QUEUE_INVALID)
-    num_evts = odp_queue_deq_multi (inq, evt_tbl, VLIB_FRAME_SIZE);
-  else
-    num_evts =
-      odp_schedule_multi (NULL, sched_wait, evt_tbl, VLIB_FRAME_SIZE);
+  while (num_evts < VLIB_FRAME_SIZE)
+    {
+      if (inq != ODP_QUEUE_INVALID)
+	i = odp_queue_deq_multi (inq, &evt_tbl[num_evts],
+				 VLIB_FRAME_SIZE - num_evts);
+      else
+	i = odp_schedule_multi (NULL, sched_wait, &evt_tbl[num_evts],
+				VLIB_FRAME_SIZE - num_evts);
+      if (i <= 0)
+	break;
+      num_evts += i;
+    }
 
   /* convert events to packets, discarding any non-packet events */
   for (i = 0; i < num_evts; ++i)
@@ -107,14 +94,14 @@ odp_packet_queue_mode (odp_pktio_t pktio, u32 mode, odp_packet_t pkt_tbl[])
     }
 
   return num_pkts;
-
 }
 
-always_inline int
+int
 odp_packet_burst_mode (odp_pktio_t pktio, odp_pktin_queue_t pktin,
 		       odp_packet_t pkt_tbl[])
 {
-  u32 num_pkts;
+  u32 num_pkts = 0;
+  int ret;
 
   if (odp_pktin_queue (pktio, &pktin, 1) != 1)
     {
@@ -122,10 +109,50 @@ odp_packet_burst_mode (odp_pktio_t pktio, odp_pktin_queue_t pktin,
       return -1;
     }
 
-  num_pkts = odp_pktin_recv (pktin, pkt_tbl, VLIB_FRAME_SIZE);
+  while (num_pkts < VLIB_FRAME_SIZE)
+    {
+      ret = odp_pktin_recv (pktin, &pkt_tbl[num_pkts],
+			    VLIB_FRAME_SIZE - num_pkts);
+      if (ret <= 0)
+	break;
+      num_pkts += ret;
+    }
 
   return num_pkts;
+}
+
+always_inline int
+vlib_buffer_is_ip4 (vlib_buffer_t * b)
+{
+  ethernet_header_t *h = (ethernet_header_t *) vlib_buffer_get_current (b);
+  return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP4));
+}
+
+always_inline int
+vlib_buffer_is_ip6 (vlib_buffer_t * b)
+{
+  ethernet_header_t *h = (ethernet_header_t *) vlib_buffer_get_current (b);
+  return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6));
+}
 
+always_inline int
+vlib_buffer_is_mpls (vlib_buffer_t * b)
+{
+  ethernet_header_t *h = (ethernet_header_t *) vlib_buffer_get_current (b);
+  return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS));
+}
+
+always_inline u32
+odp_rx_next_from_etype (void *mb, vlib_buffer_t * b0)
+{
+  if (PREDICT_TRUE (vlib_buffer_is_ip4 (b0)))
+    return VNET_DEVICE_INPUT_NEXT_IP4_INPUT;
+  else if (PREDICT_TRUE (vlib_buffer_is_ip6 (b0)))
+    return VNET_DEVICE_INPUT_NEXT_IP6_INPUT;
+  else if (PREDICT_TRUE (vlib_buffer_is_mpls (b0)))
+    return VNET_DEVICE_INPUT_NEXT_MPLS_INPUT;
+  else
+    return VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
 }
 
 always_inline uword
@@ -134,32 +161,12 @@ odp_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 {
   u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
   uword n_trace = vlib_get_trace_count (vm, node);
-  odp_packet_main_t *om = &odp_packet_main;
   u32 n_rx_packets = 0;
   u32 n_rx_bytes = 0;
   u32 *to_next = 0;
-  u32 n_free_bufs;
-  u32 thread_index = vlib_get_thread_index ();
   odp_pktin_queue_t pktin = { 0 };
-  odp_packet_t pkt, pkt_tbl[VLIB_FRAME_SIZE];
+  odp_packet_t pkt_tbl[VLIB_FRAME_SIZE];
   u32 pkts = 0, pkts_ok = 0;
-  u32 n_buffer_bytes = vlib_buffer_free_list_buffer_size (vm,
-							  VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
-
-  if (oif->per_interface_next_index != ~0)
-    next_index = oif->per_interface_next_index;
-
-  n_free_bufs = vec_len (om->rx_buffers[thread_index]);
-  if (PREDICT_FALSE (n_free_bufs < VLIB_FRAME_SIZE))
-    {
-      vec_validate (om->rx_buffers[thread_index],
-		    VLIB_FRAME_SIZE + n_free_bufs - 1);
-      n_free_bufs +=
-	vlib_buffer_alloc (vm, &om->rx_buffers[thread_index][n_free_bufs],
-			   VLIB_FRAME_SIZE);
-      _vec_len (om->rx_buffers[thread_index]) = n_free_bufs;
-
-    }
 
   if ((oif->mode == (APPL_MODE_PKT_QUEUE)) ||
       (oif->mode == (APPL_MODE_PKT_SCHED)))
@@ -178,84 +185,52 @@ odp_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
       pkts_ok = drop_err_pkts (pkt_tbl, pkts);
       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 
-      while ((i < pkts_ok) && (n_left_to_next) && (n_free_bufs))
+      while ((i < pkts_ok) && (n_left_to_next))
 	{
-	  vlib_buffer_t *first_b0 = 0;
-	  u32 offset = 0;
-	  u32 bi0 = 0, first_bi0 = 0, prev_bi0;
-	  uint8_t *data_buf;
-	  pkt = pkt_tbl[i];
-	  u32 data_len = odp_packet_len (pkt);
-	  data_buf = malloc (data_len);
-	  memset (data_buf, 0, data_len);
-	  odp_packet_copy_to_mem (pkt, 0, data_len, data_buf);
-
-	  while (data_len && n_free_bufs)
-	    {
-	      vlib_buffer_t *b0;
-	      /* grab free buffer */
-	      u32 last_empty_buffer =
-		vec_len (om->rx_buffers[thread_index]) - 1;
-	      prev_bi0 = bi0;
-	      bi0 = om->rx_buffers[thread_index][last_empty_buffer];
-	      b0 = vlib_get_buffer (vm, bi0);
-	      _vec_len (om->rx_buffers[thread_index]) = last_empty_buffer;
-	      n_free_bufs--;
-	      /* copy data */
-	      u32 bytes_to_copy =
-		data_len > n_buffer_bytes ? n_buffer_bytes : data_len;
-	      b0->current_data = 0;
-	      clib_memcpy (vlib_buffer_get_current (b0),
-			   (u8 *) data_buf + offset, bytes_to_copy);
-
-	      /* fill buffer header */
-	      b0->current_length = bytes_to_copy;
-
-	      if (offset == 0)
-		{
-		  b0->total_length_not_including_first_buffer = 0;
-		  b0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
-		  vnet_buffer (b0)->sw_if_index[VLIB_RX] = oif->sw_if_index;
-		  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-		  first_bi0 = bi0;
-		  first_b0 = vlib_get_buffer (vm, first_bi0);
-		}
-	      else
-		{
-		  buffer_add_to_chain (vm, bi0, first_bi0, prev_bi0);
-		}
-
-	      offset += bytes_to_copy;
-	      data_len -= bytes_to_copy;
-	    }
+	  u32 bi0 = 0;
+	  vlib_buffer_t *b0;
+
+	  b0 = (vlib_buffer_t *) odp_packet_user_area (pkt_tbl[i]);
+	  bi0 = vlib_get_buffer_index (vm, b0);
+	  b0->l2_priv_data = pkt_tbl[i];
+
+	  b0->current_length = odp_packet_len (pkt_tbl[i]);
+	  b0->current_data = 0;
+	  b0->total_length_not_including_first_buffer = 0;
+	  b0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
+	  vnet_buffer (b0)->sw_if_index[VLIB_RX] = oif->sw_if_index;
+	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+
+	  if (PREDICT_FALSE (oif->per_interface_next_index != ~0))
+	    next0 = oif->per_interface_next_index;
+	  else
+	    next0 = odp_rx_next_from_etype (pkt_tbl[i], b0);
+
+	  vlib_buffer_advance (b0, device_input_next_node_advance[next0]);
+
 	  /* trace */
-	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (first_b0);
+	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
 	  if (PREDICT_FALSE (n_trace > 0))
 	    {
 	      odp_packet_input_trace_t *tr;
-	      vlib_trace_buffer (vm, node, next0, first_b0, 0);
+	      vlib_trace_buffer (vm, node, next0, b0, 0);
 	      vlib_set_trace_count (vm, node, --n_trace);
-	      tr = vlib_add_trace (vm, node, first_b0, sizeof (*tr));
+	      tr = vlib_add_trace (vm, node, b0, sizeof (*tr));
 	      tr->next_index = next0;
 	      tr->hw_if_index = oif->hw_if_index;
 	    }
 
-	  /* redirect if feature path enabled */
-	  vnet_feature_start_device_input_x1 (oif->sw_if_index, &next0,
-					      first_b0);
+	  n_left_to_next--;
+	  to_next[0] = bi0;
+	  to_next += 1;
 
 	  /* enque and take next packet */
 	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
-					   n_left_to_next, first_bi0, next0);
+					   n_left_to_next, bi0, next0);
 
 	  /* next packet */
 	  n_rx_packets++;
-	  n_rx_bytes += odp_packet_len (pkt);
-	  to_next[0] = first_bi0;
-	  to_next += 1;
-	  n_left_to_next--;
-	  free (data_buf);
-	  odp_packet_free (pkt_tbl[i]);
+	  n_rx_bytes += odp_packet_len (pkt_tbl[i]);
 	  i++;
 	}
 
@@ -270,7 +245,6 @@ odp_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 				   n_rx_packets, n_rx_bytes);
 
   return n_rx_packets;
-
 }
 
 static uword