1 files changed, 173 insertions, 192 deletions

diff --git a/src/vnet/udp/udp_input.c b/src/vnet/udp/udp_input.c
index e5a82429a27..bd4e75710f5 100644
--- a/src/vnet/udp/udp_input.c
+++ b/src/vnet/udp/udp_input.c
@@ -80,226 +80,207 @@ always_inline uword
 udp46_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 		    vlib_frame_t * frame, u8 is_ip4)
 {
-  u32 n_left_from, *from, *to_next;
-  u32 next_index, errors;
+  u32 n_left_from, *from;
+  u32 errors, *first_buffer;
   u32 my_thread_index = vm->thread_index;
 
-  from = vlib_frame_vector_args (frame);
+  from = first_buffer = vlib_frame_vector_args (frame);
   n_left_from = frame->n_vectors;
-  next_index = node->cached_next_index;
 
   while (n_left_from > 0)
     {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
-
-      while (n_left_from > 0 && n_left_to_next > 0)
+      u32 bi0, fib_index0;
+      vlib_buffer_t *b0;
+      u32 error0 = UDP_ERROR_ENQUEUED;
+      udp_header_t *udp0;
+      ip4_header_t *ip40;
+      ip6_header_t *ip60;
+      u8 *data0;
+      session_t *s0;
+      udp_connection_t *uc0, *child0, *new_uc0;
+      transport_connection_t *tc0;
+      int wrote0;
+      void *rmt_addr, *lcl_addr;
+      session_dgram_hdr_t hdr0;
+      u8 queue_event = 1;
+
+      /* speculatively enqueue b0 to the current next frame */
+      bi0 = from[0];
+      from += 1;
+      n_left_from -= 1;
+
+      b0 = vlib_get_buffer (vm, bi0);
+
+      /* udp_local hands us a pointer to the udp data */
+      data0 = vlib_buffer_get_current (b0);
+      udp0 = (udp_header_t *) (data0 - sizeof (*udp0));
+      fib_index0 = vnet_buffer (b0)->ip.fib_index;
+
+      if (is_ip4)
 	{
-	  u32 bi0, fib_index0;
-	  vlib_buffer_t *b0;
-	  u32 next0 = UDP_INPUT_NEXT_DROP;
-	  u32 error0 = UDP_ERROR_ENQUEUED;
-	  udp_header_t *udp0;
-	  ip4_header_t *ip40;
-	  ip6_header_t *ip60;
-	  u8 *data0;
-	  session_t *s0;
-	  udp_connection_t *uc0, *child0, *new_uc0;
-	  transport_connection_t *tc0;
-	  int wrote0;
-	  void *rmt_addr, *lcl_addr;
-	  session_dgram_hdr_t hdr0;
-	  u8 queue_event = 1;
-
-	  /* speculatively enqueue b0 to the current next frame */
-	  bi0 = from[0];
-	  to_next[0] = bi0;
-	  from += 1;
-	  to_next += 1;
-	  n_left_from -= 1;
-	  n_left_to_next -= 1;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-
-	  /* udp_local hands us a pointer to the udp data */
-	  data0 = vlib_buffer_get_current (b0);
-	  udp0 = (udp_header_t *) (data0 - sizeof (*udp0));
-	  fib_index0 = vnet_buffer (b0)->ip.fib_index;
-
-	  if (is_ip4)
-	    {
-	      /* TODO: must fix once udp_local does ip options correctly */
-	      ip40 = (ip4_header_t *) (((u8 *) udp0) - sizeof (*ip40));
-	      s0 = session_lookup_safe4 (fib_index0, &ip40->dst_address,
-					 &ip40->src_address, udp0->dst_port,
-					 udp0->src_port, TRANSPORT_PROTO_UDP);
-	      lcl_addr = &ip40->dst_address;
-	      rmt_addr = &ip40->src_address;
+	  /* TODO: must fix once udp_local does ip options correctly */
+	  ip40 = (ip4_header_t *) (((u8 *) udp0) - sizeof (*ip40));
+	  s0 = session_lookup_safe4 (fib_index0, &ip40->dst_address,
+				     &ip40->src_address, udp0->dst_port,
+				     udp0->src_port, TRANSPORT_PROTO_UDP);
+	  lcl_addr = &ip40->dst_address;
+	  rmt_addr = &ip40->src_address;
 
-	    }
-	  else
-	    {
-	      ip60 = (ip6_header_t *) (((u8 *) udp0) - sizeof (*ip60));
-	      s0 = session_lookup_safe6 (fib_index0, &ip60->dst_address,
-					 &ip60->src_address, udp0->dst_port,
-					 udp0->src_port, TRANSPORT_PROTO_UDP);
-	      lcl_addr = &ip60->dst_address;
-	      rmt_addr = &ip60->src_address;
-	    }
+	}
+      else
+	{
+	  ip60 = (ip6_header_t *) (((u8 *) udp0) - sizeof (*ip60));
+	  s0 = session_lookup_safe6 (fib_index0, &ip60->dst_address,
+				     &ip60->src_address, udp0->dst_port,
+				     udp0->src_port, TRANSPORT_PROTO_UDP);
+	  lcl_addr = &ip60->dst_address;
+	  rmt_addr = &ip60->src_address;
+	}
 
-	  if (PREDICT_FALSE (!s0))
-	    {
-	      error0 = UDP_ERROR_NO_LISTENER;
-	      goto trace0;
-	    }
+      if (PREDICT_FALSE (!s0))
+	{
+	  error0 = UDP_ERROR_NO_LISTENER;
+	  goto trace0;
+	}
 
-	  if (s0->session_state == SESSION_STATE_OPENED)
+      if (s0->session_state == SESSION_STATE_OPENED)
+	{
+	  /* TODO optimization: move cl session to right thread
+	   * However, since such a move would affect the session handle,
+	   * which we pass 'raw' to the app, we'd also have notify the
+	   * app of the change or change the way we pass handles to apps.
+	   */
+	  tc0 = session_get_transport (s0);
+	  uc0 = udp_get_connection_from_transport (tc0);
+	  if (uc0->flags & UDP_CONN_F_CONNECTED)
 	    {
-	      /* TODO optimization: move cl session to right thread
-	       * However, since such a move would affect the session handle,
-	       * which we pass 'raw' to the app, we'd also have notify the
-	       * app of the change or change the way we pass handles to apps.
-	       */
-	      tc0 = session_get_transport (s0);
-	      uc0 = udp_get_connection_from_transport (tc0);
-	      if (uc0->flags & UDP_CONN_F_CONNECTED)
+	      if (s0->thread_index != vlib_get_thread_index ())
 		{
-		  if (s0->thread_index != vlib_get_thread_index ())
-		    {
-		      /*
-		       * Clone the transport. It will be cleaned up with the
-		       * session once we notify the session layer.
-		       */
-		      new_uc0 =
-			udp_connection_clone_safe (s0->connection_index,
-						   s0->thread_index);
-		      ASSERT (s0->session_index == new_uc0->c_s_index);
-
-		      /*
-		       * Drop the 'lock' on pool resize
-		       */
-		      session_pool_remove_peeker (s0->thread_index);
-		      session_dgram_connect_notify (&new_uc0->connection,
-						    s0->thread_index, &s0);
-		      tc0 = &new_uc0->connection;
-		      uc0 = new_uc0;
-		      queue_event = 0;
-		    }
-		  else
-		    s0->session_state = SESSION_STATE_READY;
+		  /*
+		   * Clone the transport. It will be cleaned up with the
+		   * session once we notify the session layer.
+		   */
+		  new_uc0 =
+		    udp_connection_clone_safe (s0->connection_index,
+					       s0->thread_index);
+		  ASSERT (s0->session_index == new_uc0->c_s_index);
+
+		  /*
+		   * Drop the 'lock' on pool resize
+		   */
+		  session_pool_remove_peeker (s0->thread_index);
+		  session_dgram_connect_notify (&new_uc0->connection,
+						s0->thread_index, &s0);
+		  tc0 = &new_uc0->connection;
+		  uc0 = new_uc0;
+		  queue_event = 0;
 		}
+	      else
+		s0->session_state = SESSION_STATE_READY;
 	    }
-	  else if (s0->session_state == SESSION_STATE_READY)
-	    {
-	      tc0 = session_get_transport (s0);
-	      uc0 = udp_get_connection_from_transport (tc0);
-	    }
-	  else if (s0->session_state == SESSION_STATE_LISTENING)
+	}
+      else if (s0->session_state == SESSION_STATE_READY)
+	{
+	  tc0 = session_get_transport (s0);
+	  uc0 = udp_get_connection_from_transport (tc0);
+	}
+      else if (s0->session_state == SESSION_STATE_LISTENING)
+	{
+	  tc0 = listen_session_get_transport (s0);
+	  uc0 = udp_get_connection_from_transport (tc0);
+	  if (uc0->flags & UDP_CONN_F_CONNECTED)
 	    {
-	      tc0 = listen_session_get_transport (s0);
-	      uc0 = udp_get_connection_from_transport (tc0);
-	      if (uc0->flags & UDP_CONN_F_CONNECTED)
+	      child0 = udp_connection_alloc (my_thread_index);
+	      if (is_ip4)
 		{
-		  child0 = udp_connection_alloc (my_thread_index);
-		  if (is_ip4)
-		    {
-		      ip_set (&child0->c_lcl_ip, &ip40->dst_address, 1);
-		      ip_set (&child0->c_rmt_ip, &ip40->src_address, 1);
-		    }
-		  else
-		    {
-		      ip_set (&child0->c_lcl_ip, &ip60->dst_address, 0);
-		      ip_set (&child0->c_rmt_ip, &ip60->src_address, 0);
-		    }
-		  child0->c_lcl_port = udp0->dst_port;
-		  child0->c_rmt_port = udp0->src_port;
-		  child0->c_is_ip4 = is_ip4;
-		  child0->c_fib_index = tc0->fib_index;
-		  child0->flags |= UDP_CONN_F_CONNECTED;
-
-		  if (session_stream_accept (&child0->connection,
-					     tc0->s_index, tc0->thread_index,
-					     1))
-		    {
-		      error0 = UDP_ERROR_CREATE_SESSION;
-		      goto trace0;
-		    }
-		  s0 = session_get (child0->c_s_index,
-				    child0->c_thread_index);
-		  s0->session_state = SESSION_STATE_READY;
-		  tc0 = &child0->connection;
-		  uc0 = udp_get_connection_from_transport (tc0);
-		  error0 = UDP_ERROR_LISTENER;
+		  ip_set (&child0->c_lcl_ip, &ip40->dst_address, 1);
+		  ip_set (&child0->c_rmt_ip, &ip40->src_address, 1);
 		}
+	      else
+		{
+		  ip_set (&child0->c_lcl_ip, &ip60->dst_address, 0);
+		  ip_set (&child0->c_rmt_ip, &ip60->src_address, 0);
+		}
+	      child0->c_lcl_port = udp0->dst_port;
+	      child0->c_rmt_port = udp0->src_port;
+	      child0->c_is_ip4 = is_ip4;
+	      child0->c_fib_index = tc0->fib_index;
+	      child0->flags |= UDP_CONN_F_CONNECTED;
+
+	      if (session_stream_accept (&child0->connection,
+					 tc0->s_index, tc0->thread_index, 1))
+		{
+		  error0 = UDP_ERROR_CREATE_SESSION;
+		  goto trace0;
+		}
+	      s0 = session_get (child0->c_s_index, child0->c_thread_index);
+	      s0->session_state = SESSION_STATE_READY;
+	      tc0 = &child0->connection;
+	      uc0 = udp_get_connection_from_transport (tc0);
+	      error0 = UDP_ERROR_LISTENER;
 	    }
-	  else
-	    {
-	      error0 = UDP_ERROR_NOT_READY;
-	      goto trace0;
-	    }
+	}
+      else
+	{
+	  error0 = UDP_ERROR_NOT_READY;
+	  goto trace0;
+	}
 
 
-	  if (svm_fifo_max_enqueue_prod (s0->rx_fifo)
-	      < b0->current_length + sizeof (session_dgram_hdr_t))
-	    {
-	      error0 = UDP_ERROR_FIFO_FULL;
-	      goto trace0;
-	    }
-	  hdr0.data_length = b0->current_length;
-	  hdr0.data_offset = 0;
-	  ip_set (&hdr0.lcl_ip, lcl_addr, is_ip4);
-	  ip_set (&hdr0.rmt_ip, rmt_addr, is_ip4);
-	  hdr0.lcl_port = udp0->dst_port;
-	  hdr0.rmt_port = udp0->src_port;
-	  hdr0.is_ip4 = is_ip4;
-
-	  clib_spinlock_lock (&uc0->rx_lock);
-	  /* If session is owned by another thread and rx event needed,
-	   * enqueue event now while we still have the peeker lock */
-	  if (s0->thread_index != my_thread_index)
-	    {
-	      wrote0 = session_enqueue_dgram_connection (s0, &hdr0, b0,
-							 TRANSPORT_PROTO_UDP,
-							 /* queue event */ 0);
-	      if (queue_event && !svm_fifo_has_event (s0->rx_fifo))
-		session_enqueue_notify (s0);
-	    }
-	  else
-	    {
-	      wrote0 = session_enqueue_dgram_connection (s0, &hdr0, b0,
-							 TRANSPORT_PROTO_UDP,
-							 queue_event);
-	    }
-	  clib_spinlock_unlock (&uc0->rx_lock);
-	  ASSERT (wrote0 > 0);
-
-	  if (s0->session_state != SESSION_STATE_LISTENING)
-	    session_pool_remove_peeker (s0->thread_index);
+      if (svm_fifo_max_enqueue_prod (s0->rx_fifo)
+	  < b0->current_length + sizeof (session_dgram_hdr_t))
+	{
+	  error0 = UDP_ERROR_FIFO_FULL;
+	  goto trace0;
+	}
+      hdr0.data_length = b0->current_length;
+      hdr0.data_offset = 0;
+      ip_set (&hdr0.lcl_ip, lcl_addr, is_ip4);
+      ip_set (&hdr0.rmt_ip, rmt_addr, is_ip4);
+      hdr0.lcl_port = udp0->dst_port;
+      hdr0.rmt_port = udp0->src_port;
+      hdr0.is_ip4 = is_ip4;
+
+      clib_spinlock_lock (&uc0->rx_lock);
+      /* If session is owned by another thread and rx event needed,
+       * enqueue event now while we still have the peeker lock */
+      if (s0->thread_index != my_thread_index)
+	{
+	  wrote0 = session_enqueue_dgram_connection (s0, &hdr0, b0,
+						     TRANSPORT_PROTO_UDP,
+						     /* queue event */ 0);
+	  if (queue_event && !svm_fifo_has_event (s0->rx_fifo))
+	    session_enqueue_notify (s0);
+	}
+      else
+	{
+	  wrote0 = session_enqueue_dgram_connection (s0, &hdr0, b0,
+						     TRANSPORT_PROTO_UDP,
+						     queue_event);
+	}
+      clib_spinlock_unlock (&uc0->rx_lock);
+      ASSERT (wrote0 > 0);
 
-	trace0:
+      if (s0->session_state != SESSION_STATE_LISTENING)
+	session_pool_remove_peeker (s0->thread_index);
 
-	  b0->error = node->errors[error0];
+    trace0:
 
-	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
-			     && (b0->flags & VLIB_BUFFER_IS_TRACED)))
-	    {
-	      udp_input_trace_t *t = vlib_add_trace (vm, node, b0,
-						     sizeof (*t));
+      b0->error = node->errors[error0];
 
-	      t->connection = s0 ? s0->connection_index : ~0;
-	      t->disposition = error0;
-	      t->thread_index = my_thread_index;
-	    }
+      if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
+			 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+	{
+	  udp_input_trace_t *t = vlib_add_trace (vm, node, b0,
+						 sizeof (*t));
 
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, next0);
+	  t->connection = s0 ? s0->connection_index : ~0;
+	  t->disposition = error0;
+	  t->thread_index = my_thread_index;
 	}
-
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
     }
 
+  vlib_buffer_free (vm, first_buffer, frame->n_vectors);
   errors = session_main_flush_enqueue_events (TRANSPORT_PROTO_UDP,
 					      my_thread_index);
   udp_input_inc_counter (vm, is_ip4, UDP_ERROR_EVENT_FIFO_FULL, errors);