ip4: perf optimization in the ip4-input node

Change-Id: I9ddbbf8ce0d7307b9eb82ccd0c51f84e479ffd23
Signed-off-by: Damjan Marion <damarion@cisco.com>

2 files changed, 76 insertions, 83 deletions

diff --git a/vnet/vnet/ip/ip4_forward.c b/vnet/vnet/ip/ip4_forward.c
index c743139c5b0..6a9e066027c 100644
--- a/vnet/vnet/ip/ip4_forward.c
+++ b/vnet/vnet/ip/ip4_forward.c
@@ -824,11 +824,11 @@ ip4_sw_interface_enable_disable (u32 sw_if_index, u32 is_enable)
       if (0 != --im->ip_enabled_by_sw_if_index[sw_if_index])
 	return;
     }
-  vnet_feature_enable_disable ("ip4-unicast", "ip4-lookup", sw_if_index,
-			       is_enable, 0, 0);
+  vnet_feature_enable_disable ("ip4-unicast", "ip4-drop", sw_if_index,
+			       !is_enable, 0, 0);
 
-  vnet_feature_enable_disable ("ip4-multicast", "ip4-lookup-multicast",
-			       sw_if_index, is_enable, 0, 0);
+  vnet_feature_enable_disable ("ip4-multicast", "ip4-drop", sw_if_index,
+			       !is_enable, 0, 0);
 
 }
 
@@ -932,6 +932,7 @@ VNET_FEATURE_ARC_INIT (ip4_unicast, static) =
 {
   .arc_name = "ip4-unicast",
   .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
+  .end_node = "ip4-lookup",
   .arc_index_ptr = &ip4_main.lookup_main.ucast_feature_arc_index,
 };
 
@@ -1018,6 +1019,7 @@ VNET_FEATURE_ARC_INIT (ip4_multicast, static) =
 {
   .arc_name = "ip4-multicast",
   .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
+  .end_node = "ip4-lookup-multicast",
   .arc_index_ptr = &ip4_main.lookup_main.mcast_feature_arc_index,
 };
 
diff --git a/vnet/vnet/ip/ip4_input.c b/vnet/vnet/ip/ip4_input.c
index 69b1e5a83d2..1cf5e0b8517 100644
--- a/vnet/vnet/ip/ip4_input.c
+++ b/vnet/vnet/ip/ip4_input.c
@@ -109,8 +109,8 @@ ip4_input_inline (vlib_main_t * vm,
 	{
 	  vlib_buffer_t *p0, *p1;
 	  ip4_header_t *ip0, *ip1;
-	  u32 sw_if_index0, pi0, ip_len0, cur_len0, next0 = 0;
-	  u32 sw_if_index1, pi1, ip_len1, cur_len1, next1 = 0;
+	  u32 sw_if_index0, pi0, ip_len0, cur_len0, next0;
+	  u32 sw_if_index1, pi1, ip_len1, cur_len1, next1;
 	  i32 len_diff0, len_diff1;
 	  u8 error0, error1, arc0, arc1;
 
@@ -144,12 +144,33 @@ ip4_input_inline (vlib_main_t * vm,
 	  sw_if_index0 = vnet_buffer (p0)->sw_if_index[VLIB_RX];
 	  sw_if_index1 = vnet_buffer (p1)->sw_if_index[VLIB_RX];
 
-	  arc0 =
-	    ip4_address_is_multicast (&ip0->dst_address) ?
-	    lm->mcast_feature_arc_index : lm->ucast_feature_arc_index;
-	  arc1 =
-	    ip4_address_is_multicast (&ip1->dst_address) ?
-	    lm->mcast_feature_arc_index : lm->ucast_feature_arc_index;
+	  error0 = error1 = IP4_ERROR_NONE;
+
+	  if (PREDICT_FALSE (ip4_address_is_multicast (&ip0->dst_address)))
+	    {
+	      arc0 = lm->mcast_feature_arc_index;
+	      next0 = IP4_INPUT_NEXT_LOOKUP_MULTICAST;
+	    }
+	  else
+	    {
+	      arc0 = lm->ucast_feature_arc_index;
+	      next0 = IP4_INPUT_NEXT_LOOKUP;
+	      if (PREDICT_FALSE (ip0->ttl < 1))
+		error0 = IP4_ERROR_TIME_EXPIRED;
+	    }
+
+	  if (PREDICT_FALSE (ip4_address_is_multicast (&ip1->dst_address)))
+	    {
+	      arc1 = lm->mcast_feature_arc_index;
+	      next1 = IP4_INPUT_NEXT_LOOKUP_MULTICAST;
+	    }
+	  else
+	    {
+	      arc1 = lm->ucast_feature_arc_index;
+	      next1 = IP4_INPUT_NEXT_LOOKUP;
+	      if (PREDICT_FALSE (ip1->ttl < 1))
+		error1 = IP4_ERROR_TIME_EXPIRED;
+	    }
 
 	  vnet_buffer (p0)->ip.adj_index[VLIB_RX] = ~0;
 	  vnet_buffer (p1)->ip.adj_index[VLIB_RX] = ~0;
@@ -160,23 +181,14 @@ ip4_input_inline (vlib_main_t * vm,
 	  vlib_increment_simple_counter (cm, cpu_index, sw_if_index0, 1);
 	  vlib_increment_simple_counter (cm, cpu_index, sw_if_index1, 1);
 
-	  error0 = error1 = IP4_ERROR_NONE;
+	  /* Punt packets with options or wrong version. */
+	  if (PREDICT_FALSE (ip0->ip_version_and_header_length != 0x45))
+	    error0 = (ip0->ip_version_and_header_length & 0xf) != 5 ?
+	      IP4_ERROR_OPTIONS : IP4_ERROR_VERSION;
 
-	  /* Punt packets with options. */
-	  error0 =
-	    (ip0->ip_version_and_header_length & 0xf) !=
-	    5 ? IP4_ERROR_OPTIONS : error0;
-	  error1 =
-	    (ip1->ip_version_and_header_length & 0xf) !=
-	    5 ? IP4_ERROR_OPTIONS : error1;
-
-	  /* Version != 4?  Drop it. */
-	  error0 =
-	    (ip0->ip_version_and_header_length >> 4) !=
-	    4 ? IP4_ERROR_VERSION : error0;
-	  error1 =
-	    (ip1->ip_version_and_header_length >> 4) !=
-	    4 ? IP4_ERROR_VERSION : error1;
+	  if (PREDICT_FALSE (ip1->ip_version_and_header_length != 0x45))
+	    error1 = (ip1->ip_version_and_header_length & 0xf) != 5 ?
+	      IP4_ERROR_OPTIONS : IP4_ERROR_VERSION;
 
 	  /* Verify header checksum. */
 	  if (verify_checksum)
@@ -186,31 +198,17 @@ ip4_input_inline (vlib_main_t * vm,
 	      ip4_partial_header_checksum_x1 (ip0, sum0);
 	      ip4_partial_header_checksum_x1 (ip1, sum1);
 
-	      error0 =
-		0xffff !=
-		ip_csum_fold (sum0) ? IP4_ERROR_BAD_CHECKSUM : error0;
-	      error1 =
-		0xffff !=
-		ip_csum_fold (sum1) ? IP4_ERROR_BAD_CHECKSUM : error1;
+	      error0 = 0xffff != ip_csum_fold (sum0) ?
+		IP4_ERROR_BAD_CHECKSUM : error0;
+	      error1 = 0xffff != ip_csum_fold (sum1) ?
+		IP4_ERROR_BAD_CHECKSUM : error1;
 	    }
 
 	  /* Drop fragmentation offset 1 packets. */
-	  error0 =
-	    ip4_get_fragment_offset (ip0) ==
-	    1 ? IP4_ERROR_FRAGMENT_OFFSET_ONE : error0;
-	  error1 =
-	    ip4_get_fragment_offset (ip1) ==
-	    1 ? IP4_ERROR_FRAGMENT_OFFSET_ONE : error1;
-
-	  /* TTL < 1? Drop it. */
-	  error0 = (ip0->ttl < 1
-		    && arc0 ==
-		    lm->ucast_feature_arc_index) ? IP4_ERROR_TIME_EXPIRED :
-	    error0;
-	  error1 = (ip1->ttl < 1
-		    && arc1 ==
-		    lm->ucast_feature_arc_index) ? IP4_ERROR_TIME_EXPIRED :
-	    error1;
+	  error0 = ip4_get_fragment_offset (ip0) == 1 ?
+	    IP4_ERROR_FRAGMENT_OFFSET_ONE : error0;
+	  error1 = ip4_get_fragment_offset (ip1) == 1 ?
+	    IP4_ERROR_FRAGMENT_OFFSET_ONE : error1;
 
 	  /* Verify lengths. */
 	  ip_len0 = clib_net_to_host_u16 (ip0->length);
@@ -242,10 +240,8 @@ ip4_input_inline (vlib_main_t * vm,
 		  next0 = IP4_INPUT_NEXT_ICMP_ERROR;
 		}
 	      else
-		next0 =
-		  error0 !=
-		  IP4_ERROR_OPTIONS ? IP4_INPUT_NEXT_DROP :
-		  IP4_INPUT_NEXT_PUNT;
+		next0 = error0 != IP4_ERROR_OPTIONS ?
+		  IP4_INPUT_NEXT_DROP : IP4_INPUT_NEXT_PUNT;
 	    }
 	  if (PREDICT_FALSE (error1 != IP4_ERROR_NONE))
 	    {
@@ -257,10 +253,8 @@ ip4_input_inline (vlib_main_t * vm,
 		  next1 = IP4_INPUT_NEXT_ICMP_ERROR;
 		}
 	      else
-		next1 =
-		  error1 !=
-		  IP4_ERROR_OPTIONS ? IP4_INPUT_NEXT_DROP :
-		  IP4_INPUT_NEXT_PUNT;
+		next1 = error1 != IP4_ERROR_OPTIONS ?
+		  IP4_INPUT_NEXT_DROP : IP4_INPUT_NEXT_PUNT;
 	    }
 
 	  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
@@ -271,7 +265,7 @@ ip4_input_inline (vlib_main_t * vm,
 	{
 	  vlib_buffer_t *p0;
 	  ip4_header_t *ip0;
-	  u32 sw_if_index0, pi0, ip_len0, cur_len0, next0 = 0;
+	  u32 sw_if_index0, pi0, ip_len0, cur_len0, next0;
 	  i32 len_diff0;
 	  u8 error0, arc0;
 
@@ -287,25 +281,30 @@ ip4_input_inline (vlib_main_t * vm,
 
 	  sw_if_index0 = vnet_buffer (p0)->sw_if_index[VLIB_RX];
 
-	  arc0 =
-	    ip4_address_is_multicast (&ip0->dst_address) ?
-	    lm->mcast_feature_arc_index : lm->ucast_feature_arc_index;
+	  error0 = IP4_ERROR_NONE;
+
+	  if (PREDICT_FALSE (ip4_address_is_multicast (&ip0->dst_address)))
+	    {
+	      arc0 = lm->mcast_feature_arc_index;
+	      next0 = IP4_INPUT_NEXT_LOOKUP_MULTICAST;
+	    }
+	  else
+	    {
+	      arc0 = lm->ucast_feature_arc_index;
+	      next0 = IP4_INPUT_NEXT_LOOKUP;
+	      if (PREDICT_FALSE (ip0->ttl < 1))
+		error0 = IP4_ERROR_TIME_EXPIRED;
+	    }
+
 	  vnet_buffer (p0)->ip.adj_index[VLIB_RX] = ~0;
 	  vnet_feature_arc_start (arc0, sw_if_index0, &next0, p0);
 
 	  vlib_increment_simple_counter (cm, cpu_index, sw_if_index0, 1);
 
-	  error0 = IP4_ERROR_NONE;
-
-	  /* Punt packets with options. */
-	  error0 =
-	    (ip0->ip_version_and_header_length & 0xf) !=
-	    5 ? IP4_ERROR_OPTIONS : error0;
-
-	  /* Version != 4?  Drop it. */
-	  error0 =
-	    (ip0->ip_version_and_header_length >> 4) !=
-	    4 ? IP4_ERROR_VERSION : error0;
+	  /* Punt packets with options or wrong version. */
+	  if (PREDICT_FALSE (ip0->ip_version_and_header_length != 0x45))
+	    error0 = (ip0->ip_version_and_header_length & 0xf) != 5 ?
+	      IP4_ERROR_OPTIONS : IP4_ERROR_VERSION;
 
 	  /* Verify header checksum. */
 	  if (verify_checksum)
@@ -323,12 +322,6 @@ ip4_input_inline (vlib_main_t * vm,
 	    ip4_get_fragment_offset (ip0) ==
 	    1 ? IP4_ERROR_FRAGMENT_OFFSET_ONE : error0;
 
-	  /* TTL < 1? Drop it. */
-	  error0 = (ip0->ttl < 1
-		    && arc0 ==
-		    lm->ucast_feature_arc_index) ? IP4_ERROR_TIME_EXPIRED :
-	    error0;
-
 	  /* Verify lengths. */
 	  ip_len0 = clib_net_to_host_u16 (ip0->length);
 
@@ -350,10 +343,8 @@ ip4_input_inline (vlib_main_t * vm,
 		  next0 = IP4_INPUT_NEXT_ICMP_ERROR;
 		}
 	      else
-		next0 =
-		  error0 !=
-		  IP4_ERROR_OPTIONS ? IP4_INPUT_NEXT_DROP :
-		  IP4_INPUT_NEXT_PUNT;
+		next0 = error0 != IP4_ERROR_OPTIONS ?
+		  IP4_INPUT_NEXT_DROP : IP4_INPUT_NEXT_PUNT;
 	    }
 
 	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,