ip4-input node rework

Gain is around 6 clocks per packet (22 to 16).

Change-Id: Ia6f4293ea9062368a9a6b235c650591dbc0707d0
Signed-off-by: Damjan Marion <damarion@cisco.com>

1 files changed, 156 insertions, 0 deletions

diff --git a/src/vnet/ip/ip4_input.h b/src/vnet/ip/ip4_input.h
index 600d6937dd8..e0873039990 100644
--- a/src/vnet/ip/ip4_input.h
+++ b/src/vnet/ip/ip4_input.h
@@ -57,6 +57,162 @@ typedef enum
 } ip4_input_next_t;
 
 always_inline void
+ip4_input_check_x4 (vlib_main_t * vm,
+		    vlib_node_runtime_t * error_node,
+		    vlib_buffer_t ** p, ip4_header_t ** ip,
+		    u16 * next, int verify_checksum)
+{
+  u8 error0, error1, error2, error3;
+  u32 ip_len0, cur_len0;
+  u32 ip_len1, cur_len1;
+  u32 ip_len2, cur_len2;
+  u32 ip_len3, cur_len3;
+  i32 len_diff0, len_diff1, len_diff2, len_diff3;
+
+  error0 = error1 = error2 = error3 = IP4_ERROR_NONE;
+
+  /* Punt packets with options or wrong version. */
+  if (PREDICT_FALSE (ip[0]->ip_version_and_header_length != 0x45))
+    error0 = (ip[0]->ip_version_and_header_length & 0xf) != 5 ?
+      IP4_ERROR_OPTIONS : IP4_ERROR_VERSION;
+
+  if (PREDICT_FALSE (ip[1]->ip_version_and_header_length != 0x45))
+    error1 = (ip[1]->ip_version_and_header_length & 0xf) != 5 ?
+      IP4_ERROR_OPTIONS : IP4_ERROR_VERSION;
+
+  if (PREDICT_FALSE (ip[2]->ip_version_and_header_length != 0x45))
+    error2 = (ip[2]->ip_version_and_header_length & 0xf) != 5 ?
+      IP4_ERROR_OPTIONS : IP4_ERROR_VERSION;
+
+  if (PREDICT_FALSE (ip[3]->ip_version_and_header_length != 0x45))
+    error3 = (ip[3]->ip_version_and_header_length & 0xf) != 5 ?
+      IP4_ERROR_OPTIONS : IP4_ERROR_VERSION;
+
+  if (PREDICT_FALSE (ip[0]->ttl < 1))
+    error0 = IP4_ERROR_TIME_EXPIRED;
+  if (PREDICT_FALSE (ip[1]->ttl < 1))
+    error1 = IP4_ERROR_TIME_EXPIRED;
+  if (PREDICT_FALSE (ip[2]->ttl < 1))
+    error2 = IP4_ERROR_TIME_EXPIRED;
+  if (PREDICT_FALSE (ip[3]->ttl < 1))
+    error3 = IP4_ERROR_TIME_EXPIRED;
+
+  /* Verify header checksum. */
+  if (verify_checksum)
+    {
+      ip_csum_t sum0, sum1, sum2, sum3;
+
+      ip4_partial_header_checksum_x1 (ip[0], sum0);
+      ip4_partial_header_checksum_x1 (ip[1], sum1);
+      ip4_partial_header_checksum_x1 (ip[2], sum2);
+      ip4_partial_header_checksum_x1 (ip[3], sum3);
+
+      error0 = 0xffff != ip_csum_fold (sum0) ?
+	IP4_ERROR_BAD_CHECKSUM : error0;
+      error1 = 0xffff != ip_csum_fold (sum1) ?
+	IP4_ERROR_BAD_CHECKSUM : error1;
+      error2 = 0xffff != ip_csum_fold (sum2) ?
+	IP4_ERROR_BAD_CHECKSUM : error2;
+      error3 = 0xffff != ip_csum_fold (sum3) ?
+	IP4_ERROR_BAD_CHECKSUM : error3;
+    }
+
+  /* Drop fragmentation offset 1 packets. */
+  error0 = ip4_get_fragment_offset (ip[0]) == 1 ?
+    IP4_ERROR_FRAGMENT_OFFSET_ONE : error0;
+  error1 = ip4_get_fragment_offset (ip[1]) == 1 ?
+    IP4_ERROR_FRAGMENT_OFFSET_ONE : error1;
+  error2 = ip4_get_fragment_offset (ip[2]) == 1 ?
+    IP4_ERROR_FRAGMENT_OFFSET_ONE : error2;
+  error3 = ip4_get_fragment_offset (ip[3]) == 1 ?
+    IP4_ERROR_FRAGMENT_OFFSET_ONE : error3;
+
+  /* Verify lengths. */
+  ip_len0 = clib_net_to_host_u16 (ip[0]->length);
+  ip_len1 = clib_net_to_host_u16 (ip[1]->length);
+  ip_len2 = clib_net_to_host_u16 (ip[2]->length);
+  ip_len3 = clib_net_to_host_u16 (ip[3]->length);
+
+  /* IP length must be at least minimal IP header. */
+  error0 = ip_len0 < sizeof (ip[0][0]) ? IP4_ERROR_TOO_SHORT : error0;
+  error1 = ip_len1 < sizeof (ip[1][0]) ? IP4_ERROR_TOO_SHORT : error1;
+  error2 = ip_len2 < sizeof (ip[2][0]) ? IP4_ERROR_TOO_SHORT : error2;
+  error3 = ip_len3 < sizeof (ip[3][0]) ? IP4_ERROR_TOO_SHORT : error3;
+
+  cur_len0 = vlib_buffer_length_in_chain (vm, p[0]);
+  cur_len1 = vlib_buffer_length_in_chain (vm, p[1]);
+  cur_len2 = vlib_buffer_length_in_chain (vm, p[2]);
+  cur_len3 = vlib_buffer_length_in_chain (vm, p[3]);
+
+  len_diff0 = cur_len0 - ip_len0;
+  len_diff1 = cur_len1 - ip_len1;
+  len_diff2 = cur_len2 - ip_len2;
+  len_diff3 = cur_len3 - ip_len3;
+
+  error0 = len_diff0 < 0 ? IP4_ERROR_BAD_LENGTH : error0;
+  error1 = len_diff1 < 0 ? IP4_ERROR_BAD_LENGTH : error1;
+  error2 = len_diff2 < 0 ? IP4_ERROR_BAD_LENGTH : error2;
+  error3 = len_diff3 < 0 ? IP4_ERROR_BAD_LENGTH : error3;
+
+  if (PREDICT_FALSE (error0 != IP4_ERROR_NONE))
+    {
+      if (error0 == IP4_ERROR_TIME_EXPIRED)
+	{
+	  icmp4_error_set_vnet_buffer (p[0], ICMP4_time_exceeded,
+				       ICMP4_time_exceeded_ttl_exceeded_in_transit,
+				       0);
+	  next[0] = IP4_INPUT_NEXT_ICMP_ERROR;
+	}
+      else
+	next[0] = error0 != IP4_ERROR_OPTIONS ?
+	  IP4_INPUT_NEXT_DROP : IP4_INPUT_NEXT_PUNT;
+      p[0]->error = error_node->errors[error0];
+    }
+  if (PREDICT_FALSE (error1 != IP4_ERROR_NONE))
+    {
+      if (error1 == IP4_ERROR_TIME_EXPIRED)
+	{
+	  icmp4_error_set_vnet_buffer (p[1], ICMP4_time_exceeded,
+				       ICMP4_time_exceeded_ttl_exceeded_in_transit,
+				       0);
+	  next[1] = IP4_INPUT_NEXT_ICMP_ERROR;
+	}
+      else
+	next[1] = error1 != IP4_ERROR_OPTIONS ?
+	  IP4_INPUT_NEXT_DROP : IP4_INPUT_NEXT_PUNT;
+      p[1]->error = error_node->errors[error1];
+    }
+  if (PREDICT_FALSE (error2 != IP4_ERROR_NONE))
+    {
+      if (error2 == IP4_ERROR_TIME_EXPIRED)
+	{
+	  icmp4_error_set_vnet_buffer (p[2], ICMP4_time_exceeded,
+				       ICMP4_time_exceeded_ttl_exceeded_in_transit,
+				       0);
+	  next[2] = IP4_INPUT_NEXT_ICMP_ERROR;
+	}
+      else
+	next[2] = error2 != IP4_ERROR_OPTIONS ?
+	  IP4_INPUT_NEXT_DROP : IP4_INPUT_NEXT_PUNT;
+      p[2]->error = error_node->errors[error2];
+    }
+  if (PREDICT_FALSE (error3 != IP4_ERROR_NONE))
+    {
+      if (error3 == IP4_ERROR_TIME_EXPIRED)
+	{
+	  icmp4_error_set_vnet_buffer (p[3], ICMP4_time_exceeded,
+				       ICMP4_time_exceeded_ttl_exceeded_in_transit,
+				       0);
+	  next[3] = IP4_INPUT_NEXT_ICMP_ERROR;
+	}
+      else
+	next[3] = error3 != IP4_ERROR_OPTIONS ?
+	  IP4_INPUT_NEXT_DROP : IP4_INPUT_NEXT_PUNT;
+      p[3]->error = error_node->errors[error3];
+    }
+}
+
+always_inline void
 ip4_input_check_x2 (vlib_main_t * vm,
 		    vlib_node_runtime_t * error_node,
 		    vlib_buffer_t * p0, vlib_buffer_t * p1,