ipsec: VPP-1316 calculate IP/TCP/UDP inner checksums

Calculate IP/TCP/UDP checksums in software before adding authentication.

Change-Id: I3e121cb00aeba667764f39ade8d62170f18f8b6b
Signed-off-by: Klement Sekera <ksekera@cisco.com>

6 files changed, 180 insertions, 195 deletions

diff --git a/src/vnet.am b/src/vnet.am
index cecc2e2ff31..95b94c3c09c 100644
--- a/src/vnet.am
+++ b/src/vnet.am
@@ -451,7 +451,6 @@ libvnet_la_SOURCES +=				\
  vnet/ipsec/ipsec_input.c			\
  vnet/ipsec/ipsec_if.c				\
  vnet/ipsec/ipsec_if_in.c			\
- vnet/ipsec/ipsec_if_out.c			\
  vnet/ipsec/esp_format.c			\
  vnet/ipsec/esp_encrypt.c			\
  vnet/ipsec/esp_decrypt.c			\
diff --git a/src/vnet/ipsec/ah_encrypt.c b/src/vnet/ipsec/ah_encrypt.c
index 6619d872013..898c0f27547 100644
--- a/src/vnet/ipsec/ah_encrypt.c
+++ b/src/vnet/ipsec/ah_encrypt.c
@@ -263,19 +263,17 @@ ah_encrypt_node_fn (vlib_main_t * vm,
 
 	  u8 sig[64];
 	  memset (sig, 0, sizeof (sig));
-	  u8 *digest = NULL;
-	  {
-	    digest = vlib_buffer_get_current (i_b0) + ip_hdr_size + icv_size;
-	    memset (digest, 0, icv_size);
-	  }
+	  u8 *digest =
+	    vlib_buffer_get_current (i_b0) + ip_hdr_size + icv_size;
+	  memset (digest, 0, icv_size);
 
-	  hmac_calc (sa0->integ_alg, sa0->integ_key,
-		     sa0->integ_key_len,
-		     (u8 *) vlib_buffer_get_current (i_b0),
-		     i_b0->current_length, sig, sa0->use_esn, sa0->seq_hi);
-
-	  memcpy (digest, (char *) &sig[0], 12);
+	  unsigned size = hmac_calc (sa0->integ_alg, sa0->integ_key,
+				     sa0->integ_key_len,
+				     vlib_buffer_get_current (i_b0),
+				     i_b0->current_length, sig, sa0->use_esn,
+				     sa0->seq_hi);
 
+	  memcpy (digest, sig, size);
 	  if (PREDICT_FALSE (is_ipv6))
 	    {
 	    }
@@ -287,7 +285,7 @@ ah_encrypt_node_fn (vlib_main_t * vm,
 	    }
 
 	  if (transport_mode)
-	    vlib_buffer_advance (i_b0, -sizeof (ethernet_header_t));;
+	    vlib_buffer_advance (i_b0, -sizeof (ethernet_header_t));
 
 	trace:
 	  if (PREDICT_FALSE (i_b0->flags & VLIB_BUFFER_IS_TRACED))
diff --git a/src/vnet/ipsec/ipsec.h b/src/vnet/ipsec/ipsec.h
index 404756a418f..e457e33944e 100644
--- a/src/vnet/ipsec/ipsec.h
+++ b/src/vnet/ipsec/ipsec.h
@@ -310,7 +310,6 @@ extern vlib_node_registration_t esp_encrypt_node;
 extern vlib_node_registration_t esp_decrypt_node;
 extern vlib_node_registration_t ah_encrypt_node;
 extern vlib_node_registration_t ah_decrypt_node;
-extern vlib_node_registration_t ipsec_if_output_node;
 extern vlib_node_registration_t ipsec_if_input_node;
 
 
@@ -328,7 +327,6 @@ int ipsec_set_sa_key (vlib_main_t * vm, ipsec_sa_t * sa_update);
 
 u32 ipsec_get_sa_index_by_sa_id (u32 sa_id);
 u8 ipsec_is_sa_used (u32 sa_index);
-u8 *format_ipsec_if_output_trace (u8 * s, va_list * args);
 u8 *format_ipsec_policy_action (u8 * s, va_list * args);
 u8 *format_ipsec_crypto_alg (u8 * s, va_list * args);
 u8 *format_ipsec_integ_alg (u8 * s, va_list * args);
diff --git a/src/vnet/ipsec/ipsec_if.c b/src/vnet/ipsec/ipsec_if.c
index e7536b2756e..a7dbcbadb16 100644
--- a/src/vnet/ipsec/ipsec_if.c
+++ b/src/vnet/ipsec/ipsec_if.c
@@ -34,14 +34,128 @@ format_ipsec_name (u8 * s, va_list * args)
   return format (s, "ipsec%d", t->show_instance);
 }
 
+/* Statistics (not really errors) */
+#define foreach_ipsec_if_tx_error    \
+_(TX, "good packets transmitted")
+
+static char *ipsec_if_tx_error_strings[] = {
+#define _(sym,string) string,
+  foreach_ipsec_if_tx_error
+#undef _
+};
+
+typedef enum
+{
+#define _(sym,str) IPSEC_IF_OUTPUT_ERROR_##sym,
+  foreach_ipsec_if_tx_error
+#undef _
+    IPSEC_IF_TX_N_ERROR,
+} ipsec_if_tx_error_t;
+
+typedef struct
+{
+  u32 spi;
+  u32 seq;
+} ipsec_if_tx_trace_t;
+
+u8 *
+format_ipsec_if_tx_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  ipsec_if_tx_trace_t *t = va_arg (*args, ipsec_if_tx_trace_t *);
+
+  s = format (s, "IPSec: spi %u seq %u", t->spi, t->seq);
+  return s;
+}
+
 static uword
-dummy_interface_tx (vlib_main_t * vm,
-		    vlib_node_runtime_t * node, vlib_frame_t * frame)
+ipsec_if_tx_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
+		     vlib_frame_t * from_frame)
 {
-  clib_warning ("you shouldn't be here, leaking buffers...");
-  return frame->n_vectors;
+  ipsec_main_t *im = &ipsec_main;
+  vnet_main_t *vnm = im->vnet_main;
+  vnet_interface_main_t *vim = &vnm->interface_main;
+  u32 *from, *to_next = 0, next_index;
+  u32 n_left_from, sw_if_index0, last_sw_if_index = ~0;
+  u32 thread_index = vlib_get_thread_index ();
+  u32 n_bytes = 0, n_packets = 0;
+
+  from = vlib_frame_vector_args (from_frame);
+  n_left_from = from_frame->n_vectors;
+  next_index = node->cached_next_index;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	  u32 bi0, next0, len0;
+	  vlib_buffer_t *b0;
+	  ipsec_tunnel_if_t *t0;
+	  vnet_hw_interface_t *hi0;
+
+	  bi0 = to_next[0] = from[0];
+	  from += 1;
+	  n_left_from -= 1;
+	  to_next += 1;
+	  n_left_to_next -= 1;
+	  b0 = vlib_get_buffer (vm, bi0);
+	  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX];
+	  hi0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
+	  t0 = pool_elt_at_index (im->tunnel_interfaces, hi0->dev_instance);
+	  vnet_buffer (b0)->ipsec.sad_index = t0->output_sa_index;
+	  next0 = IPSEC_OUTPUT_NEXT_ESP_ENCRYPT;
+
+	  len0 = vlib_buffer_length_in_chain (vm, b0);
+
+	  if (PREDICT_TRUE (sw_if_index0 == last_sw_if_index))
+	    {
+	      n_packets++;
+	      n_bytes += len0;
+	    }
+	  else
+	    {
+	      vlib_increment_combined_counter (vim->combined_sw_if_counters +
+					       VNET_INTERFACE_COUNTER_TX,
+					       thread_index, sw_if_index0,
+					       n_packets, n_bytes);
+	      last_sw_if_index = sw_if_index0;
+	      n_packets = 1;
+	      n_bytes = len0;
+	    }
+
+	  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+	    {
+	      ipsec_if_tx_trace_t *tr =
+		vlib_add_trace (vm, node, b0, sizeof (*tr));
+	      ipsec_sa_t *sa0 =
+		pool_elt_at_index (im->sad, t0->output_sa_index);
+	      tr->spi = sa0->spi;
+	      tr->seq = sa0->seq;
+	    }
+
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
+					   n_left_to_next, bi0, next0);
+	}
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+  if (last_sw_if_index != ~0)
+    {
+      vlib_increment_combined_counter (vim->combined_sw_if_counters +
+				       VNET_INTERFACE_COUNTER_TX,
+				       thread_index,
+				       last_sw_if_index, n_packets, n_bytes);
+    }
+
+  return from_frame->n_vectors;
 }
 
+
 static clib_error_t *
 ipsec_admin_up_down_function (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
 {
@@ -113,13 +227,16 @@ ipsec_admin_up_down_function (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
   return /* no error */ 0;
 }
 
+
 /* *INDENT-OFF* */
 VNET_DEVICE_CLASS (ipsec_device_class, static) =
 {
   .name = "IPSec",
   .format_device_name = format_ipsec_name,
-  .format_tx_trace = format_ipsec_if_output_trace,
-  .tx_function = dummy_interface_tx,
+  .format_tx_trace = format_ipsec_if_tx_trace,
+  .tx_function = ipsec_if_tx_node_fn,
+  .tx_function_n_errors = IPSEC_IF_TX_N_ERROR,
+  .tx_function_error_strings = ipsec_if_tx_error_strings,
   .admin_up_down_function = ipsec_admin_up_down_function,
 };
 /* *INDENT-ON* */
@@ -162,6 +279,7 @@ ipsec_add_del_tunnel_if_internal (vnet_main_t * vnm,
   uword *p;
   ipsec_sa_t *sa;
   u32 dev_instance;
+  u32 slot;
 
   u64 key = (u64) args->remote_ip.as_u32 << 32 | (u64) args->remote_spi;
   p = hash_get (im->ipsec_if_pool_index_by_key, key);
@@ -247,7 +365,13 @@ ipsec_add_del_tunnel_if_internal (vnet_main_t * vnm,
 					     t - im->tunnel_interfaces);
 
       hi = vnet_get_hw_interface (vnm, hw_if_index);
-      hi->output_node_index = ipsec_if_output_node.index;
+
+      slot = vlib_node_add_named_next_with_slot
+	(vnm->vlib_main, hi->tx_node_index, "esp-encrypt",
+	 IPSEC_OUTPUT_NEXT_ESP_ENCRYPT);
+
+      ASSERT (slot == IPSEC_OUTPUT_NEXT_ESP_ENCRYPT);
+
       t->hw_if_index = hw_if_index;
 
       vnet_feature_enable_disable ("interface-output", "ipsec-if-output",
diff --git a/src/vnet/ipsec/ipsec_if_out.c b/src/vnet/ipsec/ipsec_if_out.c
deleted file mode 100644
index cab6ff3a278..00000000000
--- a/src/vnet/ipsec/ipsec_if_out.c
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * ipsec_if_out.c : IPSec interface output node
- *
- * Copyright (c) 2015 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <vnet/vnet.h>
-#include <vnet/api_errno.h>
-#include <vnet/ip/ip.h>
-
-#include <vnet/ipsec/ipsec.h>
-
-/* Statistics (not really errors) */
-#define foreach_ipsec_if_output_error    \
-_(TX, "good packets transmitted")
-
-static char *ipsec_if_output_error_strings[] = {
-#define _(sym,string) string,
-  foreach_ipsec_if_output_error
-#undef _
-};
-
-typedef enum
-{
-#define _(sym,str) IPSEC_IF_OUTPUT_ERROR_##sym,
-  foreach_ipsec_if_output_error
-#undef _
-    IPSEC_IF_OUTPUT_N_ERROR,
-} ipsec_if_output_error_t;
-
-
-typedef struct
-{
-  u32 spi;
-  u32 seq;
-} ipsec_if_output_trace_t;
-
-u8 *
-format_ipsec_if_output_trace (u8 * s, va_list * args)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  ipsec_if_output_trace_t *t = va_arg (*args, ipsec_if_output_trace_t *);
-
-  s = format (s, "IPSec: spi %u seq %u", t->spi, t->seq);
-  return s;
-}
-
-static uword
-ipsec_if_output_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
-			 vlib_frame_t * from_frame)
-{
-  ipsec_main_t *im = &ipsec_main;
-  vnet_main_t *vnm = im->vnet_main;
-  vnet_interface_main_t *vim = &vnm->interface_main;
-  u32 *from, *to_next = 0, next_index;
-  u32 n_left_from, sw_if_index0, last_sw_if_index = ~0;
-  u32 thread_index = vlib_get_thread_index ();
-  u32 n_bytes = 0, n_packets = 0;
-
-  from = vlib_frame_vector_args (from_frame);
-  n_left_from = from_frame->n_vectors;
-  next_index = node->cached_next_index;
-
-  while (n_left_from > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
-
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-	  u32 bi0, next0, len0;
-	  vlib_buffer_t *b0;
-	  ipsec_tunnel_if_t *t0;
-	  vnet_hw_interface_t *hi0;
-
-	  bi0 = to_next[0] = from[0];
-	  from += 1;
-	  n_left_from -= 1;
-	  to_next += 1;
-	  n_left_to_next -= 1;
-	  b0 = vlib_get_buffer (vm, bi0);
-	  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX];
-	  hi0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
-	  t0 = pool_elt_at_index (im->tunnel_interfaces, hi0->dev_instance);
-	  vnet_buffer (b0)->ipsec.sad_index = t0->output_sa_index;
-	  next0 = im->esp_encrypt_next_index;
-
-	  len0 = vlib_buffer_length_in_chain (vm, b0);
-
-	  if (PREDICT_TRUE (sw_if_index0 == last_sw_if_index))
-	    {
-	      n_packets++;
-	      n_bytes += len0;
-	    }
-	  else
-	    {
-	      vlib_increment_combined_counter (vim->combined_sw_if_counters +
-					       VNET_INTERFACE_COUNTER_TX,
-					       thread_index, sw_if_index0,
-					       n_packets, n_bytes);
-	      last_sw_if_index = sw_if_index0;
-	      n_packets = 1;
-	      n_bytes = len0;
-	    }
-
-	  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
-	    {
-	      ipsec_if_output_trace_t *tr =
-		vlib_add_trace (vm, node, b0, sizeof (*tr));
-	      ipsec_sa_t *sa0 =
-		pool_elt_at_index (im->sad, t0->output_sa_index);
-	      tr->spi = sa0->spi;
-	      tr->seq = sa0->seq;
-	    }
-
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
-					   n_left_to_next, bi0, next0);
-	}
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-
-  if (last_sw_if_index != ~0)
-    {
-      vlib_increment_combined_counter (vim->combined_sw_if_counters +
-				       VNET_INTERFACE_COUNTER_TX,
-				       thread_index,
-				       last_sw_if_index, n_packets, n_bytes);
-    }
-
-  vlib_node_increment_counter (vm, ipsec_if_output_node.index,
-			       IPSEC_IF_OUTPUT_ERROR_TX,
-			       from_frame->n_vectors);
-
-  return from_frame->n_vectors;
-}
-
-/* *INDENT-OFF* */
-VLIB_REGISTER_NODE (ipsec_if_output_node) = {
-  .function = ipsec_if_output_node_fn,
-  .name = "ipsec-if-output",
-  .vector_size = sizeof (u32),
-  .format_trace = format_ipsec_if_output_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-
-  .n_errors = ARRAY_LEN(ipsec_if_output_error_strings),
-  .error_strings = ipsec_if_output_error_strings,
-
-  .sibling_of = "ipsec-output-ip4",
-};
-/* *INDENT-ON* */
-
-VLIB_NODE_FUNCTION_MULTIARCH (ipsec_if_output_node, ipsec_if_output_node_fn)
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/ipsec/ipsec_output.c b/src/vnet/ipsec/ipsec_output.c
index 4bfbf603072..a62c0a53458 100644
--- a/src/vnet/ipsec/ipsec_output.c
+++ b/src/vnet/ipsec/ipsec_output.c
@@ -190,6 +190,7 @@ ipsec_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
   vlib_frame_t *f = 0;
   u32 spd_index0 = ~0;
   ipsec_spd_t *spd0 = 0;
+  int bogus;
   u64 nc_protect = 0, nc_bypass = 0, nc_discard = 0, nc_nomatch = 0;
 
   from = vlib_frame_vector_args (from_frame);
@@ -204,6 +205,7 @@ ipsec_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
       ip6_header_t *ip6_0 = 0;
       udp_header_t *udp0;
       u32 iph_offset = 0;
+      tcp_header_t *tcp0;
 
       bi0 = from[0];
       b0 = vlib_get_buffer (vm, bi0);
@@ -269,6 +271,7 @@ ipsec_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 					  clib_net_to_host_u16
 					  (udp0->dst_port));
 	}
+      tcp0 = (void *) udp0;
 
       if (PREDICT_TRUE (p0 != NULL))
 	{
@@ -282,18 +285,53 @@ ipsec_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	      else
 		next_node_index = im->ah_encrypt_node_index;
 	      vnet_buffer (b0)->ipsec.sad_index = p0->sa_index;
-	      vlib_buffer_advance (b0, iph_offset);
 	      p0->counter.packets++;
 	      if (is_ipv6)
 		{
 		  p0->counter.bytes +=
 		    clib_net_to_host_u16 (ip6_0->payload_length);
 		  p0->counter.bytes += sizeof (ip6_header_t);
+		  if (PREDICT_FALSE
+		      (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM))
+		    {
+		      tcp0->checksum =
+			ip6_tcp_udp_icmp_compute_checksum (vm, b0, ip6_0,
+							   &bogus);
+		      b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+		    }
+		  if (PREDICT_FALSE
+		      (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
+		    {
+		      udp0->checksum =
+			ip6_tcp_udp_icmp_compute_checksum (vm, b0, ip6_0,
+							   &bogus);
+		      b0->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+		    }
 		}
 	      else
 		{
 		  p0->counter.bytes += clib_net_to_host_u16 (ip0->length);
+		  if (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
+		    {
+		      ip0->checksum = ip4_header_checksum (ip0);
+		      b0->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+		    }
+		  if (PREDICT_FALSE
+		      (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM))
+		    {
+		      tcp0->checksum =
+			ip4_tcp_udp_compute_checksum (vm, b0, ip0);
+		      b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+		    }
+		  if (PREDICT_FALSE
+		      (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
+		    {
+		      udp0->checksum =
+			ip4_tcp_udp_compute_checksum (vm, b0, ip0);
+		      b0->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+		    }
 		}
+	      vlib_buffer_advance (b0, iph_offset);
 	    }
 	  else if (p0->policy == IPSEC_POLICY_ACTION_BYPASS)
 	    {