IPSEC: SPD counters in the stats sgement

- return the stats_index of each SPD in the create API call
- no ip_any in the API as this creates 2 SPD entries. client must add both v4 and v6 explicitly
- only one pool of SPD entries (rhter than one per-SPD) to support this
- no packets/bytes in the dump API. Polling the stats segment is much more efficient
  (if the SA lifetime is based on packet/bytes)
- emit the policy index in the packet trace and CLI commands.

Change-Id: I7eaf52c9d0495fa24450facf55229941279b8569
Signed-off-by: Neale Ranns <nranns@cisco.com>

12 files changed, 411 insertions, 505 deletions

diff --git a/src/vnet/ipsec/ipsec.api b/src/vnet/ipsec/ipsec.api
index 92c39acefd6..54cc76bb9d6 100644
--- a/src/vnet/ipsec/ipsec.api
+++ b/src/vnet/ipsec/ipsec.api
@@ -96,7 +96,6 @@ typedef ipsec_spd_entry
   u8 protocol;
 
   // Selector
-  u8 is_ip_any;
   vl_api_address_t remote_address_start;
   vl_api_address_t remote_address_stop;
   vl_api_address_t local_address_start;
@@ -115,7 +114,7 @@ typedef ipsec_spd_entry
     @param is_add - add SPD if non-zero, else delete
     @param entry - Description of the entry to add/dell
 */
-autoreply define ipsec_spd_entry_add_del
+define ipsec_spd_entry_add_del
 {
   u32 client_index;
   u32 context;
@@ -123,6 +122,19 @@ autoreply define ipsec_spd_entry_add_del
   vl_api_ipsec_spd_entry_t entry;
 };
 
+/** \brief IPsec: Reply Add/delete Security Policy Database entry
+
+    @param context - sender context, to match reply w/ request
+    @param retval - success/fail rutrun code
+    @param stat_index - An index for the policy in the stats segment @ /net/ipec/policy
+*/
+define ipsec_spd_entry_add_del_reply
+{
+  u32 context;
+  i32 retval;
+  u32 stat_index;
+};
+
 /** \brief Dump IPsec all SPD IDs
     @param client_index - opaque cookie to identify the sender
     @param context - sender context, to match reply w/ request
@@ -165,8 +177,6 @@ define ipsec_spd_dump {
 define ipsec_spd_details {
     u32 context;
     vl_api_ipsec_spd_entry_t entry;
-    u64 bytes;
-    u64 packets;
 };
 
 /*
diff --git a/src/vnet/ipsec/ipsec.h b/src/vnet/ipsec/ipsec.h
index fd709675b8e..ed14a5bcbe1 100644
--- a/src/vnet/ipsec/ipsec.h
+++ b/src/vnet/ipsec/ipsec.h
@@ -119,7 +119,10 @@ typedef struct
 {
   /* pool of tunnel instances */
   ipsec_spd_t *spds;
+  /* Pool of security associations */
   ipsec_sa_t *sad;
+  /* pool of policies */
+  ipsec_policy_t *policies;
 
   /* pool of tunnel interfaces */
   ipsec_tunnel_if_t *tunnel_interfaces;
@@ -175,9 +178,6 @@ typedef struct
   u32 ah_default_backend;
   /* index of default esp backend */
   u32 esp_default_backend;
-
-  /* helper for sort function */
-  ipsec_spd_t *spd_to_sort;
 } ipsec_main_t;
 
 extern ipsec_main_t ipsec_main;
diff --git a/src/vnet/ipsec/ipsec_api.c b/src/vnet/ipsec/ipsec_api.c
index eb78dbdd319..06f954622c7 100644
--- a/src/vnet/ipsec/ipsec_api.c
+++ b/src/vnet/ipsec/ipsec_api.c
@@ -143,6 +143,7 @@ static void vl_api_ipsec_spd_entry_add_del_t_handler
   vlib_main_t *vm __attribute__ ((unused)) = vlib_get_main ();
   vl_api_ipsec_spd_entry_add_del_reply_t *rmp;
   ip46_type_t itype;
+  u32 stat_index;
   int rv;
 
 #if WITH_LIBSSL > 0
@@ -181,22 +182,22 @@ static void vl_api_ipsec_spd_entry_add_del_t_handler
     }
   p.sa_id = ntohl (mp->entry.sa_id);
 
-  rv = ipsec_add_del_policy (vm, &p, mp->is_add);
+  rv = ipsec_add_del_policy (vm, &p, mp->is_add, &stat_index);
   if (rv)
     goto out;
 
-  if (mp->entry.is_ip_any)
-    {
-      p.is_ipv6 = 1;
-      rv = ipsec_add_del_policy (vm, &p, mp->is_add);
-    }
 #else
   rv = VNET_API_ERROR_UNIMPLEMENTED;
   goto out;
 #endif
 
 out:
-  REPLY_MACRO (VL_API_IPSEC_SPD_ENTRY_ADD_DEL_REPLY);
+  /* *INDENT-OFF* */
+  REPLY_MACRO2 (VL_API_IPSEC_SPD_ENTRY_ADD_DEL_REPLY,
+  ({
+    rmp->stat_index = ntohl(stat_index);
+  }));
+  /* *INDENT-ON* */
 }
 
 static int
@@ -340,6 +341,7 @@ send_ipsec_spds_details (ipsec_spd_t * spd, vl_api_registration_t * reg,
 			 u32 context)
 {
   vl_api_ipsec_spds_details_t *mp;
+  u32 n_policies = 0;
 
   mp = vl_msg_api_alloc (sizeof (*mp));
   clib_memset (mp, 0, sizeof (*mp));
@@ -347,7 +349,10 @@ send_ipsec_spds_details (ipsec_spd_t * spd, vl_api_registration_t * reg,
   mp->context = context;
 
   mp->spd_id = htonl (spd->id);
-  mp->npolicies = htonl (pool_len (spd->policies));
+#define _(s, n) n_policies += vec_len (spd->policies[IPSEC_SPD_POLICY_##s]);
+  foreach_ipsec_spd_policy_type
+#undef _
+    mp->npolicies = htonl (n_policies);
 
   vl_api_send_msg (reg, (u8 *) mp);
 }
@@ -420,9 +425,6 @@ send_ipsec_spd_details (ipsec_policy_t * p, vl_api_registration_t * reg,
   mp->entry.policy = ipsec_spd_action_encode (p->policy);
   mp->entry.sa_id = htonl (p->sa_id);
 
-  mp->bytes = clib_host_to_net_u64 (p->counter.bytes);
-  mp->packets = clib_host_to_net_u64 (p->counter.packets);
-
   vl_api_send_msg (reg, (u8 *) mp);
 }
 
@@ -431,10 +433,11 @@ vl_api_ipsec_spd_dump_t_handler (vl_api_ipsec_spd_dump_t * mp)
 {
   vl_api_registration_t *reg;
   ipsec_main_t *im = &ipsec_main;
+  ipsec_spd_policy_t ptype;
   ipsec_policy_t *policy;
   ipsec_spd_t *spd;
   uword *p;
-  u32 spd_index;
+  u32 spd_index, *ii;
 #if WITH_LIBSSL > 0
   reg = vl_api_client_index_to_registration (mp->client_index);
   if (!reg)
@@ -448,12 +451,15 @@ vl_api_ipsec_spd_dump_t_handler (vl_api_ipsec_spd_dump_t * mp)
   spd = pool_elt_at_index (im->spds, spd_index);
 
   /* *INDENT-OFF* */
-  pool_foreach (policy, spd->policies,
-  ({
-    if (mp->sa_id == ~(0) || ntohl (mp->sa_id) == policy->sa_id)
-      send_ipsec_spd_details (policy, reg,
-                              mp->context);}
-    ));
+  FOR_EACH_IPSEC_SPD_POLICY_TYPE(ptype) {
+    vec_foreach(ii, spd->policies[ptype])
+      {
+        policy = pool_elt_at_index(im->policies, *ii);
+
+        if (mp->sa_id == ~(0) || ntohl (mp->sa_id) == policy->sa_id)
+          send_ipsec_spd_details (policy, reg, mp->context);
+      }
+  }
   /* *INDENT-ON* */
 #else
   clib_warning ("unimplemented");
diff --git a/src/vnet/ipsec/ipsec_cli.c b/src/vnet/ipsec/ipsec_cli.c
index f3a6158e850..4bc14372042 100644
--- a/src/vnet/ipsec/ipsec_cli.c
+++ b/src/vnet/ipsec/ipsec_cli.c
@@ -258,9 +258,8 @@ ipsec_policy_add_del_command_fn (vlib_main_t * vm,
 {
   unformat_input_t _line_input, *line_input = &_line_input;
   ipsec_policy_t p;
-  int is_add = 0;
-  int is_ip_any = 1;
-  u32 tmp, tmp2;
+  int rv, is_add = 0;
+  u32 tmp, tmp2, stat_index;
   clib_error_t *error = NULL;
 
   clib_memset (&p, 0, sizeof (p));
@@ -304,24 +303,22 @@ ipsec_policy_add_del_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "local-ip-range %U - %U",
 			 unformat_ip4_address, &p.laddr.start.ip4,
 			 unformat_ip4_address, &p.laddr.stop.ip4))
-	is_ip_any = 0;
+	;
       else if (unformat (line_input, "remote-ip-range %U - %U",
 			 unformat_ip4_address, &p.raddr.start.ip4,
 			 unformat_ip4_address, &p.raddr.stop.ip4))
-	is_ip_any = 0;
+	;
       else if (unformat (line_input, "local-ip-range %U - %U",
 			 unformat_ip6_address, &p.laddr.start.ip6,
 			 unformat_ip6_address, &p.laddr.stop.ip6))
 	{
 	  p.is_ipv6 = 1;
-	  is_ip_any = 0;
 	}
       else if (unformat (line_input, "remote-ip-range %U - %U",
 			 unformat_ip6_address, &p.raddr.start.ip6,
 			 unformat_ip6_address, &p.raddr.stop.ip6))
 	{
 	  p.is_ipv6 = 1;
-	  is_ip_any = 0;
 	}
       else if (unformat (line_input, "local-port-range %u - %u", &tmp, &tmp2))
 	{
@@ -363,12 +360,12 @@ ipsec_policy_add_del_command_fn (vlib_main_t * vm,
 	  goto done;
 	}
     }
-  ipsec_add_del_policy (vm, &p, is_add);
-  if (is_ip_any)
-    {
-      p.is_ipv6 = 1;
-      ipsec_add_del_policy (vm, &p, is_add);
-    }
+  rv = ipsec_add_del_policy (vm, &p, is_add, &stat_index);
+
+  if (!rv)
+    vlib_cli_output (vm, "policy-index:%d", stat_index);
+  else
+    vlib_cli_output (vm, "error:%d", rv);
 
 done:
   unformat_free (line_input);
@@ -451,11 +448,9 @@ static clib_error_t *
 show_ipsec_command_fn (vlib_main_t * vm,
 		       unformat_input_t * input, vlib_cli_command_t * cmd)
 {
-  ipsec_spd_t *spd;
   ipsec_sa_t *sa;
-  ipsec_policy_t *p;
   ipsec_main_t *im = &ipsec_main;
-  u32 *i;
+  u32 i;
   ipsec_tunnel_if_t *t;
   vnet_hw_interface_t *hi;
   u8 *protocol = NULL;
@@ -494,174 +489,8 @@ show_ipsec_command_fn (vlib_main_t * vm,
   /* *INDENT-ON* */
 
   /* *INDENT-OFF* */
-  pool_foreach (spd, im->spds, ({
-    vlib_cli_output(vm, "spd %u", spd->id);
-
-    vlib_cli_output(vm, " outbound policies");
-    vec_foreach(i, spd->ipv4_outbound_policies)
-      {
-        p = pool_elt_at_index(spd->policies, *i);
-	vec_reset_length(protocol);
-	vec_reset_length(policy);
-	if (p->protocol) {
-	  protocol = format(protocol, "%U", format_ip_protocol, p->protocol);
-	} else {
-	  protocol = format(protocol, "any");
-	}
-	if (p->policy == IPSEC_POLICY_ACTION_PROTECT) {
-	  policy = format(policy, " sa %u", p->sa_id);
-	}
-
-        vlib_cli_output(vm, "  priority %d action %U protocol %v%v",
-                        p->priority, format_ipsec_policy_action, p->policy,
-			protocol, policy);
-        vlib_cli_output(vm, "   local addr range %U - %U port range %u - %u",
-                        format_ip4_address, &p->laddr.start.ip4,
-                        format_ip4_address, &p->laddr.stop.ip4,
-                        p->lport.start, p->lport.stop);
-        vlib_cli_output(vm, "   remote addr range %U - %U port range %u - %u",
-                        format_ip4_address, &p->raddr.start.ip4,
-                        format_ip4_address, &p->raddr.stop.ip4,
-                        p->rport.start, p->rport.stop);
-        vlib_cli_output(vm, "   packets %u bytes %u", p->counter.packets,
-                        p->counter.bytes);
-      };
-    vec_foreach(i, spd->ipv6_outbound_policies)
-      {
-        p = pool_elt_at_index(spd->policies, *i);
-	vec_reset_length(protocol);
-	vec_reset_length(policy);
-	if (p->protocol) {
-	  protocol = format(protocol, "%U", format_ip_protocol, p->protocol);
-	} else {
-	  protocol = format(protocol, "any");
-	}
-	if (p->policy == IPSEC_POLICY_ACTION_PROTECT) {
-	  policy = format(policy, " sa %u", p->sa_id);
-	}
-        vlib_cli_output(vm, "  priority %d action %U protocol %v%v",
-                        p->priority, format_ipsec_policy_action, p->policy,
-			protocol, policy);
-        vlib_cli_output(vm, "   local addr range %U - %U port range %u - %u",
-                        format_ip6_address, &p->laddr.start.ip6,
-                        format_ip6_address, &p->laddr.stop.ip6,
-                        p->lport.start, p->lport.stop);
-        vlib_cli_output(vm, "   remote addr range %U - %U port range %u - %u",
-                        format_ip6_address, &p->raddr.start.ip6,
-                        format_ip6_address, &p->raddr.stop.ip6,
-                        p->rport.start, p->rport.stop);
-        vlib_cli_output(vm, "   packets %u bytes %u", p->counter.packets,
-                        p->counter.bytes);
-      };
-    vlib_cli_output(vm, " inbound policies");
-    vec_foreach(i, spd->ipv4_inbound_protect_policy_indices)
-      {
-        p = pool_elt_at_index(spd->policies, *i);
-	vec_reset_length(protocol);
-	vec_reset_length(policy);
-	if (p->protocol) {
-	  protocol = format(protocol, "%U", format_ip_protocol, p->protocol);
-	} else {
-	  protocol = format(protocol, "any");
-	}
-	if (p->policy == IPSEC_POLICY_ACTION_PROTECT) {
-	  policy = format(policy, " sa %u", p->sa_id);
-	}
-        vlib_cli_output(vm, "  priority %d action %U protocol %v%v",
-                        p->priority, format_ipsec_policy_action, p->policy,
-			protocol, policy);
-        vlib_cli_output(vm, "   local addr range %U - %U port range %u - %u",
-                        format_ip4_address, &p->laddr.start.ip4,
-                        format_ip4_address, &p->laddr.stop.ip4,
-                        p->lport.start, p->lport.stop);
-        vlib_cli_output(vm, "   remote addr range %U - %U port range %u - %u",
-                        format_ip4_address, &p->raddr.start.ip4,
-                        format_ip4_address, &p->raddr.stop.ip4,
-                        p->rport.start, p->rport.stop);
-        vlib_cli_output(vm, "   packets %u bytes %u", p->counter.packets,
-                        p->counter.bytes);
-      };
-    vec_foreach(i, spd->ipv4_inbound_policy_discard_and_bypass_indices)
-      {
-        p = pool_elt_at_index(spd->policies, *i);
-	vec_reset_length(protocol);
-	vec_reset_length(policy);
-	if (p->protocol) {
-	  protocol = format(protocol, "%U", format_ip_protocol, p->protocol);
-	} else {
-	  protocol = format(protocol, "any");
-	}
-	if (p->policy == IPSEC_POLICY_ACTION_PROTECT) {
-	  policy = format(policy, " sa %u", p->sa_id);
-	}
-        vlib_cli_output(vm, "  priority %d action %U protocol %v%v",
-                        p->priority, format_ipsec_policy_action, p->policy,
-			protocol, policy);
-        vlib_cli_output(vm, "   local addr range %U - %U port range %u - %u",
-                        format_ip4_address, &p->laddr.start.ip4,
-                        format_ip4_address, &p->laddr.stop.ip4,
-                        p->lport.start, p->lport.stop);
-        vlib_cli_output(vm, "   remote addr range %U - %U port range %u - %u",
-                        format_ip4_address, &p->raddr.start.ip4,
-                        format_ip4_address, &p->raddr.stop.ip4,
-                        p->rport.start, p->rport.stop);
-        vlib_cli_output(vm, "   packets %u bytes %u", p->counter.packets,
-                        p->counter.bytes);
-      };
-    vec_foreach(i, spd->ipv6_inbound_protect_policy_indices)
-      {
-        p = pool_elt_at_index(spd->policies, *i);
-	vec_reset_length(protocol);
-	vec_reset_length(policy);
-	if (p->protocol) {
-	  protocol = format(protocol, "%U", format_ip_protocol, p->protocol);
-	} else {
-	  protocol = format(protocol, "any");
-	}
-	if (p->policy == IPSEC_POLICY_ACTION_PROTECT) {
-	  policy = format(policy, " sa %u", p->sa_id);
-	}
-        vlib_cli_output(vm, "  priority %d action %U protocol %v%v",
-                        p->priority, format_ipsec_policy_action, p->policy,
-			protocol, policy);
-        vlib_cli_output(vm, "   local addr range %U - %U port range %u - %u",
-                        format_ip6_address, &p->laddr.start.ip6,
-                        format_ip6_address, &p->laddr.stop.ip6,
-                        p->lport.start, p->lport.stop);
-        vlib_cli_output(vm, "   remote addr range %U - %U port range %u - %u",
-                        format_ip6_address, &p->raddr.start.ip6,
-                        format_ip6_address, &p->raddr.stop.ip6,
-                        p->rport.start, p->rport.stop);
-        vlib_cli_output(vm, "   packets %u bytes %u", p->counter.packets,
-                        p->counter.bytes);
-      };
-    vec_foreach(i, spd->ipv6_inbound_policy_discard_and_bypass_indices)
-      {
-        p = pool_elt_at_index(spd->policies, *i);
-	vec_reset_length(protocol);
-	vec_reset_length(policy);
-	if (p->protocol) {
-	  protocol = format(protocol, "%U", format_ip_protocol, p->protocol);
-	} else {
-	  protocol = format(protocol, "any");
-	}
-	if (p->policy == IPSEC_POLICY_ACTION_PROTECT) {
-	  policy = format(policy, " sa %u", p->sa_id);
-	}
-        vlib_cli_output(vm, "  priority %d action %U protocol %v%v",
-                        p->priority, format_ipsec_policy_action, p->policy,
-			protocol, policy);
-        vlib_cli_output(vm, "   local addr range %U - %U port range %u - %u",
-                        format_ip6_address, &p->laddr.start.ip6,
-                        format_ip6_address, &p->laddr.stop.ip6,
-                        p->lport.start, p->lport.stop);
-        vlib_cli_output(vm, "   remote addr range %U - %U port range %u - %u",
-                        format_ip6_address, &p->raddr.start.ip6,
-                        format_ip6_address, &p->raddr.stop.ip6,
-                        p->rport.start, p->rport.stop);
-        vlib_cli_output(vm, "   packets %u bytes %u", p->counter.packets,
-                        p->counter.bytes);
-      };
+  pool_foreach_index (i, im->spds, ({
+    vlib_cli_output(vm, "%U", format_ipsec_spd, i);
   }));
   /* *INDENT-ON* */
 
@@ -866,19 +695,9 @@ clear_ipsec_counters_command_fn (vlib_main_t * vm,
 				 unformat_input_t * input,
 				 vlib_cli_command_t * cmd)
 {
-  ipsec_main_t *im = &ipsec_main;
-  ipsec_spd_t *spd;
-  ipsec_policy_t *p;
-
-  /* *INDENT-OFF* */
-  pool_foreach (spd, im->spds, ({
-    pool_foreach(p, spd->policies, ({
-      p->counter.packets = p->counter.bytes = 0;
-    }));
-  }));
-  /* *INDENT-ON* */
+  vlib_clear_combined_counters (&ipsec_spd_policy_counters);
 
-  return 0;
+  return (NULL);
 }
 
 /* *INDENT-OFF* */
diff --git a/src/vnet/ipsec/ipsec_format.c b/src/vnet/ipsec/ipsec_format.c
index 38aed79a155..cbd67239680 100644
--- a/src/vnet/ipsec/ipsec_format.c
+++ b/src/vnet/ipsec/ipsec_format.c
@@ -132,6 +132,82 @@ format_ipsec_replay_window (u8 * s, va_list * args)
   return s;
 }
 
+u8 *
+format_ipsec_policy (u8 * s, va_list * args)
+{
+  u32 pi = va_arg (*args, u32);
+  ipsec_main_t *im = &ipsec_main;
+  ipsec_policy_t *p;
+  vlib_counter_t counts;
+
+  p = pool_elt_at_index (im->policies, pi);
+
+  s = format (s, "  [%d] priority %d action %U protocol ",
+	      pi, p->priority, format_ipsec_policy_action, p->policy);
+  if (p->protocol)
+    {
+      s = format (s, "%U", format_ip_protocol, p->protocol);
+    }
+  else
+    {
+      s = format (s, "any");
+    }
+  if (p->policy == IPSEC_POLICY_ACTION_PROTECT)
+    {
+      s = format (s, " sa %u", p->sa_id);
+    }
+  if (p->is_ipv6)
+    {
+      s = format (s, "\n     local addr range %U - %U port range %u - %u",
+		  format_ip6_address, &p->laddr.start.ip6,
+		  format_ip6_address, &p->laddr.stop.ip6,
+		  p->lport.start, p->lport.stop);
+      s = format (s, "\n     remote addr range %U - %U port range %u - %u",
+		  format_ip6_address, &p->raddr.start.ip6,
+		  format_ip6_address, &p->raddr.stop.ip6,
+		  p->rport.start, p->rport.stop);
+    }
+  else
+    {
+      s = format (s, "\n     local addr range %U - %U port range %u - %u",
+		  format_ip4_address, &p->laddr.start.ip4,
+		  format_ip4_address, &p->laddr.stop.ip4,
+		  p->lport.start, p->lport.stop);
+      s = format (s, "\n     remote addr range %U - %U port range %u - %u",
+		  format_ip4_address, &p->raddr.start.ip4,
+		  format_ip4_address, &p->raddr.stop.ip4,
+		  p->rport.start, p->rport.stop);
+    }
+  vlib_get_combined_counter (&ipsec_spd_policy_counters, pi, &counts);
+  s = format (s, "\n     packets %u bytes %u", counts.packets, counts.bytes);
+
+  return (s);
+}
+
+u8 *
+format_ipsec_spd (u8 * s, va_list * args)
+{
+  u32 si = va_arg (*args, u32);
+  ipsec_main_t *im = &ipsec_main;
+  ipsec_spd_t *spd;
+  u32 *i;
+
+  spd = pool_elt_at_index (im->spds, si);
+
+  s = format (s, "spd %u", spd->id);
+
+#define _(v, n)                                                 \
+  s = format (s, "\n %s:", n);                                  \
+  vec_foreach(i, spd->policies[IPSEC_SPD_POLICY_##v])           \
+  {                                                             \
+    s = format (s, "\n %U", format_ipsec_policy, *i);           \
+  }
+  foreach_ipsec_spd_policy_type;
+#undef _
+
+  return (s);
+}
+
 /*
  * fd.io coding-style-patch-verification: ON
  *
diff --git a/src/vnet/ipsec/ipsec_input.c b/src/vnet/ipsec/ipsec_input.c
index 1f3d6d01a93..e3fd4aa551d 100644
--- a/src/vnet/ipsec/ipsec_input.c
+++ b/src/vnet/ipsec/ipsec_input.c
@@ -44,7 +44,9 @@ static char *ipsec_input_error_strings[] = {
 
 typedef struct
 {
+  ip_protocol_t proto;
   u32 spd;
+  u32 policy_index;
   u32 sa_id;
   u32 spi;
   u32 seq;
@@ -58,23 +60,10 @@ format_ipsec_input_trace (u8 * s, va_list * args)
   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
   ipsec_input_trace_t *t = va_arg (*args, ipsec_input_trace_t *);
 
-  if (t->spi == 0 && t->seq == 0)
-    {
-      s = format (s, "esp: no esp packet");
-      return s;
-    }
+  s = format (s, "%U: sa_id %u spd %u policy %d spi %u seq %u",
+	      format_ip_protocol, t->proto, t->sa_id,
+	      t->spd, t->policy_index, t->spi, t->seq);
 
-  if (t->sa_id != 0)
-    {
-      s =
-	format (s, "esp: sa_id %u spd %u spi %u seq %u", t->sa_id, t->spd,
-		t->spi, t->seq);
-    }
-  else
-    {
-      s =
-	format (s, "esp: no sa spd %u spi %u seq %u", t->spd, t->spi, t->seq);
-    }
   return s;
 }
 
@@ -86,9 +75,9 @@ ipsec_input_protect_policy_match (ipsec_spd_t * spd, u32 sa, u32 da, u32 spi)
   ipsec_sa_t *s;
   u32 *i;
 
-  vec_foreach (i, spd->ipv4_inbound_protect_policy_indices)
+  vec_foreach (i, spd->policies[IPSEC_SPD_POLICY_IP4_INBOUND_PROTECT])
   {
-    p = pool_elt_at_index (spd->policies, *i);
+    p = pool_elt_at_index (im->policies, *i);
     s = pool_elt_at_index (im->sad, p->sa_index);
 
     if (spi != s->spi)
@@ -142,9 +131,9 @@ ipsec6_input_protect_policy_match (ipsec_spd_t * spd,
   ipsec_sa_t *s;
   u32 *i;
 
-  vec_foreach (i, spd->ipv6_inbound_protect_policy_indices)
+  vec_foreach (i, spd->policies[IPSEC_SPD_POLICY_IP6_INBOUND_PROTECT])
   {
-    p = pool_elt_at_index (spd->policies, *i);
+    p = pool_elt_at_index (im->policies, *i);
     s = pool_elt_at_index (im->sad, p->sa_index);
 
     if (spi != s->spi)
@@ -178,11 +167,12 @@ VLIB_NODE_FN (ipsec4_input_node) (vlib_main_t * vm,
 				  vlib_node_runtime_t * node,
 				  vlib_frame_t * from_frame)
 {
-  u32 n_left_from, *from, next_index, *to_next;
+  u32 n_left_from, *from, next_index, *to_next, thread_index;
   ipsec_main_t *im = &ipsec_main;
 
   from = vlib_frame_vector_args (from_frame);
   n_left_from = from_frame->n_vectors;
+  thread_index = vm->thread_index;
 
   next_index = node->cached_next_index;
 
@@ -194,7 +184,7 @@ VLIB_NODE_FN (ipsec4_input_node) (vlib_main_t * vm,
 
       while (n_left_from > 0 && n_left_to_next > 0)
 	{
-	  u32 bi0, next0;
+	  u32 bi0, next0, pi0;
 	  vlib_buffer_t *b0;
 	  ip4_header_t *ip0;
 	  esp_header_t *esp0;
@@ -249,37 +239,43 @@ VLIB_NODE_FN (ipsec4_input_node) (vlib_main_t * vm,
 						     clib_net_to_host_u32
 						     (esp0->spi));
 
-	      if (PREDICT_TRUE (p0 != 0))
+	      if (PREDICT_TRUE (p0 != NULL))
 		{
-		  p0->counter.packets++;
-		  p0->counter.bytes += clib_net_to_host_u16 (ip0->length);
+		  pi0 = p0 - im->policies;
+		  vlib_increment_combined_counter
+		    (&ipsec_spd_policy_counters,
+		     thread_index, pi0, 1,
+		     clib_net_to_host_u16 (ip0->length));
+
 		  vnet_buffer (b0)->ipsec.sad_index = p0->sa_index;
 		  vnet_buffer (b0)->ipsec.flags = 0;
 		  next0 = im->esp4_decrypt_next_index;
 		  vlib_buffer_advance (b0, ((u8 *) esp0 - (u8 *) ip0));
 		  goto trace0;
 		}
+	      else
+		{
+		  pi0 = ~0;
+		};
 
 	      /* FIXME bypass and discard */
 	    trace0:
-	      if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+	      if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE) &&
+		  PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
 		{
 		  ipsec_input_trace_t *tr =
 		    vlib_add_trace (vm, node, b0, sizeof (*tr));
-		  if (ip0->protocol == IP_PROTOCOL_IPSEC_ESP ||
-		      ip0->protocol == IP_PROTOCOL_UDP)
-		    {
-		      if (p0)
-			tr->sa_id = p0->sa_id;
-		      tr->spi = clib_host_to_net_u32 (esp0->spi);
-		      tr->seq = clib_host_to_net_u32 (esp0->seq);
-		      tr->spd = spd0->id;
-		    }
-		}
 
+		  tr->proto = ip0->protocol;
+		  if (p0)
+		    tr->sa_id = p0->sa_id;
+		  tr->spi = clib_net_to_host_u32 (esp0->spi);
+		  tr->seq = clib_net_to_host_u32 (esp0->seq);
+		  tr->spd = spd0->id;
+		  tr->policy_index = pi0;
+		}
 	    }
 
-
 	  if (PREDICT_TRUE (ip0->protocol == IP_PROTOCOL_IPSEC_AH))
 	    {
 	      ah0 = (ah_header_t *) ((u8 *) ip0 + ip4_header_bytes (ip0));
@@ -295,27 +291,35 @@ VLIB_NODE_FN (ipsec4_input_node) (vlib_main_t * vm,
 
 	      if (PREDICT_TRUE (p0 != 0))
 		{
-		  p0->counter.packets++;
-		  p0->counter.bytes += clib_net_to_host_u16 (ip0->length);
+		  pi0 = p0 - im->policies;
+		  vlib_increment_combined_counter
+		    (&ipsec_spd_policy_counters,
+		     thread_index, pi0, 1,
+		     clib_net_to_host_u16 (ip0->length));
 		  vnet_buffer (b0)->ipsec.sad_index = p0->sa_index;
 		  vnet_buffer (b0)->ipsec.flags = 0;
 		  next0 = im->ah4_decrypt_next_index;
 		  goto trace1;
 		}
+	      else
+		{
+		  pi0 = ~0;
+		}
 	      /* FIXME bypass and discard */
 	    trace1:
-	      if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+	      if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE) &&
+		  PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
 		{
 		  ipsec_input_trace_t *tr =
 		    vlib_add_trace (vm, node, b0, sizeof (*tr));
-		  if (ip0->protocol == IP_PROTOCOL_IPSEC_ESP)
-		    {
-		      if (p0)
-			tr->sa_id = p0->sa_id;
-		      tr->spi = clib_host_to_net_u32 (ah0->spi);
-		      tr->seq = clib_host_to_net_u32 (ah0->seq_no);
-		      tr->spd = spd0->id;
-		    }
+
+		  tr->proto = ip0->protocol;
+		  if (p0)
+		    tr->sa_id = p0->sa_id;
+		  tr->spi = clib_net_to_host_u32 (ah0->spi);
+		  tr->seq = clib_net_to_host_u32 (ah0->seq_no);
+		  tr->spd = spd0->id;
+		  tr->policy_index = pi0;
 		}
 	    }
 
@@ -359,11 +363,12 @@ VLIB_NODE_FN (ipsec6_input_node) (vlib_main_t * vm,
 				  vlib_node_runtime_t * node,
 				  vlib_frame_t * from_frame)
 {
-  u32 n_left_from, *from, next_index, *to_next;
+  u32 n_left_from, *from, next_index, *to_next, thread_index;
   ipsec_main_t *im = &ipsec_main;
 
   from = vlib_frame_vector_args (from_frame);
   n_left_from = from_frame->n_vectors;
+  thread_index = vm->thread_index;
 
   next_index = node->cached_next_index;
 
@@ -375,7 +380,7 @@ VLIB_NODE_FN (ipsec6_input_node) (vlib_main_t * vm,
 
       while (n_left_from > 0 && n_left_to_next > 0)
 	{
-	  u32 bi0, next0;
+	  u32 bi0, next0, pi0;
 	  vlib_buffer_t *b0;
 	  ip6_header_t *ip0;
 	  esp_header_t *esp0;
@@ -420,16 +425,22 @@ VLIB_NODE_FN (ipsec6_input_node) (vlib_main_t * vm,
 
 	      if (PREDICT_TRUE (p0 != 0))
 		{
-		  p0->counter.packets++;
-		  p0->counter.bytes +=
-		    clib_net_to_host_u16 (ip0->payload_length);
-		  p0->counter.bytes += header_size;
+		  pi0 = p0 - im->policies;
+		  vlib_increment_combined_counter
+		    (&ipsec_spd_policy_counters,
+		     thread_index, pi0, 1,
+		     clib_net_to_host_u16 (ip0->payload_length) +
+		     header_size);
 		  vnet_buffer (b0)->ipsec.sad_index = p0->sa_index;
 		  vnet_buffer (b0)->ipsec.flags = 0;
 		  next0 = im->esp6_decrypt_next_index;
 		  vlib_buffer_advance (b0, header_size);
 		  goto trace0;
 		}
+	      else
+		{
+		  pi0 = ~0;
+		}
 	    }
 	  else if (ip0->protocol == IP_PROTOCOL_IPSEC_AH)
 	    {
@@ -441,30 +452,36 @@ VLIB_NODE_FN (ipsec6_input_node) (vlib_main_t * vm,
 
 	      if (PREDICT_TRUE (p0 != 0))
 		{
-		  p0->counter.packets++;
-		  p0->counter.bytes +=
-		    clib_net_to_host_u16 (ip0->payload_length);
-		  p0->counter.bytes += header_size;
+		  pi0 = p0 - im->policies;
+		  vlib_increment_combined_counter
+		    (&ipsec_spd_policy_counters,
+		     thread_index, pi0, 1,
+		     clib_net_to_host_u16 (ip0->payload_length) +
+		     header_size);
 		  vnet_buffer (b0)->ipsec.sad_index = p0->sa_index;
 		  vnet_buffer (b0)->ipsec.flags = 0;
 		  next0 = im->ah6_decrypt_next_index;
 		  goto trace0;
 		}
+	      else
+		{
+		  pi0 = ~0;
+		}
 	    }
 
 	trace0:
-	  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+	  if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE) &&
+	      PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
 	    {
 	      ipsec_input_trace_t *tr =
 		vlib_add_trace (vm, node, b0, sizeof (*tr));
-	      if (ip0->protocol == IP_PROTOCOL_IPSEC_ESP)
-		{
-		  if (p0)
-		    tr->sa_id = p0->sa_id;
-		  tr->spi = clib_host_to_net_u32 (esp0->spi);
-		  tr->seq = clib_host_to_net_u32 (esp0->seq);
-		  tr->spd = spd0->id;
-		}
+
+	      if (p0)
+		tr->sa_id = p0->sa_id;
+	      tr->proto = ip0->protocol;
+	      tr->spi = clib_net_to_host_u32 (esp0->spi);
+	      tr->seq = clib_net_to_host_u32 (esp0->seq);
+	      tr->spd = spd0->id;
 	    }
 
 	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
diff --git a/src/vnet/ipsec/ipsec_output.c b/src/vnet/ipsec/ipsec_output.c
index 2ab98e7e140..40561269237 100644
--- a/src/vnet/ipsec/ipsec_output.c
+++ b/src/vnet/ipsec/ipsec_output.c
@@ -48,6 +48,7 @@ static char *ipsec_output_error_strings[] = {
 typedef struct
 {
   u32 spd_id;
+  u32 policy_id;
 } ipsec_output_trace_t;
 
 /* packet trace format function */
@@ -58,14 +59,8 @@ format_ipsec_output_trace (u8 * s, va_list * args)
   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
   ipsec_output_trace_t *t = va_arg (*args, ipsec_output_trace_t *);
 
-  if (t->spd_id != ~0)
-    {
-      s = format (s, "spd %u ", t->spd_id);
-    }
-  else
-    {
-      s = format (s, "no spd");
-    }
+  s = format (s, "spd %u policy %d", t->spd_id, t->policy_id);
+
   return s;
 }
 
@@ -73,15 +68,16 @@ always_inline ipsec_policy_t *
 ipsec_output_policy_match (ipsec_spd_t * spd, u8 pr, u32 la, u32 ra, u16 lp,
 			   u16 rp)
 {
+  ipsec_main_t *im = &ipsec_main;
   ipsec_policy_t *p;
   u32 *i;
 
   if (!spd)
     return 0;
 
-  vec_foreach (i, spd->ipv4_outbound_policies)
+  vec_foreach (i, spd->policies[IPSEC_SPD_POLICY_IP4_OUTBOUND])
   {
-    p = pool_elt_at_index (spd->policies, *i);
+    p = pool_elt_at_index (im->policies, *i);
     if (PREDICT_FALSE (p->protocol && (p->protocol != pr)))
       continue;
 
@@ -134,15 +130,16 @@ ipsec6_output_policy_match (ipsec_spd_t * spd,
 			    ip6_address_t * la,
 			    ip6_address_t * ra, u16 lp, u16 rp, u8 pr)
 {
+  ipsec_main_t *im = &ipsec_main;
   ipsec_policy_t *p;
   u32 *i;
 
   if (!spd)
     return 0;
 
-  vec_foreach (i, spd->ipv6_outbound_policies)
+  vec_foreach (i, spd->policies[IPSEC_SPD_POLICY_IP6_OUTBOUND])
   {
-    p = pool_elt_at_index (spd->policies, *i);
+    p = pool_elt_at_index (im->policies, *i);
     if (PREDICT_FALSE (p->protocol && (p->protocol != pr)))
       continue;
 
@@ -181,7 +178,7 @@ ipsec_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 {
   ipsec_main_t *im = &ipsec_main;
 
-  u32 *from, *to_next = 0;
+  u32 *from, *to_next = 0, thread_index;
   u32 n_left_from, sw_if_index0, last_sw_if_index = (u32) ~ 0;
   u32 next_node_index = (u32) ~ 0, last_next_node_index = (u32) ~ 0;
   vlib_frame_t *f = 0;
@@ -192,10 +189,11 @@ ipsec_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 
   from = vlib_frame_vector_args (from_frame);
   n_left_from = from_frame->n_vectors;
+  thread_index = vm->thread_index;
 
   while (n_left_from > 0)
     {
-      u32 bi0;
+      u32 bi0, pi0;
       vlib_buffer_t *b0;
       ipsec_policy_t *p0;
       ip4_header_t *ip0;
@@ -203,6 +201,7 @@ ipsec_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
       udp_header_t *udp0;
       u32 iph_offset = 0;
       tcp_header_t *tcp0;
+      u64 bytes0;
 
       bi0 = from[0];
       b0 = vlib_get_buffer (vm, bi0);
@@ -271,6 +270,21 @@ ipsec_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 
       if (PREDICT_TRUE (p0 != NULL))
 	{
+	  pi0 = p0 - im->policies;
+
+	  vlib_prefetch_combined_counter (&ipsec_spd_policy_counters,
+					  thread_index, pi0);
+
+	  if (is_ipv6)
+	    {
+	      bytes0 = clib_net_to_host_u16 (ip6_0->payload_length);
+	      bytes0 += sizeof (ip6_header_t);
+	    }
+	  else
+	    {
+	      bytes0 = clib_net_to_host_u16 (ip0->length);
+	    }
+
 	  if (p0->policy == IPSEC_POLICY_ACTION_PROTECT)
 	    {
 	      ipsec_sa_t *sa = 0;
@@ -286,12 +300,9 @@ ipsec_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	      else
 		next_node_index = im->ah4_encrypt_node_index;
 	      vnet_buffer (b0)->ipsec.sad_index = p0->sa_index;
-	      p0->counter.packets++;
+
 	      if (is_ipv6)
 		{
-		  p0->counter.bytes +=
-		    clib_net_to_host_u16 (ip6_0->payload_length);
-		  p0->counter.bytes += sizeof (ip6_header_t);
 		  if (PREDICT_FALSE
 		      (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM))
 		    {
@@ -311,7 +322,6 @@ ipsec_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 		}
 	      else
 		{
-		  p0->counter.bytes += clib_net_to_host_u16 (ip0->length);
 		  if (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
 		    {
 		      ip0->checksum = ip4_header_checksum (ip0);
@@ -338,37 +348,18 @@ ipsec_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	    {
 	      nc_bypass++;
 	      next_node_index = get_next_output_feature_node_index (b0, node);
-	      p0->counter.packets++;
-	      if (is_ipv6)
-		{
-		  p0->counter.bytes +=
-		    clib_net_to_host_u16 (ip6_0->payload_length);
-		  p0->counter.bytes += sizeof (ip6_header_t);
-		}
-	      else
-		{
-		  p0->counter.bytes += clib_net_to_host_u16 (ip0->length);
-		}
 	    }
 	  else
 	    {
 	      nc_discard++;
-	      p0->counter.packets++;
-	      if (is_ipv6)
-		{
-		  p0->counter.bytes +=
-		    clib_net_to_host_u16 (ip6_0->payload_length);
-		  p0->counter.bytes += sizeof (ip6_header_t);
-		}
-	      else
-		{
-		  p0->counter.bytes += clib_net_to_host_u16 (ip0->length);
-		}
 	      next_node_index = im->error_drop_node_index;
 	    }
+	  vlib_increment_combined_counter
+	    (&ipsec_spd_policy_counters, thread_index, pi0, 1, bytes0);
 	}
       else
 	{
+	  pi0 = ~0;
 	  nc_nomatch++;
 	  next_node_index = im->error_drop_node_index;
 	}
@@ -397,12 +388,14 @@ ipsec_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
       to_next += 1;
       f->n_vectors++;
 
-      if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+      if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE) &&
+	  PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
 	{
 	  ipsec_output_trace_t *tr =
 	    vlib_add_trace (vm, node, b0, sizeof (*tr));
 	  if (spd0)
 	    tr->spd_id = spd0->id;
+	  tr->policy_id = pi0;
 	}
     }
 
diff --git a/src/vnet/ipsec/ipsec_sa.c b/src/vnet/ipsec/ipsec_sa.c
index a76197b9f50..d439b4d46d3 100644
--- a/src/vnet/ipsec/ipsec_sa.c
+++ b/src/vnet/ipsec/ipsec_sa.c
@@ -86,19 +86,16 @@ u8
 ipsec_is_sa_used (u32 sa_index)
 {
   ipsec_main_t *im = &ipsec_main;
-  ipsec_spd_t *spd;
-  ipsec_policy_t *p;
   ipsec_tunnel_if_t *t;
+  ipsec_policy_t *p;
 
   /* *INDENT-OFF* */
-  pool_foreach(spd, im->spds, ({
-    pool_foreach(p, spd->policies, ({
-      if (p->policy == IPSEC_POLICY_ACTION_PROTECT)
-        {
-          if (p->sa_index == sa_index)
-            return 1;
-        }
-    }));
+  pool_foreach(p, im->policies, ({
+     if (p->policy == IPSEC_POLICY_ACTION_PROTECT)
+       {
+         if (p->sa_index == sa_index)
+           return 1;
+       }
   }));
 
   pool_foreach(t, im->tunnel_interfaces, ({
diff --git a/src/vnet/ipsec/ipsec_spd.c b/src/vnet/ipsec/ipsec_spd.c
index 7e17bb91fdb..19525b206a6 100644
--- a/src/vnet/ipsec/ipsec_spd.c
+++ b/src/vnet/ipsec/ipsec_spd.c
@@ -42,12 +42,10 @@ ipsec_add_del_spd (vlib_main_t * vm, u32 spd_id, int is_add)
       }));
       /* *INDENT-ON* */
       hash_unset (im->spd_index_by_spd_id, spd_id);
-      pool_free (spd->policies);
-      vec_free (spd->ipv4_outbound_policies);
-      vec_free (spd->ipv6_outbound_policies);
-      vec_free (spd->ipv4_inbound_protect_policy_indices);
-      vec_free (spd->ipv4_inbound_policy_discard_and_bypass_indices);
-      pool_put (im->spds, spd);
+#define _(s,v) vec_free(spd->policies[IPSEC_SPD_POLICY_##s]);
+      foreach_ipsec_spd_policy_type
+#undef _
+	pool_put (im->spds, spd);
     }
   else				/* create new SPD */
     {
diff --git a/src/vnet/ipsec/ipsec_spd.h b/src/vnet/ipsec/ipsec_spd.h
index 854076ece6d..dd09041b046 100644
--- a/src/vnet/ipsec/ipsec_spd.h
+++ b/src/vnet/ipsec/ipsec_spd.h
@@ -17,23 +17,34 @@
 
 #include <vlib/vlib.h>
 
-struct ipsec_policy_t_;
+#define foreach_ipsec_spd_policy_type                 \
+  _(IP4_OUTBOUND, "ip4-outbound")                     \
+  _(IP6_OUTBOUND, "ip6-outbound")                     \
+  _(IP4_INBOUND_PROTECT, "ip4-inbound-protect")       \
+  _(IP6_INBOUND_PROTECT, "ip6-inbound-protect")       \
+  _(IP4_INBOUND_BYPASS,  "ip4-inbound-bypass")        \
+  _(IP6_INBOUND_BYPASS,  "ip6-inbound-bypass")
+
+typedef enum ipsec_spd_policy_t_
+{
+#define _(s,v) IPSEC_SPD_POLICY_##s,
+  foreach_ipsec_spd_policy_type
+#undef _
+    IPSEC_SPD_POLICY_N_TYPES,
+} ipsec_spd_policy_t;
+
+#define FOR_EACH_IPSEC_SPD_POLICY_TYPE(_t)      \
+  for (_t = 0; _t < IPSEC_SPD_POLICY_N_TYPES; _t++)
 
 /**
  * @brief A Secruity Policy Database
  */
-typedef struct ipsec_spd_t_
+typedef struct
 {
+  /** the User's ID for this policy */
   u32 id;
-  /* pool of policies */
-  struct ipsec_policy_t_ *policies;
-  /* vectors of policy indices */
-  u32 *ipv4_outbound_policies;
-  u32 *ipv6_outbound_policies;
-  u32 *ipv4_inbound_protect_policy_indices;
-  u32 *ipv4_inbound_policy_discard_and_bypass_indices;
-  u32 *ipv6_inbound_protect_policy_indices;
-  u32 *ipv6_inbound_policy_discard_and_bypass_indices;
+  /** vectors for each of the policy types */
+  u32 *policies[IPSEC_SPD_POLICY_N_TYPES];
 } ipsec_spd_t;
 
 /**
@@ -47,6 +58,8 @@ extern int ipsec_add_del_spd (vlib_main_t * vm, u32 spd_id, int is_add);
 extern int ipsec_set_interface_spd (vlib_main_t * vm,
 				    u32 sw_if_index, u32 spd_id, int is_add);
 
+extern u8 *format_ipsec_spd (u8 * s, va_list * args);
+
 #endif /* __IPSEC_SPD_H__ */
 
 /*
diff --git a/src/vnet/ipsec/ipsec_spd_policy.c b/src/vnet/ipsec/ipsec_spd_policy.c
index 0a576595a5a..5ad147b934f 100644
--- a/src/vnet/ipsec/ipsec_spd_policy.c
+++ b/src/vnet/ipsec/ipsec_spd_policy.c
@@ -15,16 +15,81 @@
 
 #include <vnet/ipsec/ipsec.h>
 
+/**
+ * @brief
+ * Policy packet & bytes counters
+ */
+vlib_combined_counter_main_t ipsec_spd_policy_counters = {
+  .name = "policy",
+  .stat_segment_name = "/net/ipsec/policy",
+};
+
+static int
+ipsec_policy_is_equal (ipsec_policy_t * p1, ipsec_policy_t * p2)
+{
+  if (p1->priority != p2->priority)
+    return 0;
+  if (p1->is_outbound != p2->is_outbound)
+    return (0);
+  if (p1->policy != p2->policy)
+    return (0);
+  if (p1->sa_id != p2->sa_id)
+    return (0);
+  if (p1->protocol != p2->protocol)
+    return (0);
+  if (p1->lport.start != p2->lport.start)
+    return (0);
+  if (p1->lport.stop != p2->lport.stop)
+    return (0);
+  if (p1->rport.start != p2->rport.start)
+    return (0);
+  if (p1->rport.stop != p2->rport.stop)
+    return (0);
+  if (p1->is_ipv6 != p2->is_ipv6)
+    return (0);
+  if (p2->is_ipv6)
+    {
+      if (p1->laddr.start.ip6.as_u64[0] != p2->laddr.start.ip6.as_u64[0])
+	return (0);
+      if (p1->laddr.start.ip6.as_u64[1] != p2->laddr.start.ip6.as_u64[1])
+	return (0);
+      if (p1->laddr.stop.ip6.as_u64[0] != p2->laddr.stop.ip6.as_u64[0])
+	return (0);
+      if (p1->laddr.stop.ip6.as_u64[1] != p2->laddr.stop.ip6.as_u64[1])
+	return (0);
+      if (p1->raddr.start.ip6.as_u64[0] != p2->raddr.start.ip6.as_u64[0])
+	return (0);
+      if (p1->raddr.start.ip6.as_u64[1] != p2->raddr.start.ip6.as_u64[1])
+	return (0);
+      if (p1->raddr.stop.ip6.as_u64[0] != p2->raddr.stop.ip6.as_u64[0])
+	return (0);
+      if (p1->laddr.stop.ip6.as_u64[1] != p2->laddr.stop.ip6.as_u64[1])
+	return (0);
+    }
+  else
+    {
+      if (p1->laddr.start.ip4.as_u32 != p2->laddr.start.ip4.as_u32)
+	return (0);
+      if (p1->laddr.stop.ip4.as_u32 != p2->laddr.stop.ip4.as_u32)
+	return (0);
+      if (p1->raddr.start.ip4.as_u32 != p2->raddr.start.ip4.as_u32)
+	return (0);
+      if (p1->raddr.stop.ip4.as_u32 != p2->raddr.stop.ip4.as_u32)
+	return (0);
+    }
+  return (1);
+}
+
 static int
 ipsec_spd_entry_sort (void *a1, void *a2)
 {
+  ipsec_main_t *im = &ipsec_main;
   u32 *id1 = a1;
   u32 *id2 = a2;
-  ipsec_spd_t *spd = ipsec_main.spd_to_sort;
   ipsec_policy_t *p1, *p2;
 
-  p1 = pool_elt_at_index (spd->policies, *id1);
-  p2 = pool_elt_at_index (spd->policies, *id2);
+  p1 = pool_elt_at_index (im->policies, *id1);
+  p2 = pool_elt_at_index (im->policies, *id2);
   if (p1 && p2)
     return p2->priority - p1->priority;
 
@@ -32,13 +97,14 @@ ipsec_spd_entry_sort (void *a1, void *a2)
 }
 
 int
-ipsec_add_del_policy (vlib_main_t * vm, ipsec_policy_t * policy, int is_add)
+ipsec_add_del_policy (vlib_main_t * vm,
+		      ipsec_policy_t * policy, int is_add, u32 * stat_index)
 {
   ipsec_main_t *im = &ipsec_main;
   ipsec_spd_t *spd = 0;
   ipsec_policy_t *vp;
-  uword *p;
   u32 spd_index;
+  uword *p;
 
   clib_warning ("policy-id %u priority %d is_outbound %u", policy->id,
 		policy->priority, policy->is_outbound);
@@ -65,24 +131,30 @@ ipsec_add_del_policy (vlib_main_t * vm, ipsec_policy_t * policy, int is_add)
     {
       u32 policy_index;
 
-      pool_get (spd->policies, vp);
+      pool_get (im->policies, vp);
       clib_memcpy (vp, policy, sizeof (*vp));
-      policy_index = vp - spd->policies;
+      policy_index = vp - im->policies;
 
-      ipsec_main.spd_to_sort = spd;
+      vlib_validate_combined_counter (&ipsec_spd_policy_counters,
+				      policy_index);
+      vlib_zero_combined_counter (&ipsec_spd_policy_counters, policy_index);
 
       if (policy->is_outbound)
 	{
 	  if (policy->is_ipv6)
 	    {
-	      vec_add1 (spd->ipv6_outbound_policies, policy_index);
-	      vec_sort_with_function (spd->ipv6_outbound_policies,
+	      vec_add1 (spd->policies[IPSEC_SPD_POLICY_IP6_OUTBOUND],
+			policy_index);
+	      vec_sort_with_function (spd->policies
+				      [IPSEC_SPD_POLICY_IP6_OUTBOUND],
 				      ipsec_spd_entry_sort);
 	    }
 	  else
 	    {
-	      vec_add1 (spd->ipv4_outbound_policies, policy_index);
-	      vec_sort_with_function (spd->ipv4_outbound_policies,
+	      vec_add1 (spd->policies[IPSEC_SPD_POLICY_IP4_OUTBOUND],
+			policy_index);
+	      vec_sort_with_function (spd->policies
+				      [IPSEC_SPD_POLICY_IP4_OUTBOUND],
 				      ipsec_spd_entry_sort);
 	    }
 	}
@@ -92,19 +164,20 @@ ipsec_add_del_policy (vlib_main_t * vm, ipsec_policy_t * policy, int is_add)
 	    {
 	      if (policy->policy == IPSEC_POLICY_ACTION_PROTECT)
 		{
-		  vec_add1 (spd->ipv6_inbound_protect_policy_indices,
+		  vec_add1 (spd->policies
+			    [IPSEC_SPD_POLICY_IP6_INBOUND_PROTECT],
 			    policy_index);
-		  vec_sort_with_function
-		    (spd->ipv6_inbound_protect_policy_indices,
-		     ipsec_spd_entry_sort);
+		  vec_sort_with_function (spd->policies
+					  [IPSEC_SPD_POLICY_IP6_INBOUND_PROTECT],
+					  ipsec_spd_entry_sort);
 		}
 	      else
 		{
 		  vec_add1
-		    (spd->ipv6_inbound_policy_discard_and_bypass_indices,
+		    (spd->policies[IPSEC_SPD_POLICY_IP6_INBOUND_BYPASS],
 		     policy_index);
 		  vec_sort_with_function
-		    (spd->ipv6_inbound_policy_discard_and_bypass_indices,
+		    (spd->policies[IPSEC_SPD_POLICY_IP6_INBOUND_BYPASS],
 		     ipsec_spd_entry_sort);
 		}
 	    }
@@ -112,146 +185,45 @@ ipsec_add_del_policy (vlib_main_t * vm, ipsec_policy_t * policy, int is_add)
 	    {
 	      if (policy->policy == IPSEC_POLICY_ACTION_PROTECT)
 		{
-		  vec_add1 (spd->ipv4_inbound_protect_policy_indices,
+		  vec_add1 (spd->policies
+			    [IPSEC_SPD_POLICY_IP4_INBOUND_PROTECT],
 			    policy_index);
-		  vec_sort_with_function
-		    (spd->ipv4_inbound_protect_policy_indices,
-		     ipsec_spd_entry_sort);
+		  vec_sort_with_function (spd->policies
+					  [IPSEC_SPD_POLICY_IP4_INBOUND_PROTECT],
+					  ipsec_spd_entry_sort);
 		}
 	      else
 		{
 		  vec_add1
-		    (spd->ipv4_inbound_policy_discard_and_bypass_indices,
+		    (spd->policies[IPSEC_SPD_POLICY_IP4_INBOUND_BYPASS],
 		     policy_index);
 		  vec_sort_with_function
-		    (spd->ipv4_inbound_policy_discard_and_bypass_indices,
+		    (spd->policies[IPSEC_SPD_POLICY_IP4_INBOUND_BYPASS],
 		     ipsec_spd_entry_sort);
 		}
 	    }
 	}
-
-      ipsec_main.spd_to_sort = NULL;
+      *stat_index = policy_index;
     }
   else
     {
-      u32 i, j;
-      /* *INDENT-OFF* */
-      pool_foreach_index(i, spd->policies, ({
-        vp = pool_elt_at_index(spd->policies, i);
-        if (vp->priority != policy->priority)
-          continue;
-        if (vp->is_outbound != policy->is_outbound)
-          continue;
-        if (vp->policy != policy->policy)
-          continue;
-        if (vp->sa_id != policy->sa_id)
-          continue;
-        if (vp->protocol != policy->protocol)
-          continue;
-        if (vp->lport.start != policy->lport.start)
-          continue;
-        if (vp->lport.stop != policy->lport.stop)
-          continue;
-        if (vp->rport.start != policy->rport.start)
-          continue;
-        if (vp->rport.stop != policy->rport.stop)
-          continue;
-        if (vp->is_ipv6 != policy->is_ipv6)
-          continue;
-        if (policy->is_ipv6)
-          {
-            if (vp->laddr.start.ip6.as_u64[0] != policy->laddr.start.ip6.as_u64[0])
-              continue;
-            if (vp->laddr.start.ip6.as_u64[1] != policy->laddr.start.ip6.as_u64[1])
-              continue;
-            if (vp->laddr.stop.ip6.as_u64[0] != policy->laddr.stop.ip6.as_u64[0])
-              continue;
-            if (vp->laddr.stop.ip6.as_u64[1] != policy->laddr.stop.ip6.as_u64[1])
-              continue;
-            if (vp->raddr.start.ip6.as_u64[0] != policy->raddr.start.ip6.as_u64[0])
-              continue;
-            if (vp->raddr.start.ip6.as_u64[1] != policy->raddr.start.ip6.as_u64[1])
-              continue;
-            if (vp->raddr.stop.ip6.as_u64[0] != policy->raddr.stop.ip6.as_u64[0])
-              continue;
-           if (vp->laddr.stop.ip6.as_u64[1] != policy->laddr.stop.ip6.as_u64[1])
-              continue;
-           if (policy->is_outbound)
-             {
-               vec_foreach_index(j, spd->ipv6_outbound_policies) {
-                 if (vec_elt(spd->ipv6_outbound_policies, j) == i) {
-                   vec_del1 (spd->ipv6_outbound_policies, j);
-                   break;
-                 }
-               }
-             }
-           else
-             {
-               if (policy->policy == IPSEC_POLICY_ACTION_PROTECT)
-                 {
-                   vec_foreach_index(j, spd->ipv6_inbound_protect_policy_indices) {
-                     if (vec_elt(spd->ipv6_inbound_protect_policy_indices, j) == i) {
-                       vec_del1 (spd->ipv6_inbound_protect_policy_indices, j);
-                       break;
-                     }
-                   }
-                 }
-               else
-                 {
-                   vec_foreach_index(j, spd->ipv6_inbound_policy_discard_and_bypass_indices) {
-                     if (vec_elt(spd->ipv6_inbound_policy_discard_and_bypass_indices, j) == i) {
-                       vec_del1 (spd->ipv6_inbound_policy_discard_and_bypass_indices, j);
-                       break;
-                     }
-                   }
-                 }
-             }
-          }
-        else
-          {
-            if (vp->laddr.start.ip4.as_u32 != policy->laddr.start.ip4.as_u32)
-              continue;
-            if (vp->laddr.stop.ip4.as_u32 != policy->laddr.stop.ip4.as_u32)
-              continue;
-            if (vp->raddr.start.ip4.as_u32 != policy->raddr.start.ip4.as_u32)
-              continue;
-            if (vp->raddr.stop.ip4.as_u32 != policy->raddr.stop.ip4.as_u32)
-              continue;
-            if (policy->is_outbound)
-              {
-                vec_foreach_index(j, spd->ipv4_outbound_policies) {
-                  if (vec_elt(spd->ipv4_outbound_policies, j) == i) {
-                    vec_del1 (spd->ipv4_outbound_policies, j);
-                    break;
-                  }
-                }
-              }
-            else
-              {
-                if (policy->policy == IPSEC_POLICY_ACTION_PROTECT)
-                  {
-                    vec_foreach_index(j, spd->ipv4_inbound_protect_policy_indices) {
-                      if (vec_elt(spd->ipv4_inbound_protect_policy_indices, j) == i) {
-                        vec_del1 (spd->ipv4_inbound_protect_policy_indices, j);
-                        break;
-                      }
-                    }
-                  }
-                else
-                  {
-                    vec_foreach_index(j, spd->ipv4_inbound_policy_discard_and_bypass_indices) {
-                      if (vec_elt(spd->ipv4_inbound_policy_discard_and_bypass_indices, j) == i) {
-                        vec_del1 (spd->ipv4_inbound_policy_discard_and_bypass_indices, j);
-                        break;
-                      }
-                    }
-                  }
-              }
-          }
-          pool_put (spd->policies, vp);
-          break;
-      }));
-      /* *INDENT-ON* */
+      ipsec_spd_policy_t ptype;
+      u32 ii;
+
+      FOR_EACH_IPSEC_SPD_POLICY_TYPE (ptype)
+      {
+	vec_foreach_index (ii, (spd->policies[ptype]))
+	{
+	  vp = pool_elt_at_index (im->policies, spd->policies[ptype][ii]);
+	  if (ipsec_policy_is_equal (vp, policy))
+	    {
+	      vec_del1 (spd->policies[ptype], ii);
+	      pool_put (im->policies, vp);
+	      goto done;
+	    }
+	}
+      }
+    done:;
     }
 
   return 0;
diff --git a/src/vnet/ipsec/ipsec_spd_policy.h b/src/vnet/ipsec/ipsec_spd_policy.h
index fee059e0e56..40fad342978 100644
--- a/src/vnet/ipsec/ipsec_spd_policy.h
+++ b/src/vnet/ipsec/ipsec_spd_policy.h
@@ -43,6 +43,12 @@ typedef struct
 } port_range_t;
 
 /**
+ * @brief
+ * Policy packet & bytes counters
+ */
+extern vlib_combined_counter_main_t ipsec_spd_policy_counters;
+
+/**
  * @brief A Secruity Policy. An entry in an SPD
  */
 typedef struct ipsec_policy_t_
@@ -63,17 +69,16 @@ typedef struct ipsec_policy_t_
   ipsec_policy_action_t policy;
   u32 sa_id;
   u32 sa_index;
-
-  // Counter
-  vlib_counter_t counter;
 } ipsec_policy_t;
 
 /**
  * @brief Add/Delete a SPD
  */
 extern int ipsec_add_del_policy (vlib_main_t * vm,
-				 ipsec_policy_t * policy, int is_add);
+				 ipsec_policy_t * policy,
+				 int is_add, u32 * stat_index);
 
+extern u8 *format_ipsec_policy (u8 * s, va_list * args);
 extern u8 *format_ipsec_policy_action (u8 * s, va_list * args);
 extern uword unformat_ipsec_policy_action (unformat_input_t * input,
 					   va_list * args);