ipsec: Use bihash for tunnel lookup

Type: improvement

Change-Id: I0c82722dfce990345fe6eeecdb335678543367e0
Signed-off-by: Neale Ranns <nranns@cisco.com>

8 files changed, 282 insertions, 123 deletions

diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c
index d6ef8c842e7..8c65560c3dd 100644
--- a/src/vnet/ip/ip6_forward.c
+++ b/src/vnet/ip/ip6_forward.c
@@ -55,6 +55,7 @@
 #include <vppinfra/bihash_template.c>
 #endif
 #include <vnet/ip/ip6_forward.h>
+#include <vnet/ipsec/ipsec_tun.h>
 #include <vnet/interface_output.h>
 
 /* Flag used by IOAM code. Classifier sets it pop-hop-by-hop checks it */
diff --git a/src/vnet/ipsec/ipsec.h b/src/vnet/ipsec/ipsec.h
index 3d84ad3f6ad..fe44b80b321 100644
--- a/src/vnet/ipsec/ipsec.h
+++ b/src/vnet/ipsec/ipsec.h
@@ -26,6 +26,10 @@
 #include <vnet/ipsec/ipsec_spd_policy.h>
 #include <vnet/ipsec/ipsec_sa.h>
 
+#include <vppinfra/bihash_8_8.h>
+
+#include <vppinfra/bihash_24_8.h>
+
 typedef clib_error_t *(*add_del_sa_sess_cb_t) (u32 sa_index, u8 is_add);
 typedef clib_error_t *(*check_support_cb_t) (ipsec_sa_t * sa);
 typedef clib_error_t *(*enable_disable_cb_t) (int is_enable);
@@ -125,8 +129,9 @@ typedef struct
   uword *ipsec6_if_pool_index_by_key;
   uword *ipsec_if_real_dev_by_show_dev;
   uword *ipsec_if_by_sw_if_index;
-  uword *tun4_protect_by_key;
-  uword *tun6_protect_by_key;
+
+  clib_bihash_8_8_t tun4_protect_by_key;
+  clib_bihash_24_8_t tun6_protect_by_key;
 
   /* node indices */
   u32 error_drop_node_index;
diff --git a/src/vnet/ipsec/ipsec_cli.c b/src/vnet/ipsec/ipsec_cli.c
index 0d1ab033aec..937e0f6b2e4 100644
--- a/src/vnet/ipsec/ipsec_cli.c
+++ b/src/vnet/ipsec/ipsec_cli.c
@@ -1037,6 +1037,28 @@ VLIB_CLI_COMMAND (ipsec_tun_protect_show_node, static) =
 };
 /* *INDENT-ON* */
 
+static int
+ipsec_tun_protect4_hash_show_one (clib_bihash_kv_8_8_t * kv, void *arg)
+{
+  ipsec4_tunnel_kv_t *ikv = (ipsec4_tunnel_kv_t *) kv;
+  vlib_main_t *vm = arg;
+
+  vlib_cli_output (vm, " %U", format_ipsec4_tunnel_kv, ikv);
+
+  return (BIHASH_WALK_CONTINUE);
+}
+
+static int
+ipsec_tun_protect6_hash_show_one (clib_bihash_kv_24_8_t * kv, void *arg)
+{
+  ipsec6_tunnel_kv_t *ikv = (ipsec6_tunnel_kv_t *) kv;
+  vlib_main_t *vm = arg;
+
+  vlib_cli_output (vm, " %U", format_ipsec6_tunnel_kv, ikv);
+
+  return (BIHASH_WALK_CONTINUE);
+}
+
 static clib_error_t *
 ipsec_tun_protect_hash_show (vlib_main_t * vm,
 			     unformat_input_t * input,
@@ -1045,33 +1067,15 @@ ipsec_tun_protect_hash_show (vlib_main_t * vm,
   ipsec_main_t *im = &ipsec_main;
 
   {
-    ipsec_tun_lkup_result_t value;
-    ipsec4_tunnel_key_t key;
-
     vlib_cli_output (vm, "IPv4:");
 
-    /* *INDENT-OFF* */
-    hash_foreach(key.as_u64, value.as_u64, im->tun4_protect_by_key,
-    ({
-      vlib_cli_output (vm, " %U", format_ipsec4_tunnel_key, &key);
-      vlib_cli_output (vm, "  tun:%d sa:%d", value.tun_index, value.sa_index);
-    }));
-    /* *INDENT-ON* */
-  }
-
-  {
-    ipsec_tun_lkup_result_t value;
-    ipsec6_tunnel_key_t *key;
+    clib_bihash_foreach_key_value_pair_8_8
+      (&im->tun4_protect_by_key, ipsec_tun_protect4_hash_show_one, vm);
 
     vlib_cli_output (vm, "IPv6:");
 
-    /* *INDENT-OFF* */
-    hash_foreach_mem(key, value.as_u64, im->tun6_protect_by_key,
-    ({
-      vlib_cli_output (vm, " %U", format_ipsec6_tunnel_key, key);
-      vlib_cli_output (vm, "  tun:%d sa:%d", value.tun_index, value.sa_index);
-    }));
-    /* *INDENT-ON* */
+    clib_bihash_foreach_key_value_pair_24_8
+      (&im->tun6_protect_by_key, ipsec_tun_protect6_hash_show_one, vm);
   }
 
   return NULL;
diff --git a/src/vnet/ipsec/ipsec_format.c b/src/vnet/ipsec/ipsec_format.c
index e3c6f22bf00..8fdc844aec3 100644
--- a/src/vnet/ipsec/ipsec_format.c
+++ b/src/vnet/ipsec/ipsec_format.c
@@ -396,27 +396,33 @@ format_ipsec_tun_protect (u8 * s, va_list * args)
 }
 
 u8 *
-format_ipsec4_tunnel_key (u8 * s, va_list * args)
+format_ipsec4_tunnel_kv (u8 * s, va_list * args)
 {
-  ipsec4_tunnel_key_t *key = va_arg (*args, ipsec4_tunnel_key_t *);
+  ipsec4_tunnel_kv_t *kv = va_arg (*args, ipsec4_tunnel_kv_t *);
+  ip4_address_t ip;
+  u32 spi;
 
-  s = format (s, "remote:%U spi:%u (0x%08x)",
-	      format_ip4_address, &key->remote_ip,
-	      clib_net_to_host_u32 (key->spi),
-	      clib_net_to_host_u32 (key->spi));
+  ipsec4_tunnel_extract_key (kv, &ip, &spi);
+
+  s = format (s, "remote:%U spi:%u (0x%08x) sa:%d tun:%d",
+	      format_ip4_address, &ip,
+	      clib_net_to_host_u32 (spi),
+	      clib_net_to_host_u32 (spi),
+	      kv->value.sa_index, kv->value.tun_index);
 
   return (s);
 }
 
 u8 *
-format_ipsec6_tunnel_key (u8 * s, va_list * args)
+format_ipsec6_tunnel_kv (u8 * s, va_list * args)
 {
-  ipsec6_tunnel_key_t *key = va_arg (*args, ipsec6_tunnel_key_t *);
+  ipsec6_tunnel_kv_t *kv = va_arg (*args, ipsec6_tunnel_kv_t *);
 
-  s = format (s, "remote:%U spi:%u (0x%08x)",
-	      format_ip6_address, &key->remote_ip,
-	      clib_net_to_host_u32 (key->spi),
-	      clib_net_to_host_u32 (key->spi));
+  s = format (s, "remote:%U spi:%u (0x%08x) sa:%d tun:%d",
+	      format_ip6_address, &kv->key.remote_ip,
+	      clib_net_to_host_u32 (kv->key.spi),
+	      clib_net_to_host_u32 (kv->key.spi),
+	      kv->value.sa_index, kv->value.tun_index);
 
   return (s);
 }
diff --git a/src/vnet/ipsec/ipsec_tun.c b/src/vnet/ipsec/ipsec_tun.c
index 9d352e75075..e2e1a3e405b 100644
--- a/src/vnet/ipsec/ipsec_tun.c
+++ b/src/vnet/ipsec/ipsec_tun.c
@@ -23,6 +23,9 @@
 #include <vnet/adj/adj_midchain.h>
 #include <vnet/teib/teib.h>
 
+#define IPSEC_TUN_DEFAULT_HASH_NUM_BUCKETS (64 * 1024)
+#define IPSEC_TUN_DEFAULT_HASH_MEMORY_SIZE 512 << 20
+
 /**
  * The logger
  */
@@ -228,20 +231,40 @@ ipsec_tun_protect_rx_db_add (ipsec_main_t * im,
        */
       if (ip46_address_is_ip4 (&itp->itp_crypto.dst))
         {
-          ipsec4_tunnel_key_t key = {
-            .remote_ip = itp->itp_crypto.dst.ip4,
-            .spi = clib_host_to_net_u32 (sa->spi),
+          ipsec4_tunnel_kv_t key = {
+            .value = res,
           };
-          hash_set (im->tun4_protect_by_key, key.as_u64, res.as_u64);
+          clib_bihash_kv_8_8_t *bkey = (clib_bihash_kv_8_8_t*)&key;
+
+          ipsec4_tunnel_mk_key(&key, &itp->itp_crypto.dst.ip4,
+                               clib_host_to_net_u32 (sa->spi));
+
+          if (!im->tun4_protect_by_key.nbuckets)
+              clib_bihash_init_8_8 (&im->tun4_protect_by_key,
+			"IPSec IPv4 tunnels",
+			IPSEC_TUN_DEFAULT_HASH_NUM_BUCKETS,
+			IPSEC_TUN_DEFAULT_HASH_MEMORY_SIZE);
+
+          clib_bihash_add_del_8_8 (&im->tun4_protect_by_key, bkey, 1);
           ipsec_tun_register_nodes(AF_IP4);
         }
       else
         {
-          ipsec6_tunnel_key_t key = {
-            .remote_ip = itp->itp_crypto.dst.ip6,
-            .spi = clib_host_to_net_u32 (sa->spi),
+          ipsec6_tunnel_kv_t key = {
+            .key = {
+              .remote_ip = itp->itp_crypto.dst.ip6,
+              .spi = clib_host_to_net_u32 (sa->spi),
+            },
+            .value = res,
           };
-          hash_set_mem_alloc (&im->tun6_protect_by_key, &key, res.as_u64);
+          clib_bihash_kv_24_8_t *bkey = (clib_bihash_kv_24_8_t*)&key;
+
+          if (!im->tun4_protect_by_key.nbuckets)
+            clib_bihash_init_24_8 (&im->tun6_protect_by_key,
+                                   "IPSec IPv6 tunnels",
+                                   IPSEC_TUN_DEFAULT_HASH_NUM_BUCKETS,
+                                   IPSEC_TUN_DEFAULT_HASH_MEMORY_SIZE);
+          clib_bihash_add_del_24_8 (&im->tun6_protect_by_key, bkey, 1);
           ipsec_tun_register_nodes(AF_IP6);
         }
   }))
@@ -330,25 +353,31 @@ ipsec_tun_protect_rx_db_remove (ipsec_main_t * im,
   ({
     if (ip46_address_is_ip4 (&itp->itp_crypto.dst))
       {
-        ipsec4_tunnel_key_t key = {
-          .remote_ip = itp->itp_crypto.dst.ip4,
-          .spi = clib_host_to_net_u32 (sa->spi),
-        };
-        if (hash_get(im->tun4_protect_by_key, key.as_u64))
-          {
-            hash_unset (im->tun4_protect_by_key, key.as_u64);
-            ipsec_tun_unregister_nodes(AF_IP4);
-          }
+          ipsec4_tunnel_kv_t key;
+          clib_bihash_kv_8_8_t res, *bkey = (clib_bihash_kv_8_8_t*)&key;
+
+          ipsec4_tunnel_mk_key(&key, &itp->itp_crypto.dst.ip4,
+                               clib_host_to_net_u32 (sa->spi));
+
+          if (!clib_bihash_search_8_8 (&im->tun4_protect_by_key, bkey, &res))
+            {
+              clib_bihash_add_del_8_8 (&im->tun4_protect_by_key, bkey, 0);
+              ipsec_tun_unregister_nodes(AF_IP4);
+            }
       }
     else
       {
-        ipsec6_tunnel_key_t key = {
-          .remote_ip = itp->itp_crypto.dst.ip6,
-          .spi = clib_host_to_net_u32 (sa->spi),
+        ipsec6_tunnel_kv_t key = {
+          .key = {
+            .remote_ip = itp->itp_crypto.dst.ip6,
+            .spi = clib_host_to_net_u32 (sa->spi),
+          },
         };
-        if (hash_get_mem(im->tun6_protect_by_key, &key))
+        clib_bihash_kv_24_8_t res, *bkey = (clib_bihash_kv_24_8_t*)&key;
+
+        if (!clib_bihash_search_24_8 (&im->tun6_protect_by_key, bkey, &res))
           {
-            hash_unset_mem_free (&im->tun6_protect_by_key, &key);
+            clib_bihash_add_del_24_8 (&im->tun6_protect_by_key, bkey, 0);
             ipsec_tun_unregister_nodes(AF_IP6);
           }
       }
@@ -933,6 +962,20 @@ const static teib_vft_t ipsec_tun_teib_vft = {
   .nv_deleted = ipsec_tun_teib_entry_deleted,
 };
 
+void
+ipsec_tun_table_init (ip_address_family_t af, uword table_size, u32 n_buckets)
+{
+  ipsec_main_t *im;
+
+  im = &ipsec_main;
+
+  if (AF_IP4 == af)
+    clib_bihash_init_8_8 (&im->tun4_protect_by_key,
+			  "IPSec IPv4 tunnels", n_buckets, table_size);
+  else
+    clib_bihash_init_24_8 (&im->tun6_protect_by_key,
+			   "IPSec IPv6 tunnels", n_buckets, table_size);
+}
 
 clib_error_t *
 ipsec_tunnel_protect_init (vlib_main_t * vm)
@@ -940,10 +983,14 @@ ipsec_tunnel_protect_init (vlib_main_t * vm)
   ipsec_main_t *im;
 
   im = &ipsec_main;
-  im->tun6_protect_by_key = hash_create_mem (0,
-					     sizeof (ipsec6_tunnel_key_t),
-					     sizeof (u64));
-  im->tun4_protect_by_key = hash_create (0, sizeof (u64));
+  clib_bihash_init_24_8 (&im->tun6_protect_by_key,
+			 "IPSec IPv6 tunnels",
+			 IPSEC_TUN_DEFAULT_HASH_NUM_BUCKETS,
+			 IPSEC_TUN_DEFAULT_HASH_MEMORY_SIZE);
+  clib_bihash_init_8_8 (&im->tun4_protect_by_key,
+			"IPSec IPv4 tunnels",
+			IPSEC_TUN_DEFAULT_HASH_NUM_BUCKETS,
+			IPSEC_TUN_DEFAULT_HASH_MEMORY_SIZE);
 
   /* set up feature nodes to drop outbound packets with no crypto alg set */
   im->esp4_no_crypto_tun_node_index =
@@ -967,6 +1014,55 @@ ipsec_tunnel_protect_init (vlib_main_t * vm)
 
 VLIB_INIT_FUNCTION (ipsec_tunnel_protect_init);
 
+static clib_error_t *
+ipsec_config (vlib_main_t * vm, unformat_input_t * input)
+{
+  unformat_input_t sub_input;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "ip4 %U", unformat_vlib_cli_sub_input, &sub_input))
+	{
+	  uword table_size = ~0;
+	  u32 n_buckets = ~0;
+
+	  while (unformat_check_input (&sub_input) != UNFORMAT_END_OF_INPUT)
+	    {
+	      if (unformat (&sub_input, "num-buckets %u", &n_buckets))
+		;
+	      else
+		return clib_error_return (0, "unknown input `%U'",
+					  format_unformat_error, &sub_input);
+	    }
+
+	  ipsec_tun_table_init (AF_IP4, table_size, n_buckets);
+	}
+      else if (unformat (input, "ip6 %U", unformat_vlib_cli_sub_input,
+			 &sub_input))
+	{
+	  uword table_size = ~0;
+	  u32 n_buckets = ~0;
+
+	  while (unformat_check_input (&sub_input) != UNFORMAT_END_OF_INPUT)
+	    {
+	      if (unformat (&sub_input, "num-buckets %u", &n_buckets))
+		;
+	      else
+		return clib_error_return (0, "unknown input `%U'",
+					  format_unformat_error, &sub_input);
+	    }
+
+	  ipsec_tun_table_init (AF_IP6, table_size, n_buckets);
+	}
+      else
+	return clib_error_return (0, "unknown input `%U'",
+				  format_unformat_error, input);
+    }
+
+  return 0;
+}
+
+VLIB_CONFIG_FUNCTION (ipsec_config, "ipsec");
 
 /*
  * fd.io coding-style-patch-verification: ON
diff --git a/src/vnet/ipsec/ipsec_tun.h b/src/vnet/ipsec/ipsec_tun.h
index c5fbe59f2b6..0d911a2876c 100644
--- a/src/vnet/ipsec/ipsec_tun.h
+++ b/src/vnet/ipsec/ipsec_tun.h
@@ -17,35 +17,72 @@
 
 #include <vnet/ipsec/ipsec.h>
 
-/* *INDENT-OFF* */
-typedef CLIB_PACKED(struct {
+/**
+ * result of a lookup in the protection bihash
+ */
+typedef struct ipsec_tun_lkup_result_t_
+{
+  union
+  {
+    struct
+    {
+      u32 tun_index;
+      u32 sa_index;
+    };
+    u64 as_u64;
+  };
+} ipsec_tun_lkup_result_t;
+
+typedef struct ipsec4_tunnel_kv_t
+{
   /*
    * Key fields: remote ip and spi on incoming packet
    * all fields in NET byte order
    */
-  union {
-    struct {
-      ip4_address_t remote_ip;
-      u32 spi;
-    };
-    u64 as_u64;
-  };
-}) ipsec4_tunnel_key_t;
-/* *INDENT-ON* */
+  u64 key;
+  ipsec_tun_lkup_result_t value;
+} __clib_packed ipsec4_tunnel_kv_t;
+
+STATIC_ASSERT_SIZEOF (ipsec4_tunnel_kv_t, sizeof (clib_bihash_kv_8_8_t));
+STATIC_ASSERT_OFFSET_OF (ipsec4_tunnel_kv_t, value,
+			 STRUCT_OFFSET_OF (clib_bihash_kv_8_8_t, value));
+
+static inline void
+ipsec4_tunnel_mk_key (ipsec4_tunnel_kv_t * k,
+		      const ip4_address_t * ip, u32 spi)
+{
+  k->key = (((u64) ip->as_u32) << 32 | spi);
+}
+
+static inline void
+ipsec4_tunnel_extract_key (const ipsec4_tunnel_kv_t * k,
+			   ip4_address_t * ip, u32 * spi)
+{
+  *spi = (u32) k->key;
+  (*ip).as_u32 = k->key >> 32;
+}
 
-/* *INDENT-OFF* */
-typedef CLIB_PACKED(struct {
+typedef struct ipsec6_tunnel_kv_t_
+{
   /*
    * Key fields: remote ip and spi on incoming packet
    * all fields in NET byte order
    */
-  ip6_address_t remote_ip;
-  u32 spi;
-}) ipsec6_tunnel_key_t;
-/* *INDENT-ON* */
+  struct
+  {
+    ip6_address_t remote_ip;
+    u32 spi;
+    u32 __pad;
+  } key;
+  ipsec_tun_lkup_result_t value;
+} __clib_packed ipsec6_tunnel_kv_t;
 
-extern u8 *format_ipsec4_tunnel_key (u8 * s, va_list * args);
-extern u8 *format_ipsec6_tunnel_key (u8 * s, va_list * args);
+STATIC_ASSERT_SIZEOF (ipsec6_tunnel_kv_t, sizeof (clib_bihash_kv_24_8_t));
+STATIC_ASSERT_OFFSET_OF (ipsec6_tunnel_kv_t, value,
+			 STRUCT_OFFSET_OF (clib_bihash_kv_24_8_t, value));
+
+extern u8 *format_ipsec4_tunnel_kv (u8 * s, va_list * args);
+extern u8 *format_ipsec6_tunnel_kv (u8 * s, va_list * args);
 
 #define foreach_ipsec_protect_flags \
   _(L2, 1, "l2")                    \
@@ -136,6 +173,9 @@ extern u8 *format_ipsec_tun_protect_index (u8 * s, va_list * args);
 extern void ipsec_tun_register_nodes (ip_address_family_t af);
 extern void ipsec_tun_unregister_nodes (ip_address_family_t af);
 
+extern void ipsec_tun_table_init (ip_address_family_t af,
+				  uword table_size, u32 n_buckets);
+
 // FIXME
 extern vlib_node_registration_t ipsec4_tun_input_node;
 extern vlib_node_registration_t ipsec6_tun_input_node;
@@ -145,19 +185,6 @@ extern vlib_node_registration_t ipsec6_tun_input_node;
  */
 extern ipsec_tun_protect_t *ipsec_tun_protect_pool;
 
-typedef struct ipsec_tun_lkup_result_t_
-{
-  union
-  {
-    struct
-    {
-      u32 tun_index;
-      u32 sa_index;
-    };
-    u64 as_u64;
-  };
-} ipsec_tun_lkup_result_t;
-
 always_inline ipsec_tun_protect_t *
 ipsec_tun_protect_get (u32 index)
 {
diff --git a/src/vnet/ipsec/ipsec_tun_in.c b/src/vnet/ipsec/ipsec_tun_in.c
index fd0c3b08f19..804c7299c6f 100644
--- a/src/vnet/ipsec/ipsec_tun_in.c
+++ b/src/vnet/ipsec/ipsec_tun_in.c
@@ -62,12 +62,11 @@ typedef struct
 {
   union
   {
-    ipsec4_tunnel_key_t key4;
-    ipsec6_tunnel_key_t key6;
+    ipsec4_tunnel_kv_t kv4;
+    ipsec6_tunnel_kv_t kv6;
   };
   u8 is_ip6;
   u32 seq;
-  u32 sa_index;
 } ipsec_tun_protect_input_trace_t;
 
 static u8 *
@@ -79,11 +78,11 @@ format_ipsec_tun_protect_input_trace (u8 * s, va_list * args)
     va_arg (*args, ipsec_tun_protect_input_trace_t *);
 
   if (t->is_ip6)
-    s = format (s, "IPSec: %U seq %u sa %d",
-		format_ipsec6_tunnel_key, &t->key6, t->seq, t->sa_index);
+    s = format (s, "IPSec: %U seq %u",
+		format_ipsec6_tunnel_kv, &t->kv6, t->seq);
   else
     s = format (s, "IPSec: %U seq %u sa %d",
-		format_ipsec4_tunnel_key, &t->key4, t->seq, t->sa_index);
+		format_ipsec4_tunnel_kv, &t->kv4, t->seq);
   return s;
 }
 
@@ -148,8 +147,8 @@ ipsec_tun_protect_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
   ipsec_tun_lkup_result_t last_result = {
     .tun_index = ~0
   };
-  ipsec4_tunnel_key_t last_key4;
-  ipsec6_tunnel_key_t last_key6;
+  ipsec4_tunnel_kv_t last_key4;
+  ipsec6_tunnel_kv_t last_key6;
 
   vlib_combined_counter_main_t *rx_counter;
   vlib_combined_counter_main_t *drop_counter;
@@ -158,7 +157,7 @@ ipsec_tun_protect_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
   if (is_ip6)
     clib_memset (&last_key6, 0xff, sizeof (last_key6));
   else
-    last_key4.as_u64 = ~0;
+    last_key4.key = ~0;
 
   rx_counter = vim->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX;
   drop_counter = vim->combined_sw_if_counters + VNET_INTERFACE_COUNTER_DROP;
@@ -167,8 +166,10 @@ ipsec_tun_protect_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
     {
       u32 sw_if_index0, len0, hdr_sz0;
       ipsec_tun_lkup_result_t itr0;
-      ipsec4_tunnel_key_t key40;
-      ipsec6_tunnel_key_t key60;
+      clib_bihash_kv_24_8_t bkey60;
+      clib_bihash_kv_8_8_t bkey40;
+      ipsec4_tunnel_kv_t *key40;
+      ipsec6_tunnel_kv_t *key60;
       ip4_header_t *ip40;
       ip6_header_t *ip60;
       esp_header_t *esp0;
@@ -177,6 +178,9 @@ ipsec_tun_protect_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
       ip40 =
 	(ip4_header_t *) (b[0]->data + vnet_buffer (b[0])->l3_hdr_offset);
 
+      key60 = (ipsec6_tunnel_kv_t *) & bkey60;
+      key40 = (ipsec4_tunnel_kv_t *) & bkey40;
+
       if (is_ip6)
 	{
 	  ip60 = (ip6_header_t *) ip40;
@@ -221,21 +225,24 @@ ipsec_tun_protect_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 
       if (is_ip6)
 	{
-	  key60.remote_ip = ip60->src_address;
-	  key60.spi = esp0->spi;
+	  key60->key.remote_ip = ip60->src_address;
+	  key60->key.spi = esp0->spi;
+	  key60->key.__pad = 0;
 
-	  if (memcmp (&key60, &last_key6, sizeof (last_key6)) == 0)
+	  if (memcmp (key60, &last_key6, sizeof (last_key6)) == 0)
 	    {
 	      itr0 = last_result;
 	    }
 	  else
 	    {
-	      uword *p = hash_get_mem (im->tun6_protect_by_key, &key60);
-	      if (p)
+	      int rv =
+		clib_bihash_search_inline_24_8 (&im->tun6_protect_by_key,
+						&bkey60);
+	      if (!rv)
 		{
-		  itr0.as_u64 = p[0];
+		  itr0.as_u64 = bkey60.value;
 		  last_result = itr0;
-		  clib_memcpy_fast (&last_key6, &key60, sizeof (key60));
+		  clib_memcpy_fast (&last_key6, key60, sizeof (last_key6));
 		}
 	      else
 		{
@@ -247,21 +254,22 @@ ipsec_tun_protect_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	}
       else
 	{
-	  key40.remote_ip = ip40->src_address;
-	  key40.spi = esp0->spi;
+	  ipsec4_tunnel_mk_key (key40, &ip40->src_address, esp0->spi);
 
-	  if (key40.as_u64 == last_key4.as_u64)
+	  if (key40->key == last_key4.key)
 	    {
 	      itr0 = last_result;
 	    }
 	  else
 	    {
-	      uword *p = hash_get (im->tun4_protect_by_key, key40.as_u64);
-	      if (p)
+	      int rv =
+		clib_bihash_search_inline_8_8 (&im->tun4_protect_by_key,
+					       &bkey40);
+	      if (!rv)
 		{
-		  itr0.as_u64 = p[0];
+		  itr0.as_u64 = bkey40.value;
 		  last_result = itr0;
-		  last_key4.as_u64 = key40.as_u64;
+		  last_key4.key = key40->key;
 		}
 	      else
 		{
@@ -310,7 +318,8 @@ ipsec_tun_protect_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	      n_bytes = len0;
 	    }
 
-	  next[0] = im->esp4_decrypt_tun_next_index;	//IPSEC_TUN_PROTECT_NEXT_DECRYPT;
+	  //IPSEC_TUN_PROTECT_NEXT_DECRYPT;
+	  next[0] = im->esp4_decrypt_tun_next_index;
 	}
     trace00:
       if (PREDICT_FALSE (is_trace))
@@ -320,13 +329,12 @@ ipsec_tun_protect_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	      ipsec_tun_protect_input_trace_t *tr =
 		vlib_add_trace (vm, node, b[0], sizeof (*tr));
 	      if (is_ip6)
-		clib_memcpy (&tr->key6, &key60, sizeof (tr->key6));
+		clib_memcpy (&tr->kv6, &bkey60, sizeof (tr->kv6));
 	      else
-		clib_memcpy (&tr->key4, &key40, sizeof (tr->key4));
+		clib_memcpy (&tr->kv4, &bkey40, sizeof (tr->kv4));
 	      tr->is_ip6 = is_ip6;
 	      tr->seq = (len0 >= sizeof (*esp0) ?
 			 clib_host_to_net_u32 (esp0->seq) : ~0);
-	      tr->sa_index = vnet_buffer (b[0])->ipsec.sad_index;
 	    }
 	}
 
diff --git a/src/vpp/conf/startup.conf b/src/vpp/conf/startup.conf
index eb2cf09c889..32bfea04e91 100644
--- a/src/vpp/conf/startup.conf
+++ b/src/vpp/conf/startup.conf
@@ -211,3 +211,15 @@ cpu {
     ## l2fib hash table number of buckets. Must be power of 2.
     #  num-buckets 524288
 # }
+
+## ipsec
+# {
+   # ip4 {
+   ## ipsec for ipv4 tunnel lookup hash number of buckets.
+   #  num-buckets 524288
+   # }
+   # ip6 {
+   ## ipsec for ipv6 tunnel lookup hash number of buckets.
+   #  num-buckets 524288
+   # }
+# }