VPP-158: VPP crashes in IKEv2 code when running multithreaded

Change tunnel interface creation to be done from the main thread instead
of a worker thread by calling vl_api_rpc_call_main_thread.

Make per-thread copies of volatile elements in ikev2_main.

Change-Id: I4cda8aaa392a04c2aea2d50a52a07933cf40c016
Signed-off-by: Matthew Smith <mgsmith@netgate.com>

6 files changed, 222 insertions, 148 deletions

diff --git a/vnet/vnet/ipsec/ikev2.c b/vnet/vnet/ipsec/ikev2.c
index 17b297b8217..8455c1263d6 100644
--- a/vnet/vnet/ipsec/ikev2.c
+++ b/vnet/vnet/ipsec/ikev2.c
@@ -292,15 +292,16 @@ static void
 ikev2_delete_sa(ikev2_sa_t *sa)
 {
   ikev2_main_t * km = &ikev2_main;
+  u32 cpu_index = os_get_cpu_number();
   uword * p;
 
   ikev2_sa_free_all_vec(sa);
 
-  p = hash_get(km->sa_by_rspi, sa->rspi);
+  p = hash_get(km->per_thread_data[cpu_index].sa_by_rspi, sa->rspi);
   if (p)
     {
-      hash_unset(km->sa_by_rspi, sa->rspi);
-      pool_put(km->sas, sa);
+      hash_unset(km->per_thread_data[cpu_index].sa_by_rspi, sa->rspi);
+      pool_put(km->per_thread_data[cpu_index].sas, sa);
     }
 }
 
@@ -603,24 +604,25 @@ ikev2_initial_contact_cleanup (ikev2_sa_t * sa)
   ikev2_sa_t * tmp;
   u32 i, * delete = 0;
   ikev2_child_sa_t * c;
+  u32 cpu_index = os_get_cpu_number();
 
   if (!sa->initial_contact)
     return;
 
   /* find old IKE SAs with the same authenticated identity */
-  pool_foreach (tmp, km->sas, ({
+  pool_foreach (tmp, km->per_thread_data[cpu_index].sas, ({
         if (tmp->i_id.type != sa->i_id.type ||
             vec_len(tmp->i_id.data) != vec_len(sa->i_id.data) ||
             memcmp(sa->i_id.data, tmp->i_id.data, vec_len(sa->i_id.data)))
           continue;
 
         if (sa->rspi != tmp->rspi)
-          vec_add1(delete, tmp - km->sas);
+          vec_add1(delete, tmp - km->per_thread_data[cpu_index].sas);
   }));
 
   for (i = 0; i < vec_len(delete); i++)
     {
-      tmp = pool_elt_at_index(km->sas, delete[i]);
+      tmp = pool_elt_at_index(km->per_thread_data[cpu_index].sas, delete[i]);
       vec_foreach(c, tmp->childs)
         ikev2_delete_tunnel_interface(km->vnet_main, tmp, c);
       ikev2_delete_sa(tmp);
@@ -1153,7 +1155,6 @@ ikev2_create_tunnel_interface(vnet_main_t * vnm, ikev2_sa_t *sa, ikev2_child_sa_
 {
   ipsec_add_del_tunnel_args_t a;
   ikev2_sa_transform_t * tr;
-  u32 hw_if_index;
   u8  encr_type = 0;
 
   if (!child->r_proposals)
@@ -1162,6 +1163,7 @@ ikev2_create_tunnel_interface(vnet_main_t * vnm, ikev2_sa_t *sa, ikev2_child_sa_
       return 1;
     }
 
+  memset(&a, 0, sizeof(a));
   a.is_add = 1;
   a.local_ip.as_u32 = sa->raddr.as_u32;
   a.remote_ip.as_u32 = sa->iaddr.as_u32;
@@ -1224,36 +1226,21 @@ ikev2_create_tunnel_interface(vnet_main_t * vnm, ikev2_sa_t *sa, ikev2_child_sa_
       return 1;
     }
 
-  hw_if_index = ipsec_add_del_tunnel_if(vnm, &a);
-  if (hw_if_index == VNET_API_ERROR_INVALID_VALUE)
-    {
-      clib_warning("create tunnel interface failed remote-ip %U remote-spi %u",
-                   format_ip4_address, &sa->raddr, child->r_proposals[0].spi);
-      ikev2_set_state(sa, IKEV2_STATE_DELETED);
-      return hw_if_index;
-    }
-
   ikev2_calc_child_keys(sa, child);
 
-  ipsec_set_interface_key(vnm, hw_if_index,
-                          IPSEC_IF_SET_KEY_TYPE_LOCAL_CRYPTO,
-                          encr_type,
-                          child->sk_er);
-
-  ipsec_set_interface_key(vnm, hw_if_index,
-                          IPSEC_IF_SET_KEY_TYPE_REMOTE_CRYPTO,
-                          encr_type,
-                          child->sk_ei);
+  a.integ_alg = IPSEC_INTEG_ALG_SHA1_96;
+  a.local_integ_key_len = vec_len(child->sk_ar);
+  clib_memcpy(a.local_integ_key, child->sk_ar, a.local_integ_key_len);
+  a.remote_integ_key_len = vec_len(child->sk_ai);
+  clib_memcpy(a.remote_integ_key, child->sk_ai, a.remote_integ_key_len);
 
-  ipsec_set_interface_key(vnm, hw_if_index,
-                          IPSEC_IF_SET_KEY_TYPE_LOCAL_INTEG,
-                          IPSEC_INTEG_ALG_SHA1_96,
-                          child->sk_ar);
+  a.crypto_alg = encr_type;
+  a.local_crypto_key_len = vec_len(child->sk_er);
+  clib_memcpy(a.local_crypto_key, child->sk_er, a.local_crypto_key_len);
+  a.remote_crypto_key_len = vec_len(child->sk_ei);
+  clib_memcpy(a.remote_crypto_key, child->sk_ei, a.remote_crypto_key_len);
 
-  ipsec_set_interface_key(vnm, hw_if_index,
-                          IPSEC_IF_SET_KEY_TYPE_REMOTE_INTEG,
-                          IPSEC_INTEG_ALG_SHA1_96,
-                          child->sk_ai);
+  ipsec_add_del_tunnel_if(&a);
 
   return 0;
 }
@@ -1272,7 +1259,8 @@ ikev2_delete_tunnel_interface(vnet_main_t * vnm, ikev2_sa_t *sa, ikev2_child_sa_
   a.local_spi = child->i_proposals[0].spi;
   a.remote_spi = child->r_proposals[0].spi;
 
-  return ipsec_add_del_tunnel_if(vnm, &a);
+  ipsec_add_del_tunnel_if(&a);
+  return 0;
 }
 
 static u32
@@ -1505,8 +1493,9 @@ ikev2_retransmit_sa_init (ike_header_t * ike,
 {
   ikev2_main_t * km = &ikev2_main;
   ikev2_sa_t * sa;
+  u32 cpu_index = os_get_cpu_number();
 
-  pool_foreach (sa, km->sas, ({
+  pool_foreach (sa, km->per_thread_data[cpu_index].sas, ({
     if (sa->ispi == clib_net_to_host_u64(ike->ispi) &&
         sa->iaddr.as_u32 == iaddr.as_u32 &&
         sa->raddr.as_u32 == raddr.as_u32)
@@ -1617,6 +1606,7 @@ ikev2_node_fn (vlib_main_t * vm,
   u32 n_left_from, * from, * to_next;
   ikev2_next_t next_index;
   ikev2_main_t * km = &ikev2_main;
+  u32 cpu_index = os_get_cpu_number();
 
   from = vlib_frame_vector_args (frame);
   n_left_from = frame->n_vectors;
@@ -1710,9 +1700,11 @@ ikev2_node_fn (vlib_main_t * vm,
                   if (sa0->state == IKEV2_STATE_SA_INIT)
                     {
                       /* add SA to the pool */
-                      pool_get (km->sas, sa0);
+                      pool_get (km->per_thread_data[cpu_index].sas, sa0);
                       clib_memcpy(sa0, &sa, sizeof(*sa0));
-                      hash_set (km->sa_by_rspi, sa0->rspi, sa0 - km->sas);
+                      hash_set (km->per_thread_data[cpu_index].sa_by_rspi,
+                                sa0->rspi,
+                                sa0 - km->per_thread_data[cpu_index].sas);
                     }
                   else
                     {
@@ -1723,10 +1715,12 @@ ikev2_node_fn (vlib_main_t * vm,
           else if (ike0->exchange == IKEV2_EXCHANGE_IKE_AUTH)
             {
               uword * p;
-              p = hash_get(km->sa_by_rspi, clib_net_to_host_u64(ike0->rspi));
+              p = hash_get(km->per_thread_data[cpu_index].sa_by_rspi,
+                           clib_net_to_host_u64(ike0->rspi));
               if (p)
                 {
-                  sa0 = pool_elt_at_index (km->sas, p[0]);
+                  sa0 = pool_elt_at_index (km->per_thread_data[cpu_index].sas,
+                                           p[0]);
 
                   r = ikev2_retransmit_resp(sa0, ike0);
                   if (r == 1)
@@ -1761,10 +1755,12 @@ ikev2_node_fn (vlib_main_t * vm,
           else if (ike0->exchange == IKEV2_EXCHANGE_INFORMATIONAL)
             {
               uword * p;
-              p = hash_get(km->sa_by_rspi, clib_net_to_host_u64(ike0->rspi));
+              p = hash_get(km->per_thread_data[cpu_index].sa_by_rspi,
+                           clib_net_to_host_u64(ike0->rspi));
               if (p)
                 {
-                  sa0 = pool_elt_at_index (km->sas, p[0]);
+                  sa0 = pool_elt_at_index (km->per_thread_data[cpu_index].sas,
+                                           p[0]);
 
                   r = ikev2_retransmit_resp(sa0, ike0);
                   if (r == 1)
@@ -1814,10 +1810,12 @@ ikev2_node_fn (vlib_main_t * vm,
           else if (ike0->exchange == IKEV2_EXCHANGE_CREATE_CHILD_SA)
             {
               uword * p;
-              p = hash_get(km->sa_by_rspi, clib_net_to_host_u64(ike0->rspi));
+              p = hash_get(km->per_thread_data[cpu_index].sa_by_rspi,
+                           clib_net_to_host_u64(ike0->rspi));
               if (p)
                 {
-                  sa0 = pool_elt_at_index (km->sas, p[0]);
+                  sa0 = pool_elt_at_index (km->per_thread_data[cpu_index].sas,
+                                           p[0]);
 
                   r = ikev2_retransmit_resp(sa0, ike0);
                   if (r == 1)
@@ -2092,6 +2090,8 @@ ikev2_init (vlib_main_t * vm)
 {
   ikev2_main_t * km = &ikev2_main;
   clib_error_t * error;
+  vlib_thread_main_t * tm = vlib_get_thread_main();
+  int thread_id;
 
   memset (km, 0, sizeof (ikev2_main_t));
   km->vnet_main = vnet_get_main();
@@ -2099,9 +2099,15 @@ ikev2_init (vlib_main_t * vm)
 
   ikev2_crypto_init(km);
 
-  km->sa_by_rspi = hash_create (0, sizeof (uword));
   mhash_init_vec_string (&km->profile_index_by_name, sizeof (uword));
 
+  vec_validate(km->per_thread_data, tm->n_vlib_mains-1);
+  for (thread_id = 0; thread_id < tm->n_vlib_mains - 1; thread_id++)
+    {
+      km->per_thread_data[thread_id].sa_by_rspi =
+          hash_create (0, sizeof (uword));
+    }
+
   if ((error = vlib_call_init_function (vm, ikev2_cli_init)))
     return error;
 
diff --git a/vnet/vnet/ipsec/ikev2_cli.c b/vnet/vnet/ipsec/ikev2_cli.c
index e42f16d8f18..c8c8ea140b5 100644
--- a/vnet/vnet/ipsec/ikev2_cli.c
+++ b/vnet/vnet/ipsec/ikev2_cli.c
@@ -50,106 +50,109 @@ show_ikev2_sa_command_fn (vlib_main_t * vm,
                           vlib_cli_command_t * cmd)
 {
   ikev2_main_t * km = &ikev2_main;
+  ikev2_main_per_thread_data_t * tkm;
   ikev2_sa_t * sa;
   ikev2_ts_t * ts;
   ikev2_child_sa_t * child;
   ikev2_sa_transform_t * tr;
 
-  pool_foreach (sa, km->sas, ({
-    u8 * s = 0;
-    vlib_cli_output(vm, " iip %U ispi %lx rip %U rspi %lx",
-                    format_ip4_address, &sa->iaddr, sa->ispi,
-                    format_ip4_address, &sa->raddr, sa->rspi);
-
-     tr = ikev2_sa_get_td_for_type(sa->r_proposals, IKEV2_TRANSFORM_TYPE_ENCR);
-     s = format(s, "%U ", format_ikev2_sa_transform, tr);
-
-     tr = ikev2_sa_get_td_for_type(sa->r_proposals, IKEV2_TRANSFORM_TYPE_PRF);
-     s = format(s, "%U ", format_ikev2_sa_transform, tr);
-
-     tr = ikev2_sa_get_td_for_type(sa->r_proposals, IKEV2_TRANSFORM_TYPE_INTEG);
-     s = format(s, "%U ", format_ikev2_sa_transform, tr);
-
-     tr = ikev2_sa_get_td_for_type(sa->r_proposals, IKEV2_TRANSFORM_TYPE_DH);
-     s = format(s, "%U ", format_ikev2_sa_transform, tr);
-
-    vlib_cli_output(vm, " %v", s);
-    vec_free(s);
-
-    vlib_cli_output(vm, "  nonce i:%U\n        r:%U",
-                    format_hex_bytes, sa->i_nonce,  vec_len(sa->i_nonce),
-                    format_hex_bytes, sa->r_nonce,  vec_len(sa->r_nonce));
-
-    vlib_cli_output(vm, "  SK_d    %U",
-                    format_hex_bytes, sa->sk_d,  vec_len(sa->sk_d));
-    vlib_cli_output(vm, "  SK_a  i:%U\n        r:%U",
-                    format_hex_bytes, sa->sk_ai, vec_len(sa->sk_ai),
-                    format_hex_bytes, sa->sk_ar, vec_len(sa->sk_ar));
-    vlib_cli_output(vm, "  SK_e  i:%U\n        r:%U",
-                    format_hex_bytes, sa->sk_ei, vec_len(sa->sk_ei),
-                    format_hex_bytes, sa->sk_er, vec_len(sa->sk_er));
-    vlib_cli_output(vm, "  SK_p  i:%U\n        r:%U",
-                    format_hex_bytes, sa->sk_pi, vec_len(sa->sk_pi),
-                    format_hex_bytes, sa->sk_pr, vec_len(sa->sk_pr));
-
-    vlib_cli_output(vm, "  identifier (i) %U",
-                    format_ikev2_id_type_and_data, &sa->i_id);
-    vlib_cli_output(vm, "  identifier (r) %U",
-                    format_ikev2_id_type_and_data, &sa->r_id);
-
-    vec_foreach(child, sa->childs)
-      {
-        vlib_cli_output(vm, "  child sa %u:", child - sa->childs);
-
-        tr = ikev2_sa_get_td_for_type(child->r_proposals, IKEV2_TRANSFORM_TYPE_ENCR);
-        s = format(s, "%U ", format_ikev2_sa_transform, tr);
-
-        tr = ikev2_sa_get_td_for_type(child->r_proposals, IKEV2_TRANSFORM_TYPE_INTEG);
-        s = format(s, "%U ", format_ikev2_sa_transform, tr);
-
-        tr = ikev2_sa_get_td_for_type(child->r_proposals, IKEV2_TRANSFORM_TYPE_ESN);
-        s = format(s, "%U ", format_ikev2_sa_transform, tr);
-
-        vlib_cli_output(vm, "    %v", s);
-        vec_free(s);
-
-        vlib_cli_output(vm, "    spi(i) %lx spi(r) %lx",
-                        child->i_proposals ? child->i_proposals[0].spi : 0,
-                        child->r_proposals ? child->r_proposals[0].spi : 0);
-
-        vlib_cli_output(vm, "    SK_e  i:%U\n          r:%U",
-                        format_hex_bytes, child->sk_ei, vec_len(child->sk_ei),
-                        format_hex_bytes, child->sk_er, vec_len(child->sk_er));
-        vlib_cli_output(vm, "    SK_a  i:%U\n          r:%U",
-                        format_hex_bytes, child->sk_ai, vec_len(child->sk_ai),
-                        format_hex_bytes, child->sk_ar, vec_len(child->sk_ar));
-        vlib_cli_output(vm, "    traffic selectors (i):");
-        vec_foreach(ts, child->tsi)
-          {
-            vlib_cli_output(vm, "      %u type %u protocol_id %u addr "
-                            "%U - %U port %u - %u",
-                            ts - child->tsi,
-                            ts->ts_type, ts->protocol_id,
-                            format_ip4_address, &ts->start_addr,
-                            format_ip4_address, &ts->end_addr,
-                            clib_net_to_host_u16( ts->start_port),
-                            clib_net_to_host_u16( ts->end_port));
-          }
-        vlib_cli_output(vm, "    traffic selectors (r):");
-        vec_foreach(ts, child->tsr)
-          {
-            vlib_cli_output(vm, "      %u type %u protocol_id %u addr "
-                            "%U - %U port %u - %u",
-                            ts - child->tsr,
-                            ts->ts_type, ts->protocol_id,
-                            format_ip4_address, &ts->start_addr,
-                            format_ip4_address, &ts->end_addr,
-                            clib_net_to_host_u16( ts->start_port),
-                            clib_net_to_host_u16( ts->end_port));
-          }
-      }
-    vlib_cli_output(vm, "");
-  }));
+  vec_foreach(tkm, km->per_thread_data) {
+    pool_foreach (sa, tkm->sas, ({
+      u8 * s = 0;
+      vlib_cli_output(vm, " iip %U ispi %lx rip %U rspi %lx",
+                      format_ip4_address, &sa->iaddr, sa->ispi,
+                      format_ip4_address, &sa->raddr, sa->rspi);
+  
+       tr = ikev2_sa_get_td_for_type(sa->r_proposals, IKEV2_TRANSFORM_TYPE_ENCR);
+       s = format(s, "%U ", format_ikev2_sa_transform, tr);
+  
+       tr = ikev2_sa_get_td_for_type(sa->r_proposals, IKEV2_TRANSFORM_TYPE_PRF);
+       s = format(s, "%U ", format_ikev2_sa_transform, tr);
+  
+       tr = ikev2_sa_get_td_for_type(sa->r_proposals, IKEV2_TRANSFORM_TYPE_INTEG);
+       s = format(s, "%U ", format_ikev2_sa_transform, tr);
+  
+       tr = ikev2_sa_get_td_for_type(sa->r_proposals, IKEV2_TRANSFORM_TYPE_DH);
+       s = format(s, "%U ", format_ikev2_sa_transform, tr);
+  
+      vlib_cli_output(vm, " %v", s);
+      vec_free(s);
+  
+      vlib_cli_output(vm, "  nonce i:%U\n        r:%U",
+                      format_hex_bytes, sa->i_nonce,  vec_len(sa->i_nonce),
+                      format_hex_bytes, sa->r_nonce,  vec_len(sa->r_nonce));
+  
+      vlib_cli_output(vm, "  SK_d    %U",
+                      format_hex_bytes, sa->sk_d,  vec_len(sa->sk_d));
+      vlib_cli_output(vm, "  SK_a  i:%U\n        r:%U",
+                      format_hex_bytes, sa->sk_ai, vec_len(sa->sk_ai),
+                      format_hex_bytes, sa->sk_ar, vec_len(sa->sk_ar));
+      vlib_cli_output(vm, "  SK_e  i:%U\n        r:%U",
+                      format_hex_bytes, sa->sk_ei, vec_len(sa->sk_ei),
+                      format_hex_bytes, sa->sk_er, vec_len(sa->sk_er));
+      vlib_cli_output(vm, "  SK_p  i:%U\n        r:%U",
+                      format_hex_bytes, sa->sk_pi, vec_len(sa->sk_pi),
+                      format_hex_bytes, sa->sk_pr, vec_len(sa->sk_pr));
+  
+      vlib_cli_output(vm, "  identifier (i) %U",
+                      format_ikev2_id_type_and_data, &sa->i_id);
+      vlib_cli_output(vm, "  identifier (r) %U",
+                      format_ikev2_id_type_and_data, &sa->r_id);
+  
+      vec_foreach(child, sa->childs)
+        {
+          vlib_cli_output(vm, "  child sa %u:", child - sa->childs);
+  
+          tr = ikev2_sa_get_td_for_type(child->r_proposals, IKEV2_TRANSFORM_TYPE_ENCR);
+          s = format(s, "%U ", format_ikev2_sa_transform, tr);
+  
+          tr = ikev2_sa_get_td_for_type(child->r_proposals, IKEV2_TRANSFORM_TYPE_INTEG);
+          s = format(s, "%U ", format_ikev2_sa_transform, tr);
+  
+          tr = ikev2_sa_get_td_for_type(child->r_proposals, IKEV2_TRANSFORM_TYPE_ESN);
+          s = format(s, "%U ", format_ikev2_sa_transform, tr);
+  
+          vlib_cli_output(vm, "    %v", s);
+          vec_free(s);
+  
+          vlib_cli_output(vm, "    spi(i) %lx spi(r) %lx",
+                          child->i_proposals ? child->i_proposals[0].spi : 0,
+                          child->r_proposals ? child->r_proposals[0].spi : 0);
+  
+          vlib_cli_output(vm, "    SK_e  i:%U\n          r:%U",
+                          format_hex_bytes, child->sk_ei, vec_len(child->sk_ei),
+                          format_hex_bytes, child->sk_er, vec_len(child->sk_er));
+          vlib_cli_output(vm, "    SK_a  i:%U\n          r:%U",
+                          format_hex_bytes, child->sk_ai, vec_len(child->sk_ai),
+                          format_hex_bytes, child->sk_ar, vec_len(child->sk_ar));
+          vlib_cli_output(vm, "    traffic selectors (i):");
+          vec_foreach(ts, child->tsi)
+            {
+              vlib_cli_output(vm, "      %u type %u protocol_id %u addr "
+                              "%U - %U port %u - %u",
+                              ts - child->tsi,
+                              ts->ts_type, ts->protocol_id,
+                              format_ip4_address, &ts->start_addr,
+                              format_ip4_address, &ts->end_addr,
+                              clib_net_to_host_u16( ts->start_port),
+                              clib_net_to_host_u16( ts->end_port));
+            }
+          vlib_cli_output(vm, "    traffic selectors (r):");
+          vec_foreach(ts, child->tsr)
+            {
+              vlib_cli_output(vm, "      %u type %u protocol_id %u addr "
+                              "%U - %U port %u - %u",
+                              ts - child->tsr,
+                              ts->ts_type, ts->protocol_id,
+                              format_ip4_address, &ts->start_addr,
+                              format_ip4_address, &ts->end_addr,
+                              clib_net_to_host_u16( ts->start_port),
+                              clib_net_to_host_u16( ts->end_port));
+            }
+        }
+      vlib_cli_output(vm, "");
+    }));
+  }
   return 0;
 }
 
diff --git a/vnet/vnet/ipsec/ikev2_priv.h b/vnet/vnet/ipsec/ikev2_priv.h
index 4f05a60a621..731bb15e6b2 100644
--- a/vnet/vnet/ipsec/ikev2_priv.h
+++ b/vnet/vnet/ipsec/ikev2_priv.h
@@ -213,14 +213,18 @@ typedef struct {
     /* pool of IKEv2 Security Associations */
     ikev2_sa_t * sas;
 
+    /* hash */
+    uword * sa_by_rspi;
+} ikev2_main_per_thread_data_t;
+
+typedef struct {
     /* pool of IKEv2 profiles */
     ikev2_profile_t * profiles;
 
     /* vector of supported transform types */
     ikev2_sa_transform_t * supported_transforms;
 
-    /* hashes */
-    uword * sa_by_rspi;
+    /* hash */
     mhash_t profile_index_by_name;
 
     /* local private key */
@@ -229,6 +233,9 @@ typedef struct {
     /* convenience */
     vlib_main_t * vlib_main;
     vnet_main_t * vnet_main;
+
+    ikev2_main_per_thread_data_t * per_thread_data;
+
 } ikev2_main_t;
 
 ikev2_main_t ikev2_main;
diff --git a/vnet/vnet/ipsec/ipsec.h b/vnet/vnet/ipsec/ipsec.h
index 8cd2b41c92f..564a8e12fb8 100644
--- a/vnet/vnet/ipsec/ipsec.h
+++ b/vnet/vnet/ipsec/ipsec.h
@@ -107,6 +107,16 @@ typedef struct {
   ip4_address_t local_ip, remote_ip;
   u32 local_spi;
   u32 remote_spi;
+  ipsec_crypto_alg_t crypto_alg;
+  u8 local_crypto_key_len;
+  u8 local_crypto_key[128];
+  u8 remote_crypto_key_len;
+  u8 remote_crypto_key[128];
+  ipsec_integ_alg_t integ_alg;
+  u8 local_integ_key_len;
+  u8 local_integ_key[128];
+  u8 remote_integ_key_len;
+  u8 remote_integ_key[128];
 } ipsec_add_del_tunnel_args_t;
 
 typedef enum {
@@ -225,7 +235,8 @@ uword unformat_ipsec_policy_action (unformat_input_t * input, va_list * args);
 uword unformat_ipsec_crypto_alg (unformat_input_t * input, va_list * args);
 uword unformat_ipsec_integ_alg (unformat_input_t * input, va_list * args);
 
-u32 ipsec_add_del_tunnel_if (vnet_main_t * vnm, ipsec_add_del_tunnel_args_t * args);
+/*u32 ipsec_add_del_tunnel_if (vnet_main_t * vnm, ipsec_add_del_tunnel_args_t * args); */
+int ipsec_add_del_tunnel_if (ipsec_add_del_tunnel_args_t * args);
 int ipsec_set_interface_key(vnet_main_t * vnm, u32 hw_if_index, ipsec_if_set_key_type_t type, u8 alg, u8 * key);
 
 
diff --git a/vnet/vnet/ipsec/ipsec_cli.c b/vnet/vnet/ipsec/ipsec_cli.c
index 0205d8bc877..1d19e537b88 100644
--- a/vnet/vnet/ipsec/ipsec_cli.c
+++ b/vnet/vnet/ipsec/ipsec_cli.c
@@ -585,12 +585,11 @@ create_ipsec_tunnel_command_fn (vlib_main_t * vm,
 {
   unformat_input_t _line_input, * line_input = &_line_input;
   ipsec_add_del_tunnel_args_t a;
-  ipsec_main_t *im = &ipsec_main;
   int rv;
   u32 num_m_args = 0;
+
+  memset(&a, 0, sizeof(a));
   a.is_add = 1;
-  a.anti_replay = 0;
-  a.esn = 0;
 
   /* Get a line of input. */
   if (! unformat_user (input, unformat_line_input, line_input))
@@ -616,7 +615,7 @@ create_ipsec_tunnel_command_fn (vlib_main_t * vm,
   if (num_m_args < 4)
       return clib_error_return (0, "mandatory argument(s) missing");
 
-  rv = ipsec_add_del_tunnel_if (im->vnet_main, &a);
+  rv = ipsec_add_del_tunnel_if (&a);
 
  switch(rv)
     {
diff --git a/vnet/vnet/ipsec/ipsec_if.c b/vnet/vnet/ipsec/ipsec_if.c
index 30631b60587..7a85fb16781 100644
--- a/vnet/vnet/ipsec/ipsec_if.c
+++ b/vnet/vnet/ipsec/ipsec_if.c
@@ -21,6 +21,8 @@
 
 #include <vnet/ipsec/ipsec.h>
 
+void vl_api_rpc_call_main_thread (void *fp, u8 * data, u32 data_length);
+
 static u8 * format_ipsec_name (u8 * s, va_list * args)
 {
   u32 dev_instance = va_arg (*args, u32);
@@ -46,8 +48,30 @@ VNET_HW_INTERFACE_CLASS (ipsec_hw_class) = {
   .name = "IPSec",
 };
 
-u32
-ipsec_add_del_tunnel_if (vnet_main_t * vnm, ipsec_add_del_tunnel_args_t * args)
+
+static int
+ipsec_add_del_tunnel_if_internal (vnet_main_t * vnm,
+                                  ipsec_add_del_tunnel_args_t * args);
+
+static int
+ipsec_add_del_tunnel_if_rpc_callback (ipsec_add_del_tunnel_args_t *a)
+{
+  vnet_main_t * vnm = vnet_get_main();
+  ASSERT(os_get_cpu_number() == 0);
+
+  return ipsec_add_del_tunnel_if_internal(vnm, a);
+}
+
+int
+ipsec_add_del_tunnel_if (ipsec_add_del_tunnel_args_t * args)
+{
+  vl_api_rpc_call_main_thread (ipsec_add_del_tunnel_if_rpc_callback,
+                               (u8 *) args, sizeof(*args));
+  return 0;
+}
+
+int
+ipsec_add_del_tunnel_if_internal (vnet_main_t * vnm, ipsec_add_del_tunnel_args_t * args)
 {
   ipsec_tunnel_if_t * t;
   ipsec_main_t * im = &ipsec_main;
@@ -77,6 +101,18 @@ ipsec_add_del_tunnel_if (vnet_main_t * vnm, ipsec_add_del_tunnel_args_t * args)
       sa->is_tunnel = 1;
       sa->use_esn = args->esn;
       sa->use_anti_replay = args->anti_replay;
+      sa->integ_alg = args->integ_alg;
+      if (args->remote_integ_key_len <= sizeof(args->remote_integ_key))
+        {
+          sa->integ_key_len = args->remote_integ_key_len;
+          clib_memcpy(sa->integ_key, args->remote_integ_key, args->remote_integ_key_len);
+        }
+      sa->crypto_alg = args->crypto_alg;
+      if (args->remote_crypto_key_len <= sizeof(args->remote_crypto_key))
+        {
+          sa->crypto_key_len = args->remote_crypto_key_len;
+          clib_memcpy(sa->crypto_key, args->remote_crypto_key, args->remote_crypto_key_len);
+        }
 
       pool_get (im->sad, sa);
       memset (sa, 0, sizeof (*sa));
@@ -88,6 +124,18 @@ ipsec_add_del_tunnel_if (vnet_main_t * vnm, ipsec_add_del_tunnel_args_t * args)
       sa->seq = 1;
       sa->use_esn = args->esn;
       sa->use_anti_replay = args->anti_replay;
+      sa->integ_alg = args->integ_alg;
+      if (args->local_integ_key_len <= sizeof(args->local_integ_key))
+        {
+          sa->integ_key_len = args->local_integ_key_len;
+          clib_memcpy(sa->integ_key, args->local_integ_key, args->local_integ_key_len);
+        }
+      sa->crypto_alg = args->crypto_alg;
+      if (args->local_crypto_key_len <= sizeof(args->local_crypto_key))
+        {
+          sa->crypto_key_len = args->local_crypto_key_len;
+          clib_memcpy(sa->crypto_key, args->local_crypto_key, args->local_crypto_key_len);
+        }
 
       hash_set (im->ipsec_if_pool_index_by_key, key, t - im->tunnel_interfaces);