7 files changed, 302 insertions, 56 deletions

diff --git a/src/vnet/session/application.c b/src/vnet/session/application.c
index 583c4b055ee..82c890f56ce 100644
--- a/src/vnet/session/application.c
+++ b/src/vnet/session/application.c
@@ -591,8 +591,6 @@ application_free (application_t * app)
   if (application_is_builtin (app))
     application_name_table_del (app);
   vec_free (app->name);
-  vec_free (app->tls_cert);
-  vec_free (app->tls_key);
   pool_put (app_main.app_pool, app);
 }
 
@@ -1305,24 +1303,20 @@ application_get_segment_manager_properties (u32 app_index)
 clib_error_t *
 vnet_app_add_tls_cert (vnet_app_add_tls_cert_args_t * a)
 {
-  application_t *app;
-  app = application_get (a->app_index);
-  if (!app)
-    return clib_error_return_code (0, VNET_API_ERROR_APPLICATION_NOT_ATTACHED,
-				   0, "app %u doesn't exist", a->app_index);
-  app->tls_cert = vec_dup (a->cert);
+  /* Deprected, will be remove after 20.01 */
+  app_cert_key_pair_t *ckpair;
+  ckpair = app_cert_key_pair_get_default ();
+  ckpair->cert = vec_dup (a->cert);
   return 0;
 }
 
 clib_error_t *
 vnet_app_add_tls_key (vnet_app_add_tls_key_args_t * a)
 {
-  application_t *app;
-  app = application_get (a->app_index);
-  if (!app)
-    return clib_error_return_code (0, VNET_API_ERROR_APPLICATION_NOT_ATTACHED,
-				   0, "app %u doesn't exist", a->app_index);
-  app->tls_key = vec_dup (a->key);
+  /* Deprected, will be remove after 20.01 */
+  app_cert_key_pair_t *ckpair;
+  ckpair = app_cert_key_pair_get_default ();
+  ckpair->key = vec_dup (a->key);
   return 0;
 }
 
@@ -1376,6 +1370,22 @@ application_format_connects (application_t * app, int verbose)
 }
 
 u8 *
+format_cert_key_pair (u8 * s, va_list * args)
+{
+  app_cert_key_pair_t *ckpair = va_arg (*args, app_cert_key_pair_t *);
+  int key_len = 0, cert_len = 0;
+  cert_len = vec_len (ckpair->cert);
+  key_len = vec_len (ckpair->key);
+  if (ckpair->cert_key_index == 0)
+    s = format (s, "DEFAULT (cert:%d, key:%d)", cert_len, key_len);
+  else
+    s =
+      format (s, "%d (cert:%d, key:%d)", ckpair->cert_key_index, cert_len,
+	      key_len);
+  return s;
+}
+
+u8 *
 format_application (u8 * s, va_list * args)
 {
   application_t *app = va_arg (*args, application_t *);
@@ -1460,6 +1470,21 @@ application_format_all_clients (vlib_main_t * vm, int verbose)
 }
 
 static clib_error_t *
+show_certificate_command_fn (vlib_main_t * vm, unformat_input_t * input,
+			     vlib_cli_command_t * cmd)
+{
+  app_cert_key_pair_t *ckpair;
+  session_cli_return_if_not_enabled ();
+
+  /* *INDENT-OFF* */
+  pool_foreach (ckpair, app_main.cert_key_pair_store, ({
+    vlib_cli_output (vm, "%U", format_cert_key_pair, ckpair);
+  }));
+  /* *INDENT-ON* */
+  return 0;
+}
+
+static clib_error_t *
 show_app_command_fn (vlib_main_t * vm, unformat_input_t * input,
 		     vlib_cli_command_t * cmd)
 {
@@ -1521,13 +1546,112 @@ show_app_command_fn (vlib_main_t * vm, unformat_input_t * input,
   return 0;
 }
 
+/*
+ * Certificate store
+ *
+ */
+
+static app_cert_key_pair_t *
+app_cert_key_pair_alloc ()
+{
+  app_cert_key_pair_t *ckpair;
+  pool_get (app_main.cert_key_pair_store, ckpair);
+  clib_memset (ckpair, 0, sizeof (*ckpair));
+  ckpair->cert_key_index = ckpair - app_main.cert_key_pair_store;
+  return ckpair;
+}
+
+app_cert_key_pair_t *
+app_cert_key_pair_get_if_valid (u32 index)
+{
+  if (pool_is_free_index (app_main.cert_key_pair_store, index))
+    return 0;
+  return app_cert_key_pair_get (index);
+}
+
+app_cert_key_pair_t *
+app_cert_key_pair_get (u32 index)
+{
+  return pool_elt_at_index (app_main.cert_key_pair_store, index);
+}
+
+app_cert_key_pair_t *
+app_cert_key_pair_get_default ()
+{
+  /* To maintain legacy bapi */
+  return app_cert_key_pair_get (0);
+}
+
+int
+vnet_app_add_cert_key_pair (vnet_app_add_cert_key_pair_args_t * a)
+{
+  app_cert_key_pair_t *ckpair = app_cert_key_pair_alloc ();
+  ckpair->cert = vec_dup (a->cert);
+  ckpair->key = vec_dup (a->key);
+  a->index = ckpair->cert_key_index;
+  return 0;
+}
+
+int
+vent_app_add_cert_key_interest (u32 index, u32 app_index)
+{
+  app_cert_key_pair_t *ckpair;
+  if (!(ckpair = app_cert_key_pair_get_if_valid (index)))
+    return -1;
+  vec_add1 (ckpair->app_interests, app_index);
+  return 0;
+}
+
+int
+vnet_app_del_cert_key_pair (u32 index)
+{
+  app_cert_key_pair_t *ckpair;
+  application_t *app;
+  u32 *app_index;
+
+  if (!(ckpair = app_cert_key_pair_get_if_valid (index)))
+    return (VNET_API_ERROR_INVALID_VALUE);
+
+  vec_foreach (app_index, ckpair->app_interests)
+  {
+    if ((app = application_get_if_valid (*app_index))
+	&& app->cb_fns.app_cert_key_pair_delete_callback)
+      app->cb_fns.app_cert_key_pair_delete_callback (ckpair);
+  }
+
+  vec_free (ckpair->cert);
+  vec_free (ckpair->key);
+  pool_put (app_main.cert_key_pair_store, ckpair);
+  return 0;
+}
+
+clib_error_t *
+cert_key_pair_store_init (vlib_main_t * vm)
+{
+  /* Add a certificate with index 0 to support legacy apis */
+  (void) app_cert_key_pair_alloc ();
+  return 0;
+}
+
 /* *INDENT-OFF* */
+VLIB_INIT_FUNCTION (cert_key_pair_store_init) =
+{
+  .runs_after = VLIB_INITS("unix_physmem_init"),
+};
+
 VLIB_CLI_COMMAND (show_app_command, static) =
 {
   .path = "show app",
   .short_help = "show app [server|client] [verbose]",
   .function = show_app_command_fn,
 };
+
+VLIB_CLI_COMMAND (show_certificate_command, static) =
+{
+  .path = "show app certificate",
+  .short_help = "list app certs and keys present in store",
+  .function = show_certificate_command_fn,
+};
 /* *INDENT-ON* */
 
 /*
diff --git a/src/vnet/session/application.h b/src/vnet/session/application.h
index 9ec1055bbbc..a853c3cb73a 100644
--- a/src/vnet/session/application.h
+++ b/src/vnet/session/application.h
@@ -111,16 +111,6 @@ typedef struct application_
   /** Pool of listeners for the app */
   app_listener_t *listeners;
 
-  /*
-   * TLS & QUIC Specific
-   */
-
-  /** Certificate to be used for listen sessions */
-  u8 *tls_cert;
-
-  /** PEM encoded key */
-  u8 *tls_key;
-
   /** Preferred tls engine */
   u8 tls_engine;
 
@@ -144,6 +134,11 @@ typedef struct app_main_
    * Hash table of builtin apps by name
    */
   uword *app_by_name;
+
+  /**
+   * Pool from which we allocate certificates (key, cert)
+   */
+  app_cert_key_pair_t *cert_key_pair_store;
 } app_main_t;
 
 typedef struct app_init_args_
@@ -284,6 +279,9 @@ int vnet_app_worker_add_del (vnet_app_worker_add_del_args_t * a);
 
 uword unformat_application_proto (unformat_input_t * input, va_list * args);
 
+app_cert_key_pair_t *app_cert_key_pair_get (u32 index);
+app_cert_key_pair_t *app_cert_key_pair_get_if_valid (u32 index);
+app_cert_key_pair_t *app_cert_key_pair_get_default ();
 
 /* Needed while we support both bapi and mq ctrl messages */
 int mq_send_session_bound_cb (u32 app_wrk_index, u32 api_context,
diff --git a/src/vnet/session/application_interface.h b/src/vnet/session/application_interface.h
index 17f7ef209e5..fa6206a5279 100644
--- a/src/vnet/session/application_interface.h
+++ b/src/vnet/session/application_interface.h
@@ -21,6 +21,14 @@
 #include <vnet/tls/tls_test.h>
 #include <svm/fifo_segment.h>
 
+typedef struct certificate_
+{
+  u32 *app_interests;		/* vec of application index asking for deletion cb */
+  u32 cert_key_index;		/* index in cert & key pool */
+  u8 *key;
+  u8 *cert;
+} app_cert_key_pair_t;
+
 typedef struct _stream_session_cb_vft
 {
   /** Notify server of new segment */
@@ -57,6 +65,9 @@ typedef struct _stream_session_cb_vft
   /** Direct TX callback for built-in application */
   int (*builtin_app_tx_callback) (session_t * session);
 
+  /** Cert and key pair delete notification */
+  int (*app_cert_key_pair_delete_callback) (app_cert_key_pair_t * ckpair);
+
 } session_cb_vft_t;
 
 #define foreach_app_init_args			\
@@ -158,6 +169,13 @@ typedef enum tls_engine_type_
   TLS_N_ENGINES
 } tls_engine_type_t;
 
+typedef struct _vnet_app_add_cert_key_pair_args_
+{
+  u8 *cert;
+  u8 *key;
+  u32 index;
+} vnet_app_add_cert_key_pair_args_t;
+
 /* Application attach options */
 typedef enum
 {
@@ -236,6 +254,9 @@ int vnet_disconnect_session (vnet_disconnect_args_t * a);
 
 clib_error_t *vnet_app_add_tls_cert (vnet_app_add_tls_cert_args_t * a);
 clib_error_t *vnet_app_add_tls_key (vnet_app_add_tls_key_args_t * a);
+int vnet_app_add_cert_key_pair (vnet_app_add_cert_key_pair_args_t * a);
+int vnet_app_del_cert_key_pair (u32 index);
+int vent_app_add_cert_key_interest (u32 index, u32 app_index);	/* Ask for app cb on pair deletion */
 
 typedef struct app_session_transport_
 {
@@ -273,6 +294,7 @@ typedef struct session_listen_msg_
   u8 proto;
   u8 is_ip4;
   ip46_address_t ip;
+  u32 ckpair_index;
 } __clib_packed session_listen_msg_t;
 
 typedef struct session_listen_uri_msg_
@@ -345,6 +367,7 @@ typedef struct session_connect_msg_
   u8 hostname_len;
   u8 hostname[16];
   u64 parent_handle;
+  u32 ckpair_index;
 } __clib_packed session_connect_msg_t;
 
 typedef struct session_connect_uri_msg_
diff --git a/src/vnet/session/session.api b/src/vnet/session/session.api
index 6f208ff5b0e..33e53419a8d 100644
--- a/src/vnet/session/session.api
+++ b/src/vnet/session/session.api
@@ -108,7 +108,46 @@ define app_attach_reply {
     u64 segment_handle;
 };
 
+/** \brief Add certificate and key
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param engine - crypto engine
+    @param cert_len - cert length (comes first)
+    @param certkey_len - cert and key length
+    @param certkey - cert & key data (due to API limitation)
+*/
+define app_add_cert_key_pair {
+    u32 client_index;
+    u32 context;
+    u16 cert_len;
+    u16 certkey_len;
+    u8 certkey[certkey_len];
+};
+
+/** \brief Add certificate and key
+    @param context - sender context, to match reply w/ request
+    @param retval - return code for the request
+    @param index - index in certificate store
+*/
+define app_add_cert_key_pair_reply {
+    u32 context;
+    i32 retval;
+    u32 index;
+};
+
+/** \brief Delete certificate and key
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param index - index in certificate store
+*/
+autoreply define app_del_cert_key_pair {
+    u32 client_index;
+    u32 context;
+    u32 index;
+};
+
 /** \brief Application add TLS certificate
+ 	### WILL BE DEPRECATED POST 20.01 ###
     @param client_index - opaque cookie to identify the sender
     @param context - sender context, to match reply w/ request
     @param cert_len - certificate length
@@ -123,6 +162,7 @@ autoreply define application_tls_cert_add {
 };
 
 /** \brief Application add TLS key
+ 	### WILL BE DEPRECATED POST 20.01 ###
     @param client_index - opaque cookie to identify the sender
     @param context - sender context, to match reply w/ request
     @param key_len - certificate length
diff --git a/src/vnet/session/session_api.c b/src/vnet/session/session_api.c
index c55aab33541..c17d98c0517 100755
--- a/src/vnet/session/session_api.c
+++ b/src/vnet/session/session_api.c
@@ -59,6 +59,8 @@ _(SESSION_RULE_ADD_DEL, session_rule_add_del)				\
 _(SESSION_RULES_DUMP, session_rules_dump)				\
 _(APPLICATION_TLS_CERT_ADD, application_tls_cert_add)			\
 _(APPLICATION_TLS_KEY_ADD, application_tls_key_add)			\
+_(APP_ADD_CERT_KEY_PAIR, app_add_cert_key_pair)				\
+_(APP_DEL_CERT_KEY_PAIR, app_del_cert_key_pair)				\
 _(APP_WORKER_ADD_DEL, app_worker_add_del)				\
 
 static int
@@ -1059,7 +1061,7 @@ vl_api_app_worker_add_del_t_handler (vl_api_app_worker_add_del_t * mp)
   application_t *app;
   u8 fd_flags = 0;
 
-  if (!session_main_is_enabled ())
+  if (session_main_is_enabled () == 0)
     {
       rv = VNET_API_ERROR_FEATURE_DISABLED;
       goto done;
@@ -1138,7 +1140,7 @@ vl_api_app_namespace_add_del_t_handler (vl_api_app_namespace_add_del_t * mp)
   u32 appns_index = 0;
   u8 *ns_id = 0;
   int rv = 0;
-  if (!session_main_is_enabled ())
+  if (session_main_is_enabled () == 0)
     {
       rv = VNET_API_ERROR_FEATURE_DISABLED;
       goto done;
@@ -1356,16 +1358,84 @@ vl_api_session_rules_dump_t_handler (vl_api_one_map_server_dump_t * mp)
 }
 
 static void
+vl_api_app_add_cert_key_pair_t_handler (vl_api_app_add_cert_key_pair_t * mp)
+{
+  vl_api_app_add_cert_key_pair_reply_t *rmp;
+  vnet_app_add_cert_key_pair_args_t _a, *a = &_a;
+  u32 certkey_len, key_len, cert_len;
+  int rv = 0;
+  if (session_main_is_enabled () == 0)
+    {
+      rv = VNET_API_ERROR_FEATURE_DISABLED;
+      goto done;
+    }
+
+  cert_len = clib_net_to_host_u16 (mp->cert_len);
+  if (cert_len > 10000)
+    {
+      rv = VNET_API_ERROR_INVALID_VALUE;
+      goto done;
+    }
+
+  certkey_len = clib_net_to_host_u16 (mp->certkey_len);
+  if (certkey_len < cert_len)
+    {
+      rv = VNET_API_ERROR_INVALID_VALUE;
+      goto done;
+    }
+
+  key_len = certkey_len - cert_len;
+  if (key_len > 10000)
+    {
+      rv = VNET_API_ERROR_INVALID_VALUE;
+      goto done;
+    }
+
+  clib_memset (a, 0, sizeof (*a));
+  vec_validate (a->cert, cert_len);
+  vec_validate (a->key, key_len);
+  clib_memcpy_fast (a->cert, mp->certkey, cert_len);
+  clib_memcpy_fast (a->key, mp->certkey + cert_len, key_len);
+  rv = vnet_app_add_cert_key_pair (a);
+  vec_free (a->cert);
+  vec_free (a->key);
+
+done:
+  /* *INDENT-OFF* */
+  REPLY_MACRO2 (VL_API_APP_ADD_CERT_KEY_PAIR_REPLY, ({
+    if (!rv)
+      rmp->index = a->index;
+  }));
+  /* *INDENT-ON* */
+}
+
+static void
+vl_api_app_del_cert_key_pair_t_handler (vl_api_app_del_cert_key_pair_t * mp)
+{
+  vl_api_app_del_cert_key_pair_reply_t *rmp;
+  int rv = 0;
+  if (session_main_is_enabled () == 0)
+    {
+      rv = VNET_API_ERROR_FEATURE_DISABLED;
+      goto done;
+    }
+  rv = vnet_app_del_cert_key_pair (mp->index);
+
+done:
+  REPLY_MACRO (VL_API_APP_ADD_CERT_KEY_PAIR_REPLY);
+}
+
+/* ### WILL BE DEPRECATED POST 20.01 ### */
+static void
 vl_api_application_tls_cert_add_t_handler (vl_api_application_tls_cert_add_t *
 					   mp)
 {
-  vl_api_app_namespace_add_del_reply_t *rmp;
-  vnet_app_add_tls_cert_args_t _a, *a = &_a;
-  clib_error_t *error;
+  vl_api_application_tls_cert_add_reply_t *rmp;
+  app_cert_key_pair_t *ckpair;
   application_t *app;
   u32 cert_len;
   int rv = 0;
-  if (!session_main_is_enabled ())
+  if (session_main_is_enabled () == 0)
     {
       rv = VNET_API_ERROR_FEATURE_DISABLED;
       goto done;
@@ -1375,37 +1445,31 @@ vl_api_application_tls_cert_add_t_handler (vl_api_application_tls_cert_add_t *
       rv = VNET_API_ERROR_APPLICATION_NOT_ATTACHED;
       goto done;
     }
-  clib_memset (a, 0, sizeof (*a));
-  a->app_index = app->app_index;
   cert_len = clib_net_to_host_u16 (mp->cert_len);
   if (cert_len > 10000)
     {
       rv = VNET_API_ERROR_INVALID_VALUE;
       goto done;
     }
-  vec_validate (a->cert, cert_len);
-  clib_memcpy_fast (a->cert, mp->cert, cert_len);
-  if ((error = vnet_app_add_tls_cert (a)))
-    {
-      rv = clib_error_get_code (error);
-      clib_error_report (error);
-    }
-  vec_free (a->cert);
+  ckpair = app_cert_key_pair_get_default ();
+  vec_validate (ckpair->cert, cert_len);
+  clib_memcpy_fast (ckpair->cert, mp->cert, cert_len);
+
 done:
   REPLY_MACRO (VL_API_APPLICATION_TLS_CERT_ADD_REPLY);
 }
 
+/* ### WILL BE DEPRECATED POST 20.01 ### */
 static void
 vl_api_application_tls_key_add_t_handler (vl_api_application_tls_key_add_t *
 					  mp)
 {
-  vl_api_app_namespace_add_del_reply_t *rmp;
-  vnet_app_add_tls_key_args_t _a, *a = &_a;
-  clib_error_t *error;
+  vl_api_application_tls_key_add_reply_t *rmp;
+  app_cert_key_pair_t *ckpair;
   application_t *app;
   u32 key_len;
   int rv = 0;
-  if (!session_main_is_enabled ())
+  if (session_main_is_enabled () == 0)
     {
       rv = VNET_API_ERROR_FEATURE_DISABLED;
       goto done;
@@ -1415,22 +1479,15 @@ vl_api_application_tls_key_add_t_handler (vl_api_application_tls_key_add_t *
       rv = VNET_API_ERROR_APPLICATION_NOT_ATTACHED;
       goto done;
     }
-  clib_memset (a, 0, sizeof (*a));
-  a->app_index = app->app_index;
   key_len = clib_net_to_host_u16 (mp->key_len);
   if (key_len > 10000)
     {
       rv = VNET_API_ERROR_INVALID_VALUE;
       goto done;
     }
-  vec_validate (a->key, key_len);
-  clib_memcpy_fast (a->key, mp->key, key_len);
-  if ((error = vnet_app_add_tls_key (a)))
-    {
-      rv = clib_error_get_code (error);
-      clib_error_report (error);
-    }
-  vec_free (a->key);
+  ckpair = app_cert_key_pair_get_default ();
+  vec_validate (ckpair->key, key_len);
+  clib_memcpy_fast (ckpair->key, mp->key, key_len);
 done:
   REPLY_MACRO (VL_API_APPLICATION_TLS_KEY_ADD_REPLY);
 }
diff --git a/src/vnet/session/session_node.c b/src/vnet/session/session_node.c
index e2e98eba19f..fde1931aaf9 100644
--- a/src/vnet/session/session_node.c
+++ b/src/vnet/session/session_node.c
@@ -54,6 +54,7 @@ session_mq_listen_handler (void *data)
   a->sep.fib_index = mp->vrf;
   a->sep.sw_if_index = ENDPOINT_INVALID_INDEX;
   a->sep.transport_proto = mp->proto;
+  a->sep_ext.ckpair_index = mp->ckpair_index;
   a->app_index = app->app_index;
   a->wrk_map_index = mp->wrk_index;
 
@@ -112,6 +113,7 @@ session_mq_connect_handler (void *data)
   a->sep.peer.fib_index = mp->vrf;
   a->sep.peer.sw_if_index = ENDPOINT_INVALID_INDEX;
   a->sep_ext.parent_handle = mp->parent_handle;
+  a->sep_ext.ckpair_index = mp->ckpair_index;
   if (mp->hostname_len)
     {
       vec_validate (a->sep_ext.hostname, mp->hostname_len - 1);
@@ -311,7 +313,7 @@ session_mq_reset_reply_handler (void *data)
   app_wrk = app_worker_get (s->app_wrk_index);
   if (!app_wrk || app_wrk->app_index != app->app_index)
     {
-      clib_warning ("App % does not own handle 0x%lx!", app->app_index,
+      clib_warning ("App %u does not own handle 0x%lx!", app->app_index,
 		    mp->handle);
       return;
     }
diff --git a/src/vnet/session/session_types.h b/src/vnet/session/session_types.h
index 03f7096c4b8..bb309f2f31c 100644
--- a/src/vnet/session/session_types.h
+++ b/src/vnet/session/session_types.h
@@ -44,6 +44,7 @@ typedef struct _session_endpoint_cfg
   u8 original_tp;
   u8 *hostname;
   u64 parent_handle;
+  u32 ckpair_index;
 } session_endpoint_cfg_t;
 
 #define SESSION_IP46_ZERO			\
@@ -83,7 +84,8 @@ typedef struct _session_endpoint_cfg
   .app_wrk_index = ENDPOINT_INVALID_INDEX,	\
   .opaque = ENDPOINT_INVALID_INDEX,		\
   .hostname = 0,				\
-  .parent_handle = SESSION_INVALID_HANDLE	\
+  .parent_handle = SESSION_INVALID_HANDLE,	\
+  .ckpair_index = 0				\
 }
 
 #define session_endpoint_to_transport(_sep) ((transport_endpoint_t *)_sep)