8 files changed, 185 insertions, 168 deletions

diff --git a/src/plugins/hs_apps/echo_client.c b/src/plugins/hs_apps/echo_client.c
index ba9785cd547..128503279b7 100644
--- a/src/plugins/hs_apps/echo_client.c
+++ b/src/plugins/hs_apps/echo_client.c
@@ -630,6 +630,8 @@ static session_cb_vft_t echo_clients = {
 static clib_error_t *
 echo_clients_attach (u8 * appns_id, u64 appns_flags, u64 appns_secret)
 {
+  vnet_app_add_tls_cert_args_t _a_cert, *a_cert = &_a_cert;
+  vnet_app_add_tls_key_args_t _a_key, *a_key = &_a_key;
   u32 prealloc_fifos, segment_size = 256 << 20;
   echo_client_main_t *ecm = &echo_client_main;
   vnet_app_attach_args_t _a, *a = &_a;
@@ -671,6 +673,18 @@ echo_clients_attach (u8 * appns_id, u64 appns_flags, u64 appns_secret)
     return clib_error_return (0, "attach returned %d", rv);
 
   ecm->app_index = a->app_index;
+
+  clib_memset (a_cert, 0, sizeof (*a_cert));
+  a_cert->app_index = a->app_index;
+  vec_validate (a_cert->cert, test_srv_crt_rsa_len);
+  clib_memcpy_fast (a_cert->cert, test_srv_crt_rsa, test_srv_crt_rsa_len);
+  vnet_app_add_tls_cert (a_cert);
+
+  clib_memset (a_key, 0, sizeof (*a_key));
+  a_key->app_index = a->app_index;
+  vec_validate (a_key->key, test_srv_key_rsa_len);
+  clib_memcpy_fast (a_key->key, test_srv_key_rsa, test_srv_key_rsa_len);
+  vnet_app_add_tls_key (a_key);
   return 0;
 }
 
diff --git a/src/plugins/quic/quic.c b/src/plugins/quic/quic.c
index de7ea536094..3cc26d9a351 100644
--- a/src/plugins/quic/quic.c
+++ b/src/plugins/quic/quic.c
@@ -32,13 +32,115 @@
 
 static char *quic_error_strings[] = {
 #define quic_error(n,s) s,
-#include "quic_error.def"
+#include <plugins/quic/quic_error.def>
 #undef quic_error
 };
 
 static quic_main_t quic_main;
 static void quic_update_timer (quic_ctx_t * ctx);
 static int quic_check_quic_session_connected (quic_ctx_t * ctx);
+static quicly_stream_open_t on_stream_open;
+static quicly_closed_by_peer_t on_closed_by_peer;
+static quicly_now_t quicly_vpp_now_cb;
+
+static int
+quic_store_quicly_ctx (application_t * app, u32 ckpair_index,
+		       u8 crypto_engine)
+{
+  quic_main_t *qm = &quic_main;
+  quicly_context_t *quicly_ctx;
+  ptls_iovec_t key_vec;
+  app_cert_key_pair_t *ckpair;
+  u64 max_enq;
+  if (app->quicly_ctx)
+    return 0;
+
+  if (crypto_engine == CRYPTO_ENGINE_NONE)
+    {
+      QUIC_DBG (2, "No crypto engine specified, using %d", crypto_engine);
+      crypto_engine = qm->default_crypto_engine;
+    }
+  if (!clib_bitmap_get (qm->available_crypto_engines, crypto_engine))
+    {
+      QUIC_DBG (1, "Quic does not support crypto engine %d", crypto_engine);
+      return VNET_API_ERROR_MISSING_CERT_KEY;
+    }
+
+  quicly_ctx_data_t *quicly_ctx_data =
+    clib_mem_alloc (sizeof (quicly_ctx_data_t));
+  clib_memset (quicly_ctx_data, 0, sizeof (*quicly_ctx_data));	/* picotls depends on this */
+  quicly_ctx = &quicly_ctx_data->quicly_ctx;
+  ptls_context_t *ptls_ctx = &quicly_ctx_data->ptls_ctx;
+  ptls_ctx->random_bytes = ptls_openssl_random_bytes;
+  ptls_ctx->get_time = &ptls_get_time;
+  ptls_ctx->key_exchanges = ptls_openssl_key_exchanges;
+  ptls_ctx->cipher_suites = qm->quic_ciphers[crypto_engine];
+  ptls_ctx->certificates.list = NULL;
+  ptls_ctx->certificates.count = 0;
+  ptls_ctx->esni = NULL;
+  ptls_ctx->on_client_hello = NULL;
+  ptls_ctx->emit_certificate = NULL;
+  ptls_ctx->sign_certificate = NULL;
+  ptls_ctx->verify_certificate = NULL;
+  ptls_ctx->ticket_lifetime = 86400;
+  ptls_ctx->max_early_data_size = 8192;
+  ptls_ctx->hkdf_label_prefix__obsolete = NULL;
+  ptls_ctx->require_dhe_on_psk = 1;
+  ptls_ctx->encrypt_ticket = &qm->session_cache.super;
+
+  app->quicly_ctx = (u64 *) quicly_ctx;
+  clib_memcpy (quicly_ctx, &quicly_spec_context, sizeof (quicly_context_t));
+
+  quicly_ctx->max_packet_size = QUIC_MAX_PACKET_SIZE;
+  quicly_ctx->tls = ptls_ctx;
+  quicly_ctx->stream_open = &on_stream_open;
+  quicly_ctx->closed_by_peer = &on_closed_by_peer;
+  quicly_ctx->now = &quicly_vpp_now_cb;
+  quicly_amend_ptls_context (quicly_ctx->tls);
+
+  quicly_ctx->transport_params.max_data = QUIC_INT_MAX;
+  quicly_ctx->transport_params.max_streams_uni = (uint64_t) 1 << 60;
+  quicly_ctx->transport_params.max_streams_bidi = (uint64_t) 1 << 60;
+
+  /* max_enq is FIFO_SIZE - 1 */
+  max_enq = app->sm_properties.rx_fifo_size - 1;
+  quicly_ctx->transport_params.max_stream_data.bidi_local = max_enq;
+  max_enq = app->sm_properties.tx_fifo_size - 1;
+  quicly_ctx->transport_params.max_stream_data.bidi_remote = max_enq;
+  quicly_ctx->transport_params.max_stream_data.uni = QUIC_INT_MAX;
+
+  quicly_ctx->tls->random_bytes (quicly_ctx_data->cid_key, 16);
+  quicly_ctx_data->cid_key[16] = 0;
+  key_vec = ptls_iovec_init (quicly_ctx_data->cid_key,
+			     strlen (quicly_ctx_data->cid_key));
+  quicly_ctx->cid_encryptor =
+    quicly_new_default_cid_encryptor (&ptls_openssl_bfecb,
+				      &ptls_openssl_aes128ecb,
+				      &ptls_openssl_sha256, key_vec);
+
+  ckpair = app_cert_key_pair_get_if_valid (ckpair_index);
+  if (!ckpair || !ckpair->key || !ckpair->cert)
+    {
+      QUIC_DBG (1, "Wrong ckpair id %d\n", ckpair_index);
+      goto error;
+    }
+  if (load_bio_private_key (quicly_ctx->tls, (char *) ckpair->key))
+    {
+      QUIC_DBG (1, "failed to read private key from app configuration\n");
+      goto error;
+    }
+  if (load_bio_certificate_chain (quicly_ctx->tls, (char *) ckpair->cert))
+    {
+      QUIC_DBG (1, "failed to load certificate\n");
+      goto error;
+    }
+  return 0;
+
+error:
+  clib_mem_free (quicly_ctx_data);
+  return VNET_API_ERROR_MISSING_CERT_KEY;
+}
+
 
 /*  Helper functions */
 
@@ -719,9 +821,6 @@ quic_on_closed_by_peer (quicly_closed_by_peer_t * self, quicly_conn_t * conn,
   session_transport_closing_notify (&ctx->connection);
 }
 
-static quicly_stream_open_t on_stream_open = { quic_on_stream_open };
-static quicly_closed_by_peer_t on_closed_by_peer = { quic_on_closed_by_peer };
-
 /* Timer handling */
 
 static int64_t
@@ -737,8 +836,6 @@ quic_get_time (quicly_now_t * self)
   return quic_get_thread_time (thread_index);
 }
 
-static quicly_now_t quicly_vpp_now_cb = { quic_get_time };
-
 static u32
 quic_set_time_now (u32 thread_index)
 {
@@ -842,149 +939,6 @@ quic_expired_timers_dispatch (u32 * expired_timers)
     }
 }
 
-static int
-quic_encrypt_ticket_cb (ptls_encrypt_ticket_t * _self, ptls_t * tls,
-			int is_encrypt, ptls_buffer_t * dst, ptls_iovec_t src)
-{
-  quic_session_cache_t *self = (void *) _self;
-  int ret;
-
-  if (is_encrypt)
-    {
-
-      /* replace the cached entry along with a newly generated session id */
-      clib_mem_free (self->data.base);
-      if ((self->data.base = clib_mem_alloc (src.len)) == NULL)
-	return PTLS_ERROR_NO_MEMORY;
-
-      ptls_get_context (tls)->random_bytes (self->id, sizeof (self->id));
-      clib_memcpy (self->data.base, src.base, src.len);
-      self->data.len = src.len;
-
-      /* store the session id in buffer */
-      if ((ret = ptls_buffer_reserve (dst, sizeof (self->id))) != 0)
-	return ret;
-      clib_memcpy (dst->base + dst->off, self->id, sizeof (self->id));
-      dst->off += sizeof (self->id);
-
-    }
-  else
-    {
-
-      /* check if session id is the one stored in cache */
-      if (src.len != sizeof (self->id))
-	return PTLS_ERROR_SESSION_NOT_FOUND;
-      if (clib_memcmp (self->id, src.base, sizeof (self->id)) != 0)
-	return PTLS_ERROR_SESSION_NOT_FOUND;
-
-      /* return the cached value */
-      if ((ret = ptls_buffer_reserve (dst, self->data.len)) != 0)
-	return ret;
-      clib_memcpy (dst->base + dst->off, self->data.base, self->data.len);
-      dst->off += self->data.len;
-    }
-
-  return 0;
-}
-
-static int
-quic_store_quicly_ctx (application_t * app, u32 ckpair_index,
-		       u8 crypto_engine)
-{
-  quic_main_t *qm = &quic_main;
-  quicly_context_t *quicly_ctx;
-  ptls_iovec_t key_vec;
-  app_cert_key_pair_t *ckpair;
-  u64 max_enq;
-  if (app->quicly_ctx)
-    return 0;
-
-  if (crypto_engine == CRYPTO_ENGINE_NONE)
-    {
-      QUIC_DBG (2, "No crypto engine specified, using %d", crypto_engine);
-      crypto_engine = qm->default_crypto_engine;
-    }
-  if (!clib_bitmap_get (qm->available_crypto_engines, crypto_engine))
-    {
-      QUIC_ERR ("Quic does not support crypto engine %d", crypto_engine);
-      return VNET_API_ERROR_MISSING_CERT_KEY;
-    }
-
-  quicly_ctx_data_t *quicly_ctx_data =
-    clib_mem_alloc (sizeof (quicly_ctx_data_t));
-  clib_memset (quicly_ctx_data, 0, sizeof (*quicly_ctx_data));	/* picotls depends on this */
-  quicly_ctx = &quicly_ctx_data->quicly_ctx;
-  ptls_context_t *ptls_ctx = &quicly_ctx_data->ptls_ctx;
-  ptls_ctx->random_bytes = ptls_openssl_random_bytes;
-  ptls_ctx->get_time = &ptls_get_time;
-  ptls_ctx->key_exchanges = ptls_openssl_key_exchanges;
-  ptls_ctx->cipher_suites = qm->quic_ciphers[crypto_engine];
-  ptls_ctx->certificates.list = NULL;
-  ptls_ctx->certificates.count = 0;
-  ptls_ctx->esni = NULL;
-  ptls_ctx->on_client_hello = NULL;
-  ptls_ctx->emit_certificate = NULL;
-  ptls_ctx->sign_certificate = NULL;
-  ptls_ctx->verify_certificate = NULL;
-  ptls_ctx->ticket_lifetime = 86400;
-  ptls_ctx->max_early_data_size = 8192;
-  ptls_ctx->hkdf_label_prefix__obsolete = NULL;
-  ptls_ctx->require_dhe_on_psk = 1;
-  ptls_ctx->encrypt_ticket = &qm->session_cache.super;
-
-  app->quicly_ctx = (u64 *) quicly_ctx;
-  clib_memcpy (quicly_ctx, &quicly_spec_context, sizeof (quicly_context_t));
-
-  quicly_ctx->max_packet_size = QUIC_MAX_PACKET_SIZE;
-  quicly_ctx->tls = ptls_ctx;
-  quicly_ctx->stream_open = &on_stream_open;
-  quicly_ctx->closed_by_peer = &on_closed_by_peer;
-  quicly_ctx->now = &quicly_vpp_now_cb;
-  quicly_amend_ptls_context (quicly_ctx->tls);
-
-  quicly_ctx->transport_params.max_data = QUIC_INT_MAX;
-  quicly_ctx->transport_params.max_streams_uni = (uint64_t) 1 << 60;
-  quicly_ctx->transport_params.max_streams_bidi = (uint64_t) 1 << 60;
-
-  /* max_enq is FIFO_SIZE - 1 */
-  max_enq = app->sm_properties.rx_fifo_size - 1;
-  quicly_ctx->transport_params.max_stream_data.bidi_local = max_enq;
-  max_enq = app->sm_properties.tx_fifo_size - 1;
-  quicly_ctx->transport_params.max_stream_data.bidi_remote = max_enq;
-  quicly_ctx->transport_params.max_stream_data.uni = QUIC_INT_MAX;
-
-  quicly_ctx->tls->random_bytes (quicly_ctx_data->cid_key, 16);
-  quicly_ctx_data->cid_key[16] = 0;
-  key_vec = ptls_iovec_init (quicly_ctx_data->cid_key,
-			     strlen (quicly_ctx_data->cid_key));
-  quicly_ctx->cid_encryptor =
-    quicly_new_default_cid_encryptor (&ptls_openssl_bfecb,
-				      &ptls_openssl_aes128ecb,
-				      &ptls_openssl_sha256, key_vec);
-
-  ckpair = app_cert_key_pair_get_if_valid (ckpair_index);
-  if (!ckpair || !ckpair->key || !ckpair->cert)
-    {
-      QUIC_ERR ("Wrong ckpair id %d\n", ckpair_index);
-      goto error;
-    }
-  if (load_bio_private_key (quicly_ctx->tls, (char *) ckpair->key))
-    {
-      QUIC_ERR ("failed to read private key from app configuration\n");
-      goto error;
-    }
-  if (load_bio_certificate_chain (quicly_ctx->tls, (char *) ckpair->cert))
-    {
-      QUIC_ERR ("failed to load certificate\n");
-      goto error;
-    }
-  return 0;
-
-error:
-  clib_mem_free (quicly_ctx_data);
-  return VNET_API_ERROR_MISSING_CERT_KEY;
-}
-
 /* Transport proto functions */
 
 static int
@@ -1099,11 +1053,12 @@ quic_connect_connection (session_endpoint_cfg_t * sep)
   app_worker_t *app_wrk;
   application_t *app;
   u32 ctx_index;
+  u32 thread_index = vlib_get_thread_index ();
   int error;
 
   clib_memset (cargs, 0, sizeof (*cargs));
-  ctx_index = quic_ctx_alloc (vlib_get_thread_index ());
-  ctx = quic_ctx_get (ctx_index, vlib_get_thread_index ());
+  ctx_index = quic_ctx_alloc (thread_index);
+  ctx = quic_ctx_get (ctx_index, thread_index);
   ctx->parent_app_wrk_id = sep->app_wrk_index;
   ctx->c_s_index = QUIC_SESSION_INVALID;
   ctx->c_c_index = ctx_index;
@@ -1215,6 +1170,7 @@ quic_start_listen (u32 quic_listen_session_index, transport_endpoint_t * tep)
   quic_ctx_t *lctx;
   u32 lctx_index;
   app_listener_t *app_listener;
+  int rv;
 
   sep = (session_endpoint_cfg_t *) tep;
   app_wrk = app_worker_get (sep->app_wrk_index);
@@ -1232,8 +1188,8 @@ quic_start_listen (u32 quic_listen_session_index, transport_endpoint_t * tep)
   args->app_index = qm->app_index;
   args->sep_ext = *sep;
   args->sep_ext.ns_index = app->ns_index;
-  if (vnet_listen (args))
-    return -1;
+  if ((rv = vnet_listen (args)))
+    return rv;
 
   lctx_index = quic_ctx_alloc (0);
   udp_handle = args->handle;
@@ -1480,7 +1436,6 @@ quic_receive_connection (void *arg)
   QUIC_DBG (2, "Received conn %u (now %u)", temp_ctx->c_thread_index,
 	    new_ctx_id);
 
-
   clib_memcpy (new_ctx, temp_ctx, sizeof (quic_ctx_t));
   clib_mem_free (temp_ctx);
 
@@ -2150,6 +2105,10 @@ static const transport_proto_vft_t quic_proto = {
 };
 /* *INDENT-ON* */
 
+static quicly_stream_open_t on_stream_open = { quic_on_stream_open };
+static quicly_closed_by_peer_t on_closed_by_peer = { quic_on_closed_by_peer };
+static quicly_now_t quicly_vpp_now_cb = { quic_get_time };
+
 static void
 quic_register_cipher_suite (crypto_engine_type_t type,
 			    ptls_cipher_suite_t ** ciphers)
diff --git a/src/plugins/quic/quic.h b/src/plugins/quic/quic.h
index 29d5224cdec..56817bc0352 100644
--- a/src/plugins/quic/quic.h
+++ b/src/plugins/quic/quic.h
@@ -165,7 +165,7 @@ typedef struct quic_stream_data_
 {
   u32 ctx_id;
   u32 thread_index;
-  u32 app_rx_data_len;		/* bytes received, to be read by external app */
+  u32 app_rx_data_len;		/**< bytes received, to be read by external app */
 } quic_stream_data_t;
 
 typedef struct quic_worker_ctx_
@@ -203,21 +203,16 @@ typedef struct quic_main_
   u32 app_index;
   quic_ctx_t **ctx_pool;
   quic_worker_ctx_t *wrk_ctx;
-  clib_bihash_16_8_t connection_hash;	/* quic connection id -> conn handle */
+  clib_bihash_16_8_t connection_hash;	/**< quic connection id -> conn handle */
   f64 tstamp_ticks_per_clock;
 
-  ptls_cipher_suite_t ***quic_ciphers;	/* available ciphers by crypto engine */
-  uword *available_crypto_engines;	/* Bitmap for registered engines */
-  u8 default_crypto_engine;	/* Used if you do connect with CRYPTO_ENGINE_NONE (0) */
+  ptls_cipher_suite_t ***quic_ciphers;	/**< available ciphers by crypto engine */
+  uword *available_crypto_engines;	/**< Bitmap for registered engines */
+  u8 default_crypto_engine;		/**< Used if you do connect with CRYPTO_ENGINE_NONE (0) */
 
-  quic_session_cache_t session_cache;
-
-  /*
-   * Config
-   */
-  quicly_context_t quicly_ctx;
   ptls_handshake_properties_t hs_properties;
   quicly_cid_plaintext_t next_cid;
+  quic_session_cache_t session_cache;
 
   u32 udp_fifo_size;
   u32 udp_fifo_prealloc;
diff --git a/src/plugins/quic/quic_crypto.c b/src/plugins/quic/quic_crypto.c
index c30e68c34ab..2223ab66eab 100644
--- a/src/plugins/quic/quic_crypto.c
+++ b/src/plugins/quic/quic_crypto.c
@@ -322,6 +322,51 @@ ptls_cipher_suite_t *quic_crypto_cipher_suites[] =
   NULL
 };
 
+int
+quic_encrypt_ticket_cb (ptls_encrypt_ticket_t * _self, ptls_t * tls,
+			int is_encrypt, ptls_buffer_t * dst, ptls_iovec_t src)
+{
+  quic_session_cache_t *self = (void *) _self;
+  int ret;
+
+  if (is_encrypt)
+    {
+
+      /* replace the cached entry along with a newly generated session id */
+      clib_mem_free (self->data.base);
+      if ((self->data.base = clib_mem_alloc (src.len)) == NULL)
+	return PTLS_ERROR_NO_MEMORY;
+
+      ptls_get_context (tls)->random_bytes (self->id, sizeof (self->id));
+      clib_memcpy (self->data.base, src.base, src.len);
+      self->data.len = src.len;
+
+      /* store the session id in buffer */
+      if ((ret = ptls_buffer_reserve (dst, sizeof (self->id))) != 0)
+	return ret;
+      clib_memcpy (dst->base + dst->off, self->id, sizeof (self->id));
+      dst->off += sizeof (self->id);
+
+    }
+  else
+    {
+
+      /* check if session id is the one stored in cache */
+      if (src.len != sizeof (self->id))
+	return PTLS_ERROR_SESSION_NOT_FOUND;
+      if (clib_memcmp (self->id, src.base, sizeof (self->id)) != 0)
+	return PTLS_ERROR_SESSION_NOT_FOUND;
+
+      /* return the cached value */
+      if ((ret = ptls_buffer_reserve (dst, self->data.len)) != 0)
+	return ret;
+      clib_memcpy (dst->base + dst->off, self->data.base, self->data.len);
+      dst->off += self->data.len;
+    }
+
+  return 0;
+}
+
 /*
  * fd.io coding-style-patch-verification: ON
  *
diff --git a/src/plugins/quic/quic_crypto.h b/src/plugins/quic/quic_crypto.h
index 625d838c938..ff74fac4086 100644
--- a/src/plugins/quic/quic_crypto.h
+++ b/src/plugins/quic/quic_crypto.h
@@ -20,6 +20,10 @@
 
 extern ptls_cipher_suite_t *quic_crypto_cipher_suites[];
 
+int quic_encrypt_ticket_cb (ptls_encrypt_ticket_t * _self, ptls_t * tls,
+			    int is_encrypt, ptls_buffer_t * dst,
+			    ptls_iovec_t src);
+
 #endif /* __included_vpp_quic_crypto_h__ */
 
 /*
diff --git a/src/vnet/session/application.c b/src/vnet/session/application.c
index 7c3293abc54..42c5136a70b 100644
--- a/src/vnet/session/application.c
+++ b/src/vnet/session/application.c
@@ -1432,8 +1432,6 @@ format_crypto_context (u8 * s, va_list * args)
     format (s, "[0x%x][sub%d,ckpair%x]", crctx->ctx_index,
 	    crctx->n_subscribers, crctx->ckpair_index);
   s = format (s, "[%U]", format_crypto_engine, crctx->crypto_engine);
-  if (crctx->stale)
-    s = format (s, " -- DELETED");
   return s;
 }
 
diff --git a/src/vnet/session/application.h b/src/vnet/session/application.h
index 4a963826fec..e53c8ed8598 100644
--- a/src/vnet/session/application.h
+++ b/src/vnet/session/application.h
@@ -115,6 +115,9 @@ typedef struct application_
   u8 tls_engine;
 
   u64 *quicly_ctx;
+  /** quic initialization vector */
+  char quic_iv[17];
+  u8 quic_iv_set;
 
 } application_t;
 
diff --git a/src/vnet/session/application_interface.h b/src/vnet/session/application_interface.h
index a865b081ca2..3a4f9928d04 100644
--- a/src/vnet/session/application_interface.h
+++ b/src/vnet/session/application_interface.h
@@ -182,7 +182,6 @@ typedef struct crypto_ctx_
   u32 n_subscribers;		/**< refcount of sessions using said context */
   u32 ckpair_index;		/**< certificate & key */
   u8 crypto_engine;
-  u8 stale;			/**< Marked invalid for re-use (aka ckpair deleted) */
 } crypto_context_t;
 
 /* Application attach options */