hsa: make vpp_echo use mq instead of bapi

Type: refactor

Change-Id: I4f370b09e22dbbc8920272df9a042dae04825bfc
Signed-off-by: Nathan Skrzypczak <nathan.skrzypczak@gmail.com>

9 files changed, 286 insertions, 193 deletions

diff --git a/src/plugins/hs_apps/sapi/vpp_echo.c b/src/plugins/hs_apps/sapi/vpp_echo.c
index f0aa6a9e050..44e6d1fb36f 100644
--- a/src/plugins/hs_apps/sapi/vpp_echo.c
+++ b/src/plugins/hs_apps/sapi/vpp_echo.c
@@ -450,6 +450,16 @@ echo_data_thread_fn (void *arg)
 }
 
 static void
+session_unlisten_handler (session_unlisten_msg_t * mp)
+{
+  echo_session_t *listen_session;
+  echo_main_t *em = &echo_main;
+  listen_session = pool_elt_at_index (em->sessions, em->listen_session_index);
+  em->proto_cb_vft->cleanup_cb (listen_session, 0 /* parent_died */ );
+  em->state = STATE_DISCONNECTED;
+}
+
+static void
 session_bound_handler (session_bound_msg_t * mp)
 {
   echo_main_t *em = &echo_main;
@@ -542,9 +552,10 @@ session_connected_handler (session_connected_msg_t * mp)
 
   if (mp->retval)
     {
-      ECHO_FAIL (ECHO_FAIL_SESSION_CONNECT,
-		 "connection failed with code: %U", format_api_error,
-		 clib_net_to_host_u32 (mp->retval));
+      if (em->proto_cb_vft->connected_cb)
+	em->
+	  proto_cb_vft->connected_cb ((session_connected_bundled_msg_t *) mp,
+				      listener_index, 1 /* is_failed */ );
       return;
     }
 
@@ -597,7 +608,11 @@ session_disconnected_handler (session_disconnected_msg_t * mp)
   echo_session_t *s;
   ECHO_LOG (1, "passive close session 0x%lx", mp->handle);
   if (!(s = echo_get_session_from_handle (em, mp->handle)))
-    return;
+    {
+      ECHO_FAIL (ECHO_FAIL_SESSION_DISCONNECT, "Session 0x%lx not found",
+		 mp->handle);
+      return;
+    }
   em->proto_cb_vft->disconnected_cb (mp, s);
 
   app_alloc_ctrl_evt_to_vpp (s->vpp_evt_q, app_evt,
@@ -618,7 +633,11 @@ session_reset_handler (session_reset_msg_t * mp)
   echo_session_t *s = 0;
   ECHO_LOG (1, "Reset session 0x%lx", mp->handle);
   if (!(s = echo_get_session_from_handle (em, mp->handle)))
-    return;
+    {
+      ECHO_FAIL (ECHO_FAIL_SESSION_RESET, "Session 0x%lx not found",
+		 mp->handle);
+      return;
+    }
   em->proto_cb_vft->reset_cb (mp, s);
 
   app_alloc_ctrl_evt_to_vpp (s->vpp_evt_q, app_evt,
@@ -635,20 +654,18 @@ handle_mq_event (session_event_t * e)
   switch (e->event_type)
     {
     case SESSION_CTRL_EVT_BOUND:
-      session_bound_handler ((session_bound_msg_t *) e->data);
-      break;
+      return session_bound_handler ((session_bound_msg_t *) e->data);
     case SESSION_CTRL_EVT_ACCEPTED:
-      session_accepted_handler ((session_accepted_msg_t *) e->data);
-      break;
+      return session_accepted_handler ((session_accepted_msg_t *) e->data);
     case SESSION_CTRL_EVT_CONNECTED:
-      session_connected_handler ((session_connected_msg_t *) e->data);
-      break;
+      return session_connected_handler ((session_connected_msg_t *) e->data);
     case SESSION_CTRL_EVT_DISCONNECTED:
-      session_disconnected_handler ((session_disconnected_msg_t *) e->data);
-      break;
+      return session_disconnected_handler ((session_disconnected_msg_t *)
+					   e->data);
     case SESSION_CTRL_EVT_RESET:
-      session_reset_handler ((session_reset_msg_t *) e->data);
-      break;
+      return session_reset_handler ((session_reset_msg_t *) e->data);
+    case SESSION_CTRL_EVT_UNLISTEN_REPLY:
+      return session_unlisten_handler ((session_unlisten_msg_t *) e->data);
     case SESSION_IO_EVT_RX:
       break;
     default:
@@ -710,7 +727,7 @@ echo_mq_thread_fn (void *arg)
   vec_validate (msg_vec, em->evt_q_size);
   vec_reset_length (msg_vec);
   wait_for_state_change (em, STATE_ATTACHED, 0);
-  mq = em->our_event_queue;
+  mq = em->app_mq;
   if (em->state < STATE_ATTACHED || !mq)
     {
       ECHO_FAIL (ECHO_FAIL_APP_ATTACH, "Application failed to attach");
@@ -747,7 +764,7 @@ clients_run (echo_main_t * em)
   u64 i;
   echo_notify_event (em, ECHO_EVT_FIRST_QCONNECT);
   for (i = 0; i < em->n_connects; i++)
-    echo_send_connect (em->uri, SESSION_INVALID_INDEX);
+    echo_send_connect (SESSION_INVALID_HANDLE, SESSION_INVALID_INDEX);
   wait_for_state_change (em, STATE_READY, 0);
   ECHO_LOG (1, "App is ready");
   echo_process_rpcs (em);
@@ -756,12 +773,14 @@ clients_run (echo_main_t * em)
 static void
 server_run (echo_main_t * em)
 {
+  echo_session_t *ls;
   echo_send_listen (em);
   wait_for_state_change (em, STATE_READY, 0);
   ECHO_LOG (1, "App is ready");
   echo_process_rpcs (em);
   /* Cleanup */
-  echo_send_unbind (em);
+  ls = pool_elt_at_index (em->sessions, em->listen_session_index);
+  echo_send_unbind (em, ls);
   if (wait_for_state_change (em, STATE_DISCONNECTED, TIMEOUT))
     {
       ECHO_FAIL (ECHO_FAIL_SERVER_DISCONNECT_TIMEOUT,
@@ -896,6 +915,11 @@ echo_process_opts (int argc, char **argv)
 	}
       else if (unformat (a, "nthreads %d", &em->n_rx_threads))
 	;
+      else
+	if (unformat
+	    (a, "crypto %U", echo_unformat_crypto_engine,
+	     &em->crypto_ctx_engine))
+	;
       else if (unformat (a, "appns %_%v%_", &em->appns_id))
 	;
       else if (unformat (a, "all-scope"))
@@ -1024,6 +1048,7 @@ main (int argc, char **argv)
   em->tx_buf_size = 1 << 20;
   em->data_source = ECHO_INVALID_DATA_SOURCE;
   em->uri = format (0, "%s%c", "tcp://0.0.0.0/1234", 0);
+  em->crypto_ctx_engine = TLS_ENGINE_NONE;
   echo_set_each_proto_defaults_before_opts (em);
   echo_process_opts (argc, argv);
   echo_process_uri (em);
@@ -1077,18 +1102,36 @@ main (int argc, char **argv)
   echo_notify_event (em, ECHO_EVT_START);
 
   echo_send_attach (em);
-  if (wait_for_state_change (em, STATE_ATTACHED, TIMEOUT))
+  if (wait_for_state_change (em, STATE_ATTACHED_NO_CERT, TIMEOUT))
     {
       ECHO_FAIL (ECHO_FAIL_ATTACH_TO_VPP,
 		 "Couldn't attach to vpp, did you run <session enable> ?");
       goto exit_on_error;
     }
+
+  if (em->crypto_ctx_engine == TLS_ENGINE_NONE)
+    /* when no crypto engine specified, dont expect crypto ctx */
+    em->state = STATE_ATTACHED;
+  else
+    {
+      ECHO_LOG (1, "Adding crypto context %U", echo_format_crypto_engine,
+		em->crypto_ctx_engine);
+      echo_send_add_crypto_ctx (em);
+      if (wait_for_state_change (em, STATE_ATTACHED, TIMEOUT))
+	{
+	  ECHO_FAIL (ECHO_FAIL_APP_ATTACH,
+		     "Couldn't add crypto context to vpp\n");
+	  exit (1);
+	}
+    }
+
   if (pthread_create (&em->mq_thread_handle,
 		      NULL /*attr */ , echo_mq_thread_fn, 0))
     {
       ECHO_FAIL (ECHO_FAIL_PTHREAD_CREATE, "pthread create errored");
       goto exit_on_error;
     }
+
   for (i = 0; i < em->n_rx_threads; i++)
     if (pthread_create (&em->data_thread_handles[i],
 			NULL /*attr */ , echo_data_thread_fn, (void *) i))
diff --git a/src/plugins/hs_apps/sapi/vpp_echo_bapi.c b/src/plugins/hs_apps/sapi/vpp_echo_bapi.c
index 1bfcb366b25..a565a91899a 100644
--- a/src/plugins/hs_apps/sapi/vpp_echo_bapi.c
+++ b/src/plugins/hs_apps/sapi/vpp_echo_bapi.c
@@ -27,14 +27,11 @@
 void
 echo_send_attach (echo_main_t * em)
 {
-  vl_api_application_attach_t *bmp;
-  vl_api_application_tls_cert_add_t *cert_mp;
-  vl_api_application_tls_key_add_t *key_mp;
-
+  vl_api_app_attach_t *bmp;
   bmp = vl_msg_api_alloc (sizeof (*bmp));
   clib_memset (bmp, 0, sizeof (*bmp));
 
-  bmp->_vl_msg_id = ntohs (VL_API_APPLICATION_ATTACH);
+  bmp->_vl_msg_id = ntohs (VL_API_APP_ATTACH);
   bmp->client_index = em->my_client_index;
   bmp->context = ntohl (0xfeedface);
   bmp->options[APP_OPTIONS_FLAGS] = APP_OPTIONS_FLAGS_ACCEPT_REDIRECT;
@@ -54,6 +51,27 @@ echo_send_attach (echo_main_t * em)
       bmp->options[APP_OPTIONS_NAMESPACE_SECRET] = em->appns_secret;
     }
   vl_msg_api_send_shmem (em->vl_input_queue, (u8 *) & bmp);
+}
+
+void
+echo_send_detach (echo_main_t * em)
+{
+  vl_api_application_detach_t *bmp;
+  bmp = vl_msg_api_alloc (sizeof (*bmp));
+  clib_memset (bmp, 0, sizeof (*bmp));
+
+  bmp->_vl_msg_id = ntohs (VL_API_APPLICATION_DETACH);
+  bmp->client_index = em->my_client_index;
+  bmp->context = ntohl (0xfeedface);
+
+  vl_msg_api_send_shmem (em->vl_input_queue, (u8 *) & bmp);
+}
+
+void
+echo_send_add_crypto_ctx (echo_main_t * em)
+{
+  vl_api_application_tls_cert_add_t *cert_mp;
+  vl_api_application_tls_key_add_t *key_mp;
 
   cert_mp = vl_msg_api_alloc (sizeof (*cert_mp) + test_srv_crt_rsa_len);
   clib_memset (cert_mp, 0, sizeof (*cert_mp));
@@ -75,72 +93,83 @@ echo_send_attach (echo_main_t * em)
 }
 
 void
-echo_send_detach (echo_main_t * em)
-{
-  vl_api_application_detach_t *bmp;
-  bmp = vl_msg_api_alloc (sizeof (*bmp));
-  clib_memset (bmp, 0, sizeof (*bmp));
-
-  bmp->_vl_msg_id = ntohs (VL_API_APPLICATION_DETACH);
-  bmp->client_index = em->my_client_index;
-  bmp->context = ntohl (0xfeedface);
-  vl_msg_api_send_shmem (em->vl_input_queue, (u8 *) & bmp);
-}
-
-void
 echo_send_listen (echo_main_t * em)
 {
-  vl_api_bind_uri_t *bmp;
-  bmp = vl_msg_api_alloc (sizeof (*bmp));
-  clib_memset (bmp, 0, sizeof (*bmp));
-
-  bmp->_vl_msg_id = ntohs (VL_API_BIND_URI);
-  bmp->client_index = em->my_client_index;
-  bmp->context = ntohl (0xfeedface);
-  memcpy (bmp->uri, em->uri, vec_len (em->uri));
-  vl_msg_api_send_shmem (em->vl_input_queue, (u8 *) & bmp);
+  app_session_evt_t _app_evt, *app_evt = &_app_evt;
+  session_listen_msg_t *mp;
+  svm_msg_q_t *mq = em->ctrl_mq;
+
+  app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_LISTEN);
+  mp = (session_listen_msg_t *) app_evt->evt->data;
+  memset (mp, 0, sizeof (*mp));
+  mp->client_index = em->my_client_index;
+  mp->context = ntohl (0xfeedface);
+  mp->wrk_index = 0;
+  mp->is_ip4 = em->uri_elts.is_ip4;
+  clib_memcpy_fast (&mp->ip, &em->uri_elts.ip, sizeof (mp->ip));
+  mp->port = em->uri_elts.port;
+  mp->proto = em->uri_elts.transport_proto;
+  app_send_ctrl_evt_to_vpp (mq, app_evt);
 }
 
 void
-echo_send_unbind (echo_main_t * em)
+echo_send_unbind (echo_main_t * em, echo_session_t * s)
 {
-  vl_api_unbind_uri_t *ump;
-
-  ump = vl_msg_api_alloc (sizeof (*ump));
-  clib_memset (ump, 0, sizeof (*ump));
-
-  ump->_vl_msg_id = ntohs (VL_API_UNBIND_URI);
-  ump->client_index = em->my_client_index;
-  memcpy (ump->uri, em->uri, vec_len (em->uri));
-  vl_msg_api_send_shmem (em->vl_input_queue, (u8 *) & ump);
+  app_session_evt_t _app_evt, *app_evt = &_app_evt;
+  session_unlisten_msg_t *mp;
+  svm_msg_q_t *mq = em->ctrl_mq;
+
+  app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_UNLISTEN);
+  mp = (session_unlisten_msg_t *) app_evt->evt->data;
+  memset (mp, 0, sizeof (*mp));
+  mp->client_index = em->my_client_index;
+  mp->wrk_index = 0;
+  mp->handle = s->vpp_session_handle;
+  mp->context = 0;
+  app_send_ctrl_evt_to_vpp (mq, app_evt);
 }
 
 void
-echo_send_connect (u8 * uri, u32 opaque)
+echo_send_connect (u64 parent_session_handle, u32 opaque)
 {
   echo_main_t *em = &echo_main;
-  vl_api_connect_uri_t *cmp;
-  cmp = vl_msg_api_alloc (sizeof (*cmp));
-  clib_memset (cmp, 0, sizeof (*cmp));
-  cmp->_vl_msg_id = ntohs (VL_API_CONNECT_URI);
-  cmp->client_index = em->my_client_index;
-  cmp->context = ntohl (opaque);
-  memcpy (cmp->uri, uri, vec_len (uri));
-  vl_msg_api_send_shmem (em->vl_input_queue, (u8 *) & cmp);
+  app_session_evt_t _app_evt, *app_evt = &_app_evt;
+  session_connect_msg_t *mp;
+  svm_msg_q_t *mq = em->ctrl_mq;
+
+  app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_CONNECT);
+  mp = (session_connect_msg_t *) app_evt->evt->data;
+  memset (mp, 0, sizeof (*mp));
+  mp->client_index = em->my_client_index;
+  mp->context = ntohl (opaque);
+  mp->wrk_index = 0;
+  mp->is_ip4 = em->uri_elts.is_ip4;
+  clib_memcpy_fast (&mp->ip, &em->uri_elts.ip, sizeof (mp->ip));
+  mp->port = em->uri_elts.port;
+  mp->proto = em->uri_elts.transport_proto;
+  mp->parent_handle = parent_session_handle;
+  app_send_ctrl_evt_to_vpp (mq, app_evt);
 }
 
 void
 echo_send_disconnect_session (u64 handle, u32 opaque)
 {
   echo_main_t *em = &echo_main;
-  vl_api_disconnect_session_t *dmp;
-  dmp = vl_msg_api_alloc (sizeof (*dmp));
-  clib_memset (dmp, 0, sizeof (*dmp));
-  dmp->_vl_msg_id = ntohs (VL_API_DISCONNECT_SESSION);
-  dmp->client_index = em->my_client_index;
-  dmp->handle = handle;
-  ECHO_LOG (1, "Disconnect session 0x%lx", dmp->handle);
-  vl_msg_api_send_shmem (em->vl_input_queue, (u8 *) & dmp);
+  echo_session_t *s;
+  app_session_evt_t _app_evt, *app_evt = &_app_evt;
+  session_disconnect_msg_t *mp;
+  svm_msg_q_t *mq = em->ctrl_mq;
+
+  app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_DISCONNECT);
+  mp = (session_disconnect_msg_t *) app_evt->evt->data;
+  memset (mp, 0, sizeof (*mp));
+  mp->client_index = em->my_client_index;
+  mp->handle = handle;
+  app_send_ctrl_evt_to_vpp (mq, app_evt);
+
+  if (!(s = echo_get_session_from_handle (em, mp->handle)))
+    return;
+  em->proto_cb_vft->sent_disconnect_cb (s);
 }
 
 /*
@@ -187,8 +216,51 @@ echo_segment_handle_add_del (echo_main_t * em, u64 segment_handle, u8 add)
  */
 
 static void
-vl_api_application_attach_reply_t_handler (vl_api_application_attach_reply_t *
-					   mp)
+  vl_api_application_tls_cert_add_reply_t_handler
+  (vl_api_application_tls_cert_add_reply_t * mp)
+{
+  echo_main_t *em = &echo_main;
+  if (mp->retval)
+    {
+      ECHO_FAIL (ECHO_FAIL_VL_API_TLS_CERT_ADD_REPLY,
+		 "tls cert add returned %d",
+		 clib_net_to_host_u32 (mp->retval));
+      return;
+    }
+  /* No concurrency here, only bapi thread writes */
+  if (em->state != STATE_ATTACHED_NO_CERT
+      && em->state != STATE_ATTACHED_ONE_CERT)
+    {
+      ECHO_FAIL (ECHO_FAIL_VL_API_TLS_CERT_ADD_REPLY, "Wrong state");
+      return;
+    }
+  em->state++;
+}
+
+static void
+  vl_api_application_tls_key_add_reply_t_handler
+  (vl_api_application_tls_key_add_reply_t * mp)
+{
+  echo_main_t *em = &echo_main;
+  if (mp->retval)
+    {
+      ECHO_FAIL (ECHO_FAIL_VL_API_TLS_KEY_ADD_REPLY,
+		 "tls key add returned %d",
+		 clib_net_to_host_u32 (mp->retval));
+      return;
+    }
+  /* No concurrency here, only bapi thread writes */
+  if (em->state != STATE_ATTACHED_NO_CERT
+      && em->state != STATE_ATTACHED_ONE_CERT)
+    {
+      ECHO_FAIL (ECHO_FAIL_VL_API_TLS_CERT_ADD_REPLY, "Wrong state");
+      return;
+    }
+  em->state++;
+}
+
+static void
+vl_api_app_attach_reply_t_handler (vl_api_app_attach_reply_t * mp)
 {
   echo_main_t *em = &echo_main;
   int *fds = 0, i;
@@ -211,9 +283,9 @@ vl_api_application_attach_reply_t_handler (vl_api_application_attach_reply_t *
       return;
     }
 
-  ASSERT (mp->app_event_queue_address);
-  em->our_event_queue = uword_to_pointer (mp->app_event_queue_address,
-					  svm_msg_q_t *);
+  ASSERT (mp->app_mq);
+  em->app_mq = uword_to_pointer (mp->app_mq, svm_msg_q_t *);
+  em->ctrl_mq = uword_to_pointer (mp->vpp_ctrl_mq, svm_msg_q_t *);
 
   if (mp->n_fds)
     {
@@ -243,7 +315,7 @@ vl_api_application_attach_reply_t_handler (vl_api_application_attach_reply_t *
 	    goto failed;
 	  }
       if (mp->fd_flags & SESSION_FD_F_MQ_EVENTFD)
-	svm_msg_q_set_consumer_eventfd (em->our_event_queue, fds[n_fds++]);
+	svm_msg_q_set_consumer_eventfd (em->app_mq, fds[n_fds++]);
 
       vec_free (fds);
     }
@@ -261,7 +333,7 @@ vl_api_application_attach_reply_t_handler (vl_api_application_attach_reply_t *
   echo_segment_handle_add_del (em, segment_handle, 1 /* add */ );
   ECHO_LOG (1, "Mapped segment 0x%lx", segment_handle);
 
-  em->state = STATE_ATTACHED;
+  em->state = STATE_ATTACHED_NO_CERT;
   return;
 failed:
   for (i = clib_max (n_fds - 1, 0); i < vec_len (fds); i++)
@@ -282,7 +354,6 @@ vl_api_application_detach_reply_t_handler (vl_api_application_detach_reply_t *
   echo_main.state = STATE_DETACHED;
 }
 
-
 static void
 vl_api_unmap_segment_t_handler (vl_api_unmap_segment_t * mp)
 {
@@ -344,90 +415,13 @@ failed:
   vec_free (fds);
 }
 
-static void
-vl_api_bind_uri_reply_t_handler (vl_api_bind_uri_reply_t * mp)
-{
-  if (mp->retval)
-    {
-      ECHO_FAIL (ECHO_FAIL_VL_API_BIND_URI_REPLY, "bind failed: %U",
-		 format_api_error, clib_net_to_host_u32 (mp->retval));
-    }
-}
-
-static void
-vl_api_unbind_uri_reply_t_handler (vl_api_unbind_uri_reply_t * mp)
-{
-  echo_session_t *listen_session;
-  echo_main_t *em = &echo_main;
-  if (mp->retval != 0)
-    {
-      ECHO_FAIL (ECHO_FAIL_VL_API_UNBIND_REPLY, "unbind_uri returned %d",
-		 ntohl (mp->retval));
-      return;
-    }
-  listen_session = pool_elt_at_index (em->sessions, em->listen_session_index);
-  em->proto_cb_vft->cleanup_cb (listen_session, 0 /* parent_died */ );
-  em->state = STATE_DISCONNECTED;
-}
-
-static void
-vl_api_disconnect_session_reply_t_handler (vl_api_disconnect_session_reply_t *
-					   mp)
-{
-  echo_main_t *em = &echo_main;
-  echo_session_t *s;
-
-  if (mp->retval)
-    {
-      ECHO_FAIL (ECHO_FAIL_VL_API_DISCONNECT_SESSION_REPLY,
-		 "vpp complained about disconnect: %d", ntohl (mp->retval));
-      return;
-    }
-
-  ECHO_LOG (1, "Got disonnected reply for session 0x%lx", mp->handle);
-  if (!(s = echo_get_session_from_handle (em, mp->handle)))
-    return;
-  em->proto_cb_vft->disconnected_reply_cb (s);
-}
-
-static void
-  vl_api_application_tls_cert_add_reply_t_handler
-  (vl_api_application_tls_cert_add_reply_t * mp)
-{
-  if (mp->retval)
-    ECHO_FAIL (ECHO_FAIL_VL_API_TLS_CERT_ADD_REPLY,
-	       "failed to add application tls cert");
-}
-
-static void
-  vl_api_application_tls_key_add_reply_t_handler
-  (vl_api_application_tls_key_add_reply_t * mp)
-{
-  if (mp->retval)
-    ECHO_FAIL (ECHO_FAIL_VL_API_TLS_KEY_ADD_REPLY,
-	       "failed to add application tls key");
-}
-
-static void
-vl_api_connect_uri_reply_t_handler (vl_api_connect_uri_reply_t * mp)
-{
-  echo_main_t *em = &echo_main;
-  if (mp->retval && (em->proto_cb_vft->connected_cb))
-    em->proto_cb_vft->connected_cb ((session_connected_bundled_msg_t *) mp,
-				    mp->context, 1 /* is_failed */ );
-}
-
-#define foreach_quic_echo_msg                                           \
-_(BIND_URI_REPLY, bind_uri_reply)                                       \
-_(UNBIND_URI_REPLY, unbind_uri_reply)                                   \
-_(DISCONNECT_SESSION_REPLY, disconnect_session_reply)                   \
-_(APPLICATION_ATTACH_REPLY, application_attach_reply)                   \
-_(APPLICATION_DETACH_REPLY, application_detach_reply)                   \
-_(MAP_ANOTHER_SEGMENT, map_another_segment)                             \
-_(UNMAP_SEGMENT, unmap_segment)                                         \
-_(APPLICATION_TLS_CERT_ADD_REPLY, application_tls_cert_add_reply)       \
-_(APPLICATION_TLS_KEY_ADD_REPLY, application_tls_key_add_reply)         \
-_(CONNECT_URI_REPLY, connect_uri_reply)         \
+#define foreach_quic_echo_msg                              \
+_(APP_ATTACH_REPLY, app_attach_reply)                      \
+_(APPLICATION_DETACH_REPLY, application_detach_reply)      \
+_(MAP_ANOTHER_SEGMENT, map_another_segment)                \
+_(APPLICATION_TLS_CERT_ADD_REPLY, application_tls_cert_add_reply)              \
+_(APPLICATION_TLS_KEY_ADD_REPLY, application_tls_key_add_reply)              \
+_(UNMAP_SEGMENT, unmap_segment)
 
 void
 echo_api_hookup (echo_main_t * em)
diff --git a/src/plugins/hs_apps/sapi/vpp_echo_common.c b/src/plugins/hs_apps/sapi/vpp_echo_common.c
index d6ee644ea6b..0f6c2ec7135 100644
--- a/src/plugins/hs_apps/sapi/vpp_echo_common.c
+++ b/src/plugins/hs_apps/sapi/vpp_echo_common.c
@@ -174,6 +174,10 @@ echo_format_app_state (u8 * s, va_list * args)
     return format (s, "STATE_START");
   if (state == STATE_ATTACHED)
     return format (s, "STATE_ATTACHED");
+  if (state == STATE_ATTACHED_NO_CERT)
+    return format (s, "STATE_ATTACHED_NO_CERT");
+  if (state == STATE_ATTACHED_ONE_CERT)
+    return format (s, "STATE_ATTACHED_ONE_CERT");
   if (state == STATE_LISTEN)
     return format (s, "STATE_LISTEN");
   if (state == STATE_READY)
@@ -426,6 +430,40 @@ unformat_ip6_address (unformat_input_t * input, va_list * args)
   }
 }
 
+u8 *
+echo_format_crypto_engine (u8 * s, va_list * args)
+{
+  u32 state = va_arg (*args, u32);
+  if (state == TLS_ENGINE_MBEDTLS)
+    return format (s, "mbedtls");
+  if (state == TLS_ENGINE_OPENSSL)
+    return format (s, "openssl");
+  if (state == CRYPTO_ENGINE_PICOTLS)
+    return format (s, "picotls");
+  if (state == CRYPTO_ENGINE_VPP)
+    return format (s, "vpp");
+  else
+    return format (s, "unknown crypto engine");
+}
+
+uword
+echo_unformat_crypto_engine (unformat_input_t * input, va_list * args)
+{
+  u8 *a = va_arg (*args, u8 *);
+  if (unformat (input, "mbedtls"))
+    *a = TLS_ENGINE_MBEDTLS;
+  else if (unformat (input, "openssl"))
+    *a = TLS_ENGINE_OPENSSL;
+  else if (unformat (input, "picotls"))
+    *a = CRYPTO_ENGINE_PICOTLS;
+  else if (unformat (input, "vpp"))
+    *a = CRYPTO_ENGINE_VPP;
+  else
+    return 0;
+  return 1;
+}
+
+
 /*
  *
  *  End of format functions
diff --git a/src/plugins/hs_apps/sapi/vpp_echo_common.h b/src/plugins/hs_apps/sapi/vpp_echo_common.h
index 0e9eaebd345..ea20e2b9e09 100644
--- a/src/plugins/hs_apps/sapi/vpp_echo_common.h
+++ b/src/plugins/hs_apps/sapi/vpp_echo_common.h
@@ -73,8 +73,8 @@
   _(ECHO_FAIL_VL_API_DETACH_REPLY, "ECHO_FAIL_VL_API_DETACH_REPLY")     \
   _(ECHO_FAIL_VL_API_BIND_URI_REPLY, "ECHO_FAIL_VL_API_BIND_URI_REPLY") \
   _(ECHO_FAIL_VL_API_UNBIND_REPLY, "ECHO_FAIL_VL_API_UNBIND_REPLY")     \
-  _(ECHO_FAIL_VL_API_DISCONNECT_SESSION_REPLY,                          \
-    "ECHO_FAIL_VL_API_DISCONNECT_SESSION_REPLY")                        \
+  _(ECHO_FAIL_SESSION_DISCONNECT, "ECHO_FAIL_SESSION_DISCONNECT")       \
+  _(ECHO_FAIL_SESSION_RESET, "ECHO_FAIL_SESSION_RESET")                 \
   _(ECHO_FAIL_VL_API_TLS_CERT_ADD_REPLY,                                \
     "ECHO_FAIL_VL_API_TLS_CERT_ADD_REPLY")                              \
   _(ECHO_FAIL_VL_API_TLS_KEY_ADD_REPLY,                                 \
@@ -189,6 +189,8 @@ enum quic_session_state_t
 typedef enum
 {
   STATE_START,
+  STATE_ATTACHED_NO_CERT,
+  STATE_ATTACHED_ONE_CERT,
   STATE_ATTACHED,
   STATE_LISTEN,
   STATE_READY,
@@ -221,7 +223,7 @@ typedef struct echo_proto_cb_vft_
   void (*bound_uri_cb) (session_bound_msg_t * mp, echo_session_t * session);	/* Session got bound */
   void (*reset_cb) (session_reset_msg_t * mp, echo_session_t * s);	/* Received RESET on session */
   void (*disconnected_cb) (session_disconnected_msg_t * mp, echo_session_t * s);	/* Received DISCONNECT on session */
-  void (*disconnected_reply_cb) (echo_session_t * s);	/* ACK disconnect we sent to vpp */
+  void (*sent_disconnect_cb) (echo_session_t * s);	/* ACK disconnect we sent to vpp */
   void (*cleanup_cb) (echo_session_t * s, u8 parent_died);	/* Session should be cleaned up (parent listener may be dead) */
   /* Add CLI options */
   int (*process_opts_cb) (unformat_input_t * a);
@@ -249,7 +251,8 @@ typedef struct
   u32 my_client_index;		/* API client handle */
   u8 *uri;			/* The URI we're playing with */
   echo_session_t *sessions;	/* Session pool */
-  svm_msg_q_t *our_event_queue;	/* Our event queue */
+  svm_msg_q_t *app_mq;		/* Our receiveing event queue */
+  svm_msg_q_t *ctrl_mq;		/* Our control queue (towards vpp) */
   clib_time_t clib_time;	/* For deadman timers */
   u8 *socket_name;
   int i_am_master;
@@ -289,6 +292,7 @@ typedef struct
   u8 log_lvl;			/* Verbosity of the logging */
   int max_test_msg;		/* Limit the number of incorrect data messages */
   u32 evt_q_size;		/* Size of the vpp MQ (app<->vpp events) */
+  u32 crypto_ctx_engine;	/* crypto engine used */
 
   u8 *appns_id;
   u64 appns_flags;
@@ -373,16 +377,19 @@ int wait_for_state_change (echo_main_t * em, connection_state_t state,
 			   f64 timeout);
 void echo_notify_event (echo_main_t * em, echo_test_evt_t e);
 void echo_session_print_stats (echo_main_t * em, echo_session_t * session);
+u8 *echo_format_crypto_engine (u8 * s, va_list * args);
+uword echo_unformat_crypto_engine (unformat_input_t * input, va_list * args);
 
 /* Binary API */
 
 void echo_send_attach (echo_main_t * em);
 void echo_send_detach (echo_main_t * em);
 void echo_send_listen (echo_main_t * em);
-void echo_send_unbind (echo_main_t * em);
-void echo_send_connect (u8 * uri, u32 opaque);
+void echo_send_unbind (echo_main_t * em, echo_session_t * s);
+void echo_send_connect (u64 vpp_session_handle, u32 opaque);
 void echo_send_disconnect_session (u64 handle, u32 opaque);
 void echo_api_hookup (echo_main_t * em);
+void echo_send_add_crypto_ctx (echo_main_t * em);
 
 #endif /* __included_vpp_echo_common_h__ */
 
diff --git a/src/plugins/hs_apps/sapi/vpp_echo_proto_quic.c b/src/plugins/hs_apps/sapi/vpp_echo_proto_quic.c
index 5ffa7a7c70e..999cacb5353 100644
--- a/src/plugins/hs_apps/sapi/vpp_echo_proto_quic.c
+++ b/src/plugins/hs_apps/sapi/vpp_echo_proto_quic.c
@@ -54,12 +54,11 @@ quic_echo_on_connected_connect (session_connected_msg_t * mp,
 {
   echo_main_t *em = &echo_main;
   quic_echo_proto_main_t *eqm = &quic_echo_proto_main;
-  u8 *uri = format (0, "quic://session/%lu", mp->handle);
   u64 i;
 
   echo_notify_event (em, ECHO_EVT_FIRST_SCONNECT);
   for (i = 0; i < eqm->n_stream_clients; i++)
-    echo_send_rpc (em, echo_send_connect, (void *) uri, session_index);
+    echo_send_rpc (em, echo_send_connect, (void *) mp->handle, session_index);
 
   ECHO_LOG (0, "Qsession 0x%llx connected to %U:%d",
 	    mp->handle, format_ip46_address, &mp->lcl.ip,
@@ -108,12 +107,11 @@ quic_echo_on_accept_connect (session_accepted_msg_t * mp, u32 session_index)
   echo_main_t *em = &echo_main;
   quic_echo_proto_main_t *eqm = &quic_echo_proto_main;
   ECHO_LOG (1, "Accept on QSession 0x%lx %u", mp->handle);
-  u8 *uri = format (0, "quic://session/%lu", mp->handle);
   u32 i;
 
   echo_notify_event (em, ECHO_EVT_FIRST_SCONNECT);
   for (i = 0; i < eqm->n_stream_clients; i++)
-    echo_send_rpc (em, echo_send_connect, (void *) uri, session_index);
+    echo_send_rpc (em, echo_send_connect, (void *) mp->handle, session_index);
 }
 
 static void
@@ -295,19 +293,18 @@ quic_echo_retry_connect (u32 session_index)
   /* retry connect */
   echo_session_t *session;
   echo_main_t *em = &echo_main;
-  u8 *uri;
   if (session_index == SESSION_INVALID_INDEX)
     {
       ECHO_LOG (1, "Retrying connect %s", em->uri);
-      echo_send_rpc (em, echo_send_connect, (void *) em->uri,
+      echo_send_rpc (em, echo_send_connect, (void *) SESSION_INVALID_HANDLE,
 		     SESSION_INVALID_INDEX);
     }
   else
     {
       session = pool_elt_at_index (em->sessions, session_index);
-      uri = format (0, "quic://session/%lu", session->vpp_session_handle);
-      ECHO_LOG (1, "Retrying connect %s", uri);
-      echo_send_rpc (em, echo_send_connect, (void *) uri, session_index);
+      ECHO_LOG (1, "Retrying connect 0x%lx", session->vpp_session_handle);
+      echo_send_rpc (em, echo_send_connect,
+		     (void *) session->vpp_session_handle, session_index);
     }
 }
 
@@ -364,7 +361,7 @@ quic_echo_accepted_cb (session_accepted_msg_t * mp, echo_session_t * session)
 }
 
 static void
-quic_echo_disconnected_reply_cb (echo_session_t * s)
+quic_echo_sent_disconnect_cb (echo_session_t * s)
 {
   if (s->session_type == ECHO_SESSION_TYPE_STREAM)
     s->session_state = ECHO_SESSION_STATE_CLOSING;
@@ -454,6 +451,8 @@ quic_echo_set_defaults_after_opts_cb ()
   echo_main_t *em = &echo_main;
   u8 default_f_active;
 
+  if (em->crypto_ctx_engine == TLS_ENGINE_NONE)
+    em->crypto_ctx_engine = CRYPTO_ENGINE_PICOTLS;
   em->n_connects = em->n_clients;
   em->n_sessions =
     clib_max (1, eqm->n_stream_clients) * em->n_clients + em->n_clients + 1;
@@ -488,7 +487,7 @@ echo_proto_cb_vft_t quic_echo_proto_cb_vft = {
   .connected_cb = quic_echo_connected_cb,
   .accepted_cb = quic_echo_accepted_cb,
   .reset_cb = quic_echo_reset_cb,
-  .disconnected_reply_cb = quic_echo_disconnected_reply_cb,
+  .sent_disconnect_cb = quic_echo_sent_disconnect_cb,
   .cleanup_cb = quic_echo_cleanup_cb,
   .process_opts_cb = quic_echo_process_opts_cb,
   .print_usage_cb = quic_echo_print_usage_cb,
diff --git a/src/plugins/hs_apps/sapi/vpp_echo_proto_tcp.c b/src/plugins/hs_apps/sapi/vpp_echo_proto_tcp.c
index c68ffafbac6..0854d489a41 100644
--- a/src/plugins/hs_apps/sapi/vpp_echo_proto_tcp.c
+++ b/src/plugins/hs_apps/sapi/vpp_echo_proto_tcp.c
@@ -97,7 +97,7 @@ tcp_echo_accepted_cb (session_accepted_msg_t * mp, echo_session_t * session)
 }
 
 static void
-tcp_echo_disconnected_reply_cb (echo_session_t * s)
+tcp_echo_sent_disconnect_cb (echo_session_t * s)
 {
   s->session_state = ECHO_SESSION_STATE_CLOSING;
 }
@@ -122,13 +122,31 @@ tcp_echo_reset_cb (session_reset_msg_t * mp, echo_session_t * s)
   s->session_state = ECHO_SESSION_STATE_CLOSING;
 }
 
+static void
+tls_echo_set_defaults_after_opts_cb ()
+{
+  echo_main_t *em = &echo_main;
+  if (em->crypto_ctx_engine == TLS_ENGINE_NONE)
+    em->crypto_ctx_engine = TLS_ENGINE_OPENSSL;
+}
+
 echo_proto_cb_vft_t echo_tcp_proto_cb_vft = {
   .disconnected_cb = tcp_echo_disconnected_cb,
   .connected_cb = tcp_echo_connected_cb,
   .accepted_cb = tcp_echo_accepted_cb,
   .reset_cb = tcp_echo_reset_cb,
-  .disconnected_reply_cb = tcp_echo_disconnected_reply_cb,
+  .sent_disconnect_cb = tcp_echo_sent_disconnect_cb,
+  .cleanup_cb = tcp_echo_cleanup_cb,
+};
+
+echo_proto_cb_vft_t echo_tls_proto_cb_vft = {
+  .disconnected_cb = tcp_echo_disconnected_cb,
+  .connected_cb = tcp_echo_connected_cb,
+  .accepted_cb = tcp_echo_accepted_cb,
+  .reset_cb = tcp_echo_reset_cb,
+  .sent_disconnect_cb = tcp_echo_sent_disconnect_cb,
   .cleanup_cb = tcp_echo_cleanup_cb,
+  .set_defaults_after_opts_cb = tls_echo_set_defaults_after_opts_cb,
 };
 
 ECHO_REGISTER_PROTO (TRANSPORT_PROTO_TCP, echo_tcp_proto_cb_vft);
diff --git a/src/plugins/hs_apps/sapi/vpp_echo_proto_udp.c b/src/plugins/hs_apps/sapi/vpp_echo_proto_udp.c
index 357d3e08022..902b30ff1c3 100644
--- a/src/plugins/hs_apps/sapi/vpp_echo_proto_udp.c
+++ b/src/plugins/hs_apps/sapi/vpp_echo_proto_udp.c
@@ -98,7 +98,7 @@ udp_echo_accepted_cb (session_accepted_msg_t * mp, echo_session_t * session)
 }
 
 static void
-udp_echo_disconnected_reply_cb (echo_session_t * s)
+udp_echo_sent_disconnect_cb (echo_session_t * s)
 {
   s->session_state = ECHO_SESSION_STATE_CLOSING;
 }
@@ -166,7 +166,7 @@ echo_proto_cb_vft_t echo_udp_proto_cb_vft = {
   .connected_cb = udp_echo_connected_cb,
   .accepted_cb = udp_echo_accepted_cb,
   .reset_cb = udp_echo_reset_cb,
-  .disconnected_reply_cb = udp_echo_disconnected_reply_cb,
+  .sent_disconnect_cb = udp_echo_sent_disconnect_cb,
   .cleanup_cb = udp_echo_cleanup_cb,
   .bound_uri_cb = udp_echo_bound_uri_cb,
 };
diff --git a/src/plugins/quic/quic.c b/src/plugins/quic/quic.c
index 291f54608cd..e9df9ff5025 100644
--- a/src/plugins/quic/quic.c
+++ b/src/plugins/quic/quic.c
@@ -2079,7 +2079,7 @@ static const transport_proto_vft_t quic_proto = {
 /* *INDENT-ON* */
 
 static void
-quic_register_cipher_suite (quic_crypto_engine_t type,
+quic_register_cipher_suite (tls_engine_type_t type,
 			    ptls_cipher_suite_t ** ciphers)
 {
   quic_main_t *qm = &quic_main;
diff --git a/src/plugins/quic/quic.h b/src/plugins/quic/quic.h
index dacf07a14ad..312ffcba1cb 100644
--- a/src/plugins/quic/quic.h
+++ b/src/plugins/quic/quic.h
@@ -135,12 +135,6 @@ STATIC_ASSERT (offsetof (quic_ctx_t, _sctx_end_marker) <=
 	       TRANSPORT_CONN_ID_LEN,
 	       "connection data must be less than TRANSPORT_CONN_ID_LEN bytes");
 
-typedef enum quic_crypto_engine_
-{
-  CRYPTO_ENGINE_VPP,
-  CRYPTO_ENGINE_PICOTLS,
-} quic_crypto_engine_t;
-
 /* single-entry session cache */
 typedef struct quic_session_cache_
 {