tls: dtls initial implementation

Type: feature

Basic dtls transport protocol implementation that relies on openssl
wire protocol implementation. Retries/timeouts not yet supported.

To test using vcl test apps, first ensure all arp entries are properly
resolved and subsequently:

server: vcl_server -p dtls 1234
client: vcl_client -p dtls <server-ip> 1234 -U -N 2000000 -T 1460 -X

Signed-off-by: Florin Coras <fcoras@cisco.com>
Change-Id: I04b4516a8fe9ce85ba230bcdd891f33a900046ed

4 files changed, 247 insertions, 30 deletions

diff --git a/src/vnet/session/session.c b/src/vnet/session/session.c
index 169cca5efe8..85a802c3a06 100644
--- a/src/vnet/session/session.c
+++ b/src/vnet/session/session.c
@@ -1825,7 +1825,7 @@ session_main_init (vlib_main_t * vm)
   smm->evt_qs_segment_size = 1 << 20;
 #endif
 
-  smm->last_transport_proto_type = TRANSPORT_PROTO_QUIC;
+  smm->last_transport_proto_type = TRANSPORT_PROTO_DTLS;
 
   return 0;
 }
diff --git a/src/vnet/session/transport_types.h b/src/vnet/session/transport_types.h
index 2caafea0fbe..1e7d5672f1c 100644
--- a/src/vnet/session/transport_types.h
+++ b/src/vnet/session/transport_types.h
@@ -158,12 +158,13 @@ STATIC_ASSERT (STRUCT_OFFSET_OF (transport_connection_t, s_index)
 STATIC_ASSERT (sizeof (transport_connection_t) <= 128,
 	       "moved into 3rd cache line");
 
-#define foreach_transport_proto				\
-  _(TCP, "tcp", "T")					\
-  _(UDP, "udp", "U")					\
-  _(NONE, "ct", "C")					\
-  _(TLS, "tls", "J")					\
-  _(QUIC, "quic", "Q")					\
+#define foreach_transport_proto                                               \
+  _ (TCP, "tcp", "T")                                                         \
+  _ (UDP, "udp", "U")                                                         \
+  _ (NONE, "ct", "C")                                                         \
+  _ (TLS, "tls", "J")                                                         \
+  _ (QUIC, "quic", "Q")                                                       \
+  _ (DTLS, "dtls", "D")
 
 typedef enum _transport_proto
 {
diff --git a/src/vnet/tls/tls.c b/src/vnet/tls/tls.c
index 59dae88fd1f..6052ca1bf65 100644
--- a/src/vnet/tls/tls.c
+++ b/src/vnet/tls/tls.c
@@ -221,13 +221,32 @@ tls_notify_app_connected (tls_ctx_t * ctx, session_error_t err)
     }
 
   if (err)
-    goto failed;
+    {
+      /* Free app session pre-allocated when transport was established */
+      if (ctx->tls_type == TRANSPORT_PROTO_TLS)
+	session_free (session_get (ctx->c_s_index, ctx->c_thread_index));
+      goto failed;
+    }
+
+  /* For DTLS the app session is not preallocated because the underlying udp
+   * session might migrate to a different worker during the handshake */
+  if (ctx->tls_type == TRANSPORT_PROTO_DTLS)
+    {
+      session_type_t st;
+      app_session = session_alloc (ctx->c_thread_index);
+      app_session->session_state = SESSION_STATE_CREATED;
+      ctx->c_s_index = app_session->session_index;
+      st =
+	session_type_from_proto_and_ip (TRANSPORT_PROTO_DTLS, ctx->tcp_is_ip4);
+      app_session->session_type = st;
+      app_session->connection_index = ctx->tls_ctx_handle;
+    }
+  else
+    {
+      app_session = session_get (ctx->c_s_index, ctx->c_thread_index);
+    }
 
-  app_session = session_get (ctx->c_s_index, ctx->c_thread_index);
   app_session->app_wrk_index = ctx->parent_app_wrk_index;
-  app_session->connection_index = ctx->tls_ctx_handle;
-  app_session->session_type =
-    session_type_from_proto_and_ip (TRANSPORT_PROTO_TLS, ctx->tcp_is_ip4);
 
   if ((err = app_worker_init_connected (app_wrk, app_session)))
     goto failed;
@@ -247,8 +266,6 @@ tls_notify_app_connected (tls_ctx_t * ctx, session_error_t err)
   return 0;
 
 failed:
-  /* Free app session pre-allocated when transport was established */
-  session_free (session_get (ctx->c_s_index, ctx->c_thread_index));
   ctx->no_app_session = 1;
   tls_disconnect (ctx->tls_ctx_handle, vlib_get_thread_index ());
   return app_worker_connect_notify (app_wrk, 0, err,
@@ -278,6 +295,28 @@ tls_ctx_alloc (crypto_engine_type_t engine_type)
   return (((u32) engine_type << TLS_ENGINE_TYPE_SHIFT) | ctx_index);
 }
 
+static inline u32
+tls_ctx_alloc_w_thread (crypto_engine_type_t engine_type, u32 thread_index)
+{
+  u32 ctx_index;
+  ctx_index = tls_vfts[engine_type].ctx_alloc_w_thread (thread_index);
+  return (((u32) engine_type << TLS_ENGINE_TYPE_SHIFT) | ctx_index);
+}
+
+static inline u32
+tls_ctx_attach (crypto_engine_type_t engine_type, u32 thread_index, void *ctx)
+{
+  u32 ctx_index;
+  ctx_index = tls_vfts[engine_type].ctx_attach (thread_index, ctx);
+  return (((u32) engine_type << TLS_ENGINE_TYPE_SHIFT) | ctx_index);
+}
+
+static inline void *
+tls_ctx_detach (tls_ctx_t *ctx)
+{
+  return tls_vfts[ctx->tls_ctx_engine].ctx_detach (ctx);
+}
+
 static inline tls_ctx_t *
 tls_ctx_get (u32 ctx_handle)
 {
@@ -442,6 +481,10 @@ tls_app_rx_callback (session_t * tls_session)
 {
   tls_ctx_t *ctx;
 
+  /* DTLS session migrating, wait for next notification */
+  if (PREDICT_FALSE (tls_session->flags & SESSION_F_IS_MIGRATING))
+    return 0;
+
   ctx = tls_ctx_get (tls_session->opaque);
   tls_ctx_read (ctx, tls_session);
   return 0;
@@ -459,11 +502,12 @@ tls_app_tx_callback (session_t * tls_session)
 }
 
 int
-tls_session_connected_callback (u32 tls_app_index, u32 ho_ctx_index,
-				session_t * tls_session, session_error_t err)
+tls_session_connected_cb (u32 tls_app_index, u32 ho_ctx_index,
+			  session_t *tls_session, session_error_t err)
 {
   session_t *app_session;
   tls_ctx_t *ho_ctx, *ctx;
+  session_type_t st;
   u32 ctx_handle;
 
   ho_ctx = tls_ctx_half_open_get (ho_ctx_index);
@@ -496,7 +540,7 @@ tls_session_connected_callback (u32 tls_app_index, u32 ho_ctx_index,
   ctx->c_flags |= TRANSPORT_CONNECTION_F_NO_LOOKUP;
 
   TLS_DBG (1, "TCP connect for %u returned %u. New connection [%u]%x",
-	   ho_ctx_index, is_fail, vlib_get_thread_index (),
+	   ho_ctx_index, err, vlib_get_thread_index (),
 	   (ctx) ? ctx_handle : ~0);
 
   ctx->tls_session_handle = session_handle (tls_session);
@@ -508,10 +552,43 @@ tls_session_connected_callback (u32 tls_app_index, u32 ho_ctx_index,
   app_session = session_alloc (ctx->c_thread_index);
   app_session->session_state = SESSION_STATE_CREATED;
   ctx->c_s_index = app_session->session_index;
+  st = session_type_from_proto_and_ip (TRANSPORT_PROTO_TLS, ctx->tcp_is_ip4);
+  app_session->session_type = st;
+  app_session->connection_index = ctx->tls_ctx_handle;
+
+  return tls_ctx_init_client (ctx);
+}
+
+int
+dtls_session_connected_cb (u32 app_wrk_index, u32 ctx_handle, session_t *us,
+			   session_error_t err)
+{
+  tls_ctx_t *ctx;
+
+  ctx = tls_ctx_get_w_thread (ctx_handle, 1 /* udp allocs on thread 1 */);
+
+  ctx->tls_session_handle = session_handle (us);
+  ctx->c_flags |= TRANSPORT_CONNECTION_F_NO_LOOKUP;
+  us->opaque = ctx_handle;
+
+  /* We don't preallocate the app session because the udp session might
+   * actually migrate to a different worker at the end of the handshake */
 
   return tls_ctx_init_client (ctx);
 }
 
+int
+tls_session_connected_callback (u32 tls_app_index, u32 ho_ctx_index,
+				session_t *tls_session, session_error_t err)
+{
+  if (session_get_transport_proto (tls_session) == TRANSPORT_PROTO_TCP)
+    return tls_session_connected_cb (tls_app_index, ho_ctx_index, tls_session,
+				     err);
+  else
+    return dtls_session_connected_cb (tls_app_index, ho_ctx_index, tls_session,
+				      err);
+}
+
 static void
 tls_app_session_cleanup (session_t * s, session_cleanup_ntf_t ntf)
 {
@@ -531,7 +608,42 @@ tls_app_session_cleanup (session_t * s, session_cleanup_ntf_t ntf)
   tls_ctx_free (ctx);
 }
 
-/* *INDENT-OFF* */
+static void
+dtls_migrate_udp (void *arg)
+{
+  tls_ctx_t *ctx = (tls_ctx_t *) arg;
+  u32 ctx_handle, thread_index;
+  session_t *us;
+
+  thread_index = session_thread_from_handle (ctx->tls_session_handle);
+  ASSERT (thread_index == vlib_get_thread_index ());
+
+  ctx_handle = tls_ctx_attach (ctx->tls_ctx_engine, thread_index, ctx);
+  ctx = tls_ctx_get_w_thread (ctx_handle, thread_index);
+  ctx->tls_ctx_handle = ctx_handle;
+
+  us = session_get_from_handle (ctx->tls_session_handle);
+  us->opaque = ctx_handle;
+  us->flags &= ~SESSION_F_IS_MIGRATING;
+  if (svm_fifo_max_dequeue (us->tx_fifo))
+    session_send_io_evt_to_thread (us->tx_fifo, SESSION_IO_EVT_TX);
+}
+
+static void
+dtls_session_migrate_callback (session_t *us, session_handle_t new_sh)
+{
+  u32 new_thread = session_thread_from_handle (new_sh);
+  tls_ctx_t *ctx;
+
+  /* Migrate dtls context to new thread */
+  ctx = tls_ctx_get_w_thread (us->opaque, us->thread_index);
+  ctx->tls_session_handle = new_sh;
+  ctx = tls_ctx_detach (ctx);
+  ctx->is_migrated = 1;
+
+  session_send_rpc_evt_to_thread (new_thread, dtls_migrate_udp, (void *) ctx);
+}
+
 static session_cb_vft_t tls_app_cb_vft = {
   .session_accept_callback = tls_session_accept_callback,
   .session_disconnect_callback = tls_session_disconnect_callback,
@@ -541,16 +653,16 @@ static session_cb_vft_t tls_app_cb_vft = {
   .del_segment_callback = tls_del_segment_callback,
   .builtin_app_rx_callback = tls_app_rx_callback,
   .builtin_app_tx_callback = tls_app_tx_callback,
+  .session_migrate_callback = dtls_session_migrate_callback,
   .session_cleanup_callback = tls_app_session_cleanup,
 };
-/* *INDENT-ON* */
 
 int
 tls_connect (transport_endpoint_cfg_t * tep)
 {
   vnet_connect_args_t _cargs = { {}, }, *cargs = &_cargs;
-  session_endpoint_cfg_t *sep;
   crypto_engine_type_t engine_type;
+  session_endpoint_cfg_t *sep;
   tls_main_t *tm = &tls_main;
   app_worker_t *app_wrk;
   application_t *app;
@@ -574,6 +686,7 @@ tls_connect (transport_endpoint_cfg_t * tep)
   ctx->parent_app_api_context = sep->opaque;
   ctx->tcp_is_ip4 = sep->is_ip4;
   ctx->ckpair_index = sep->ckpair_index;
+  ctx->tls_type = sep->transport_proto;
   if (sep->hostname)
     {
       ctx->srv_hostname = format (0, "%v", sep->hostname);
@@ -638,6 +751,11 @@ tls_start_listen (u32 app_listener_index, transport_endpoint_t * tep)
   args->sep_ext = *sep;
   args->sep_ext.ns_index = app->ns_index;
   args->sep_ext.transport_proto = TRANSPORT_PROTO_TCP;
+  if (sep->transport_proto == TRANSPORT_PROTO_DTLS)
+    {
+      args->sep_ext.transport_proto = TRANSPORT_PROTO_UDP;
+      args->sep_ext.transport_flags = TRANSPORT_CFG_F_CONNECTED;
+    }
   if (vnet_listen (args))
     return -1;
 
@@ -656,6 +774,7 @@ tls_start_listen (u32 app_listener_index, transport_endpoint_t * tep)
   lctx->tcp_is_ip4 = sep->is_ip4;
   lctx->tls_ctx_engine = engine_type;
   lctx->ckpair_index = sep->ckpair_index;
+  lctx->tls_type = sep->transport_proto;
 
   if (tls_vfts[engine_type].ctx_start_listen (lctx))
     {
@@ -694,7 +813,7 @@ tls_stop_listen (u32 lctx_index)
   sep.fib_index = lc->fib_index;
   sep.port = lc->lcl_port;
   sep.is_ip4 = lc->is_ip4;
-  sep.transport_proto = TRANSPORT_PROTO_TLS;
+  sep.transport_proto = lctx->tls_type;
   clib_memcpy (&sep.ip, &lc->lcl_ip, sizeof (lc->lcl_ip));
   session_lookup_del_session_endpoint2 (&sep);
 
@@ -749,13 +868,15 @@ format_tls_ctx (u8 * s, va_list * args)
 {
   u32 tcp_si, tcp_ti, ctx_index, ctx_engine;
   tls_ctx_t *ctx = va_arg (*args, tls_ctx_t *);
+  char *proto;
 
+  proto = ctx->tls_type == TRANSPORT_PROTO_TLS ? "TLS" : "DTLS";
   session_parse_handle (ctx->tls_session_handle, &tcp_si, &tcp_ti);
   tls_ctx_parse_handle (ctx->tls_ctx_handle, &ctx_index, &ctx_engine);
-  s = format (s, "[%d:%d][TLS] app_wrk %u index %u engine %u tcp %d:%d",
-	      ctx->c_thread_index, ctx->c_s_index,
-	      ctx->parent_app_wrk_index, ctx_index,
-	      ctx_engine, tcp_ti, tcp_si);
+  s =
+    format (s, "[%d:%d][%s] app_wrk %u index %u engine %u ts %d:%d",
+	    ctx->c_thread_index, ctx->c_s_index, proto,
+	    ctx->parent_app_wrk_index, ctx_index, ctx_engine, tcp_ti, tcp_si);
 
   return s;
 }
@@ -766,13 +887,15 @@ format_tls_listener_ctx (u8 * s, va_list * args)
   session_t *tls_listener;
   app_listener_t *al;
   tls_ctx_t *ctx;
+  char *proto;
 
   ctx = va_arg (*args, tls_ctx_t *);
 
+  proto = ctx->tls_type == TRANSPORT_PROTO_TLS ? "TLS" : "DTLS";
   al = app_listener_get_w_handle (ctx->tls_session_handle);
   tls_listener = app_listener_get_session (al);
-  s = format (s, "[%d:%d][TLS] app_wrk %u engine %u tcp %d:%d",
-	      ctx->c_thread_index, ctx->c_s_index,
+  s = format (s, "[%d:%d][%s] app_wrk %u engine %u ts %d:%d",
+	      ctx->c_thread_index, ctx->c_s_index, proto,
 	      ctx->parent_app_wrk_index, ctx->tls_ctx_engine,
 	      tls_listener->thread_index, tls_listener->session_index);
 
@@ -926,7 +1049,6 @@ tls_enable (vlib_main_t * vm, u8 is_en)
   return 0;
 }
 
-/* *INDENT-OFF* */
 static const transport_proto_vft_t tls_proto = {
   .enable = tls_enable,
   .connect = tls_connect,
@@ -948,7 +1070,89 @@ static const transport_proto_vft_t tls_proto = {
     .service_type = TRANSPORT_SERVICE_APP,
   },
 };
-/* *INDENT-ON* */
+
+int
+dtls_connect (transport_endpoint_cfg_t *tep)
+{
+  vnet_connect_args_t _cargs = { {}, }, *cargs = &_cargs;
+  crypto_engine_type_t engine_type;
+  session_endpoint_cfg_t *sep;
+  tls_main_t *tm = &tls_main;
+  app_worker_t *app_wrk;
+  application_t *app;
+  tls_ctx_t *ctx;
+  u32 ctx_handle;
+  int rv;
+
+  sep = (session_endpoint_cfg_t *) tep;
+  app_wrk = app_worker_get (sep->app_wrk_index);
+  app = application_get (app_wrk->app_index);
+  engine_type = tls_get_engine_type (app->tls_engine);
+  if (engine_type == CRYPTO_ENGINE_NONE)
+    {
+      clib_warning ("No tls engine_type available");
+      return -1;
+    }
+
+  ctx_handle = tls_ctx_alloc_w_thread (engine_type, 1 /* because of udp */);
+  ctx = tls_ctx_get_w_thread (ctx_handle, 1);
+  ctx->parent_app_wrk_index = sep->app_wrk_index;
+  ctx->parent_app_api_context = sep->opaque;
+  ctx->tcp_is_ip4 = sep->is_ip4;
+  ctx->ckpair_index = sep->ckpair_index;
+  ctx->tls_type = sep->transport_proto;
+  ctx->tls_ctx_handle = ctx_handle;
+  if (sep->hostname)
+    {
+      ctx->srv_hostname = format (0, "%v", sep->hostname);
+      vec_terminate_c_string (ctx->srv_hostname);
+    }
+
+  app_worker_alloc_connects_segment_manager (app_wrk);
+  ctx->tls_ctx_engine = engine_type;
+
+  clib_memcpy_fast (&cargs->sep, sep, sizeof (session_endpoint_t));
+  cargs->sep.transport_proto = TRANSPORT_PROTO_UDP;
+  cargs->app_index = tm->app_index;
+  cargs->api_context = ctx_handle;
+  cargs->sep_ext.ns_index = app->ns_index;
+  cargs->sep_ext.transport_flags = TRANSPORT_CFG_F_CONNECTED;
+  if ((rv = vnet_connect (cargs)))
+    return rv;
+
+  TLS_DBG (1, "New DTLS connect request %x engine %d", ctx_handle,
+	   engine_type);
+  return 0;
+}
+
+static void
+dtls_cleanup_callback (u32 ctx_index, u32 thread_index)
+{
+  /* No op */
+}
+
+static const transport_proto_vft_t dtls_proto = {
+  .enable = 0,
+  .connect = dtls_connect,
+  .close = tls_disconnect,
+  .start_listen = tls_start_listen,
+  .stop_listen = tls_stop_listen,
+  .get_connection = tls_connection_get,
+  .get_listener = tls_listener_get,
+  .custom_tx = tls_custom_tx_callback,
+  .cleanup = dtls_cleanup_callback,
+  .format_connection = format_tls_connection,
+  .format_half_open = format_tls_half_open,
+  .format_listener = format_tls_listener,
+  .get_transport_endpoint = tls_transport_endpoint_get,
+  .get_transport_listener_endpoint = tls_transport_listener_endpoint_get,
+  .transport_options = {
+    .name = "dtls",
+    .short_name = "D",
+    .tx_type = TRANSPORT_TX_INTERNAL,
+    .service_type = TRANSPORT_SERVICE_APP,
+  },
+};
 
 void
 tls_register_engine (const tls_engine_vft_t * vft, crypto_engine_type_t type)
@@ -978,6 +1182,11 @@ tls_init (vlib_main_t * vm)
 			       FIB_PROTOCOL_IP4, ~0);
   transport_register_protocol (TRANSPORT_PROTO_TLS, &tls_proto,
 			       FIB_PROTOCOL_IP6, ~0);
+
+  transport_register_protocol (TRANSPORT_PROTO_DTLS, &dtls_proto,
+			       FIB_PROTOCOL_IP4, ~0);
+  transport_register_protocol (TRANSPORT_PROTO_DTLS, &dtls_proto,
+			       FIB_PROTOCOL_IP6, ~0);
   return 0;
 }
 
diff --git a/src/vnet/tls/tls.h b/src/vnet/tls/tls.h
index d950fe82629..2038fdff133 100644
--- a/src/vnet/tls/tls.h
+++ b/src/vnet/tls/tls.h
@@ -21,7 +21,7 @@
 #ifndef SRC_VNET_TLS_TLS_H_
 #define SRC_VNET_TLS_TLS_H_
 
-#define TLS_DEBUG 		0
+#define TLS_DEBUG		0
 #define TLS_DEBUG_LEVEL_CLIENT 	0
 #define TLS_DEBUG_LEVEL_SERVER 	0
 
@@ -49,6 +49,7 @@ typedef struct tls_cxt_id_
   u32 listener_ctx_index;
   u8 tcp_is_ip4;
   u8 tls_engine_id;
+  void *migrate_ctx;
 } tls_ctx_id_t;
 /* *INDENT-ON* */
 
@@ -73,14 +74,17 @@ typedef struct tls_ctx_
   /* Temporary storage for session open opaque. Overwritten once
    * underlying tcp connection is established */
 #define parent_app_api_context c_tls_ctx_id.parent_app_api_ctx
+#define migration_ctx	       c_tls_ctx_id.migrate_ctx
 
   u8 is_passive_close;
   u8 resume;
   u8 app_closed;
   u8 no_app_session;
+  u8 is_migrated;
   u8 *srv_hostname;
   u32 evt_index;
   u32 ckpair_index;
+  transport_proto_t tls_type;
 } tls_ctx_t;
 
 typedef struct tls_main_
@@ -104,7 +108,10 @@ typedef struct tls_main_
 typedef struct tls_engine_vft_
 {
   u32 (*ctx_alloc) (void);
+  u32 (*ctx_alloc_w_thread) (u32 thread_index);
   void (*ctx_free) (tls_ctx_t * ctx);
+  void *(*ctx_detach) (tls_ctx_t *ctx);
+  u32 (*ctx_attach) (u32 thread_index, void *ctx);
   tls_ctx_t *(*ctx_get) (u32 ctx_index);
   tls_ctx_t *(*ctx_get_w_thread) (u32 ctx_index, u8 thread_index);
   int (*ctx_init_client) (tls_ctx_t * ctx);