4 files changed, 256 insertions, 57 deletions

diff --git a/src/vnet/crypto/cli.c b/src/vnet/crypto/cli.c
index 8d523aec484..ef6371ad3dc 100644
--- a/src/vnet/crypto/cli.c
+++ b/src/vnet/crypto/cli.c
@@ -74,14 +74,24 @@ format_vnet_crypto_handlers (u8 * s, va_list * args)
       od = cm->opt_data + id;
       if (first == 0)
         s = format (s, "\n%U", format_white_space, indent);
-      s = format (s, "%-20U%-20U", format_vnet_crypto_op_type, od->type,
-		  format_vnet_crypto_engine, od->active_engine_index,s);
+      s = format (s, "%-22U%-20U", format_vnet_crypto_op_type, od->type, 0,
+		  format_vnet_crypto_engine, od->active_engine_index_simple,s);
 
       vec_foreach (e, cm->engines)
 	{
 	  if (e->ops_handlers[id] != 0)
 	    s = format (s, "%U ", format_vnet_crypto_engine, e - cm->engines);
 	}
+
+      s = format (s, "\n%U", format_white_space, indent);
+      s = format (s, "%-22U%-20U", format_vnet_crypto_op_type, od->type, 1,
+                  format_vnet_crypto_engine,
+                  od->active_engine_index_chained);
+      vec_foreach (e, cm->engines)
+	{
+	  if (e->chained_ops_handlers[id] != 0)
+	    s = format (s, "%U ", format_vnet_crypto_engine, e - cm->engines);
+	}
       first = 0;
     }
   return s;
@@ -98,7 +108,7 @@ show_crypto_handlers_command_fn (vlib_main_t * vm,
   if (unformat_user (input, unformat_line_input, line_input))
     unformat_free (line_input);
 
-  vlib_cli_output (vm, "%-20s%-20s%-20s%s", "Algo", "Type", "Active",
+  vlib_cli_output (vm, "%-20s%-22s%-20s%s", "Algo", "Type", "Active",
 		   "Candidates");
 
   for (i = 0; i < VNET_CRYPTO_N_ALGS; i++)
@@ -128,6 +138,7 @@ set_crypto_handler_command_fn (vlib_main_t * vm,
   char **args = 0, *s, **arg, *engine = 0;
   int all = 0;
   clib_error_t *error = 0;
+  crypto_op_class_type_t oct = CRYPTO_OP_BOTH;
 
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
@@ -136,6 +147,12 @@ set_crypto_handler_command_fn (vlib_main_t * vm,
     {
       if (unformat (line_input, "all"))
 	all = 1;
+      else if (unformat (line_input, "simple"))
+	oct = CRYPTO_OP_SIMPLE;
+      else if (unformat (line_input, "chained"))
+	oct = CRYPTO_OP_CHAINED;
+      else if (unformat (line_input, "both"))
+	oct = CRYPTO_OP_BOTH;
       else if (unformat (line_input, "%s", &s))
 	vec_add1 (args, s);
       else
@@ -163,7 +180,7 @@ set_crypto_handler_command_fn (vlib_main_t * vm,
       hash_foreach_mem (key, value, cm->alg_index_by_name,
       ({
         (void) value;
-        rc += vnet_crypto_set_handler (key, engine);
+        rc += vnet_crypto_set_handler2 (key, engine, oct);
       }));
       /* *INDENT-ON* */
 
@@ -174,7 +191,7 @@ set_crypto_handler_command_fn (vlib_main_t * vm,
     {
       vec_foreach (arg, args)
       {
-	rc = vnet_crypto_set_handler (arg[0], engine);
+	rc = vnet_crypto_set_handler2 (arg[0], engine, oct);
 	if (rc)
 	  {
 	    vlib_cli_output (vm, "failed to set engine %s for %s!",
@@ -195,7 +212,8 @@ done:
 VLIB_CLI_COMMAND (set_crypto_handler_command, static) =
 {
   .path = "set crypto handler",
-  .short_help = "set crypto handler cipher [cipher2 cipher3 ...] engine",
+  .short_help = "set crypto handler cipher [cipher2 cipher3 ...] engine"
+    " [simple|chained]",
   .function = set_crypto_handler_command_fn,
 };
 /* *INDENT-ON* */
diff --git a/src/vnet/crypto/crypto.c b/src/vnet/crypto/crypto.c
index 6cd12108f6c..4458bfcff4d 100644
--- a/src/vnet/crypto/crypto.c
+++ b/src/vnet/crypto/crypto.c
@@ -19,31 +19,52 @@
 
 vnet_crypto_main_t crypto_main;
 
+static_always_inline void
+crypto_set_op_status (vnet_crypto_op_t * ops[], u32 n_ops, int status)
+{
+  while (n_ops--)
+    {
+      ops[0]->status = status;
+      ops++;
+    }
+}
+
 static_always_inline u32
 vnet_crypto_process_ops_call_handler (vlib_main_t * vm,
 				      vnet_crypto_main_t * cm,
 				      vnet_crypto_op_id_t opt,
-				      vnet_crypto_op_t * ops[], u32 n_ops)
+				      vnet_crypto_op_t * ops[],
+				      vnet_crypto_op_chunk_t * chunks,
+				      u32 n_ops)
 {
+  u32 rv = 0;
   if (n_ops == 0)
     return 0;
 
-  if (cm->ops_handlers[opt] == 0)
+  if (chunks)
     {
-      while (n_ops--)
-	{
-	  ops[0]->status = VNET_CRYPTO_OP_STATUS_FAIL_NO_HANDLER;
-	  ops++;
-	}
-      return 0;
-    }
 
-  return (cm->ops_handlers[opt]) (vm, ops, n_ops);
+      if (cm->chained_ops_handlers[opt] == 0)
+	crypto_set_op_status (ops, n_ops,
+			      VNET_CRYPTO_OP_STATUS_FAIL_NO_HANDLER);
+      else
+	rv = (cm->chained_ops_handlers[opt]) (vm, ops, chunks, n_ops);
+    }
+  else
+    {
+      if (cm->ops_handlers[opt] == 0)
+	crypto_set_op_status (ops, n_ops,
+			      VNET_CRYPTO_OP_STATUS_FAIL_NO_HANDLER);
+      else
+	rv = (cm->ops_handlers[opt]) (vm, ops, n_ops);
+    }
+  return rv;
 }
 
 
-u32
-vnet_crypto_process_ops (vlib_main_t * vm, vnet_crypto_op_t ops[], u32 n_ops)
+static_always_inline u32
+vnet_crypto_process_ops_inline (vlib_main_t * vm, vnet_crypto_op_t ops[],
+				vnet_crypto_op_chunk_t * chunks, u32 n_ops)
 {
   vnet_crypto_main_t *cm = &crypto_main;
   const int op_q_size = VLIB_FRAME_SIZE;
@@ -61,7 +82,8 @@ vnet_crypto_process_ops (vlib_main_t * vm, vnet_crypto_op_t ops[], u32 n_ops)
       if (current_op_type != opt || n_op_queue >= op_q_size)
 	{
 	  rv += vnet_crypto_process_ops_call_handler (vm, cm, current_op_type,
-						      op_queue, n_op_queue);
+						      op_queue, chunks,
+						      n_op_queue);
 	  n_op_queue = 0;
 	  current_op_type = opt;
 	}
@@ -70,11 +92,24 @@ vnet_crypto_process_ops (vlib_main_t * vm, vnet_crypto_op_t ops[], u32 n_ops)
     }
 
   rv += vnet_crypto_process_ops_call_handler (vm, cm, current_op_type,
-					      op_queue, n_op_queue);
+					      op_queue, chunks, n_op_queue);
   return rv;
 }
 
 u32
+vnet_crypto_process_ops (vlib_main_t * vm, vnet_crypto_op_t ops[], u32 n_ops)
+{
+  return vnet_crypto_process_ops_inline (vm, ops, 0, n_ops);
+}
+
+u32
+vnet_crypto_process_chained_ops (vlib_main_t * vm, vnet_crypto_op_t ops[],
+				 vnet_crypto_op_chunk_t * chunks, u32 n_ops)
+{
+  return vnet_crypto_process_ops_inline (vm, ops, chunks, n_ops);
+}
+
+u32
 vnet_crypto_register_engine (vlib_main_t * vm, char *name, int prio,
 			     char *desc)
 {
@@ -91,13 +126,40 @@ vnet_crypto_register_engine (vlib_main_t * vm, char *name, int prio,
   return p - cm->engines;
 }
 
+static_always_inline void
+crypto_set_active_engine (vnet_crypto_op_data_t * od,
+			  vnet_crypto_op_id_t id, u32 ei,
+			  crypto_op_class_type_t oct)
+{
+  vnet_crypto_main_t *cm = &crypto_main;
+  vnet_crypto_engine_t *ce = vec_elt_at_index (cm->engines, ei);
+
+  if (oct == CRYPTO_OP_BOTH || oct == CRYPTO_OP_CHAINED)
+    {
+      if (ce->chained_ops_handlers[id])
+	{
+	  od->active_engine_index_chained = ei;
+	  cm->chained_ops_handlers[id] = ce->chained_ops_handlers[id];
+	}
+    }
+
+  if (oct == CRYPTO_OP_BOTH || oct == CRYPTO_OP_SIMPLE)
+    {
+      if (ce->ops_handlers[id])
+	{
+	  od->active_engine_index_simple = ei;
+	  cm->ops_handlers[id] = ce->ops_handlers[id];
+	}
+    }
+}
+
 int
-vnet_crypto_set_handler (char *alg_name, char *engine)
+vnet_crypto_set_handler2 (char *alg_name, char *engine,
+			  crypto_op_class_type_t oct)
 {
   uword *p;
   vnet_crypto_main_t *cm = &crypto_main;
   vnet_crypto_alg_data_t *ad;
-  vnet_crypto_engine_t *ce;
   int i;
 
   p = hash_get_mem (cm->alg_index_by_name, alg_name);
@@ -110,20 +172,15 @@ vnet_crypto_set_handler (char *alg_name, char *engine)
   if (!p)
     return -1;
 
-  ce = vec_elt_at_index (cm->engines, p[0]);
-
-  for (i = 0; i < VNET_CRYPTO_OP_N_TYPES; i++)
+  for (i = 0; i < VNET_CRYPTO_OP_N_TYPES; i += 2)
     {
       vnet_crypto_op_data_t *od;
       vnet_crypto_op_id_t id = ad->op_by_type[i];
       if (id == 0)
 	continue;
+
       od = cm->opt_data + id;
-      if (ce->ops_handlers[id])
-	{
-	  od->active_engine_index = p[0];
-	  cm->ops_handlers[id] = ce->ops_handlers[id];
-	}
+      crypto_set_active_engine (od, id, p[0], oct);
     }
 
   return 0;
@@ -138,34 +195,78 @@ vnet_crypto_is_set_handler (vnet_crypto_alg_t alg)
 }
 
 void
-vnet_crypto_register_ops_handler (vlib_main_t * vm, u32 engine_index,
-				  vnet_crypto_op_id_t opt,
-				  vnet_crypto_ops_handler_t * fn)
+vnet_crypto_register_ops_handler_inline (vlib_main_t * vm, u32 engine_index,
+					 vnet_crypto_op_id_t opt,
+					 vnet_crypto_ops_handler_t * fn,
+					 vnet_crypto_chained_ops_handler_t *
+					 cfn)
 {
   vnet_crypto_main_t *cm = &crypto_main;
   vnet_crypto_engine_t *ae, *e = vec_elt_at_index (cm->engines, engine_index);
   vnet_crypto_op_data_t *otd = cm->opt_data + opt;
   vec_validate_aligned (cm->ops_handlers, VNET_CRYPTO_N_OP_IDS - 1,
 			CLIB_CACHE_LINE_BYTES);
-  e->ops_handlers[opt] = fn;
+  vec_validate_aligned (cm->chained_ops_handlers, VNET_CRYPTO_N_OP_IDS - 1,
+			CLIB_CACHE_LINE_BYTES);
 
-  if (otd->active_engine_index == ~0)
+  if (fn)
     {
-      otd->active_engine_index = engine_index;
-      cm->ops_handlers[opt] = fn;
-      return;
+      e->ops_handlers[opt] = fn;
+      if (otd->active_engine_index_simple == ~0)
+	{
+	  otd->active_engine_index_simple = engine_index;
+	  cm->ops_handlers[opt] = fn;
+	}
+
+      ae = vec_elt_at_index (cm->engines, otd->active_engine_index_simple);
+      if (ae->priority < e->priority)
+	crypto_set_active_engine (otd, opt, engine_index, CRYPTO_OP_SIMPLE);
     }
-  ae = vec_elt_at_index (cm->engines, otd->active_engine_index);
-  if (ae->priority < e->priority)
+
+  if (cfn)
     {
-      otd->active_engine_index = engine_index;
-      cm->ops_handlers[opt] = fn;
+      e->chained_ops_handlers[opt] = cfn;
+      if (otd->active_engine_index_chained == ~0)
+	{
+	  otd->active_engine_index_chained = engine_index;
+	  cm->chained_ops_handlers[opt] = cfn;
+	}
+
+      ae = vec_elt_at_index (cm->engines, otd->active_engine_index_chained);
+      if (ae->priority < e->priority)
+	crypto_set_active_engine (otd, opt, engine_index, CRYPTO_OP_CHAINED);
     }
 
   return;
 }
 
 void
+vnet_crypto_register_ops_handler (vlib_main_t * vm, u32 engine_index,
+				  vnet_crypto_op_id_t opt,
+				  vnet_crypto_ops_handler_t * fn)
+{
+  vnet_crypto_register_ops_handler_inline (vm, engine_index, opt, fn, 0);
+}
+
+void
+vnet_crypto_register_chained_ops_handler (vlib_main_t * vm, u32 engine_index,
+					  vnet_crypto_op_id_t opt,
+					  vnet_crypto_chained_ops_handler_t *
+					  fn)
+{
+  vnet_crypto_register_ops_handler_inline (vm, engine_index, opt, 0, fn);
+}
+
+void
+vnet_crypto_register_ops_handlers (vlib_main_t * vm, u32 engine_index,
+				   vnet_crypto_op_id_t opt,
+				   vnet_crypto_ops_handler_t * fn,
+				   vnet_crypto_chained_ops_handler_t * cfn)
+{
+  vnet_crypto_register_ops_handler_inline (vm, engine_index, opt, fn, cfn);
+}
+
+void
 vnet_crypto_register_key_handler (vlib_main_t * vm, u32 engine_index,
 				  vnet_crypto_key_handler_t * key_handler)
 {
@@ -253,10 +354,13 @@ vnet_crypto_init_cipher_data (vnet_crypto_alg_t alg, vnet_crypto_op_id_t eid,
 {
   vnet_crypto_op_type_t eopt, dopt;
   vnet_crypto_main_t *cm = &crypto_main;
+
   cm->algs[alg].name = name;
   cm->opt_data[eid].alg = cm->opt_data[did].alg = alg;
-  cm->opt_data[eid].active_engine_index = ~0;
-  cm->opt_data[did].active_engine_index = ~0;
+  cm->opt_data[eid].active_engine_index_simple = ~0;
+  cm->opt_data[did].active_engine_index_simple = ~0;
+  cm->opt_data[eid].active_engine_index_chained = ~0;
+  cm->opt_data[did].active_engine_index_chained = ~0;
   if (is_aead)
     {
       eopt = VNET_CRYPTO_OP_TYPE_AEAD_ENCRYPT;
@@ -282,7 +386,8 @@ vnet_crypto_init_hmac_data (vnet_crypto_alg_t alg,
   cm->algs[alg].name = name;
   cm->algs[alg].op_by_type[VNET_CRYPTO_OP_TYPE_HMAC] = id;
   cm->opt_data[id].alg = alg;
-  cm->opt_data[id].active_engine_index = ~0;
+  cm->opt_data[id].active_engine_index_simple = ~0;
+  cm->opt_data[id].active_engine_index_chained = ~0;
   cm->opt_data[id].type = VNET_CRYPTO_OP_TYPE_HMAC;
   hash_set_mem (cm->alg_index_by_name, name, alg);
 }
diff --git a/src/vnet/crypto/crypto.h b/src/vnet/crypto/crypto.h
index 626e71d5030..f89ecf9f3d5 100644
--- a/src/vnet/crypto/crypto.h
+++ b/src/vnet/crypto/crypto.h
@@ -116,6 +116,13 @@ typedef enum
 } vnet_crypto_op_id_t;
 /* *INDENT-ON* */
 
+typedef enum
+{
+  CRYPTO_OP_SIMPLE,
+  CRYPTO_OP_CHAINED,
+  CRYPTO_OP_BOTH,
+} crypto_op_class_type_t;
+
 typedef struct
 {
   char *name;
@@ -124,30 +131,67 @@ typedef struct
 
 typedef struct
 {
+  u8 *src;
+  u8 *dst;
+  u32 len;
+} vnet_crypto_op_chunk_t;
+
+typedef struct
+{
   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+  uword user_data;
   vnet_crypto_op_id_t op:16;
   vnet_crypto_op_status_t status:8;
   u8 flags;
 #define VNET_CRYPTO_OP_FLAG_INIT_IV (1 << 0)
 #define VNET_CRYPTO_OP_FLAG_HMAC_CHECK (1 << 1)
-  u32 key_index;
-  u32 len;
+#define VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS (1 << 2)
+
+  union
+  {
+    u8 digest_len;
+    u8 tag_len;
+  };
   u16 aad_len;
-  u8 digest_len, tag_len;
+
+  union
+  {
+    struct
+    {
+      u8 *src;
+      u8 *dst;
+    };
+
+    /* valid if VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS is set */
+    u16 n_chunks;
+  };
+
+  union
+  {
+    u32 len;
+    /* valid if VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS is set */
+    u32 chunk_index;
+  };
+
+  u32 key_index;
   u8 *iv;
-  u8 *src;
-  u8 *dst;
   u8 *aad;
-  u8 *tag;
-  u8 *digest;
-  uword user_data;
+
+  union
+  {
+    u8 *tag;
+    u8 *digest;
+  };
 } vnet_crypto_op_t;
 
+STATIC_ASSERT_SIZEOF (vnet_crypto_op_t, CLIB_CACHE_LINE_BYTES);
+
 typedef struct
 {
   vnet_crypto_op_type_t type;
   vnet_crypto_alg_t alg;
-  u32 active_engine_index;
+  u32 active_engine_index_simple;
+  u32 active_engine_index_chained;
 } vnet_crypto_op_data_t;
 
 typedef struct
@@ -158,6 +202,11 @@ typedef struct
 
 typedef u32 vnet_crypto_key_index_t;
 
+typedef u32 (vnet_crypto_chained_ops_handler_t) (vlib_main_t * vm,
+						 vnet_crypto_op_t * ops[],
+						 vnet_crypto_op_chunk_t *
+						 chunks, u32 n_ops);
+
 typedef u32 (vnet_crypto_ops_handler_t) (vlib_main_t * vm,
 					 vnet_crypto_op_t * ops[], u32 n_ops);
 
@@ -171,6 +220,18 @@ u32 vnet_crypto_register_engine (vlib_main_t * vm, char *name, int prio,
 void vnet_crypto_register_ops_handler (vlib_main_t * vm, u32 engine_index,
 				       vnet_crypto_op_id_t opt,
 				       vnet_crypto_ops_handler_t * oph);
+
+void vnet_crypto_register_chained_ops_handler (vlib_main_t * vm,
+					       u32 engine_index,
+					       vnet_crypto_op_id_t opt,
+					       vnet_crypto_chained_ops_handler_t
+					       * oph);
+void vnet_crypto_register_ops_handlers (vlib_main_t * vm, u32 engine_index,
+					vnet_crypto_op_id_t opt,
+					vnet_crypto_ops_handler_t * fn,
+					vnet_crypto_chained_ops_handler_t *
+					cfn);
+
 void vnet_crypto_register_key_handler (vlib_main_t * vm, u32 engine_index,
 				       vnet_crypto_key_handler_t * keyh);
 
@@ -181,6 +242,8 @@ typedef struct
   int priority;
   vnet_crypto_key_handler_t *key_op_handler;
   vnet_crypto_ops_handler_t *ops_handlers[VNET_CRYPTO_N_OP_IDS];
+    vnet_crypto_chained_ops_handler_t
+    * chained_ops_handlers[VNET_CRYPTO_N_OP_IDS];
 } vnet_crypto_engine_t;
 
 typedef struct
@@ -188,6 +251,7 @@ typedef struct
   vnet_crypto_alg_data_t *algs;
   vnet_crypto_thread_t *threads;
   vnet_crypto_ops_handler_t **ops_handlers;
+  vnet_crypto_chained_ops_handler_t **chained_ops_handlers;
   vnet_crypto_op_data_t opt_data[VNET_CRYPTO_N_OP_IDS];
   vnet_crypto_engine_t *engines;
   vnet_crypto_key_t *keys;
@@ -200,10 +264,14 @@ extern vnet_crypto_main_t crypto_main;
 u32 vnet_crypto_submit_ops (vlib_main_t * vm, vnet_crypto_op_t ** jobs,
 			    u32 n_jobs);
 
+u32 vnet_crypto_process_chained_ops (vlib_main_t * vm, vnet_crypto_op_t ops[],
+				     vnet_crypto_op_chunk_t * chunks,
+				     u32 n_ops);
 u32 vnet_crypto_process_ops (vlib_main_t * vm, vnet_crypto_op_t ops[],
 			     u32 n_ops);
 
-int vnet_crypto_set_handler (char *ops_handler_name, char *engine);
+int vnet_crypto_set_handler2 (char *ops_handler_name, char *engine,
+			      crypto_op_class_type_t oct);
 int vnet_crypto_is_set_handler (vnet_crypto_alg_t alg);
 
 u32 vnet_crypto_key_add (vlib_main_t * vm, vnet_crypto_alg_t alg,
@@ -225,6 +293,7 @@ vnet_crypto_op_init (vnet_crypto_op_t * op, vnet_crypto_op_id_t type)
   op->op = type;
   op->flags = 0;
   op->key_index = ~0;
+  op->n_chunks = 0;
 }
 
 static_always_inline vnet_crypto_op_type_t
@@ -243,6 +312,12 @@ vnet_crypto_get_key (vnet_crypto_key_index_t index)
   return vec_elt_at_index (cm->keys, index);
 }
 
+static_always_inline int
+vnet_crypto_set_handler (char *alg_name, char *engine)
+{
+  return vnet_crypto_set_handler2 (alg_name, engine, CRYPTO_OP_BOTH);
+}
+
 #endif /* included_vnet_crypto_crypto_h */
 
 /*
diff --git a/src/vnet/crypto/format.c b/src/vnet/crypto/format.c
index 715941e0ee7..3210ab9eac0 100644
--- a/src/vnet/crypto/format.c
+++ b/src/vnet/crypto/format.c
@@ -54,7 +54,7 @@ format_vnet_crypto_op (u8 * s, va_list * args)
   vnet_crypto_op_id_t op = va_arg (*args, int);	// vnet_crypto_op_id_t);
   vnet_crypto_op_data_t *otd = cm->opt_data + op;
 
-  return format (s, "%U-%U", format_vnet_crypto_op_type, otd->type,
+  return format (s, "%U-%U", format_vnet_crypto_op_type, otd->type, 0,
 		 format_vnet_crypto_alg, otd->alg);
 }
 
@@ -62,6 +62,7 @@ u8 *
 format_vnet_crypto_op_type (u8 * s, va_list * args)
 {
   vnet_crypto_op_type_t opt = va_arg (*args, vnet_crypto_op_type_t);
+  int is_chained = va_arg (*args, int);
   char *strings[] = {
 #define _(n, s) [VNET_CRYPTO_OP_TYPE_##n] = s,
     foreach_crypto_op_type
@@ -71,7 +72,7 @@ format_vnet_crypto_op_type (u8 * s, va_list * args)
   if (opt >= VNET_CRYPTO_OP_N_TYPES)
     return format (s, "unknown");
 
-  return format (s, "%s", strings[opt]);
+  return format (s, "%s%s", strings[opt], is_chained ? "-chained" : "");
 }
 
 u8 *