crypto: improve key handling

Change-Id: If96f661d507305da4b96cac7b1a8f14ba90676ad
Signed-off-by: Damjan Marion <damarion@cisco.com>

8 files changed, 143 insertions, 40 deletions

diff --git a/src/plugins/crypto_ia32/aes_cbc.c b/src/plugins/crypto_ia32/aes_cbc.c
index 091f7b6a539..7befdd2b4e6 100644
--- a/src/plugins/crypto_ia32/aes_cbc.c
+++ b/src/plugins/crypto_ia32/aes_cbc.c
@@ -97,7 +97,7 @@ aesni_ops_enc_aes_cbc (vlib_main_t * vm, vnet_crypto_op_t * ops[],
   u8 dummy[8192];
   u8 *src[4] = { };
   u8 *dst[4] = { };
-  u8 *key[4] = { };
+  vnet_crypto_key_index_t key_index[4] = { ~0, ~0, ~0, ~0 };
   u32x4 dummy_mask, len = { };
   u32 i, j, count, n_left = n_ops;
   __m128i r[4] = { }, k[4][rounds + 1];
@@ -127,10 +127,13 @@ more:
 	    dst[i] = ops[0]->dst;
 	    len[i] = ops[0]->len;
 	    dummy_mask[i] = ~0;
-	    if (key[i] != ops[0]->key)
+	    if (key_index[i] != ops[0]->key_index)
 	      {
-		aes_key_expand (k[i], ops[0]->key, ks);
-		key[i] = ops[0]->key;
+		aesni_key_data_t *kd;
+		key_index[i] = ops[0]->key_index;
+		kd = (aesni_key_data_t *) cm->key_data[key_index[i]];
+		clib_memcpy_fast (k[i], kd->encrypt_key,
+				  (rounds + 1) * sizeof (__m128i));
 	      }
 	    ops[0]->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
 	    n_left--;
@@ -188,28 +191,22 @@ static_always_inline u32
 aesni_ops_dec_aes_cbc (vlib_main_t * vm, vnet_crypto_op_t * ops[],
 		       u32 n_ops, aesni_key_size_t ks)
 {
+  crypto_ia32_main_t *cm = &crypto_ia32_main;
   int rounds = AESNI_KEY_ROUNDS (ks);
   vnet_crypto_op_t *op = ops[0];
+  aesni_key_data_t *kd = (aesni_key_data_t *) cm->key_data[op->key_index];
   u32 n_left = n_ops;
-  u8 *last_key;
-  __m128i k[rounds + 1];
 
   ASSERT (n_ops >= 1);
 
-key_expand:
-  last_key = op->key;
-  aes_key_expand (k, op->key, ks);
-  aes_key_enc_to_dec (k, ks);
-
 decrypt:
-  aes_cbc_dec (k, op->src, op->dst, op->iv, op->len, rounds);
+  aes_cbc_dec (kd->decrypt_key, op->src, op->dst, op->iv, op->len, rounds);
   op->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
 
   if (--n_left)
     {
       op += 1;
-      if (last_key != op->key)
-	goto key_expand;
+      kd = (aesni_key_data_t *) cm->key_data[op->key_index];
       goto decrypt;
     }
 
diff --git a/src/plugins/crypto_ia32/aesni.h b/src/plugins/crypto_ia32/aesni.h
index 28e09fc5c51..80e30e51b33 100644
--- a/src/plugins/crypto_ia32/aesni.h
+++ b/src/plugins/crypto_ia32/aesni.h
@@ -18,6 +18,13 @@
 #ifndef __aesni_h__
 #define __aesni_h__
 
+
+typedef struct
+{
+  __m128i encrypt_key[15];
+  __m128i decrypt_key[15];
+} aesni_key_data_t;
+
 typedef enum
 {
   AESNI_KEY_128 = 0,
diff --git a/src/plugins/crypto_ia32/crypto_ia32.h b/src/plugins/crypto_ia32/crypto_ia32.h
index ccb26ab1a4d..f5f09dfd499 100644
--- a/src/plugins/crypto_ia32/crypto_ia32.h
+++ b/src/plugins/crypto_ia32/crypto_ia32.h
@@ -27,6 +27,7 @@ typedef struct
 {
   u32 crypto_engine_index;
   crypto_ia32_per_thread_data_t *per_thread_data;
+  void **key_data;
 } crypto_ia32_main_t;
 
 extern crypto_ia32_main_t crypto_ia32_main;
diff --git a/src/plugins/crypto_ia32/main.c b/src/plugins/crypto_ia32/main.c
index 9b83f8913db..8dd596b0dd6 100644
--- a/src/plugins/crypto_ia32/main.c
+++ b/src/plugins/crypto_ia32/main.c
@@ -19,9 +19,80 @@
 #include <vnet/plugin/plugin.h>
 #include <vnet/crypto/crypto.h>
 #include <crypto_ia32/crypto_ia32.h>
+#include <crypto_ia32/aesni.h>
 
 crypto_ia32_main_t crypto_ia32_main;
 
+static void
+crypto_ia32_key_handler (vlib_main_t * vm, vnet_crypto_key_op_t kop,
+			 vnet_crypto_key_index_t idx)
+{
+  vnet_crypto_key_t *key = vnet_crypto_get_key (idx);
+  crypto_ia32_main_t *cm = &crypto_ia32_main;
+  aesni_key_data_t *kd;
+
+  switch (key->alg)
+    {
+    case VNET_CRYPTO_ALG_AES_128_CBC:
+    case VNET_CRYPTO_ALG_AES_192_CBC:
+    case VNET_CRYPTO_ALG_AES_256_CBC:
+      break;
+    default:
+      return;
+      break;
+    }
+
+  if (kop == VNET_CRYPTO_KEY_OP_DEL)
+    {
+      if (idx >= vec_len (cm->key_data))
+	return;
+
+      if (cm->key_data[idx] == 0)
+	return;
+
+      clib_memset_u8 (cm->key_data[idx], 0,
+		      clib_mem_size (cm->key_data[idx]));
+      clib_mem_free (cm->key_data[idx]);
+      cm->key_data[idx] = 0;
+      return;
+    }
+
+  vec_validate_aligned (cm->key_data, idx, CLIB_CACHE_LINE_BYTES);
+
+  if (kop == VNET_CRYPTO_KEY_OP_MODIFY && cm->key_data[idx])
+    {
+      clib_memset_u8 (cm->key_data[idx], 0,
+		      clib_mem_size (cm->key_data[idx]));
+      clib_mem_free (cm->key_data[idx]);
+    }
+
+  kd = cm->key_data[idx] = clib_mem_alloc_aligned (sizeof (aesni_key_data_t),
+						   CLIB_CACHE_LINE_BYTES);
+
+  /* ADD or MODIFY */
+  switch (key->alg)
+    {
+    case VNET_CRYPTO_ALG_AES_128_CBC:
+      aes_key_expand (kd->encrypt_key, key->data, AESNI_KEY_128);
+      aes_key_expand (kd->decrypt_key, key->data, AESNI_KEY_128);
+      aes_key_enc_to_dec (kd->decrypt_key, AESNI_KEY_128);
+      break;
+    case VNET_CRYPTO_ALG_AES_192_CBC:
+      aes_key_expand (kd->encrypt_key, key->data, AESNI_KEY_192);
+      aes_key_expand (kd->decrypt_key, key->data, AESNI_KEY_192);
+      aes_key_enc_to_dec (kd->decrypt_key, AESNI_KEY_192);
+      break;
+    case VNET_CRYPTO_ALG_AES_256_CBC:
+      aes_key_expand (kd->encrypt_key, key->data, AESNI_KEY_256);
+      aes_key_expand (kd->decrypt_key, key->data, AESNI_KEY_256);
+      aes_key_enc_to_dec (kd->decrypt_key, AESNI_KEY_256);
+      break;
+    default:
+      return;
+      break;
+    }
+}
+
 clib_error_t *
 crypto_ia32_init (vlib_main_t * vm)
 {
@@ -43,6 +114,10 @@ crypto_ia32_init (vlib_main_t * vm)
       (error = crypto_ia32_aesni_cbc_init (vm)))
     goto error;
 
+  vnet_crypto_register_key_handler (vm, cm->crypto_engine_index,
+				    crypto_ia32_key_handler);
+
+
 error:
   if (error)
     vec_free (cm->per_thread_data);
diff --git a/src/plugins/crypto_ipsecmb/ipsecmb.c b/src/plugins/crypto_ipsecmb/ipsecmb.c
index 6d4d9136784..10cc32f8e6d 100644
--- a/src/plugins/crypto_ipsecmb/ipsecmb.c
+++ b/src/plugins/crypto_ipsecmb/ipsecmb.c
@@ -154,9 +154,10 @@ ipsecmb_ops_hmac_inline (vlib_main_t * vm,
   for (i = 0; i < n_ops; i++)
     {
       vnet_crypto_op_t *op = ops[i];
+      vnet_crypto_key_t *key = vnet_crypto_get_key (op->key_index);
       u8 ipad[256], opad[256];
 
-      hash_expand_keys (ptd->mgr, op->key, op->key_len,
+      hash_expand_keys (ptd->mgr, key->data, vec_len (key->data),
 			block_size, ipad, opad, fn);
 
       job = IMB_GET_NEXT_JOB (ptd->mgr);
@@ -172,7 +173,7 @@ ipsecmb_ops_hmac_inline (vlib_main_t * vm,
       job->cipher_direction = DECRYPT;
       job->chain_order = HASH_CIPHER;
 
-      job->aes_key_len_in_bytes = op->key_len;
+      job->aes_key_len_in_bytes = vec_len (key->data);
 
       job->u.HMAC._hashed_auth_key_xor_ipad = ipad;
       job->u.HMAC._hashed_auth_key_xor_opad = opad;
@@ -250,9 +251,10 @@ ipsecmb_ops_cbc_cipher_inline (vlib_main_t * vm,
       u8 aes_enc_key_expanded[EXPANDED_KEY_N_BYTES];
       u8 aes_dec_key_expanded[EXPANDED_KEY_N_BYTES];
       vnet_crypto_op_t *op = ops[i];
+      vnet_crypto_key_t *key = vnet_crypto_get_key (op->key_index);
       __m128i iv;
 
-      fn (op->key, aes_enc_key_expanded, aes_dec_key_expanded);
+      fn (key->data, aes_enc_key_expanded, aes_dec_key_expanded);
 
       job = IMB_GET_NEXT_JOB (ptd->mgr);
 
@@ -380,10 +382,11 @@ ipsecmb_ops_gcm_cipher_inline (vlib_main_t * vm,
     {
       struct gcm_key_data key_data;
       vnet_crypto_op_t *op = ops[i];
+      vnet_crypto_key_t *key = vnet_crypto_get_key (op->key_index);
       u32 nonce[3];
       __m128i iv;
 
-      fn (op->key, &key_data);
+      fn (key->data, &key_data);
 
       job = IMB_GET_NEXT_JOB (ptd->mgr);
 
diff --git a/src/plugins/crypto_openssl/main.c b/src/plugins/crypto_openssl/main.c
index ac586c2af77..fb9754f85a2 100644
--- a/src/plugins/crypto_openssl/main.c
+++ b/src/plugins/crypto_openssl/main.c
@@ -68,12 +68,13 @@ openssl_ops_enc_cbc (vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops,
   for (i = 0; i < n_ops; i++)
     {
       vnet_crypto_op_t *op = ops[i];
+      vnet_crypto_key_t *key = vnet_crypto_get_key (op->key_index);
       int out_len;
 
       if (op->flags & VNET_CRYPTO_OP_FLAG_INIT_IV)
 	RAND_bytes (op->iv, 16);
 
-      EVP_EncryptInit_ex (ctx, cipher, NULL, op->key, op->iv);
+      EVP_EncryptInit_ex (ctx, cipher, NULL, key->data, op->iv);
       EVP_EncryptUpdate (ctx, op->dst, &out_len, op->src, op->len);
       if (out_len < op->len)
 	EVP_EncryptFinal_ex (ctx, op->dst + out_len, &out_len);
@@ -93,9 +94,10 @@ openssl_ops_dec_cbc (vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops,
   for (i = 0; i < n_ops; i++)
     {
       vnet_crypto_op_t *op = ops[i];
+      vnet_crypto_key_t *key = vnet_crypto_get_key (op->key_index);
       int out_len;
 
-      EVP_DecryptInit_ex (ctx, cipher, NULL, op->key, op->iv);
+      EVP_DecryptInit_ex (ctx, cipher, NULL, key->data, op->iv);
       EVP_DecryptUpdate (ctx, op->dst, &out_len, op->src, op->len);
       if (out_len < op->len)
 	EVP_DecryptFinal_ex (ctx, op->dst + out_len, &out_len);
@@ -115,6 +117,7 @@ openssl_ops_enc_gcm (vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops,
   for (i = 0; i < n_ops; i++)
     {
       vnet_crypto_op_t *op = ops[i];
+      vnet_crypto_key_t *key = vnet_crypto_get_key (op->key_index);
       u32 nonce[3];
       int len;
 
@@ -126,7 +129,7 @@ openssl_ops_enc_gcm (vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops,
 
       EVP_EncryptInit_ex (ctx, cipher, 0, 0, 0);
       EVP_CIPHER_CTX_ctrl (ctx, EVP_CTRL_GCM_SET_IVLEN, 12, NULL);
-      EVP_EncryptInit_ex (ctx, 0, 0, op->key, (u8 *) nonce);
+      EVP_EncryptInit_ex (ctx, 0, 0, key->data, (u8 *) nonce);
       if (op->aad_len)
 	EVP_EncryptUpdate (ctx, NULL, &len, op->aad, op->aad_len);
       EVP_EncryptUpdate (ctx, op->dst, &len, op->src, op->len);
@@ -148,11 +151,12 @@ openssl_ops_dec_gcm (vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops,
   for (i = 0; i < n_ops; i++)
     {
       vnet_crypto_op_t *op = ops[i];
+      vnet_crypto_key_t *key = vnet_crypto_get_key (op->key_index);
       int len;
 
       EVP_DecryptInit_ex (ctx, cipher, 0, 0, 0);
       EVP_CIPHER_CTX_ctrl (ctx, EVP_CTRL_GCM_SET_IVLEN, op->iv_len, 0);
-      EVP_DecryptInit_ex (ctx, 0, 0, op->key, op->iv);
+      EVP_DecryptInit_ex (ctx, 0, 0, key->data, op->iv);
       if (op->aad_len)
 	EVP_DecryptUpdate (ctx, 0, &len, op->aad, op->aad_len);
       EVP_DecryptUpdate (ctx, op->dst, &len, op->src, op->len);
@@ -181,10 +185,11 @@ openssl_ops_hmac (vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops,
   for (i = 0; i < n_ops; i++)
     {
       vnet_crypto_op_t *op = ops[i];
+      vnet_crypto_key_t *key = vnet_crypto_get_key (op->key_index);
       unsigned int out_len;
       size_t sz = op->digest_len ? op->digest_len : EVP_MD_size (md);
 
-      HMAC_Init_ex (ctx, op->key, op->key_len, md, NULL);
+      HMAC_Init_ex (ctx, key->data, vec_len (key->data), md, NULL);
       HMAC_Update (ctx, op->src, op->len);
       HMAC_Final (ctx, buffer, &out_len);
 
diff --git a/src/plugins/unittest/crypto/crypto.h b/src/plugins/unittest/crypto/crypto.h
index ab9f706c081..f15e34b02f0 100644
--- a/src/plugins/unittest/crypto/crypto.h
+++ b/src/plugins/unittest/crypto/crypto.h
@@ -45,7 +45,6 @@ typedef struct
   u32 rounds;
   u32 buffer_size;
   u32 n_buffers;
-  u8 one_key;
 
   unittest_crypto_test_registration_t *test_registrations;
 } crypto_test_main_t;
diff --git a/src/plugins/unittest/crypto_test.c b/src/plugins/unittest/crypto_test.c
index 37cdc688f5e..b4f48741573 100644
--- a/src/plugins/unittest/crypto_test.c
+++ b/src/plugins/unittest/crypto_test.c
@@ -38,6 +38,7 @@ test_crypto (vlib_main_t * vm, crypto_test_main_t * tm)
   unittest_crypto_test_registration_t **rv = 0;
   vnet_crypto_alg_data_t *ad;
   vnet_crypto_op_t *ops = 0, *op;
+  vnet_crypto_key_index_t *key_indices = 0;
   u8 *computed_data = 0, *s = 0, *err = 0;
   u32 computed_data_total_len = 0, n_ops = 0;
   u32 i;
@@ -114,8 +115,10 @@ test_crypto (vlib_main_t * vm, crypto_test_main_t * tm)
 	    case VNET_CRYPTO_OP_TYPE_DECRYPT:
 	      op->iv = r->iv.data;
 	      op->iv_len = r->iv.length;
-	      op->key = r->key.data;
-	      op->key_len = r->key.length;
+	      op->key_index = vnet_crypto_key_add (vm, r->alg,
+						   r->key.data,
+						   r->key.length);
+	      vec_add1 (key_indices, op->key_index);
 	      op->len = r->plaintext.length;
 	      op->src = t == VNET_CRYPTO_OP_TYPE_ENCRYPT ?
 		r->plaintext.data : r->ciphertext.data;
@@ -126,8 +129,10 @@ test_crypto (vlib_main_t * vm, crypto_test_main_t * tm)
 	    case VNET_CRYPTO_OP_TYPE_AEAD_DECRYPT:
 	      op->iv = r->iv.data;
 	      op->iv_len = r->iv.length;
-	      op->key = r->key.data;
-	      op->key_len = r->key.length;
+	      op->key_index = vnet_crypto_key_add (vm, r->alg,
+						   r->key.data,
+						   r->key.length);
+	      vec_add1 (key_indices, op->key_index);
 	      op->aad = r->aad.data;
 	      op->aad_len = r->aad.length;
 	      op->len = r->plaintext.length;
@@ -147,8 +152,10 @@ test_crypto (vlib_main_t * vm, crypto_test_main_t * tm)
 	      op->tag_len = r->tag.length;
 	      break;
 	    case VNET_CRYPTO_OP_TYPE_HMAC:
-	      op->key = r->key.data;
-	      op->key_len = r->key.length;
+	      op->key_index = vnet_crypto_key_add (vm, r->alg,
+						   r->key.data,
+						   r->key.length);
+	      vec_add1 (key_indices, op->key_index);
 	      op->src = r->plaintext.data;
 	      op->len = r->plaintext.length;
 	      op->digest_len = r->digest.length;
@@ -250,6 +257,9 @@ test_crypto (vlib_main_t * vm, crypto_test_main_t * tm)
 			     op->digest_len);
 	}
     }
+
+  vec_foreach_index (i, key_indices)
+    vnet_crypto_key_del (vm, key_indices[i]);
   /* *INDENT-ON* */
 
   vec_free (computed_data);
@@ -269,11 +279,12 @@ test_crypto_perf (vlib_main_t * vm, crypto_test_main_t * tm)
   u32 *buffer_indices = 0;
   vnet_crypto_op_t *ops1 = 0, *ops2 = 0, *op1, *op2;
   vnet_crypto_alg_data_t *ad = vec_elt_at_index (cm->algs, tm->alg);
+  vnet_crypto_key_index_t key_index = ~0;
+  u8 key[32];
   int buffer_size = vlib_buffer_get_default_data_size (vm);
   u64 seed = clib_cpu_time_now ();
   u64 t0[5], t1[5], t2[5], n_bytes = 0;
   int i, j;
-  u8 *key;
 
   if (tm->buffer_size > buffer_size)
     return clib_error_return (0, "buffer size must be <= %u", buffer_size);
@@ -300,14 +311,19 @@ test_crypto_perf (vlib_main_t * vm, crypto_test_main_t * tm)
     }
 
   vlib_cli_output (vm, "%U: n_buffers %u buffer-size %u rounds %u "
-		   "warmup-rounds %u one-key %s",
+		   "warmup-rounds %u",
 		   format_vnet_crypto_alg, tm->alg, n_buffers, buffer_size,
-		   rounds, warmup_rounds, tm->one_key ? "yes" : "no");
+		   rounds, warmup_rounds);
   vlib_cli_output (vm, "   cpu-freq %.2f GHz",
 		   (f64) vm->clib_time.clocks_per_second * 1e-9);
 
   vnet_crypto_op_type_t ot = 0;
 
+  for (i = 0; i < sizeof (key); i++)
+    key[i] = i;
+
+  key_index = vnet_crypto_key_add (vm, tm->alg, key, sizeof (key));
+
   for (i = 0; i < VNET_CRYPTO_OP_N_TYPES; i++)
     {
       vnet_crypto_op_id_t id = ad->op_by_type[i];
@@ -322,8 +338,6 @@ test_crypto_perf (vlib_main_t * vm, crypto_test_main_t * tm)
       vlib_buffer_t *b = vlib_get_buffer (vm, buffer_indices[i]);
       op1 = ops1 + i;
       op2 = ops2 + i;
-      if (i == 0)
-	key = b->data - 32;
 
       switch (ot)
 	{
@@ -335,7 +349,7 @@ test_crypto_perf (vlib_main_t * vm, crypto_test_main_t * tm)
 			       ad->op_by_type[VNET_CRYPTO_OP_TYPE_DECRYPT]);
 	  op1->flags = VNET_CRYPTO_OP_FLAG_INIT_IV;
 	  op1->src = op2->src = op1->dst = op2->dst = b->data;
-	  op1->key = op2->key = tm->one_key ? key : b->data - 32;
+	  op1->key_index = op2->key_index = key_index;
 	  op1->iv = op2->iv = b->data - 64;
 	  n_bytes += op1->len = op2->len = buffer_size;
 	  break;
@@ -349,7 +363,7 @@ test_crypto_perf (vlib_main_t * vm, crypto_test_main_t * tm)
 			       [VNET_CRYPTO_OP_TYPE_AEAD_DECRYPT]);
 	  op1->flags = VNET_CRYPTO_OP_FLAG_INIT_IV;
 	  op1->src = op2->src = op1->dst = op2->dst = b->data;
-	  op1->key = op2->key = tm->one_key ? key : b->data - 32;
+	  op1->key_index = op2->key_index = key_index;
 	  op1->iv = op2->iv = b->data - 64;
 	  op1->aad = op2->aad = b->data - VLIB_BUFFER_PRE_DATA_SIZE;
 	  op1->aad_len = op2->aad_len = 0;
@@ -358,7 +372,7 @@ test_crypto_perf (vlib_main_t * vm, crypto_test_main_t * tm)
 	case VNET_CRYPTO_OP_TYPE_HMAC:
 	  vnet_crypto_op_init (op1, ad->op_by_type[VNET_CRYPTO_OP_TYPE_HMAC]);
 	  op1->src = b->data;
-	  op1->key = tm->one_key ? key : b->data - 32;
+	  op1->key_index = key_index;
 	  op1->iv = 0;
 	  op1->digest = b->data - VLIB_BUFFER_PRE_DATA_SIZE;
 	  op1->digest_len = 0;
@@ -418,6 +432,10 @@ test_crypto_perf (vlib_main_t * vm, crypto_test_main_t * tm)
 done:
   if (n_alloc)
     vlib_buffer_free (vm, buffer_indices, n_alloc);
+
+  if (key_index != ~0)
+    vnet_crypto_key_del (vm, key_index);
+
   vec_free (buffer_indices);
   vec_free (ops1);
   vec_free (ops2);
@@ -454,8 +472,6 @@ test_crypto_command_fn (vlib_main_t * vm,
 	;
       else if (unformat (input, "buffer-size %u", &tm->buffer_size))
 	;
-      else if (unformat (input, "one-key"))
-	tm->one_key = 1;
       else
 	return clib_error_return (0, "unknown input '%U'",
 				  format_unformat_error, input);