crypto_ia32: multiarch

Change-Id: Iead43a2b524b735a2069e611d899cd41d3a8efdc
Signed-off-by: Damjan Marion <damarion@cisco.com>

5 files changed, 70 insertions, 58 deletions

diff --git a/src/plugins/crypto_ia32/CMakeLists.txt b/src/plugins/crypto_ia32/CMakeLists.txt
index a100cdbb681..b01c57b9f6f 100644
--- a/src/plugins/crypto_ia32/CMakeLists.txt
+++ b/src/plugins/crypto_ia32/CMakeLists.txt
@@ -15,10 +15,20 @@ if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|x86_64.*|AMD64.*")
   return()
 endif()
 
-add_vpp_plugin(crypto_ia32
-  SOURCES
-  aes_cbc.c
-  main.c
-)
+add_vpp_plugin(crypto_ia32 SOURCES main.c)
 
-target_compile_options(crypto_ia32_plugin PRIVATE "-march=silvermont")
+list(APPEND VARIANTS "sse42\;-march=silvermont")
+list(APPEND VARIANTS "avx2\;-march=core-avx2")
+if(compiler_flag_march_skylake_avx512)
+  list(APPEND VARIANTS "avx512\;-march=skylake-avx512")
+endif()
+
+foreach(VARIANT ${VARIANTS})
+  list(GET VARIANT 0 v)
+  list(GET VARIANT 1 f)
+  set(l crypto_ia32_${v})
+  add_library(${l} OBJECT aes_cbc.c)
+  set_target_properties(${l} PROPERTIES POSITION_INDEPENDENT_CODE ON)
+  target_compile_options(${l} PUBLIC ${f} -O3 -Wall -fno-common)
+  target_sources(crypto_ia32_plugin PRIVATE $<TARGET_OBJECTS:${l}>)
+endforeach()
diff --git a/src/plugins/crypto_ia32/aes_cbc.c b/src/plugins/crypto_ia32/aes_cbc.c
index 7befdd2b4e6..2c08aa207ac 100644
--- a/src/plugins/crypto_ia32/aes_cbc.c
+++ b/src/plugins/crypto_ia32/aes_cbc.c
@@ -22,6 +22,12 @@
 #include <crypto_ia32/crypto_ia32.h>
 #include <crypto_ia32/aesni.h>
 
+typedef struct
+{
+  __m128i encrypt_key[15];
+  __m128i decrypt_key[15];
+} aes_cbc_key_data_t;
+
 static_always_inline void
 aes_cbc_dec (__m128i * k, u8 * src, u8 * dst, u8 * iv, int count,
 	     aesni_key_size_t rounds)
@@ -98,7 +104,8 @@ aesni_ops_enc_aes_cbc (vlib_main_t * vm, vnet_crypto_op_t * ops[],
   u8 *src[4] = { };
   u8 *dst[4] = { };
   vnet_crypto_key_index_t key_index[4] = { ~0, ~0, ~0, ~0 };
-  u32x4 dummy_mask, len = { };
+  u32x4 dummy_mask = { };
+  u32x4 len = { };
   u32 i, j, count, n_left = n_ops;
   __m128i r[4] = { }, k[4][rounds + 1];
 
@@ -129,9 +136,9 @@ more:
 	    dummy_mask[i] = ~0;
 	    if (key_index[i] != ops[0]->key_index)
 	      {
-		aesni_key_data_t *kd;
+		aes_cbc_key_data_t *kd;
 		key_index[i] = ops[0]->key_index;
-		kd = (aesni_key_data_t *) cm->key_data[key_index[i]];
+		kd = (aes_cbc_key_data_t *) cm->key_data[key_index[i]];
 		clib_memcpy_fast (k[i], kd->encrypt_key,
 				  (rounds + 1) * sizeof (__m128i));
 	      }
@@ -194,7 +201,7 @@ aesni_ops_dec_aes_cbc (vlib_main_t * vm, vnet_crypto_op_t * ops[],
   crypto_ia32_main_t *cm = &crypto_ia32_main;
   int rounds = AESNI_KEY_ROUNDS (ks);
   vnet_crypto_op_t *op = ops[0];
-  aesni_key_data_t *kd = (aesni_key_data_t *) cm->key_data[op->key_index];
+  aes_cbc_key_data_t *kd = (aes_cbc_key_data_t *) cm->key_data[op->key_index];
   u32 n_left = n_ops;
 
   ASSERT (n_ops >= 1);
@@ -206,13 +213,24 @@ decrypt:
   if (--n_left)
     {
       op += 1;
-      kd = (aesni_key_data_t *) cm->key_data[op->key_index];
+      kd = (aes_cbc_key_data_t *) cm->key_data[op->key_index];
       goto decrypt;
     }
 
   return n_ops;
 }
 
+static_always_inline void *
+aesni_cbc_key_exp (vnet_crypto_key_t * key, aesni_key_size_t ks)
+{
+  aes_cbc_key_data_t *kd;
+  kd = clib_mem_alloc_aligned (sizeof (*kd), CLIB_CACHE_LINE_BYTES);
+  aes_key_expand (kd->encrypt_key, key->data, ks);
+  aes_key_expand (kd->decrypt_key, key->data, ks);
+  aes_key_enc_to_dec (kd->decrypt_key, ks);
+  return kd;
+}
+
 #define foreach_aesni_cbc_handler_type _(128) _(192) _(256)
 
 #define _(x) \
@@ -222,6 +240,8 @@ static u32 aesni_ops_dec_aes_cbc_##x \
 static u32 aesni_ops_enc_aes_cbc_##x \
 (vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops) \
 { return aesni_ops_enc_aes_cbc (vm, ops, n_ops, AESNI_KEY_##x); } \
+static void * aesni_cbc_key_exp_##x (vnet_crypto_key_t *key) \
+{ return aesni_cbc_key_exp (key, AESNI_KEY_##x); }
 
 foreach_aesni_cbc_handler_type;
 #undef _
@@ -229,7 +249,13 @@ foreach_aesni_cbc_handler_type;
 #include <fcntl.h>
 
 clib_error_t *
-crypto_ia32_aesni_cbc_init (vlib_main_t * vm)
+#ifdef __AVX512F__
+crypto_ia32_aesni_cbc_init_avx512 (vlib_main_t * vm)
+#elif __AVX2__
+crypto_ia32_aesni_cbc_init_avx2 (vlib_main_t * vm)
+#else
+crypto_ia32_aesni_cbc_init_sse42 (vlib_main_t * vm)
+#endif
 {
   crypto_ia32_main_t *cm = &crypto_ia32_main;
   crypto_ia32_per_thread_data_t *ptd;
@@ -260,7 +286,8 @@ crypto_ia32_aesni_cbc_init (vlib_main_t * vm)
 				    aesni_ops_enc_aes_cbc_##x); \
   vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \
 				    VNET_CRYPTO_OP_AES_##x##_CBC_DEC, \
-				    aesni_ops_dec_aes_cbc_##x);
+				    aesni_ops_dec_aes_cbc_##x); \
+  cm->key_fn[VNET_CRYPTO_ALG_AES_##x##_CBC] = aesni_cbc_key_exp_##x;
   foreach_aesni_cbc_handler_type;
 #undef _
 
diff --git a/src/plugins/crypto_ia32/aesni.h b/src/plugins/crypto_ia32/aesni.h
index 80e30e51b33..ceb28451732 100644
--- a/src/plugins/crypto_ia32/aesni.h
+++ b/src/plugins/crypto_ia32/aesni.h
@@ -19,12 +19,6 @@
 #define __aesni_h__
 
 
-typedef struct
-{
-  __m128i encrypt_key[15];
-  __m128i decrypt_key[15];
-} aesni_key_data_t;
-
 typedef enum
 {
   AESNI_KEY_128 = 0,
diff --git a/src/plugins/crypto_ia32/crypto_ia32.h b/src/plugins/crypto_ia32/crypto_ia32.h
index f5f09dfd499..7acf0741ec6 100644
--- a/src/plugins/crypto_ia32/crypto_ia32.h
+++ b/src/plugins/crypto_ia32/crypto_ia32.h
@@ -18,6 +18,8 @@
 #ifndef __crypto_ia32_h__
 #define __crypto_ia32_h__
 
+typedef void *(crypto_ia32_key_fn_t) (vnet_crypto_key_t * key);
+
 typedef struct
 {
   __m128i cbc_iv[4];
@@ -27,12 +29,15 @@ typedef struct
 {
   u32 crypto_engine_index;
   crypto_ia32_per_thread_data_t *per_thread_data;
+  crypto_ia32_key_fn_t *key_fn[VNET_CRYPTO_N_ALGS];
   void **key_data;
 } crypto_ia32_main_t;
 
 extern crypto_ia32_main_t crypto_ia32_main;
 
-clib_error_t *crypto_ia32_aesni_cbc_init (vlib_main_t * vm);
+clib_error_t *crypto_ia32_aesni_cbc_init_sse42 (vlib_main_t * vm);
+clib_error_t *crypto_ia32_aesni_cbc_init_avx2 (vlib_main_t * vm);
+clib_error_t *crypto_ia32_aesni_cbc_init_avx512 (vlib_main_t * vm);
 
 #endif /* __crypto_ia32_h__ */
 
diff --git a/src/plugins/crypto_ia32/main.c b/src/plugins/crypto_ia32/main.c
index a02fc38f99e..b31c9019495 100644
--- a/src/plugins/crypto_ia32/main.c
+++ b/src/plugins/crypto_ia32/main.c
@@ -19,7 +19,6 @@
 #include <vnet/plugin/plugin.h>
 #include <vnet/crypto/crypto.h>
 #include <crypto_ia32/crypto_ia32.h>
-#include <crypto_ia32/aesni.h>
 
 crypto_ia32_main_t crypto_ia32_main;
 
@@ -29,17 +28,9 @@ crypto_ia32_key_handler (vlib_main_t * vm, vnet_crypto_key_op_t kop,
 {
   vnet_crypto_key_t *key = vnet_crypto_get_key (idx);
   crypto_ia32_main_t *cm = &crypto_ia32_main;
-  aesni_key_data_t *kd;
 
-  switch (key->alg)
-    {
-    case VNET_CRYPTO_ALG_AES_128_CBC:
-    case VNET_CRYPTO_ALG_AES_192_CBC:
-    case VNET_CRYPTO_ALG_AES_256_CBC:
-      break;
-    default:
-      return;
-    }
+  if (cm->key_fn[key->alg] == 0)
+    return;
 
   if (kop == VNET_CRYPTO_KEY_OP_DEL)
     {
@@ -65,31 +56,7 @@ crypto_ia32_key_handler (vlib_main_t * vm, vnet_crypto_key_op_t kop,
       clib_mem_free (cm->key_data[idx]);
     }
 
-  kd = cm->key_data[idx] = clib_mem_alloc_aligned (sizeof (aesni_key_data_t),
-						   CLIB_CACHE_LINE_BYTES);
-
-  /* ADD or MODIFY */
-  switch (key->alg)
-    {
-    case VNET_CRYPTO_ALG_AES_128_CBC:
-      aes_key_expand (kd->encrypt_key, key->data, AESNI_KEY_128);
-      aes_key_expand (kd->decrypt_key, key->data, AESNI_KEY_128);
-      aes_key_enc_to_dec (kd->decrypt_key, AESNI_KEY_128);
-      break;
-    case VNET_CRYPTO_ALG_AES_192_CBC:
-      aes_key_expand (kd->encrypt_key, key->data, AESNI_KEY_192);
-      aes_key_expand (kd->decrypt_key, key->data, AESNI_KEY_192);
-      aes_key_enc_to_dec (kd->decrypt_key, AESNI_KEY_192);
-      break;
-    case VNET_CRYPTO_ALG_AES_256_CBC:
-      aes_key_expand (kd->encrypt_key, key->data, AESNI_KEY_256);
-      aes_key_expand (kd->decrypt_key, key->data, AESNI_KEY_256);
-      aes_key_enc_to_dec (kd->decrypt_key, AESNI_KEY_256);
-      break;
-    default:
-      break;
-    }
-  return;
+  cm->key_data[idx] = cm->key_fn[key->alg] (key);
 }
 
 clib_error_t *
@@ -99,6 +66,9 @@ crypto_ia32_init (vlib_main_t * vm)
   vlib_thread_main_t *tm = vlib_get_thread_main ();
   clib_error_t *error = 0;
 
+  if (clib_cpu_supports_x86_aes () == 0)
+    return 0;
+
   vec_validate_aligned (cm->per_thread_data, tm->n_vlib_mains - 1,
 			CLIB_CACHE_LINE_BYTES);
 
@@ -106,8 +76,14 @@ crypto_ia32_init (vlib_main_t * vm)
     vnet_crypto_register_engine (vm, "ia32", 100,
 				 "Intel IA32 ISA Optimized Crypto");
 
-  if (clib_cpu_supports_x86_aes () &&
-      (error = crypto_ia32_aesni_cbc_init (vm)))
+  if (clib_cpu_supports_avx512f ())
+    error = crypto_ia32_aesni_cbc_init_avx512 (vm);
+  else if (clib_cpu_supports_avx2 ())
+    error = crypto_ia32_aesni_cbc_init_avx2 (vm);
+  else
+    error = crypto_ia32_aesni_cbc_init_sse42 (vm);
+
+  if (error)
     goto error;
 
   vnet_crypto_register_key_handler (vm, cm->crypto_engine_index,