diff options
| -rw-r--r-- | src/plugins/crypto_native/CMakeLists.txt | 4 | ||||
| -rw-r--r-- | src/plugins/crypto_native/aes_ctr.c | 112 | ||||
| -rw-r--r-- | src/plugins/crypto_native/crypto_native.h | 7 | ||||
| -rw-r--r-- | src/plugins/crypto_native/main.c | 39 |
4 files changed, 144 insertions, 18 deletions
diff --git a/src/plugins/crypto_native/CMakeLists.txt b/src/plugins/crypto_native/CMakeLists.txt index ba6f6cbcc28..9b6091610d9 100644 --- a/src/plugins/crypto_native/CMakeLists.txt +++ b/src/plugins/crypto_native/CMakeLists.txt @@ -23,13 +23,13 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|x86_64.*|AMD64.*") if(compiler_flag_march_alderlake) list(APPEND VARIANTS "adl\;-march=alderlake -mprefer-vector-width=256") endif() - set (COMPILE_FILES aes_cbc.c aes_gcm.c) + set (COMPILE_FILES aes_cbc.c aes_gcm.c aes_ctr.c) set (COMPILE_OPTS -Wall -fno-common -maes) endif() if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*)") list(APPEND VARIANTS "armv8\;-march=armv8.1-a+crc+crypto") - set (COMPILE_FILES aes_cbc.c aes_gcm.c) + set (COMPILE_FILES aes_cbc.c aes_gcm.c aes_ctr.c) set (COMPILE_OPTS -Wall -fno-common) endif() diff --git a/src/plugins/crypto_native/aes_ctr.c b/src/plugins/crypto_native/aes_ctr.c new file mode 100644 index 00000000000..3a219510419 --- /dev/null +++ b/src/plugins/crypto_native/aes_ctr.c @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2024 Cisco Systems, Inc. + */ + +#include <vlib/vlib.h> +#include <vnet/plugin/plugin.h> +#include <vnet/crypto/crypto.h> +#include <crypto_native/crypto_native.h> +#include <vppinfra/crypto/aes_ctr.h> + +#if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0 +#pragma GCC optimize("O3") +#endif + +static_always_inline u32 +aes_ops_aes_ctr (vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops, + vnet_crypto_op_chunk_t *chunks, aes_key_size_t ks, + int maybe_chained) +{ + crypto_native_main_t *cm = &crypto_native_main; + vnet_crypto_op_t *op = ops[0]; + aes_ctr_key_data_t *kd; + aes_ctr_ctx_t ctx; + u32 n_left = n_ops; + +next: + kd = (aes_ctr_key_data_t *) cm->key_data[op->key_index]; + + clib_aes_ctr_init (&ctx, kd, op->iv, ks); + if (op->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS) + { + vnet_crypto_op_chunk_t *chp = chunks + op->chunk_index; + for (int j = 0; j < op->n_chunks; j++, chp++) + clib_aes_ctr_transform (&ctx, chp->src, chp->dst, chp->len, ks); + } + else + clib_aes_ctr_transform (&ctx, op->src, op->dst, op->len, ks); + + op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; + + if (--n_left) + { + op += 1; + goto next; + } + + return n_ops; +} + +static_always_inline void * +aes_ctr_key_exp (vnet_crypto_key_t *key, aes_key_size_t ks) +{ + aes_ctr_key_data_t *kd; + + kd = clib_mem_alloc_aligned (sizeof (*kd), CLIB_CACHE_LINE_BYTES); + + clib_aes_ctr_key_expand (kd, key->data, ks); + + return kd; +} + +#define foreach_aes_ctr_handler_type _ (128) _ (192) _ (256) + +#define _(x) \ + static u32 aes_ops_aes_ctr_##x (vlib_main_t *vm, vnet_crypto_op_t *ops[], \ + u32 n_ops) \ + { \ + return aes_ops_aes_ctr (vm, ops, n_ops, 0, AES_KEY_##x, 0); \ + } \ + static u32 aes_ops_aes_ctr_##x##_chained ( \ + vlib_main_t *vm, vnet_crypto_op_t *ops[], vnet_crypto_op_chunk_t *chunks, \ + u32 n_ops) \ + { \ + return aes_ops_aes_ctr (vm, ops, n_ops, chunks, AES_KEY_##x, 1); \ + } \ + static void *aes_ctr_key_exp_##x (vnet_crypto_key_t *key) \ + { \ + return aes_ctr_key_exp (key, AES_KEY_##x); \ + } + +foreach_aes_ctr_handler_type; +#undef _ + +clib_error_t * +#if defined(__VAES__) && defined(__AVX512F__) +crypto_native_aes_ctr_init_icl (vlib_main_t *vm) +#elif defined(__VAES__) +crypto_native_aes_ctr_init_adl (vlib_main_t *vm) +#elif __AVX512F__ +crypto_native_aes_ctr_init_skx (vlib_main_t *vm) +#elif __AVX2__ +crypto_native_aes_ctr_init_hsw (vlib_main_t *vm) +#elif __aarch64__ +crypto_native_aes_ctr_init_neon (vlib_main_t *vm) +#else +crypto_native_aes_ctr_init_slm (vlib_main_t *vm) +#endif +{ + crypto_native_main_t *cm = &crypto_native_main; + +#define _(x) \ + vnet_crypto_register_ops_handlers ( \ + vm, cm->crypto_engine_index, VNET_CRYPTO_OP_AES_##x##_CTR_ENC, \ + aes_ops_aes_ctr_##x, aes_ops_aes_ctr_##x##_chained); \ + vnet_crypto_register_ops_handlers ( \ + vm, cm->crypto_engine_index, VNET_CRYPTO_OP_AES_##x##_CTR_DEC, \ + aes_ops_aes_ctr_##x, aes_ops_aes_ctr_##x##_chained); \ + cm->key_fn[VNET_CRYPTO_ALG_AES_##x##_CTR] = aes_ctr_key_exp_##x; + foreach_aes_ctr_handler_type; +#undef _ + return 0; +} diff --git a/src/plugins/crypto_native/crypto_native.h b/src/plugins/crypto_native/crypto_native.h index 623070c19b7..c15b8cbd1da 100644 --- a/src/plugins/crypto_native/crypto_native.h +++ b/src/plugins/crypto_native/crypto_native.h @@ -32,9 +32,10 @@ extern crypto_native_main_t crypto_native_main; #define foreach_crypto_native_march_variant \ _ (slm) _ (hsw) _ (skx) _ (icl) _ (adl) _ (neon) -#define _(v) \ -clib_error_t __clib_weak *crypto_native_aes_cbc_init_##v (vlib_main_t * vm); \ -clib_error_t __clib_weak *crypto_native_aes_gcm_init_##v (vlib_main_t * vm); \ +#define _(v) \ + clib_error_t __clib_weak *crypto_native_aes_cbc_init_##v (vlib_main_t *vm); \ + clib_error_t __clib_weak *crypto_native_aes_ctr_init_##v (vlib_main_t *vm); \ + clib_error_t __clib_weak *crypto_native_aes_gcm_init_##v (vlib_main_t *vm); foreach_crypto_native_march_variant; #undef _ diff --git a/src/plugins/crypto_native/main.c b/src/plugins/crypto_native/main.c index 718356c745f..8a59be319b9 100644 --- a/src/plugins/crypto_native/main.c +++ b/src/plugins/crypto_native/main.c @@ -97,6 +97,31 @@ crypto_native_init (vlib_main_t * vm) if (error) return error; + if (0) + ; +#if __x86_64__ + else if (crypto_native_aes_ctr_init_icl && clib_cpu_supports_vaes () && + clib_cpu_supports_avx512f ()) + error = crypto_native_aes_ctr_init_icl (vm); + else if (crypto_native_aes_ctr_init_adl && clib_cpu_supports_vaes ()) + error = crypto_native_aes_ctr_init_adl (vm); + else if (crypto_native_aes_ctr_init_skx && clib_cpu_supports_avx512f ()) + error = crypto_native_aes_ctr_init_skx (vm); + else if (crypto_native_aes_ctr_init_hsw && clib_cpu_supports_avx2 ()) + error = crypto_native_aes_ctr_init_hsw (vm); + else if (crypto_native_aes_ctr_init_slm) + error = crypto_native_aes_ctr_init_slm (vm); +#endif +#if __aarch64__ + else if (crypto_native_aes_ctr_init_neon) + error = crypto_native_aes_ctr_init_neon (vm); +#endif + else + error = clib_error_return (0, "No AES CTR implemenation available"); + + if (error) + return error; + #if __x86_64__ if (clib_cpu_supports_pclmulqdq ()) { @@ -133,26 +158,14 @@ crypto_native_init (vlib_main_t * vm) return 0; } -/* *INDENT-OFF* */ VLIB_INIT_FUNCTION (crypto_native_init) = { .runs_after = VLIB_INITS ("vnet_crypto_init"), }; -/* *INDENT-ON* */ #include <vpp/app/version.h> -/* *INDENT-OFF* */ VLIB_PLUGIN_REGISTER () = { .version = VPP_BUILD_VER, - .description = "Intel IA32 Software Crypto Engine", + .description = "Native Crypto Engine", }; -/* *INDENT-ON* */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ |