diff options
Diffstat (limited to 'src/plugins')
| -rw-r--r-- | src/plugins/crypto_ipsecmb/CMakeLists.txt | 50 | ||||
| -rw-r--r-- | src/plugins/crypto_ipsecmb/FEATURE.yaml | 11 | ||||
| -rw-r--r-- | src/plugins/crypto_ipsecmb/ipsecmb.c | 963 | ||||
| -rw-r--r-- | src/plugins/crypto_native/CMakeLists.txt | 50 | ||||
| -rw-r--r-- | src/plugins/crypto_native/FEATURE.yaml | 13 | ||||
| -rw-r--r-- | src/plugins/crypto_native/aes_cbc.c | 188 | ||||
| -rw-r--r-- | src/plugins/crypto_native/aes_ctr.c | 130 | ||||
| -rw-r--r-- | src/plugins/crypto_native/aes_gcm.c | 166 | ||||
| -rw-r--r-- | src/plugins/crypto_native/crypto_native.h | 84 | ||||
| -rw-r--r-- | src/plugins/crypto_native/main.c | 128 | ||||
| -rw-r--r-- | src/plugins/crypto_native/sha2.c | 198 | ||||
| -rw-r--r-- | src/plugins/crypto_openssl/CMakeLists.txt | 27 | ||||
| -rw-r--r-- | src/plugins/crypto_openssl/FEATURE.yaml | 14 | ||||
| -rw-r--r-- | src/plugins/crypto_openssl/crypto_openssl.h | 20 | ||||
| -rw-r--r-- | src/plugins/crypto_openssl/main.c | 719 | ||||
| -rw-r--r-- | src/plugins/crypto_sw_scheduler/main.c | 2 | ||||
| -rw-r--r-- | src/plugins/dpdk/cryptodev/cryptodev.c | 5 | ||||
| -rw-r--r-- | src/plugins/quic/quic_crypto.c | 5 |
18 files changed, 5 insertions, 2768 deletions
diff --git a/src/plugins/crypto_ipsecmb/CMakeLists.txt b/src/plugins/crypto_ipsecmb/CMakeLists.txt deleted file mode 100644 index 429343a9f3b..00000000000 --- a/src/plugins/crypto_ipsecmb/CMakeLists.txt +++ /dev/null @@ -1,50 +0,0 @@ - -# Copyright (c) 2019 Cisco Systems -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|x86_64.*|AMD64.*") - return() -endif() - -vpp_find_path(IPSECMB_INCLUDE_DIR NAMES intel-ipsec-mb.h HINTS ${IPSECMB_INCLUDE_DIR_HINT}) -vpp_find_library(IPSECMB_LIB NAMES libIPSec_MB.a HINTS ${IPSECMB_LIB_DIR_HINT}) - -if(IPSECMB_INCLUDE_DIR AND IPSECMB_LIB) - - get_filename_component(IPSECMB_LIB_DIR ${IPSECMB_LIB} DIRECTORY) - set(IPSECMB_LINK_FLAGS "${IPSECMB_LINK_FLAGS} -L${IPSECMB_LIB_DIR} -Wl,--whole-archive ${IPSECMB_LIB} -Wl,--no-whole-archive") - set(IPSECMB_LINK_FLAGS "${IPSECMB_LINK_FLAGS} -Wl,--exclude-libs,libIPSec_MB.a,-l:libIPSec_MB.a") - include_directories(${IPSECMB_INCLUDE_DIR}) - add_vpp_plugin(crypto_ipsecmb - SOURCES - ipsecmb.c - - LINK_FLAGS - ${IPSECMB_LINK_FLAGS} - ) - - file(READ "${IPSECMB_INCLUDE_DIR}/intel-ipsec-mb.h" ipsecmb_header) - string(REGEX MATCH "IMB_VERSION_STR (\"+[0-9]+\\.[0-9]+\\.[0-9]+\")" _ ${ipsecmb_header}) - string(REPLACE "\"" "" IPSECMB_VERSION ${CMAKE_MATCH_1}) - - if (${IPSECMB_VERSION} VERSION_GREATER "0.54.0") - add_definitions(-DHAVE_IPSECMB_CHACHA_POLY) - else() - message(STATUS "Intel IPSecMB ${IPSECMB_VERSION} does not support chacha20-poly1305. Disabled") - endif() - - target_compile_options(crypto_ipsecmb_plugin PRIVATE "-march=silvermont" "-maes") - message(STATUS "Intel IPSecMB found: ${IPSECMB_INCLUDE_DIR}") -else() - message(STATUS "Intel IPSecMB not found") -endif() diff --git a/src/plugins/crypto_ipsecmb/FEATURE.yaml b/src/plugins/crypto_ipsecmb/FEATURE.yaml deleted file mode 100644 index 3ca03bf5515..00000000000 --- a/src/plugins/crypto_ipsecmb/FEATURE.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -name: IPSec crypto engine provided by Intel IPSecMB library -maintainer: Neale Ranns <nranns@cisco.com> -features: - - SHA(1, 224, 256, 384, 512) - - CBC(128, 192, 256) - - GCM(128, 192, 256) - -description: "" -state: production -properties: [API, CLI, MULTITHREAD] diff --git a/src/plugins/crypto_ipsecmb/ipsecmb.c b/src/plugins/crypto_ipsecmb/ipsecmb.c deleted file mode 100644 index 064c129ba12..00000000000 --- a/src/plugins/crypto_ipsecmb/ipsecmb.c +++ /dev/null @@ -1,963 +0,0 @@ -/* - * ipsecmb.c - Intel IPSec Multi-buffer library Crypto Engine - * - * Copyright (c) 2019 Cisco Systemss - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fcntl.h> - -#include <intel-ipsec-mb.h> - -#include <vnet/vnet.h> -#include <vnet/plugin/plugin.h> -#include <vpp/app/version.h> -#include <vnet/crypto/crypto.h> -#include <vppinfra/cpu.h> - -#define HMAC_MAX_BLOCK_SIZE IMB_SHA_512_BLOCK_SIZE -#define EXPANDED_KEY_N_BYTES (16 * 15) - -typedef struct -{ - CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); - IMB_MGR *mgr; -#if IMB_VERSION_NUM >= IMB_VERSION(1, 3, 0) - IMB_JOB burst_jobs[IMB_MAX_BURST_SIZE]; -#endif -} ipsecmb_per_thread_data_t; - -typedef struct -{ - u16 data_size; - u8 block_size; - aes_gcm_pre_t aes_gcm_pre; - keyexp_t keyexp; - hash_one_block_t hash_one_block; - hash_fn_t hash_fn; -} ipsecmb_alg_data_t; - -typedef struct ipsecmb_main_t_ -{ - ipsecmb_per_thread_data_t *per_thread_data; - ipsecmb_alg_data_t alg_data[VNET_CRYPTO_N_ALGS]; - void **key_data; -} ipsecmb_main_t; - -typedef struct -{ - u8 enc_key_exp[EXPANDED_KEY_N_BYTES]; - u8 dec_key_exp[EXPANDED_KEY_N_BYTES]; -} ipsecmb_aes_key_data_t; - -static ipsecmb_main_t ipsecmb_main = { }; - -/* clang-format off */ -/* - * (Alg, JOB_HASH_ALG, fn, block-size-bytes, hash-size-bytes, digest-size-bytes) - */ -#define foreach_ipsecmb_hmac_op \ - _(SHA1, SHA_1, sha1, 64, 20, 20) \ - _(SHA224, SHA_224, sha224, 64, 32, 28) \ - _(SHA256, SHA_256, sha256, 64, 32, 32) \ - _(SHA384, SHA_384, sha384, 128, 64, 48) \ - _(SHA512, SHA_512, sha512, 128, 64, 64) - -/* - * (Alg, key-len-bits, JOB_CIPHER_MODE) - */ -#define foreach_ipsecmb_cipher_op \ - _ (AES_128_CBC, 128, CBC) \ - _ (AES_192_CBC, 192, CBC) \ - _ (AES_256_CBC, 256, CBC) \ - _ (AES_128_CTR, 128, CNTR) \ - _ (AES_192_CTR, 192, CNTR) \ - _ (AES_256_CTR, 256, CNTR) - -/* - * (Alg, key-len-bytes, iv-len-bytes) - */ -#define foreach_ipsecmb_gcm_cipher_op \ - _(AES_128_GCM, 128) \ - _(AES_192_GCM, 192) \ - _(AES_256_GCM, 256) -/* clang-format on */ -static_always_inline vnet_crypto_op_status_t -ipsecmb_status_job (IMB_STATUS status) -{ - switch (status) - { - case IMB_STATUS_COMPLETED: - return VNET_CRYPTO_OP_STATUS_COMPLETED; - case IMB_STATUS_BEING_PROCESSED: - case IMB_STATUS_COMPLETED_CIPHER: - case IMB_STATUS_COMPLETED_AUTH: - return VNET_CRYPTO_OP_STATUS_WORK_IN_PROGRESS; - case IMB_STATUS_INVALID_ARGS: - case IMB_STATUS_INTERNAL_ERROR: - case IMB_STATUS_ERROR: - return VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR; - } - ASSERT (0); - return VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR; -} - -always_inline void -ipsecmb_retire_hmac_job (IMB_JOB *job, u32 *n_fail, u32 digest_size) -{ - vnet_crypto_op_t *op = job->user_data; - u32 len = op->digest_len ? op->digest_len : digest_size; - - if (PREDICT_FALSE (IMB_STATUS_COMPLETED != job->status)) - { - op->status = ipsecmb_status_job (job->status); - *n_fail = *n_fail + 1; - return; - } - - if (op->flags & VNET_CRYPTO_OP_FLAG_HMAC_CHECK) - { - if ((memcmp (op->digest, job->auth_tag_output, len))) - { - *n_fail = *n_fail + 1; - op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC; - return; - } - } - else if (len == digest_size) - clib_memcpy_fast (op->digest, job->auth_tag_output, digest_size); - else - clib_memcpy_fast (op->digest, job->auth_tag_output, len); - - op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; -} - -#if IMB_VERSION_NUM >= IMB_VERSION(1, 3, 0) -static_always_inline u32 -ipsecmb_ops_hmac_inline (vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops, - u32 block_size, u32 hash_size, u32 digest_size, - IMB_HASH_ALG alg) -{ - ipsecmb_main_t *imbm = &ipsecmb_main; - ipsecmb_per_thread_data_t *ptd = - vec_elt_at_index (imbm->per_thread_data, vm->thread_index); - IMB_JOB *job; - u32 i, n_fail = 0, ops_index = 0; - u8 scratch[n_ops][digest_size]; - const u32 burst_sz = - (n_ops > IMB_MAX_BURST_SIZE) ? IMB_MAX_BURST_SIZE : n_ops; - - while (n_ops) - { - const u32 n = (n_ops > burst_sz) ? burst_sz : n_ops; - /* - * configure all the jobs first ... - */ - for (i = 0; i < n; i++, ops_index++) - { - vnet_crypto_op_t *op = ops[ops_index]; - const u8 *kd = (u8 *) imbm->key_data[op->key_index]; - - job = &ptd->burst_jobs[i]; - - job->src = op->src; - job->hash_start_src_offset_in_bytes = 0; - job->msg_len_to_hash_in_bytes = op->len; - job->auth_tag_output_len_in_bytes = digest_size; - job->auth_tag_output = scratch[ops_index]; - - job->u.HMAC._hashed_auth_key_xor_ipad = kd; - job->u.HMAC._hashed_auth_key_xor_opad = kd + hash_size; - job->user_data = op; - } - - /* - * submit all jobs to be processed and retire completed jobs - */ - IMB_SUBMIT_HASH_BURST_NOCHECK (ptd->mgr, ptd->burst_jobs, n, alg); - - for (i = 0; i < n; i++) - { - job = &ptd->burst_jobs[i]; - ipsecmb_retire_hmac_job (job, &n_fail, digest_size); - } - - n_ops -= n; - } - - return ops_index - n_fail; -} -#else -static_always_inline u32 -ipsecmb_ops_hmac_inline (vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops, - u32 block_size, u32 hash_size, u32 digest_size, - JOB_HASH_ALG alg) -{ - ipsecmb_main_t *imbm = &ipsecmb_main; - ipsecmb_per_thread_data_t *ptd = - vec_elt_at_index (imbm->per_thread_data, vm->thread_index); - IMB_JOB *job; - u32 i, n_fail = 0; - u8 scratch[n_ops][digest_size]; - - /* - * queue all the jobs first ... - */ - for (i = 0; i < n_ops; i++) - { - vnet_crypto_op_t *op = ops[i]; - u8 *kd = (u8 *) imbm->key_data[op->key_index]; - - job = IMB_GET_NEXT_JOB (ptd->mgr); - - job->src = op->src; - job->hash_start_src_offset_in_bytes = 0; - job->msg_len_to_hash_in_bytes = op->len; - job->hash_alg = alg; - job->auth_tag_output_len_in_bytes = digest_size; - job->auth_tag_output = scratch[i]; - - job->cipher_mode = IMB_CIPHER_NULL; - job->cipher_direction = IMB_DIR_DECRYPT; - job->chain_order = IMB_ORDER_HASH_CIPHER; - - job->u.HMAC._hashed_auth_key_xor_ipad = kd; - job->u.HMAC._hashed_auth_key_xor_opad = kd + hash_size; - job->user_data = op; - - job = IMB_SUBMIT_JOB (ptd->mgr); - - if (job) - ipsecmb_retire_hmac_job (job, &n_fail, digest_size); - } - - while ((job = IMB_FLUSH_JOB (ptd->mgr))) - ipsecmb_retire_hmac_job (job, &n_fail, digest_size); - - return n_ops - n_fail; -} -#endif - -/* clang-format off */ -#define _(a, b, c, d, e, f) \ -static_always_inline u32 \ -ipsecmb_ops_hmac_##a (vlib_main_t * vm, \ - vnet_crypto_op_t * ops[], \ - u32 n_ops) \ -{ return ipsecmb_ops_hmac_inline (vm, ops, n_ops, d, e, f, \ - IMB_AUTH_HMAC_##b); } \ - -foreach_ipsecmb_hmac_op; -#undef _ -/* clang-format on */ - -always_inline void -ipsecmb_retire_cipher_job (IMB_JOB *job, u32 *n_fail) -{ - vnet_crypto_op_t *op = job->user_data; - - if (PREDICT_FALSE (IMB_STATUS_COMPLETED != job->status)) - { - op->status = ipsecmb_status_job (job->status); - *n_fail = *n_fail + 1; - } - else - op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; -} - -#if IMB_VERSION_NUM >= IMB_VERSION(1, 3, 0) -static_always_inline u32 -ipsecmb_ops_aes_cipher_inline (vlib_main_t *vm, vnet_crypto_op_t *ops[], - u32 n_ops, u32 key_len, - IMB_CIPHER_DIRECTION direction, - IMB_CIPHER_MODE cipher_mode) -{ - ipsecmb_main_t *imbm = &ipsecmb_main; - ipsecmb_per_thread_data_t *ptd = - vec_elt_at_index (imbm->per_thread_data, vm->thread_index); - IMB_JOB *job; - u32 i, n_fail = 0, ops_index = 0; - const u32 burst_sz = - (n_ops > IMB_MAX_BURST_SIZE) ? IMB_MAX_BURST_SIZE : n_ops; - - while (n_ops) - { - const u32 n = (n_ops > burst_sz) ? burst_sz : n_ops; - - for (i = 0; i < n; i++) - { - ipsecmb_aes_key_data_t *kd; - vnet_crypto_op_t *op = ops[ops_index++]; - kd = (ipsecmb_aes_key_data_t *) imbm->key_data[op->key_index]; - - job = &ptd->burst_jobs[i]; - - job->src = op->src; - job->dst = op->dst; - job->msg_len_to_cipher_in_bytes = op->len; - job->cipher_start_src_offset_in_bytes = 0; - - job->hash_alg = IMB_AUTH_NULL; - - job->enc_keys = kd->enc_key_exp; - job->dec_keys = kd->dec_key_exp; - job->iv = op->iv; - job->iv_len_in_bytes = IMB_AES_BLOCK_SIZE; - - job->user_data = op; - } - - IMB_SUBMIT_CIPHER_BURST_NOCHECK (ptd->mgr, ptd->burst_jobs, n, - cipher_mode, direction, key_len / 8); - for (i = 0; i < n; i++) - { - job = &ptd->burst_jobs[i]; - ipsecmb_retire_cipher_job (job, &n_fail); - } - - n_ops -= n; - } - - return ops_index - n_fail; -} -#else -static_always_inline u32 -ipsecmb_ops_aes_cipher_inline (vlib_main_t *vm, vnet_crypto_op_t *ops[], - u32 n_ops, u32 key_len, - JOB_CIPHER_DIRECTION direction, - JOB_CIPHER_MODE cipher_mode) -{ - ipsecmb_main_t *imbm = &ipsecmb_main; - ipsecmb_per_thread_data_t *ptd = - vec_elt_at_index (imbm->per_thread_data, vm->thread_index); - IMB_JOB *job; - u32 i, n_fail = 0; - - for (i = 0; i < n_ops; i++) - { - ipsecmb_aes_key_data_t *kd; - vnet_crypto_op_t *op = ops[i]; - kd = (ipsecmb_aes_key_data_t *) imbm->key_data[op->key_index]; - - job = IMB_GET_NEXT_JOB (ptd->mgr); - - job->src = op->src; - job->dst = op->dst; - job->msg_len_to_cipher_in_bytes = op->len; - job->cipher_start_src_offset_in_bytes = 0; - - job->hash_alg = IMB_AUTH_NULL; - job->cipher_mode = cipher_mode; - job->cipher_direction = direction; - job->chain_order = - (direction == IMB_DIR_ENCRYPT ? IMB_ORDER_CIPHER_HASH : - IMB_ORDER_HASH_CIPHER); - - job->aes_key_len_in_bytes = key_len / 8; - job->enc_keys = kd->enc_key_exp; - job->dec_keys = kd->dec_key_exp; - job->iv = op->iv; - job->iv_len_in_bytes = IMB_AES_BLOCK_SIZE; - - job->user_data = op; - - job = IMB_SUBMIT_JOB (ptd->mgr); - - if (job) - ipsecmb_retire_cipher_job (job, &n_fail); - } - - while ((job = IMB_FLUSH_JOB (ptd->mgr))) - ipsecmb_retire_cipher_job (job, &n_fail); - - return n_ops - n_fail; -} -#endif - -/* clang-format off */ -#define _(a, b, c) \ - static_always_inline u32 ipsecmb_ops_cipher_enc_##a ( \ - vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops) \ - { \ - return ipsecmb_ops_aes_cipher_inline ( \ - vm, ops, n_ops, b, IMB_DIR_ENCRYPT, IMB_CIPHER_##c); \ - } \ - \ - static_always_inline u32 ipsecmb_ops_cipher_dec_##a ( \ - vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops) \ - { \ - return ipsecmb_ops_aes_cipher_inline ( \ - vm, ops, n_ops, b, IMB_DIR_DECRYPT, IMB_CIPHER_##c); \ - } - -foreach_ipsecmb_cipher_op; -#undef _ - -#define _(a, b) \ -static_always_inline u32 \ -ipsecmb_ops_gcm_cipher_enc_##a##_chained (vlib_main_t * vm, \ - vnet_crypto_op_t * ops[], vnet_crypto_op_chunk_t *chunks, u32 n_ops) \ -{ \ - ipsecmb_main_t *imbm = &ipsecmb_main; \ - ipsecmb_per_thread_data_t *ptd = vec_elt_at_index (imbm->per_thread_data, \ - vm->thread_index); \ - IMB_MGR *m = ptd->mgr; \ - vnet_crypto_op_chunk_t *chp; \ - u32 i, j; \ - \ - for (i = 0; i < n_ops; i++) \ - { \ - struct gcm_key_data *kd; \ - struct gcm_context_data ctx; \ - vnet_crypto_op_t *op = ops[i]; \ - \ - kd = (struct gcm_key_data *) imbm->key_data[op->key_index]; \ - ASSERT (op->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS); \ - IMB_AES##b##_GCM_INIT(m, kd, &ctx, op->iv, op->aad, op->aad_len); \ - chp = chunks + op->chunk_index; \ - for (j = 0; j < op->n_chunks; j++) \ - { \ - IMB_AES##b##_GCM_ENC_UPDATE (m, kd, &ctx, chp->dst, chp->src, \ - chp->len); \ - chp += 1; \ - } \ - IMB_AES##b##_GCM_ENC_FINALIZE(m, kd, &ctx, op->tag, op->tag_len); \ - \ - op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; \ - } \ - \ - return n_ops; \ -} \ - \ -static_always_inline u32 \ -ipsecmb_ops_gcm_cipher_enc_##a (vlib_main_t * vm, vnet_crypto_op_t * ops[], \ - u32 n_ops) \ -{ \ - ipsecmb_main_t *imbm = &ipsecmb_main; \ - ipsecmb_per_thread_data_t *ptd = vec_elt_at_index (imbm->per_thread_data, \ - vm->thread_index); \ - IMB_MGR *m = ptd->mgr; \ - u32 i; \ - \ - for (i = 0; i < n_ops; i++) \ - { \ - struct gcm_key_data *kd; \ - struct gcm_context_data ctx; \ - vnet_crypto_op_t *op = ops[i]; \ - \ - kd = (struct gcm_key_data *) imbm->key_data[op->key_index]; \ - IMB_AES##b##_GCM_ENC (m, kd, &ctx, op->dst, op->src, op->len, op->iv, \ - op->aad, op->aad_len, op->tag, op->tag_len); \ - \ - op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; \ - } \ - \ - return n_ops; \ -} \ - \ -static_always_inline u32 \ -ipsecmb_ops_gcm_cipher_dec_##a##_chained (vlib_main_t * vm, \ - vnet_crypto_op_t * ops[], vnet_crypto_op_chunk_t *chunks, u32 n_ops) \ -{ \ - ipsecmb_main_t *imbm = &ipsecmb_main; \ - ipsecmb_per_thread_data_t *ptd = vec_elt_at_index (imbm->per_thread_data, \ - vm->thread_index); \ - IMB_MGR *m = ptd->mgr; \ - vnet_crypto_op_chunk_t *chp; \ - u32 i, j, n_failed = 0; \ - \ - for (i = 0; i < n_ops; i++) \ - { \ - struct gcm_key_data *kd; \ - struct gcm_context_data ctx; \ - vnet_crypto_op_t *op = ops[i]; \ - u8 scratch[64]; \ - \ - kd = (struct gcm_key_data *) imbm->key_data[op->key_index]; \ - ASSERT (op->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS); \ - IMB_AES##b##_GCM_INIT(m, kd, &ctx, op->iv, op->aad, op->aad_len); \ - chp = chunks + op->chunk_index; \ - for (j = 0; j < op->n_chunks; j++) \ - { \ - IMB_AES##b##_GCM_DEC_UPDATE (m, kd, &ctx, chp->dst, chp->src, \ - chp->len); \ - chp += 1; \ - } \ - IMB_AES##b##_GCM_DEC_FINALIZE(m, kd, &ctx, scratch, op->tag_len); \ - \ - if ((memcmp (op->tag, scratch, op->tag_len))) \ - { \ - op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC; \ - n_failed++; \ - } \ - else \ - op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; \ - } \ - \ - return n_ops - n_failed; \ -} \ - \ -static_always_inline u32 \ -ipsecmb_ops_gcm_cipher_dec_##a (vlib_main_t * vm, vnet_crypto_op_t * ops[], \ - u32 n_ops) \ -{ \ - ipsecmb_main_t *imbm = &ipsecmb_main; \ - ipsecmb_per_thread_data_t *ptd = vec_elt_at_index (imbm->per_thread_data, \ - vm->thread_index); \ - IMB_MGR *m = ptd->mgr; \ - u32 i, n_failed = 0; \ - \ - for (i = 0; i < n_ops; i++) \ - { \ - struct gcm_key_data *kd; \ - struct gcm_context_data ctx; \ - vnet_crypto_op_t *op = ops[i]; \ - u8 scratch[64]; \ - \ - kd = (struct gcm_key_data *) imbm->key_data[op->key_index]; \ - IMB_AES##b##_GCM_DEC (m, kd, &ctx, op->dst, op->src, op->len, op->iv, \ - op->aad, op->aad_len, scratch, op->tag_len); \ - \ - if ((memcmp (op->tag, scratch, op->tag_len))) \ - { \ - op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC; \ - n_failed++; \ - } \ - else \ - op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; \ - } \ - \ - return n_ops - n_failed; \ -} -/* clang-format on */ -foreach_ipsecmb_gcm_cipher_op; -#undef _ - -#ifdef HAVE_IPSECMB_CHACHA_POLY -always_inline void -ipsecmb_retire_aead_job (IMB_JOB *job, u32 *n_fail) -{ - vnet_crypto_op_t *op = job->user_data; - u32 len = op->tag_len; - - if (PREDICT_FALSE (IMB_STATUS_COMPLETED != job->status)) - { - op->status = ipsecmb_status_job (job->status); - *n_fail = *n_fail + 1; - return; - } - - if (op->flags & VNET_CRYPTO_OP_FLAG_HMAC_CHECK) - { - if (memcmp (op->tag, job->auth_tag_output, len)) - { - *n_fail = *n_fail + 1; - op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC; - return; - } - } - - clib_memcpy_fast (op->tag, job->auth_tag_output, len); - - op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; -} - -static_always_inline u32 -ipsecmb_ops_chacha_poly (vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops, - IMB_CIPHER_DIRECTION dir) -{ - ipsecmb_main_t *imbm = &ipsecmb_main; - ipsecmb_per_thread_data_t *ptd = - vec_elt_at_index (imbm->per_thread_data, vm->thread_index); - struct IMB_JOB *job; - IMB_MGR *m = ptd->mgr; - u32 i, n_fail = 0, last_key_index = ~0; - u8 scratch[VLIB_FRAME_SIZE][16]; - u8 *key = 0; - - for (i = 0; i < n_ops; i++) - { - vnet_crypto_op_t *op = ops[i]; - - job = IMB_GET_NEXT_JOB (m); - if (last_key_index != op->key_index) - { - vnet_crypto_key_t *kd = vnet_crypto_get_key (op->key_index); - - key = kd->data; - last_key_index = op->key_index; - } - - job->cipher_direction = dir; - job->chain_order = IMB_ORDER_HASH_CIPHER; - job->cipher_mode = IMB_CIPHER_CHACHA20_POLY1305; - job->hash_alg = IMB_AUTH_CHACHA20_POLY1305; - job->enc_keys = job->dec_keys = key; - job->key_len_in_bytes = 32; - - job->u.CHACHA20_POLY1305.aad = op->aad; - job->u.CHACHA20_POLY1305.aad_len_in_bytes = op->aad_len; - job->src = op->src; - job->dst = op->dst; - - job->iv = op->iv; - job->iv_len_in_bytes = 12; - job->msg_len_to_cipher_in_bytes = job->msg_len_to_hash_in_bytes = - op->len; - job->cipher_start_src_offset_in_bytes = - job->hash_start_src_offset_in_bytes = 0; - - job->auth_tag_output = scratch[i]; - job->auth_tag_output_len_in_bytes = 16; - - job->user_data = op; - - job = IMB_SUBMIT_JOB_NOCHECK (ptd->mgr); - if (job) - ipsecmb_retire_aead_job (job, &n_fail); - - op++; - } - - while ((job = IMB_FLUSH_JOB (ptd->mgr))) - ipsecmb_retire_aead_job (job, &n_fail); - - return n_ops - n_fail; -} - -static_always_inline u32 -ipsecmb_ops_chacha_poly_enc (vlib_main_t *vm, vnet_crypto_op_t *ops[], - u32 n_ops) -{ - return ipsecmb_ops_chacha_poly (vm, ops, n_ops, IMB_DIR_ENCRYPT); -} - -static_always_inline u32 -ipsecmb_ops_chacha_poly_dec (vlib_main_t *vm, vnet_crypto_op_t *ops[], - u32 n_ops) -{ - return ipsecmb_ops_chacha_poly (vm, ops, n_ops, IMB_DIR_DECRYPT); -} - -static_always_inline u32 -ipsecmb_ops_chacha_poly_chained (vlib_main_t *vm, vnet_crypto_op_t *ops[], - vnet_crypto_op_chunk_t *chunks, u32 n_ops, - IMB_CIPHER_DIRECTION dir) -{ - ipsecmb_main_t *imbm = &ipsecmb_main; - ipsecmb_per_thread_data_t *ptd = - vec_elt_at_index (imbm->per_thread_data, vm->thread_index); - IMB_MGR *m = ptd->mgr; - u32 i, n_fail = 0, last_key_index = ~0; - u8 *key = 0; - - if (dir == IMB_DIR_ENCRYPT) - { - for (i = 0; i < n_ops; i++) - { - vnet_crypto_op_t *op = ops[i]; - struct chacha20_poly1305_context_data ctx; - vnet_crypto_op_chunk_t *chp; - u32 j; - - ASSERT (op->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS); - - if (last_key_index != op->key_index) - { - vnet_crypto_key_t *kd = vnet_crypto_get_key (op->key_index); - - key = kd->data; - last_key_index = op->key_index; - } - - IMB_CHACHA20_POLY1305_INIT (m, key, &ctx, op->iv, op->aad, - op->aad_len); - - chp = chunks + op->chunk_index; - for (j = 0; j < op->n_chunks; j++) - { - IMB_CHACHA20_POLY1305_ENC_UPDATE (m, key, &ctx, chp->dst, - chp->src, chp->len); - chp += 1; - } - - IMB_CHACHA20_POLY1305_ENC_FINALIZE (m, &ctx, op->tag, op->tag_len); - - op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; - } - } - else /* dir == IMB_DIR_DECRYPT */ - { - for (i = 0; i < n_ops; i++) - { - vnet_crypto_op_t *op = ops[i]; - struct chacha20_poly1305_context_data ctx; - vnet_crypto_op_chunk_t *chp; - u8 scratch[16]; - u32 j; - - ASSERT (op->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS); - - if (last_key_index != op->key_index) - { - vnet_crypto_key_t *kd = vnet_crypto_get_key (op->key_index); - - key = kd->data; - last_key_index = op->key_index; - } - - IMB_CHACHA20_POLY1305_INIT (m, key, &ctx, op->iv, op->aad, - op->aad_len); - - chp = chunks + op->chunk_index; - for (j = 0; j < op->n_chunks; j++) - { - IMB_CHACHA20_POLY1305_DEC_UPDATE (m, key, &ctx, chp->dst, - chp->src, chp->len); - chp += 1; - } - - IMB_CHACHA20_POLY1305_DEC_FINALIZE (m, &ctx, scratch, op->tag_len); - - if (memcmp (op->tag, scratch, op->tag_len)) - { - n_fail = n_fail + 1; - op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC; - } - else - op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; - } - } - - return n_ops - n_fail; -} - -static_always_inline u32 -ipsec_mb_ops_chacha_poly_enc_chained (vlib_main_t *vm, vnet_crypto_op_t *ops[], - vnet_crypto_op_chunk_t *chunks, - u32 n_ops) -{ - return ipsecmb_ops_chacha_poly_chained (vm, ops, chunks, n_ops, - IMB_DIR_ENCRYPT); -} - -static_always_inline u32 -ipsec_mb_ops_chacha_poly_dec_chained (vlib_main_t *vm, vnet_crypto_op_t *ops[], - vnet_crypto_op_chunk_t *chunks, - u32 n_ops) -{ - return ipsecmb_ops_chacha_poly_chained (vm, ops, chunks, n_ops, - IMB_DIR_DECRYPT); -} -#endif - -static void -crypto_ipsecmb_key_handler (vlib_main_t * vm, vnet_crypto_key_op_t kop, - vnet_crypto_key_index_t idx) -{ - ipsecmb_main_t *imbm = &ipsecmb_main; - vnet_crypto_key_t *key = vnet_crypto_get_key (idx); - ipsecmb_alg_data_t *ad = imbm->alg_data + key->alg; - u32 i; - void *kd; - - /** TODO: add linked alg support **/ - if (key->type == VNET_CRYPTO_KEY_TYPE_LINK) - return; - - if (kop == VNET_CRYPTO_KEY_OP_DEL) - { - if (idx >= vec_len (imbm->key_data)) - return; - - if (imbm->key_data[idx] == 0) - return; - - clib_mem_free_s (imbm->key_data[idx]); - imbm->key_data[idx] = 0; - return; - } - - if (ad->data_size == 0) - return; - - vec_validate_aligned (imbm->key_data, idx, CLIB_CACHE_LINE_BYTES); - - if (kop == VNET_CRYPTO_KEY_OP_MODIFY && imbm->key_data[idx]) - { - clib_mem_free_s (imbm->key_data[idx]); - } - - kd = imbm->key_data[idx] = clib_mem_alloc_aligned (ad->data_size, - CLIB_CACHE_LINE_BYTES); - - /* AES CBC key expansion */ - if (ad->keyexp) - { - ad->keyexp (key->data, ((ipsecmb_aes_key_data_t *) kd)->enc_key_exp, - ((ipsecmb_aes_key_data_t *) kd)->dec_key_exp); - return; - } - - /* AES GCM */ - if (ad->aes_gcm_pre) - { - ad->aes_gcm_pre (key->data, (struct gcm_key_data *) kd); - return; - } - - /* HMAC */ - if (ad->hash_one_block) - { - const int block_qw = HMAC_MAX_BLOCK_SIZE / sizeof (u64); - u64 pad[block_qw], key_hash[block_qw]; - - clib_memset_u8 (key_hash, 0, HMAC_MAX_BLOCK_SIZE); - if (vec_len (key->data) <= ad->block_size) - clib_memcpy_fast (key_hash, key->data, vec_len (key->data)); - else - ad->hash_fn (key->data, vec_len (key->data), key_hash); - - for (i = 0; i < block_qw; i++) - pad[i] = key_hash[i] ^ 0x3636363636363636; - ad->hash_one_block (pad, kd); - - for (i = 0; i < block_qw; i++) - pad[i] = key_hash[i] ^ 0x5c5c5c5c5c5c5c5c; - ad->hash_one_block (pad, ((u8 *) kd) + (ad->data_size / 2)); - - return; - } -} - -static clib_error_t * -crypto_ipsecmb_init (vlib_main_t * vm) -{ - ipsecmb_main_t *imbm = &ipsecmb_main; - ipsecmb_alg_data_t *ad; - ipsecmb_per_thread_data_t *ptd; - vlib_thread_main_t *tm = vlib_get_thread_main (); - IMB_MGR *m = 0; - u32 eidx; - u8 *name; - - if (!clib_cpu_supports_aes ()) - return 0; - - /* - * A priority that is better than OpenSSL but worse than VPP natvie - */ - name = format (0, "Intel(R) Multi-Buffer Crypto for IPsec Library %s%c", - IMB_VERSION_STR, 0); - eidx = vnet_crypto_register_engine (vm, "ipsecmb", 80, (char *) name); - - vec_validate_aligned (imbm->per_thread_data, tm->n_vlib_mains - 1, - CLIB_CACHE_LINE_BYTES); - - vec_foreach (ptd, imbm->per_thread_data) - { - ptd->mgr = alloc_mb_mgr (0); -#if IMB_VERSION_NUM >= IMB_VERSION(1, 3, 0) - clib_memset_u8 (ptd->burst_jobs, 0, - sizeof (IMB_JOB) * IMB_MAX_BURST_SIZE); -#endif - if (clib_cpu_supports_avx512f ()) - init_mb_mgr_avx512 (ptd->mgr); - else if (clib_cpu_supports_avx2 () && clib_cpu_supports_bmi2 ()) - init_mb_mgr_avx2 (ptd->mgr); - else - init_mb_mgr_sse (ptd->mgr); - - if (ptd == imbm->per_thread_data) - m = ptd->mgr; - } - -#define _(a, b, c, d, e, f) \ - vnet_crypto_register_ops_handler (vm, eidx, VNET_CRYPTO_OP_##a##_HMAC, \ - ipsecmb_ops_hmac_##a); \ - ad = imbm->alg_data + VNET_CRYPTO_ALG_HMAC_##a; \ - ad->block_size = d; \ - ad->data_size = e * 2; \ - ad->hash_one_block = m-> c##_one_block; \ - ad->hash_fn = m-> c; \ - - foreach_ipsecmb_hmac_op; -#undef _ -#define _(a, b, c) \ - vnet_crypto_register_ops_handler (vm, eidx, VNET_CRYPTO_OP_##a##_ENC, \ - ipsecmb_ops_cipher_enc_##a); \ - vnet_crypto_register_ops_handler (vm, eidx, VNET_CRYPTO_OP_##a##_DEC, \ - ipsecmb_ops_cipher_dec_##a); \ - ad = imbm->alg_data + VNET_CRYPTO_ALG_##a; \ - ad->data_size = sizeof (ipsecmb_aes_key_data_t); \ - ad->keyexp = m->keyexp_##b; - - foreach_ipsecmb_cipher_op; -#undef _ -#define _(a, b) \ - vnet_crypto_register_ops_handler (vm, eidx, VNET_CRYPTO_OP_##a##_ENC, \ - ipsecmb_ops_gcm_cipher_enc_##a); \ - vnet_crypto_register_ops_handler (vm, eidx, VNET_CRYPTO_OP_##a##_DEC, \ - ipsecmb_ops_gcm_cipher_dec_##a); \ - vnet_crypto_register_chained_ops_handler \ - (vm, eidx, VNET_CRYPTO_OP_##a##_ENC, \ - ipsecmb_ops_gcm_cipher_enc_##a##_chained); \ - vnet_crypto_register_chained_ops_handler \ - (vm, eidx, VNET_CRYPTO_OP_##a##_DEC, \ - ipsecmb_ops_gcm_cipher_dec_##a##_chained); \ - ad = imbm->alg_data + VNET_CRYPTO_ALG_##a; \ - ad->data_size = sizeof (struct gcm_key_data); \ - ad->aes_gcm_pre = m->gcm##b##_pre; \ - - foreach_ipsecmb_gcm_cipher_op; -#undef _ - -#ifdef HAVE_IPSECMB_CHACHA_POLY - vnet_crypto_register_ops_handler (vm, eidx, - VNET_CRYPTO_OP_CHACHA20_POLY1305_ENC, - ipsecmb_ops_chacha_poly_enc); - vnet_crypto_register_ops_handler (vm, eidx, - VNET_CRYPTO_OP_CHACHA20_POLY1305_DEC, - ipsecmb_ops_chacha_poly_dec); - vnet_crypto_register_chained_ops_handler ( - vm, eidx, VNET_CRYPTO_OP_CHACHA20_POLY1305_ENC, - ipsec_mb_ops_chacha_poly_enc_chained); - vnet_crypto_register_chained_ops_handler ( - vm, eidx, VNET_CRYPTO_OP_CHACHA20_POLY1305_DEC, - ipsec_mb_ops_chacha_poly_dec_chained); - ad = imbm->alg_data + VNET_CRYPTO_ALG_CHACHA20_POLY1305; - ad->data_size = 0; -#endif - - vnet_crypto_register_key_handler (vm, eidx, crypto_ipsecmb_key_handler); - return (NULL); -} - -VLIB_INIT_FUNCTION (crypto_ipsecmb_init) = -{ - .runs_after = VLIB_INITS ("vnet_crypto_init"), -}; - -VLIB_PLUGIN_REGISTER () = -{ - .version = VPP_BUILD_VER, - .description = "Intel IPSEC Multi-buffer Crypto Engine", -}; - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/plugins/crypto_native/CMakeLists.txt b/src/plugins/crypto_native/CMakeLists.txt deleted file mode 100644 index 5499ed4608a..00000000000 --- a/src/plugins/crypto_native/CMakeLists.txt +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) 2018 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|x86_64.*|AMD64.*") - list(APPEND VARIANTS "slm\;-march=silvermont -maes") - list(APPEND VARIANTS "hsw\;-march=haswell -maes") - if(compiler_flag_march_skylake_avx512 AND compiler_flag_mprefer_vector_width_256) - list(APPEND VARIANTS "skx\;-march=skylake-avx512 -mprefer-vector-width=256") - endif() - if(compiler_flag_march_icelake_client AND compiler_flag_mprefer_vector_width_512) - list(APPEND VARIANTS "icl\;-march=icelake-client -mprefer-vector-width=512") - endif() - if(compiler_flag_march_alderlake) - list(APPEND VARIANTS "adl\;-march=alderlake -mprefer-vector-width=256") - endif() -endif() - -if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*)") - list(APPEND VARIANTS "armv8\;-march=armv8.1-a+crc+crypto") -endif() - -set (COMPILE_FILES aes_cbc.c aes_gcm.c aes_ctr.c sha2.c) -set (COMPILE_OPTS -Wall -fno-common) - -if (NOT VARIANTS) - return() -endif() - -add_vpp_plugin(crypto_native SOURCES main.c) - -foreach(VARIANT ${VARIANTS}) - list(GET VARIANT 0 v) - list(GET VARIANT 1 f) - set(l crypto_native_${v}) - add_library(${l} OBJECT ${COMPILE_FILES}) - set_target_properties(${l} PROPERTIES POSITION_INDEPENDENT_CODE ON) - separate_arguments(f) - target_compile_options(${l} PUBLIC ${f} ${COMPILE_OPTS}) - target_sources(crypto_native_plugin PRIVATE $<TARGET_OBJECTS:${l}>) -endforeach() diff --git a/src/plugins/crypto_native/FEATURE.yaml b/src/plugins/crypto_native/FEATURE.yaml deleted file mode 100644 index d54816d673f..00000000000 --- a/src/plugins/crypto_native/FEATURE.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -name: IPSec crypto engine provided by native implementation -maintainer: Damjan Marion <damarion@cisco.com> -features: - - CBC(128, 192, 256) - - GCM(128, 192, 256) - - CTR(128, 192, 256) - - SHA(224, 256) - - HMAC-SHA(224, 256) - -description: "An implementation of a native crypto-engine" -state: production -properties: [API, CLI, MULTITHREAD] diff --git a/src/plugins/crypto_native/aes_cbc.c b/src/plugins/crypto_native/aes_cbc.c deleted file mode 100644 index c981897783f..00000000000 --- a/src/plugins/crypto_native/aes_cbc.c +++ /dev/null @@ -1,188 +0,0 @@ -/* - *------------------------------------------------------------------ - * Copyright (c) 2019 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *------------------------------------------------------------------ - */ - -#include <vlib/vlib.h> -#include <vnet/plugin/plugin.h> -#include <vnet/crypto/crypto.h> -#include <crypto_native/crypto_native.h> -#include <vppinfra/crypto/aes_cbc.h> - -#if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0 -#pragma GCC optimize ("O3") -#endif - -#define CRYPTO_NATIVE_AES_CBC_ENC_VEC_SIZE 256 - -static_always_inline u32 -aes_ops_enc_aes_cbc (vlib_main_t * vm, vnet_crypto_op_t * ops[], - u32 n_ops, aes_key_size_t ks) -{ - crypto_native_main_t *cm = &crypto_native_main; - u32 i, n_left = n_ops; - uword key_indices[CRYPTO_NATIVE_AES_CBC_ENC_VEC_SIZE] = {}; - u8 *plaintext[CRYPTO_NATIVE_AES_CBC_ENC_VEC_SIZE] = {}; - uword oplen[CRYPTO_NATIVE_AES_CBC_ENC_VEC_SIZE] = {}; - u8 *iv[CRYPTO_NATIVE_AES_CBC_ENC_VEC_SIZE] = {}; - u8 *ciphertext[CRYPTO_NATIVE_AES_CBC_ENC_VEC_SIZE] = {}; - - while (n_left) - { - i = 0; - while (n_left && i < CRYPTO_NATIVE_AES_CBC_ENC_VEC_SIZE) - { - key_indices[i] = ops[0]->key_index; - plaintext[i] = ops[0]->src; - ciphertext[i] = ops[0]->dst; - oplen[i] = ops[0]->len; - iv[i] = ops[0]->iv; - ops[0]->status = VNET_CRYPTO_OP_STATUS_COMPLETED; - - ops++; - n_left--; - i++; - } - clib_aes_cbc_encrypt_multi ((aes_cbc_key_data_t **) cm->key_data, - key_indices, plaintext, oplen, iv, ks, - ciphertext, i); - } - return n_ops; -} - - -static_always_inline u32 -aes_ops_dec_aes_cbc (vlib_main_t * vm, vnet_crypto_op_t * ops[], - u32 n_ops, aes_key_size_t ks) -{ - crypto_native_main_t *cm = &crypto_native_main; - int rounds = AES_KEY_ROUNDS (ks); - vnet_crypto_op_t *op = ops[0]; - aes_cbc_key_data_t *kd = (aes_cbc_key_data_t *) cm->key_data[op->key_index]; - u32 n_left = n_ops; - - ASSERT (n_ops >= 1); - -decrypt: -#if defined(__VAES__) && defined(__AVX512F__) - aes4_cbc_dec (kd->decrypt_key, (u8x64u *) op->src, (u8x64u *) op->dst, - (u8x16u *) op->iv, op->len, rounds); -#elif defined(__VAES__) - aes2_cbc_dec (kd->decrypt_key, (u8x32u *) op->src, (u8x32u *) op->dst, - (u8x16u *) op->iv, op->len, rounds); -#else - aes_cbc_dec (kd->decrypt_key, (u8x16u *) op->src, (u8x16u *) op->dst, - (u8x16u *) op->iv, op->len, rounds); -#endif - op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; - - if (--n_left) - { - op += 1; - kd = (aes_cbc_key_data_t *) cm->key_data[op->key_index]; - goto decrypt; - } - - return n_ops; -} - -static int -aes_cbc_cpu_probe () -{ -#if defined(__VAES__) && defined(__AVX512F__) - if (clib_cpu_supports_vaes () && clib_cpu_supports_avx512f ()) - return 50; -#elif defined(__VAES__) - if (clib_cpu_supports_vaes ()) - return 40; -#elif defined(__AVX512F__) - if (clib_cpu_supports_avx512f ()) - return 30; -#elif defined(__AVX2__) - if (clib_cpu_supports_avx2 ()) - return 20; -#elif __AES__ - if (clib_cpu_supports_aes ()) - return 10; -#elif __aarch64__ - if (clib_cpu_supports_aarch64_aes ()) - return 10; -#endif - return -1; -} - -static void * -aes_cbc_key_exp_128 (vnet_crypto_key_t *key) -{ - aes_cbc_key_data_t *kd; - kd = clib_mem_alloc_aligned (sizeof (*kd), CLIB_CACHE_LINE_BYTES); - clib_aes128_cbc_key_expand (kd, key->data); - return kd; -} - -static void * -aes_cbc_key_exp_192 (vnet_crypto_key_t *key) -{ - aes_cbc_key_data_t *kd; - kd = clib_mem_alloc_aligned (sizeof (*kd), CLIB_CACHE_LINE_BYTES); - clib_aes192_cbc_key_expand (kd, key->data); - return kd; -} - -static void * -aes_cbc_key_exp_256 (vnet_crypto_key_t *key) -{ - aes_cbc_key_data_t *kd; - kd = clib_mem_alloc_aligned (sizeof (*kd), CLIB_CACHE_LINE_BYTES); - clib_aes256_cbc_key_expand (kd, key->data); - return kd; -} - -#define foreach_aes_cbc_handler_type _ (128) _ (192) _ (256) - -#define _(x) \ - static u32 aes_ops_enc_aes_cbc_##x (vlib_main_t *vm, \ - vnet_crypto_op_t *ops[], u32 n_ops) \ - { \ - return aes_ops_enc_aes_cbc (vm, ops, n_ops, AES_KEY_##x); \ - } \ - \ - CRYPTO_NATIVE_OP_HANDLER (aes_##x##_cbc_enc) = { \ - .op_id = VNET_CRYPTO_OP_AES_##x##_CBC_ENC, \ - .fn = aes_ops_enc_aes_cbc_##x, \ - .probe = aes_cbc_cpu_probe, \ - }; \ - \ - static u32 aes_ops_dec_aes_cbc_##x (vlib_main_t *vm, \ - vnet_crypto_op_t *ops[], u32 n_ops) \ - { \ - return aes_ops_dec_aes_cbc (vm, ops, n_ops, AES_KEY_##x); \ - } \ - \ - CRYPTO_NATIVE_OP_HANDLER (aes_##x##_cbc_dec) = { \ - .op_id = VNET_CRYPTO_OP_AES_##x##_CBC_DEC, \ - .fn = aes_ops_dec_aes_cbc_##x, \ - .probe = aes_cbc_cpu_probe, \ - }; \ - \ - CRYPTO_NATIVE_KEY_HANDLER (aes_##x##_cbc) = { \ - .alg_id = VNET_CRYPTO_ALG_AES_##x##_CBC, \ - .key_fn = aes_cbc_key_exp_##x, \ - .probe = aes_cbc_cpu_probe, \ - }; - -foreach_aes_cbc_handler_type; -#undef _ - diff --git a/src/plugins/crypto_native/aes_ctr.c b/src/plugins/crypto_native/aes_ctr.c deleted file mode 100644 index d02a7b69b9d..00000000000 --- a/src/plugins/crypto_native/aes_ctr.c +++ /dev/null @@ -1,130 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 - * Copyright(c) 2024 Cisco Systems, Inc. - */ - -#include <vlib/vlib.h> -#include <vnet/plugin/plugin.h> -#include <vnet/crypto/crypto.h> -#include <crypto_native/crypto_native.h> -#include <vppinfra/crypto/aes_ctr.h> - -#if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0 -#pragma GCC optimize("O3") -#endif - -static_always_inline u32 -aes_ops_aes_ctr (vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops, - vnet_crypto_op_chunk_t *chunks, aes_key_size_t ks, - int maybe_chained) -{ - crypto_native_main_t *cm = &crypto_native_main; - vnet_crypto_op_t *op = ops[0]; - aes_ctr_key_data_t *kd; - aes_ctr_ctx_t ctx; - u32 n_left = n_ops; - -next: - kd = (aes_ctr_key_data_t *) cm->key_data[op->key_index]; - - clib_aes_ctr_init (&ctx, kd, op->iv, ks); - if (op->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS) - { - vnet_crypto_op_chunk_t *chp = chunks + op->chunk_index; - for (int j = 0; j < op->n_chunks; j++, chp++) - clib_aes_ctr_transform (&ctx, chp->src, chp->dst, chp->len, ks); - } - else - clib_aes_ctr_transform (&ctx, op->src, op->dst, op->len, ks); - - op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; - - if (--n_left) - { - op += 1; - goto next; - } - - return n_ops; -} - -static_always_inline void * -aes_ctr_key_exp (vnet_crypto_key_t *key, aes_key_size_t ks) -{ - aes_ctr_key_data_t *kd; - - kd = clib_mem_alloc_aligned (sizeof (*kd), CLIB_CACHE_LINE_BYTES); - - clib_aes_ctr_key_expand (kd, key->data, ks); - - return kd; -} - -#define foreach_aes_ctr_handler_type _ (128) _ (192) _ (256) - -#define _(x) \ - static u32 aes_ops_aes_ctr_##x (vlib_main_t *vm, vnet_crypto_op_t *ops[], \ - u32 n_ops) \ - { \ - return aes_ops_aes_ctr (vm, ops, n_ops, 0, AES_KEY_##x, 0); \ - } \ - static u32 aes_ops_aes_ctr_##x##_chained ( \ - vlib_main_t *vm, vnet_crypto_op_t *ops[], vnet_crypto_op_chunk_t *chunks, \ - u32 n_ops) \ - { \ - return aes_ops_aes_ctr (vm, ops, n_ops, chunks, AES_KEY_##x, 1); \ - } \ - static void *aes_ctr_key_exp_##x (vnet_crypto_key_t *key) \ - { \ - return aes_ctr_key_exp (key, AES_KEY_##x); \ - } - -foreach_aes_ctr_handler_type; -#undef _ - -static int -probe () -{ -#if defined(__VAES__) && defined(__AVX512F__) - if (clib_cpu_supports_vaes () && clib_cpu_supports_avx512f ()) - return 50; -#elif defined(__VAES__) - if (clib_cpu_supports_vaes ()) - return 40; -#elif defined(__AVX512F__) - if (clib_cpu_supports_avx512f ()) - return 30; -#elif defined(__AVX2__) - if (clib_cpu_supports_avx2 ()) - return 20; -#elif __AES__ - if (clib_cpu_supports_aes ()) - return 10; -#elif __aarch64__ - if (clib_cpu_supports_aarch64_aes ()) - return 10; -#endif - return -1; -} - -#define _(b) \ - CRYPTO_NATIVE_OP_HANDLER (aes_##b##_ctr_enc) = { \ - .op_id = VNET_CRYPTO_OP_AES_##b##_CTR_ENC, \ - .fn = aes_ops_aes_ctr_##b, \ - .cfn = aes_ops_aes_ctr_##b##_chained, \ - .probe = probe, \ - }; \ - \ - CRYPTO_NATIVE_OP_HANDLER (aes_##b##_ctr_dec) = { \ - .op_id = VNET_CRYPTO_OP_AES_##b##_CTR_DEC, \ - .fn = aes_ops_aes_ctr_##b, \ - .cfn = aes_ops_aes_ctr_##b##_chained, \ - .probe = probe, \ - }; \ - CRYPTO_NATIVE_KEY_HANDLER (aes_##b##_ctr) = { \ - .alg_id = VNET_CRYPTO_ALG_AES_##b##_CTR, \ - .key_fn = aes_ctr_key_exp_##b, \ - .probe = probe, \ - }; - -_ (128) _ (192) _ (256) -#undef _ diff --git a/src/plugins/crypto_native/aes_gcm.c b/src/plugins/crypto_native/aes_gcm.c deleted file mode 100644 index 220788d4e97..00000000000 --- a/src/plugins/crypto_native/aes_gcm.c +++ /dev/null @@ -1,166 +0,0 @@ -/* - *------------------------------------------------------------------ - * Copyright (c) 2019 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *------------------------------------------------------------------ - */ - -#include <vlib/vlib.h> -#include <vnet/plugin/plugin.h> -#include <vnet/crypto/crypto.h> -#include <crypto_native/crypto_native.h> -#include <vppinfra/crypto/aes_gcm.h> - -#if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0 -#pragma GCC optimize("O3") -#endif - -static_always_inline u32 -aes_ops_enc_aes_gcm (vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops, - aes_key_size_t ks) -{ - crypto_native_main_t *cm = &crypto_native_main; - vnet_crypto_op_t *op = ops[0]; - aes_gcm_key_data_t *kd; - u32 n_left = n_ops; - -next: - kd = (aes_gcm_key_data_t *) cm->key_data[op->key_index]; - aes_gcm (op->src, op->dst, op->aad, (u8 *) op->iv, op->tag, op->len, - op->aad_len, op->tag_len, kd, AES_KEY_ROUNDS (ks), - AES_GCM_OP_ENCRYPT); - op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; - - if (--n_left) - { - op += 1; - goto next; - } - - return n_ops; -} - -static_always_inline u32 -aes_ops_dec_aes_gcm (vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops, - aes_key_size_t ks) -{ - crypto_native_main_t *cm = &crypto_native_main; - vnet_crypto_op_t *op = ops[0]; - aes_gcm_key_data_t *kd; - u32 n_left = n_ops; - int rv; - -next: - kd = (aes_gcm_key_data_t *) cm->key_data[op->key_index]; - rv = aes_gcm (op->src, op->dst, op->aad, (u8 *) op->iv, op->tag, op->len, - op->aad_len, op->tag_len, kd, AES_KEY_ROUNDS (ks), - AES_GCM_OP_DECRYPT); - - if (rv) - { - op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; - } - else - { - op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC; - n_ops--; - } - - if (--n_left) - { - op += 1; - goto next; - } - - return n_ops; -} - -static_always_inline void * -aes_gcm_key_exp (vnet_crypto_key_t *key, aes_key_size_t ks) -{ - aes_gcm_key_data_t *kd; - - kd = clib_mem_alloc_aligned (sizeof (*kd), CLIB_CACHE_LINE_BYTES); - - clib_aes_gcm_key_expand (kd, key->data, ks); - - return kd; -} - -#define foreach_aes_gcm_handler_type _ (128) _ (192) _ (256) - -#define _(x) \ - static u32 aes_ops_dec_aes_gcm_##x (vlib_main_t *vm, \ - vnet_crypto_op_t *ops[], u32 n_ops) \ - { \ - return aes_ops_dec_aes_gcm (vm, ops, n_ops, AES_KEY_##x); \ - } \ - static u32 aes_ops_enc_aes_gcm_##x (vlib_main_t *vm, \ - vnet_crypto_op_t *ops[], u32 n_ops) \ - { \ - return aes_ops_enc_aes_gcm (vm, ops, n_ops, AES_KEY_##x); \ - } \ - static void *aes_gcm_key_exp_##x (vnet_crypto_key_t *key) \ - { \ - return aes_gcm_key_exp (key, AES_KEY_##x); \ - } - -foreach_aes_gcm_handler_type; -#undef _ - -static int -probe () -{ -#if defined(__VAES__) && defined(__AVX512F__) - if (clib_cpu_supports_vpclmulqdq () && clib_cpu_supports_vaes () && - clib_cpu_supports_avx512f ()) - return 50; -#elif defined(__VAES__) - if (clib_cpu_supports_vpclmulqdq () && clib_cpu_supports_vaes ()) - return 40; -#elif defined(__AVX512F__) - if (clib_cpu_supports_pclmulqdq () && clib_cpu_supports_avx512f ()) - return 30; -#elif defined(__AVX2__) - if (clib_cpu_supports_pclmulqdq () && clib_cpu_supports_avx2 ()) - return 20; -#elif __AES__ - if (clib_cpu_supports_pclmulqdq () && clib_cpu_supports_aes ()) - return 10; -#elif __aarch64__ - if (clib_cpu_supports_aarch64_aes ()) - return 10; -#endif - return -1; -} - -#define _(b) \ - CRYPTO_NATIVE_OP_HANDLER (aes_##b##_gcm_enc) = { \ - .op_id = VNET_CRYPTO_OP_AES_##b##_GCM_ENC, \ - .fn = aes_ops_enc_aes_gcm_##b, \ - .probe = probe, \ - }; \ - \ - CRYPTO_NATIVE_OP_HANDLER (aes_##b##_gcm_dec) = { \ - .op_id = VNET_CRYPTO_OP_AES_##b##_GCM_DEC, \ - .fn = aes_ops_dec_aes_gcm_##b, \ - .probe = probe, \ - }; \ - CRYPTO_NATIVE_KEY_HANDLER (aes_##b##_gcm) = { \ - .alg_id = VNET_CRYPTO_ALG_AES_##b##_GCM, \ - .key_fn = aes_gcm_key_exp_##b, \ - .probe = probe, \ - }; - -_ (128) _ (192) _ (256) -#undef _ diff --git a/src/plugins/crypto_native/crypto_native.h b/src/plugins/crypto_native/crypto_native.h deleted file mode 100644 index 3d18e8cabd0..00000000000 --- a/src/plugins/crypto_native/crypto_native.h +++ /dev/null @@ -1,84 +0,0 @@ -/* - *------------------------------------------------------------------ - * Copyright (c) 2019 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *------------------------------------------------------------------ - */ - -#ifndef __crypto_native_h__ -#define __crypto_native_h__ - -typedef void *(crypto_native_key_fn_t) (vnet_crypto_key_t * key); -typedef int (crypto_native_variant_probe_t) (); - -typedef struct crypto_native_op_handler -{ - struct crypto_native_op_handler *next; - vnet_crypto_op_id_t op_id; - vnet_crypto_ops_handler_t *fn; - vnet_crypto_chained_ops_handler_t *cfn; - crypto_native_variant_probe_t *probe; - int priority; -} crypto_native_op_handler_t; - -typedef struct crypto_native_key_handler -{ - struct crypto_native_key_handler *next; - vnet_crypto_alg_t alg_id; - crypto_native_key_fn_t *key_fn; - crypto_native_variant_probe_t *probe; - int priority; -} crypto_native_key_handler_t; - -typedef struct -{ - u32 crypto_engine_index; - crypto_native_key_fn_t *key_fn[VNET_CRYPTO_N_ALGS]; - void **key_data; - crypto_native_op_handler_t *op_handlers; - crypto_native_key_handler_t *key_handlers; -} crypto_native_main_t; - -extern crypto_native_main_t crypto_native_main; - -#define CRYPTO_NATIVE_OP_HANDLER(x) \ - static crypto_native_op_handler_t __crypto_native_op_handler_##x; \ - static void __clib_constructor __crypto_native_op_handler_cb_##x (void) \ - { \ - crypto_native_main_t *cm = &crypto_native_main; \ - int priority = __crypto_native_op_handler_##x.probe (); \ - if (priority >= 0) \ - { \ - __crypto_native_op_handler_##x.priority = priority; \ - __crypto_native_op_handler_##x.next = cm->op_handlers; \ - cm->op_handlers = &__crypto_native_op_handler_##x; \ - } \ - } \ - static crypto_native_op_handler_t __crypto_native_op_handler_##x - -#define CRYPTO_NATIVE_KEY_HANDLER(x) \ - static crypto_native_key_handler_t __crypto_native_key_handler_##x; \ - static void __clib_constructor __crypto_native_key_handler_cb_##x (void) \ - { \ - crypto_native_main_t *cm = &crypto_native_main; \ - int priority = __crypto_native_key_handler_##x.probe (); \ - if (priority >= 0) \ - { \ - __crypto_native_key_handler_##x.priority = priority; \ - __crypto_native_key_handler_##x.next = cm->key_handlers; \ - cm->key_handlers = &__crypto_native_key_handler_##x; \ - } \ - } \ - static crypto_native_key_handler_t __crypto_native_key_handler_##x -#endif /* __crypto_native_h__ */ - diff --git a/src/plugins/crypto_native/main.c b/src/plugins/crypto_native/main.c deleted file mode 100644 index 2bc0d98f196..00000000000 --- a/src/plugins/crypto_native/main.c +++ /dev/null @@ -1,128 +0,0 @@ -/* - *------------------------------------------------------------------ - * Copyright (c) 2019 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *------------------------------------------------------------------ - */ - -#include <vlib/vlib.h> -#include <vnet/plugin/plugin.h> -#include <vnet/crypto/crypto.h> -#include <crypto_native/crypto_native.h> - -crypto_native_main_t crypto_native_main; - -static void -crypto_native_key_handler (vlib_main_t * vm, vnet_crypto_key_op_t kop, - vnet_crypto_key_index_t idx) -{ - vnet_crypto_key_t *key = vnet_crypto_get_key (idx); - crypto_native_main_t *cm = &crypto_native_main; - - /** TODO: add linked alg support **/ - if (key->type == VNET_CRYPTO_KEY_TYPE_LINK) - return; - - if (cm->key_fn[key->alg] == 0) - return; - - if (kop == VNET_CRYPTO_KEY_OP_DEL) - { - if (idx >= vec_len (cm->key_data)) - return; - - if (cm->key_data[idx] == 0) - return; - - clib_mem_free_s (cm->key_data[idx]); - cm->key_data[idx] = 0; - return; - } - - vec_validate_aligned (cm->key_data, idx, CLIB_CACHE_LINE_BYTES); - - if (kop == VNET_CRYPTO_KEY_OP_MODIFY && cm->key_data[idx]) - { - clib_mem_free_s (cm->key_data[idx]); - } - - cm->key_data[idx] = cm->key_fn[key->alg] (key); -} - -clib_error_t * -crypto_native_init (vlib_main_t * vm) -{ - crypto_native_main_t *cm = &crypto_native_main; - - if (cm->op_handlers == 0) - return 0; - - cm->crypto_engine_index = - vnet_crypto_register_engine (vm, "native", 100, - "Native ISA Optimized Crypto"); - - crypto_native_op_handler_t *oh = cm->op_handlers; - crypto_native_key_handler_t *kh = cm->key_handlers; - crypto_native_op_handler_t **best_by_op_id = 0; - crypto_native_key_handler_t **best_by_alg_id = 0; - - while (oh) - { - vec_validate (best_by_op_id, oh->op_id); - - if (best_by_op_id[oh->op_id] == 0 || - best_by_op_id[oh->op_id]->priority < oh->priority) - best_by_op_id[oh->op_id] = oh; - - oh = oh->next; - } - - while (kh) - { - vec_validate (best_by_alg_id, kh->alg_id); - - if (best_by_alg_id[kh->alg_id] == 0 || - best_by_alg_id[kh->alg_id]->priority < kh->priority) - best_by_alg_id[kh->alg_id] = kh; - - kh = kh->next; - } - - vec_foreach_pointer (oh, best_by_op_id) - if (oh) - vnet_crypto_register_ops_handlers (vm, cm->crypto_engine_index, - oh->op_id, oh->fn, oh->cfn); - - vec_foreach_pointer (kh, best_by_alg_id) - if (kh) - cm->key_fn[kh->alg_id] = kh->key_fn; - - vec_free (best_by_op_id); - vec_free (best_by_alg_id); - - vnet_crypto_register_key_handler (vm, cm->crypto_engine_index, - crypto_native_key_handler); - return 0; -} - -VLIB_INIT_FUNCTION (crypto_native_init) = -{ - .runs_after = VLIB_INITS ("vnet_crypto_init"), -}; - -#include <vpp/app/version.h> - -VLIB_PLUGIN_REGISTER () = { - .version = VPP_BUILD_VER, - .description = "Native Crypto Engine", -}; diff --git a/src/plugins/crypto_native/sha2.c b/src/plugins/crypto_native/sha2.c deleted file mode 100644 index 6787f629104..00000000000 --- a/src/plugins/crypto_native/sha2.c +++ /dev/null @@ -1,198 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 - * Copyright(c) 2024 Cisco Systems, Inc. - */ - -#include <vlib/vlib.h> -#include <vnet/plugin/plugin.h> -#include <vnet/crypto/crypto.h> -#include <crypto_native/crypto_native.h> -#include <vppinfra/crypto/sha2.h> - -static_always_inline u32 -crypto_native_ops_hash_sha2 (vlib_main_t *vm, vnet_crypto_op_t *ops[], - u32 n_ops, vnet_crypto_op_chunk_t *chunks, - clib_sha2_type_t type, int maybe_chained) -{ - vnet_crypto_op_t *op = ops[0]; - clib_sha2_ctx_t ctx; - u32 n_left = n_ops; - -next: - if (op->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS) - { - vnet_crypto_op_chunk_t *chp = chunks + op->chunk_index; - clib_sha2_init (&ctx, type); - for (int j = 0; j < op->n_chunks; j++, chp++) - clib_sha2_update (&ctx, chp->src, chp->len); - clib_sha2_final (&ctx, op->digest); - } - else - clib_sha2 (type, op->src, op->len, op->digest); - - op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; - - if (--n_left) - { - op += 1; - goto next; - } - - return n_ops; -} - -static_always_inline u32 -crypto_native_ops_hmac_sha2 (vlib_main_t *vm, vnet_crypto_op_t *ops[], - u32 n_ops, vnet_crypto_op_chunk_t *chunks, - clib_sha2_type_t type) -{ - crypto_native_main_t *cm = &crypto_native_main; - vnet_crypto_op_t *op = ops[0]; - u32 n_left = n_ops; - clib_sha2_hmac_ctx_t ctx; - u8 buffer[64]; - u32 sz, n_fail = 0; - - for (; n_left; n_left--, op++) - { - clib_sha2_hmac_init ( - &ctx, type, (clib_sha2_hmac_key_data_t *) cm->key_data[op->key_index]); - if (op->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS) - { - vnet_crypto_op_chunk_t *chp = chunks + op->chunk_index; - for (int j = 0; j < op->n_chunks; j++, chp++) - clib_sha2_hmac_update (&ctx, chp->src, chp->len); - } - else - clib_sha2_hmac_update (&ctx, op->src, op->len); - - clib_sha2_hmac_final (&ctx, buffer); - - if (op->digest_len) - { - sz = op->digest_len; - if (op->flags & VNET_CRYPTO_OP_FLAG_HMAC_CHECK) - { - if ((memcmp (op->digest, buffer, sz))) - { - n_fail++; - op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC; - continue; - } - } - else - clib_memcpy_fast (op->digest, buffer, sz); - } - else - { - sz = clib_sha2_variants[type].digest_size; - if (op->flags & VNET_CRYPTO_OP_FLAG_HMAC_CHECK) - { - if ((memcmp (op->digest, buffer, sz))) - { - n_fail++; - op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC; - continue; - } - } - else - clib_memcpy_fast (op->digest, buffer, sz); - } - - op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; - } - - return n_ops - n_fail; -} - -static void * -sha2_key_add (vnet_crypto_key_t *key, clib_sha2_type_t type) -{ - clib_sha2_hmac_key_data_t *kd; - - kd = clib_mem_alloc_aligned (sizeof (*kd), CLIB_CACHE_LINE_BYTES); - clib_sha2_hmac_key_data (type, key->data, vec_len (key->data), kd); - - return kd; -} - -static int -probe () -{ -#if defined(__x86_64__) - -#if defined(__SHA__) && defined(__AVX512F__) - if (clib_cpu_supports_sha () && clib_cpu_supports_avx512f ()) - return 30; -#elif defined(__SHA__) && defined(__AVX2__) - if (clib_cpu_supports_sha () && clib_cpu_supports_avx2 ()) - return 20; -#elif defined(__SHA__) - if (clib_cpu_supports_sha ()) - return 10; -#endif - -#elif defined(__aarch64__) -#if defined(__ARM_FEATURE_SHA2) - if (clib_cpu_supports_sha2 ()) - return 10; -#endif -#endif - return -1; -} - -#define _(b) \ - static u32 crypto_native_ops_hash_sha##b ( \ - vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops) \ - { \ - return crypto_native_ops_hash_sha2 (vm, ops, n_ops, 0, CLIB_SHA2_##b, 0); \ - } \ - \ - static u32 crypto_native_ops_chained_hash_sha##b ( \ - vlib_main_t *vm, vnet_crypto_op_t *ops[], vnet_crypto_op_chunk_t *chunks, \ - u32 n_ops) \ - { \ - return crypto_native_ops_hash_sha2 (vm, ops, n_ops, chunks, \ - CLIB_SHA2_##b, 1); \ - } \ - \ - static u32 crypto_native_ops_hmac_sha##b ( \ - vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops) \ - { \ - return crypto_native_ops_hmac_sha2 (vm, ops, n_ops, 0, CLIB_SHA2_##b); \ - } \ - \ - static u32 crypto_native_ops_chained_hmac_sha##b ( \ - vlib_main_t *vm, vnet_crypto_op_t *ops[], vnet_crypto_op_chunk_t *chunks, \ - u32 n_ops) \ - { \ - return crypto_native_ops_hmac_sha2 (vm, ops, n_ops, chunks, \ - CLIB_SHA2_##b); \ - } \ - \ - static void *sha2_##b##_key_add (vnet_crypto_key_t *k) \ - { \ - return sha2_key_add (k, CLIB_SHA2_##b); \ - } \ - \ - CRYPTO_NATIVE_OP_HANDLER (crypto_native_hash_sha##b) = { \ - .op_id = VNET_CRYPTO_OP_SHA##b##_HASH, \ - .fn = crypto_native_ops_hash_sha##b, \ - .cfn = crypto_native_ops_chained_hash_sha##b, \ - .probe = probe, \ - }; \ - CRYPTO_NATIVE_OP_HANDLER (crypto_native_hmac_sha##b) = { \ - .op_id = VNET_CRYPTO_OP_SHA##b##_HMAC, \ - .fn = crypto_native_ops_hmac_sha##b, \ - .cfn = crypto_native_ops_chained_hmac_sha##b, \ - .probe = probe, \ - }; \ - CRYPTO_NATIVE_KEY_HANDLER (crypto_native_hmac_sha##b) = { \ - .alg_id = VNET_CRYPTO_ALG_HMAC_SHA##b, \ - .key_fn = sha2_##b##_key_add, \ - .probe = probe, \ - }; - -_ (224) -_ (256) - -#undef _ diff --git a/src/plugins/crypto_openssl/CMakeLists.txt b/src/plugins/crypto_openssl/CMakeLists.txt deleted file mode 100644 index 472b0ef3243..00000000000 --- a/src/plugins/crypto_openssl/CMakeLists.txt +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) 2018 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if(NOT OPENSSL_FOUND) - return() -endif() - -include_directories(${OPENSSL_INCLUDE_DIR}) -add_compile_definitions(OPENSSL_SUPPRESS_DEPRECATED) - -add_vpp_plugin(crypto_openssl - SOURCES - main.c - - LINK_LIBRARIES - ${OPENSSL_CRYPTO_LIBRARIES} -) diff --git a/src/plugins/crypto_openssl/FEATURE.yaml b/src/plugins/crypto_openssl/FEATURE.yaml deleted file mode 100644 index da0a0812595..00000000000 --- a/src/plugins/crypto_openssl/FEATURE.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -name: IPSec crypto engine provided by Openssl library -maintainer: Damjan Marion <damarion@cisco.com> -features: - - SHA(1, 224, 256, 384, 512) - - CBC(128, 192, 256) - - GCM(128, 192, 256) - - CTR(128, 192, 256) - - DES, 3DES - - MD5 - -description: "" -state: production -properties: [API, CLI, MULTITHREAD] diff --git a/src/plugins/crypto_openssl/crypto_openssl.h b/src/plugins/crypto_openssl/crypto_openssl.h deleted file mode 100644 index e16429fb5dd..00000000000 --- a/src/plugins/crypto_openssl/crypto_openssl.h +++ /dev/null @@ -1,20 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 - * Copyright (c) 2023 ARM Ltd and/or its affiliates. - */ - -#ifndef __crypto_openssl_h__ -#define __crypto_openssl_h__ - -typedef void *(crypto_openssl_ctx_fn_t) (vnet_crypto_key_t *key, - vnet_crypto_key_op_t kop, - vnet_crypto_key_index_t idx); - -typedef struct -{ - u32 crypto_engine_index; - crypto_openssl_ctx_fn_t *ctx_fn[VNET_CRYPTO_N_ALGS]; -} crypto_openssl_main_t; - -extern crypto_openssl_main_t crypto_openssl_main; - -#endif /* __crypto_openssl_h__ */ diff --git a/src/plugins/crypto_openssl/main.c b/src/plugins/crypto_openssl/main.c deleted file mode 100644 index c59b5d34a29..00000000000 --- a/src/plugins/crypto_openssl/main.c +++ /dev/null @@ -1,719 +0,0 @@ -/* - *------------------------------------------------------------------ - * Copyright (c) 2019 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *------------------------------------------------------------------ - */ - -#include <sys/syscall.h> - -#include <openssl/evp.h> -#include <openssl/hmac.h> -#include <openssl/rand.h> -#include <openssl/sha.h> - -#include <vlib/vlib.h> -#include <vnet/plugin/plugin.h> -#include <vnet/crypto/crypto.h> -#include <vpp/app/version.h> -#include <crypto_openssl/crypto_openssl.h> - -typedef struct -{ - CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); - EVP_CIPHER_CTX **evp_cipher_enc_ctx; - EVP_CIPHER_CTX **evp_cipher_dec_ctx; - HMAC_CTX **hmac_ctx; - EVP_MD_CTX *hash_ctx; -#if OPENSSL_VERSION_NUMBER < 0x10100000L - HMAC_CTX _hmac_ctx; -#endif -} openssl_per_thread_data_t; - -static openssl_per_thread_data_t *per_thread_data = 0; - -#define foreach_openssl_aes_evp_op \ - _ (cbc, DES_CBC, EVP_des_cbc, 8) \ - _ (cbc, 3DES_CBC, EVP_des_ede3_cbc, 8) \ - _ (cbc, AES_128_CBC, EVP_aes_128_cbc, 16) \ - _ (cbc, AES_192_CBC, EVP_aes_192_cbc, 16) \ - _ (cbc, AES_256_CBC, EVP_aes_256_cbc, 16) \ - _ (gcm, AES_128_GCM, EVP_aes_128_gcm, 8) \ - _ (gcm, AES_192_GCM, EVP_aes_192_gcm, 8) \ - _ (gcm, AES_256_GCM, EVP_aes_256_gcm, 8) \ - _ (cbc, AES_128_CTR, EVP_aes_128_ctr, 8) \ - _ (cbc, AES_192_CTR, EVP_aes_192_ctr, 8) \ - _ (cbc, AES_256_CTR, EVP_aes_256_ctr, 8) \ - _ (null_gmac, AES_128_NULL_GMAC, EVP_aes_128_gcm, 8) \ - _ (null_gmac, AES_192_NULL_GMAC, EVP_aes_192_gcm, 8) \ - _ (null_gmac, AES_256_NULL_GMAC, EVP_aes_256_gcm, 8) - -#define foreach_openssl_chacha20_evp_op \ - _ (chacha20_poly1305, CHACHA20_POLY1305, EVP_chacha20_poly1305, 8) - -#if OPENSSL_VERSION_NUMBER >= 0x10100000L -#define foreach_openssl_evp_op foreach_openssl_aes_evp_op \ - foreach_openssl_chacha20_evp_op -#else -#define foreach_openssl_evp_op foreach_openssl_aes_evp_op -#endif - -#ifndef EVP_CTRL_AEAD_GET_TAG -#define EVP_CTRL_AEAD_GET_TAG EVP_CTRL_GCM_GET_TAG -#endif - -#ifndef EVP_CTRL_AEAD_SET_TAG -#define EVP_CTRL_AEAD_SET_TAG EVP_CTRL_GCM_SET_TAG -#endif - -#define foreach_openssl_hash_op \ - _ (SHA1, EVP_sha1) \ - _ (SHA224, EVP_sha224) \ - _ (SHA256, EVP_sha256) \ - _ (SHA384, EVP_sha384) \ - _ (SHA512, EVP_sha512) - -#define foreach_openssl_hmac_op \ - _(MD5, EVP_md5) \ - _(SHA1, EVP_sha1) \ - _(SHA224, EVP_sha224) \ - _(SHA256, EVP_sha256) \ - _(SHA384, EVP_sha384) \ - _(SHA512, EVP_sha512) - -crypto_openssl_main_t crypto_openssl_main; - -static_always_inline u32 -openssl_ops_enc_cbc (vlib_main_t *vm, vnet_crypto_op_t *ops[], - vnet_crypto_op_chunk_t *chunks, u32 n_ops, - const EVP_CIPHER *cipher, const int iv_len) -{ - openssl_per_thread_data_t *ptd = vec_elt_at_index (per_thread_data, - vm->thread_index); - EVP_CIPHER_CTX *ctx; - vnet_crypto_op_chunk_t *chp; - u32 i, j, curr_len = 0; - u8 out_buf[VLIB_BUFFER_DEFAULT_DATA_SIZE * 5]; - - for (i = 0; i < n_ops; i++) - { - vnet_crypto_op_t *op = ops[i]; - int out_len = 0; - - ctx = ptd->evp_cipher_enc_ctx[op->key_index]; - EVP_EncryptInit_ex (ctx, NULL, NULL, NULL, op->iv); - - if (op->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS) - { - chp = chunks + op->chunk_index; - u32 offset = 0; - for (j = 0; j < op->n_chunks; j++) - { - EVP_EncryptUpdate (ctx, out_buf + offset, &out_len, chp->src, - chp->len); - curr_len = chp->len; - offset += out_len; - chp += 1; - } - if (out_len < curr_len) - EVP_EncryptFinal_ex (ctx, out_buf + offset, &out_len); - - offset = 0; - chp = chunks + op->chunk_index; - for (j = 0; j < op->n_chunks; j++) - { - clib_memcpy_fast (chp->dst, out_buf + offset, chp->len); - offset += chp->len; - chp += 1; - } - } - else - { - EVP_EncryptUpdate (ctx, op->dst, &out_len, op->src, op->len); - if (out_len < op->len) - EVP_EncryptFinal_ex (ctx, op->dst + out_len, &out_len); - } - op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; - } - return n_ops; -} - -static_always_inline u32 -openssl_ops_dec_cbc (vlib_main_t *vm, vnet_crypto_op_t *ops[], - vnet_crypto_op_chunk_t *chunks, u32 n_ops, - const EVP_CIPHER *cipher, const int iv_len) -{ - openssl_per_thread_data_t *ptd = vec_elt_at_index (per_thread_data, - vm->thread_index); - EVP_CIPHER_CTX *ctx; - vnet_crypto_op_chunk_t *chp; - u32 i, j, curr_len = 0; - u8 out_buf[VLIB_BUFFER_DEFAULT_DATA_SIZE * 5]; - - for (i = 0; i < n_ops; i++) - { - vnet_crypto_op_t *op = ops[i]; - int out_len = 0; - - ctx = ptd->evp_cipher_dec_ctx[op->key_index]; - EVP_DecryptInit_ex (ctx, NULL, NULL, NULL, op->iv); - - if (op->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS) - { - chp = chunks + op->chunk_index; - u32 offset = 0; - for (j = 0; j < op->n_chunks; j++) - { - EVP_DecryptUpdate (ctx, out_buf + offset, &out_len, chp->src, - chp->len); - curr_len = chp->len; - offset += out_len; - chp += 1; - } - if (out_len < curr_len) - EVP_DecryptFinal_ex (ctx, out_buf + offset, &out_len); - - offset = 0; - chp = chunks + op->chunk_index; - for (j = 0; j < op->n_chunks; j++) - { - clib_memcpy_fast (chp->dst, out_buf + offset, chp->len); - offset += chp->len; - chp += 1; - } - } - else - { - EVP_DecryptUpdate (ctx, op->dst, &out_len, op->src, op->len); - if (out_len < op->len) - EVP_DecryptFinal_ex (ctx, op->dst + out_len, &out_len); - } - op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; - } - return n_ops; -} - -static_always_inline u32 -openssl_ops_enc_aead (vlib_main_t *vm, vnet_crypto_op_t *ops[], - vnet_crypto_op_chunk_t *chunks, u32 n_ops, - const EVP_CIPHER *cipher, int is_gcm, int is_gmac, - const int iv_len) -{ - openssl_per_thread_data_t *ptd = vec_elt_at_index (per_thread_data, - vm->thread_index); - EVP_CIPHER_CTX *ctx; - vnet_crypto_op_chunk_t *chp; - u32 i, j; - for (i = 0; i < n_ops; i++) - { - vnet_crypto_op_t *op = ops[i]; - int len = 0; - - if (i + 2 < n_ops) - { - CLIB_PREFETCH (ops[i + 1]->src, 4 * CLIB_CACHE_PREFETCH_BYTES, LOAD); - CLIB_PREFETCH (ops[i + 1]->dst, 4 * CLIB_CACHE_PREFETCH_BYTES, - STORE); - - CLIB_PREFETCH (ops[i + 2]->src, 4 * CLIB_CACHE_PREFETCH_BYTES, LOAD); - CLIB_PREFETCH (ops[i + 2]->dst, 4 * CLIB_CACHE_PREFETCH_BYTES, - STORE); - } - - ctx = ptd->evp_cipher_enc_ctx[op->key_index]; - EVP_EncryptInit_ex (ctx, 0, 0, NULL, op->iv); - if (op->aad_len) - EVP_EncryptUpdate (ctx, NULL, &len, op->aad, op->aad_len); - if (op->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS) - { - chp = chunks + op->chunk_index; - for (j = 0; j < op->n_chunks; j++) - { - EVP_EncryptUpdate (ctx, is_gmac ? 0 : chp->dst, &len, chp->src, - chp->len); - chp += 1; - } - } - else - EVP_EncryptUpdate (ctx, is_gmac ? 0 : op->dst, &len, op->src, op->len); - EVP_EncryptFinal_ex (ctx, is_gmac ? 0 : op->dst + len, &len); - EVP_CIPHER_CTX_ctrl (ctx, EVP_CTRL_AEAD_GET_TAG, op->tag_len, op->tag); - op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; - } - return n_ops; -} - -static_always_inline u32 -openssl_ops_enc_null_gmac (vlib_main_t *vm, vnet_crypto_op_t *ops[], - vnet_crypto_op_chunk_t *chunks, u32 n_ops, - const EVP_CIPHER *cipher, const int iv_len) -{ - return openssl_ops_enc_aead (vm, ops, chunks, n_ops, cipher, - /* is_gcm */ 1, /* is_gmac */ 1, iv_len); -} - -static_always_inline u32 -openssl_ops_enc_gcm (vlib_main_t *vm, vnet_crypto_op_t *ops[], - vnet_crypto_op_chunk_t *chunks, u32 n_ops, - const EVP_CIPHER *cipher, const int iv_len) -{ - return openssl_ops_enc_aead (vm, ops, chunks, n_ops, cipher, - /* is_gcm */ 1, /* is_gmac */ 0, iv_len); -} - -static_always_inline __clib_unused u32 -openssl_ops_enc_chacha20_poly1305 (vlib_main_t *vm, vnet_crypto_op_t *ops[], - vnet_crypto_op_chunk_t *chunks, u32 n_ops, - const EVP_CIPHER *cipher, const int iv_len) -{ - return openssl_ops_enc_aead (vm, ops, chunks, n_ops, cipher, - /* is_gcm */ 0, /* is_gmac */ 0, iv_len); -} - -static_always_inline u32 -openssl_ops_dec_aead (vlib_main_t *vm, vnet_crypto_op_t *ops[], - vnet_crypto_op_chunk_t *chunks, u32 n_ops, - const EVP_CIPHER *cipher, int is_gcm, int is_gmac, - const int iv_len) -{ - openssl_per_thread_data_t *ptd = vec_elt_at_index (per_thread_data, - vm->thread_index); - EVP_CIPHER_CTX *ctx; - vnet_crypto_op_chunk_t *chp; - u32 i, j, n_fail = 0; - for (i = 0; i < n_ops; i++) - { - vnet_crypto_op_t *op = ops[i]; - int len = 0; - - ctx = ptd->evp_cipher_dec_ctx[op->key_index]; - EVP_DecryptInit_ex (ctx, 0, 0, NULL, op->iv); - if (op->aad_len) - EVP_DecryptUpdate (ctx, 0, &len, op->aad, op->aad_len); - if (op->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS) - { - chp = chunks + op->chunk_index; - for (j = 0; j < op->n_chunks; j++) - { - EVP_DecryptUpdate (ctx, is_gmac ? 0 : chp->dst, &len, chp->src, - chp->len); - chp += 1; - } - } - else - { - EVP_DecryptUpdate (ctx, is_gmac ? 0 : op->dst, &len, op->src, - op->len); - } - EVP_CIPHER_CTX_ctrl (ctx, EVP_CTRL_AEAD_SET_TAG, op->tag_len, op->tag); - - if (EVP_DecryptFinal_ex (ctx, is_gmac ? 0 : op->dst + len, &len) > 0) - op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; - else - { - n_fail++; - op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC; - } - } - return n_ops - n_fail; -} - -static_always_inline u32 -openssl_ops_dec_null_gmac (vlib_main_t *vm, vnet_crypto_op_t *ops[], - vnet_crypto_op_chunk_t *chunks, u32 n_ops, - const EVP_CIPHER *cipher, const int iv_len) -{ - return openssl_ops_dec_aead (vm, ops, chunks, n_ops, cipher, - /* is_gcm */ 1, /* is_gmac */ 1, iv_len); -} - -static_always_inline u32 -openssl_ops_dec_gcm (vlib_main_t *vm, vnet_crypto_op_t *ops[], - vnet_crypto_op_chunk_t *chunks, u32 n_ops, - const EVP_CIPHER *cipher, const int iv_len) -{ - return openssl_ops_dec_aead (vm, ops, chunks, n_ops, cipher, - /* is_gcm */ 1, /* is_gmac */ 0, iv_len); -} - -static_always_inline __clib_unused u32 -openssl_ops_dec_chacha20_poly1305 (vlib_main_t *vm, vnet_crypto_op_t *ops[], - vnet_crypto_op_chunk_t *chunks, u32 n_ops, - const EVP_CIPHER *cipher, const int iv_len) -{ - return openssl_ops_dec_aead (vm, ops, chunks, n_ops, cipher, - /* is_gcm */ 0, /* is_gmac */ 0, iv_len); -} - -static_always_inline u32 -openssl_ops_hash (vlib_main_t *vm, vnet_crypto_op_t *ops[], - vnet_crypto_op_chunk_t *chunks, u32 n_ops, const EVP_MD *md) -{ - openssl_per_thread_data_t *ptd = - vec_elt_at_index (per_thread_data, vm->thread_index); - EVP_MD_CTX *ctx = ptd->hash_ctx; - vnet_crypto_op_chunk_t *chp; - u32 md_len, i, j, n_fail = 0; - - for (i = 0; i < n_ops; i++) - { - vnet_crypto_op_t *op = ops[i]; - - EVP_DigestInit_ex (ctx, md, NULL); - if (op->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS) - { - chp = chunks + op->chunk_index; - for (j = 0; j < op->n_chunks; j++) - { - EVP_DigestUpdate (ctx, chp->src, chp->len); - chp += 1; - } - } - else - EVP_DigestUpdate (ctx, op->src, op->len); - - EVP_DigestFinal_ex (ctx, op->digest, &md_len); - op->digest_len = md_len; - op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; - } - return n_ops - n_fail; -} - -static_always_inline u32 -openssl_ops_hmac (vlib_main_t * vm, vnet_crypto_op_t * ops[], - vnet_crypto_op_chunk_t * chunks, u32 n_ops, - const EVP_MD * md) -{ - u8 buffer[64]; - openssl_per_thread_data_t *ptd = vec_elt_at_index (per_thread_data, - vm->thread_index); - HMAC_CTX *ctx; - vnet_crypto_op_chunk_t *chp; - u32 i, j, n_fail = 0; - for (i = 0; i < n_ops; i++) - { - vnet_crypto_op_t *op = ops[i]; - unsigned int out_len = 0; - size_t sz = op->digest_len ? op->digest_len : EVP_MD_size (md); - - ctx = ptd->hmac_ctx[op->key_index]; - HMAC_Init_ex (ctx, NULL, 0, NULL, NULL); - if (op->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS) - { - chp = chunks + op->chunk_index; - for (j = 0; j < op->n_chunks; j++) - { - HMAC_Update (ctx, chp->src, chp->len); - chp += 1; - } - } - else - HMAC_Update (ctx, op->src, op->len); - HMAC_Final (ctx, buffer, &out_len); - - if (op->flags & VNET_CRYPTO_OP_FLAG_HMAC_CHECK) - { - if ((memcmp (op->digest, buffer, sz))) - { - n_fail++; - op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC; - continue; - } - } - else - clib_memcpy_fast (op->digest, buffer, sz); - op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; - } - return n_ops - n_fail; -} - -static_always_inline void * -openssl_ctx_cipher (vnet_crypto_key_t *key, vnet_crypto_key_op_t kop, - vnet_crypto_key_index_t idx, const EVP_CIPHER *cipher, - int is_gcm) -{ - EVP_CIPHER_CTX *ctx; - openssl_per_thread_data_t *ptd; - - if (VNET_CRYPTO_KEY_OP_ADD == kop) - { - vec_foreach (ptd, per_thread_data) - { - vec_validate_aligned (ptd->evp_cipher_enc_ctx, idx, - CLIB_CACHE_LINE_BYTES); - vec_validate_aligned (ptd->evp_cipher_dec_ctx, idx, - CLIB_CACHE_LINE_BYTES); - - ctx = EVP_CIPHER_CTX_new (); - EVP_CIPHER_CTX_set_padding (ctx, 0); - EVP_EncryptInit_ex (ctx, cipher, NULL, NULL, NULL); - if (is_gcm) - EVP_CIPHER_CTX_ctrl (ctx, EVP_CTRL_GCM_SET_IVLEN, 12, NULL); - EVP_EncryptInit_ex (ctx, 0, 0, key->data, 0); - ptd->evp_cipher_enc_ctx[idx] = ctx; - - ctx = EVP_CIPHER_CTX_new (); - EVP_CIPHER_CTX_set_padding (ctx, 0); - EVP_DecryptInit_ex (ctx, cipher, 0, 0, 0); - if (is_gcm) - EVP_CIPHER_CTX_ctrl (ctx, EVP_CTRL_GCM_SET_IVLEN, 12, 0); - EVP_DecryptInit_ex (ctx, 0, 0, key->data, 0); - ptd->evp_cipher_dec_ctx[idx] = ctx; - } - } - else if (VNET_CRYPTO_KEY_OP_MODIFY == kop) - { - vec_foreach (ptd, per_thread_data) - { - ctx = ptd->evp_cipher_enc_ctx[idx]; - EVP_EncryptInit_ex (ctx, cipher, NULL, NULL, NULL); - if (is_gcm) - EVP_CIPHER_CTX_ctrl (ctx, EVP_CTRL_GCM_SET_IVLEN, 12, NULL); - EVP_EncryptInit_ex (ctx, 0, 0, key->data, 0); - - ctx = ptd->evp_cipher_dec_ctx[idx]; - EVP_DecryptInit_ex (ctx, cipher, 0, 0, 0); - if (is_gcm) - EVP_CIPHER_CTX_ctrl (ctx, EVP_CTRL_GCM_SET_IVLEN, 12, 0); - EVP_DecryptInit_ex (ctx, 0, 0, key->data, 0); - } - } - else if (VNET_CRYPTO_KEY_OP_DEL == kop) - { - vec_foreach (ptd, per_thread_data) - { - ctx = ptd->evp_cipher_enc_ctx[idx]; - EVP_CIPHER_CTX_free (ctx); - ptd->evp_cipher_enc_ctx[idx] = NULL; - - ctx = ptd->evp_cipher_dec_ctx[idx]; - EVP_CIPHER_CTX_free (ctx); - ptd->evp_cipher_dec_ctx[idx] = NULL; - } - } - return NULL; -} - -static_always_inline void * -openssl_ctx_hmac (vnet_crypto_key_t *key, vnet_crypto_key_op_t kop, - vnet_crypto_key_index_t idx, const EVP_MD *md) -{ - HMAC_CTX *ctx; - openssl_per_thread_data_t *ptd; - if (VNET_CRYPTO_KEY_OP_ADD == kop) - { - vec_foreach (ptd, per_thread_data) - { - vec_validate_aligned (ptd->hmac_ctx, idx, CLIB_CACHE_LINE_BYTES); -#if OPENSSL_VERSION_NUMBER >= 0x10100000L - ctx = HMAC_CTX_new (); - HMAC_Init_ex (ctx, key->data, vec_len (key->data), md, NULL); - ptd->hmac_ctx[idx] = ctx; -#else - HMAC_CTX_init (&(ptd->_hmac_ctx)); - ptd->hmac_ctx[idx] = &ptd->_hmac_ctx; -#endif - } - } - else if (VNET_CRYPTO_KEY_OP_MODIFY == kop) - { - vec_foreach (ptd, per_thread_data) - { - ctx = ptd->hmac_ctx[idx]; - HMAC_Init_ex (ctx, key->data, vec_len (key->data), md, NULL); - } - } - else if (VNET_CRYPTO_KEY_OP_DEL == kop) - { - vec_foreach (ptd, per_thread_data) - { - ctx = ptd->hmac_ctx[idx]; - HMAC_CTX_free (ctx); - ptd->hmac_ctx[idx] = NULL; - } - } - return NULL; -} - -static void -crypto_openssl_key_handler (vlib_main_t *vm, vnet_crypto_key_op_t kop, - vnet_crypto_key_index_t idx) -{ - vnet_crypto_key_t *key = vnet_crypto_get_key (idx); - crypto_openssl_main_t *cm = &crypto_openssl_main; - - /** TODO: add linked alg support **/ - if (key->type == VNET_CRYPTO_KEY_TYPE_LINK) - return; - - if (cm->ctx_fn[key->alg] == 0) - return; - - cm->ctx_fn[key->alg](key, kop, idx); -} - -#define _(m, a, b, iv) \ - static u32 openssl_ops_enc_##a (vlib_main_t *vm, vnet_crypto_op_t *ops[], \ - u32 n_ops) \ - { \ - return openssl_ops_enc_##m (vm, ops, 0, n_ops, b (), iv); \ - } \ - \ - u32 openssl_ops_dec_##a (vlib_main_t *vm, vnet_crypto_op_t *ops[], \ - u32 n_ops) \ - { \ - return openssl_ops_dec_##m (vm, ops, 0, n_ops, b (), iv); \ - } \ - \ - static u32 openssl_ops_enc_chained_##a ( \ - vlib_main_t *vm, vnet_crypto_op_t *ops[], vnet_crypto_op_chunk_t *chunks, \ - u32 n_ops) \ - { \ - return openssl_ops_enc_##m (vm, ops, chunks, n_ops, b (), iv); \ - } \ - \ - static u32 openssl_ops_dec_chained_##a ( \ - vlib_main_t *vm, vnet_crypto_op_t *ops[], vnet_crypto_op_chunk_t *chunks, \ - u32 n_ops) \ - { \ - return openssl_ops_dec_##m (vm, ops, chunks, n_ops, b (), iv); \ - } \ - static void *openssl_ctx_##a (vnet_crypto_key_t *key, \ - vnet_crypto_key_op_t kop, \ - vnet_crypto_key_index_t idx) \ - { \ - int is_gcm = ((VNET_CRYPTO_ALG_AES_128_GCM <= key->alg) && \ - (VNET_CRYPTO_ALG_AES_256_NULL_GMAC >= key->alg)) ? \ - 1 : \ - 0; \ - return openssl_ctx_cipher (key, kop, idx, b (), is_gcm); \ - } - -foreach_openssl_evp_op; -#undef _ - -#define _(a, b) \ - static u32 openssl_ops_hash_##a (vlib_main_t *vm, vnet_crypto_op_t *ops[], \ - u32 n_ops) \ - { \ - return openssl_ops_hash (vm, ops, 0, n_ops, b ()); \ - } \ - static u32 openssl_ops_hash_chained_##a ( \ - vlib_main_t *vm, vnet_crypto_op_t *ops[], vnet_crypto_op_chunk_t *chunks, \ - u32 n_ops) \ - { \ - return openssl_ops_hash (vm, ops, chunks, n_ops, b ()); \ - } - -foreach_openssl_hash_op; -#undef _ - -#define _(a, b) \ - static u32 openssl_ops_hmac_##a (vlib_main_t *vm, vnet_crypto_op_t *ops[], \ - u32 n_ops) \ - { \ - return openssl_ops_hmac (vm, ops, 0, n_ops, b ()); \ - } \ - static u32 openssl_ops_hmac_chained_##a ( \ - vlib_main_t *vm, vnet_crypto_op_t *ops[], vnet_crypto_op_chunk_t *chunks, \ - u32 n_ops) \ - { \ - return openssl_ops_hmac (vm, ops, chunks, n_ops, b ()); \ - } \ - static void *openssl_ctx_hmac_##a (vnet_crypto_key_t *key, \ - vnet_crypto_key_op_t kop, \ - vnet_crypto_key_index_t idx) \ - { \ - return openssl_ctx_hmac (key, kop, idx, b ()); \ - } - -foreach_openssl_hmac_op; -#undef _ - -clib_error_t * -crypto_openssl_init (vlib_main_t * vm) -{ - crypto_openssl_main_t *cm = &crypto_openssl_main; - vlib_thread_main_t *tm = vlib_get_thread_main (); - openssl_per_thread_data_t *ptd; - u8 seed[32]; - - if (syscall (SYS_getrandom, &seed, sizeof (seed), 0) != sizeof (seed)) - return clib_error_return_unix (0, "getrandom() failed"); - - RAND_seed (seed, sizeof (seed)); - - u32 eidx = vnet_crypto_register_engine (vm, "openssl", 50, "OpenSSL"); - cm->crypto_engine_index = eidx; - -#define _(m, a, b, iv) \ - vnet_crypto_register_ops_handlers (vm, eidx, VNET_CRYPTO_OP_##a##_ENC, \ - openssl_ops_enc_##a, \ - openssl_ops_enc_chained_##a); \ - vnet_crypto_register_ops_handlers (vm, eidx, VNET_CRYPTO_OP_##a##_DEC, \ - openssl_ops_dec_##a, \ - openssl_ops_dec_chained_##a); \ - cm->ctx_fn[VNET_CRYPTO_ALG_##a] = openssl_ctx_##a; - - foreach_openssl_evp_op; -#undef _ - -#define _(a, b) \ - vnet_crypto_register_ops_handlers (vm, eidx, VNET_CRYPTO_OP_##a##_HMAC, \ - openssl_ops_hmac_##a, \ - openssl_ops_hmac_chained_##a); \ - cm->ctx_fn[VNET_CRYPTO_ALG_HMAC_##a] = openssl_ctx_hmac_##a; - - foreach_openssl_hmac_op; -#undef _ - -#define _(a, b) \ - vnet_crypto_register_ops_handlers (vm, eidx, VNET_CRYPTO_OP_##a##_HASH, \ - openssl_ops_hash_##a, \ - openssl_ops_hash_chained_##a); - - foreach_openssl_hash_op; -#undef _ - - vec_validate_aligned (per_thread_data, tm->n_vlib_mains - 1, - CLIB_CACHE_LINE_BYTES); - - vec_foreach (ptd, per_thread_data) - { -#if OPENSSL_VERSION_NUMBER >= 0x10100000L - ptd->hash_ctx = EVP_MD_CTX_create (); -#endif - } - vnet_crypto_register_key_handler (vm, cm->crypto_engine_index, - crypto_openssl_key_handler); - return 0; -} - -VLIB_INIT_FUNCTION (crypto_openssl_init) = -{ - .runs_after = VLIB_INITS ("vnet_crypto_init"), -}; - - -VLIB_PLUGIN_REGISTER () = { - .version = VPP_BUILD_VER, - .description = "OpenSSL Crypto Engine", -}; - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/plugins/crypto_sw_scheduler/main.c b/src/plugins/crypto_sw_scheduler/main.c index 8d0430ab2f6..a594f30f823 100644 --- a/src/plugins/crypto_sw_scheduler/main.c +++ b/src/plugins/crypto_sw_scheduler/main.c @@ -51,7 +51,7 @@ crypto_sw_scheduler_set_worker_crypto (u32 worker_idx, u8 enabled) } static void -crypto_sw_scheduler_key_handler (vlib_main_t * vm, vnet_crypto_key_op_t kop, +crypto_sw_scheduler_key_handler (vnet_crypto_key_op_t kop, vnet_crypto_key_index_t idx) { crypto_sw_scheduler_main_t *cm = &crypto_sw_scheduler_main; diff --git a/src/plugins/dpdk/cryptodev/cryptodev.c b/src/plugins/dpdk/cryptodev/cryptodev.c index 43c2c879aab..0250da7cda3 100644 --- a/src/plugins/dpdk/cryptodev/cryptodev.c +++ b/src/plugins/dpdk/cryptodev/cryptodev.c @@ -327,10 +327,9 @@ cryptodev_sess_handler (vlib_main_t *vm, vnet_crypto_key_op_t kop, } /*static*/ void -cryptodev_key_handler (vlib_main_t *vm, vnet_crypto_key_op_t kop, - vnet_crypto_key_index_t idx) +cryptodev_key_handler (vnet_crypto_key_op_t kop, vnet_crypto_key_index_t idx) { - cryptodev_sess_handler (vm, kop, idx, 8); + cryptodev_sess_handler (vlib_get_main (), kop, idx, 8); } clib_error_t * diff --git a/src/plugins/quic/quic_crypto.c b/src/plugins/quic/quic_crypto.c index c5cc5a4a714..9e2c915daaa 100644 --- a/src/plugins/quic/quic_crypto.c +++ b/src/plugins/quic/quic_crypto.c @@ -130,19 +130,18 @@ quic_crypto_set_key (crypto_key_t *key) u8 thread_index = vlib_get_thread_index (); u32 key_id = quic_main.per_thread_crypto_key_indices[thread_index]; vnet_crypto_key_t *vnet_key = vnet_crypto_get_key (key_id); - vlib_main_t *vm = vlib_get_main (); vnet_crypto_engine_t *engine; vec_foreach (engine, cm->engines) if (engine->key_op_handler) - engine->key_op_handler (vm, VNET_CRYPTO_KEY_OP_DEL, key_id); + engine->key_op_handler (VNET_CRYPTO_KEY_OP_DEL, key_id); vnet_key->alg = key->algo; clib_memcpy (vnet_key->data, key->key, key->key_len); vec_foreach (engine, cm->engines) if (engine->key_op_handler) - engine->key_op_handler (vm, VNET_CRYPTO_KEY_OP_ADD, key_id); + engine->key_op_handler (VNET_CRYPTO_KEY_OP_ADD, key_id); return key_id; } |