diff options
Diffstat (limited to 'drivers/crypto/aesni_mb')
| -rw-r--r-- | drivers/crypto/aesni_mb/aesni_mb_ops.h | 89 | ||||
| -rw-r--r-- | drivers/crypto/aesni_mb/meson.build | 12 | ||||
| -rw-r--r-- | drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c | 346 | ||||
| -rw-r--r-- | drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c | 90 | ||||
| -rw-r--r-- | drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h | 27 |
5 files changed, 478 insertions, 86 deletions
diff --git a/drivers/crypto/aesni_mb/aesni_mb_ops.h b/drivers/crypto/aesni_mb/aesni_mb_ops.h index 5a1cba6c..575d6a5b 100644 --- a/drivers/crypto/aesni_mb/aesni_mb_ops.h +++ b/drivers/crypto/aesni_mb/aesni_mb_ops.h @@ -11,6 +11,15 @@ #include <intel-ipsec-mb.h> +/* + * IMB_VERSION_NUM macro was introduced in version Multi-buffer 0.50, + * so if macro is not defined, it means that the version is 0.49. + */ +#if !defined(IMB_VERSION_NUM) +#define IMB_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + (c)) +#define IMB_VERSION_NUM IMB_VERSION(0, 49, 0) +#endif + enum aesni_mb_vector_mode { RTE_AESNI_MB_NOT_SUPPORTED = 0, RTE_AESNI_MB_SSE, @@ -39,6 +48,8 @@ typedef void (*aes_cmac_sub_key_gen_t) (const void *exp_key, void *k2, void *k3); typedef void (*aes_cmac_keyexp_t) (const void *key, void *keyexp); +typedef void (*aes_gcm_keyexp_t) + (const void *key, struct gcm_key_data *keyexp); /** Multi-buffer library function pointer table */ struct aesni_mb_op_fns { @@ -86,8 +97,24 @@ struct aesni_mb_op_fns { /**< AES CMAC subkey expansions */ aes_cmac_keyexp_t aes_cmac_expkey; /**< AES CMAC key expansions */ + aes_gcm_keyexp_t aes_gcm_128; + /**< AES GCM 128 key expansions */ + aes_gcm_keyexp_t aes_gcm_192; + /**< AES GCM 192 key expansions */ + aes_gcm_keyexp_t aes_gcm_256; + /**< AES GCM 256 key expansions */ } keyexp; /**< Key expansion functions */ +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + struct { + hash_fn_t sha1; + hash_fn_t sha224; + hash_fn_t sha256; + hash_fn_t sha384; + hash_fn_t sha512; + } multi_block; + /** multi block hash functions */ +#endif } aux; /**< Auxiliary functions */ }; @@ -104,7 +131,13 @@ static const struct aesni_mb_op_fns job_ops[] = { }, .keyexp = { NULL + }, +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + .multi_block = { + NULL } +#endif + } }, [RTE_AESNI_MB_SSE] = { @@ -130,8 +163,20 @@ static const struct aesni_mb_op_fns job_ops[] = { aes_keyexp_256_sse, aes_xcbc_expand_key_sse, aes_cmac_subkey_gen_sse, - aes_keyexp_128_enc_sse + aes_keyexp_128_enc_sse, + aes_gcm_pre_128_sse, + aes_gcm_pre_192_sse, + aes_gcm_pre_256_sse + }, +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + .multi_block = { + sha1_sse, + sha224_sse, + sha256_sse, + sha384_sse, + sha512_sse } +#endif } }, [RTE_AESNI_MB_AVX] = { @@ -157,8 +202,20 @@ static const struct aesni_mb_op_fns job_ops[] = { aes_keyexp_256_avx, aes_xcbc_expand_key_avx, aes_cmac_subkey_gen_avx, - aes_keyexp_128_enc_avx + aes_keyexp_128_enc_avx, + aes_gcm_pre_128_avx_gen2, + aes_gcm_pre_192_avx_gen2, + aes_gcm_pre_256_avx_gen2 + }, +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + .multi_block = { + sha1_avx, + sha224_avx, + sha256_avx, + sha384_avx, + sha512_avx } +#endif } }, [RTE_AESNI_MB_AVX2] = { @@ -184,8 +241,20 @@ static const struct aesni_mb_op_fns job_ops[] = { aes_keyexp_256_avx2, aes_xcbc_expand_key_avx2, aes_cmac_subkey_gen_avx2, - aes_keyexp_128_enc_avx2 + aes_keyexp_128_enc_avx2, + aes_gcm_pre_128_avx_gen4, + aes_gcm_pre_192_avx_gen4, + aes_gcm_pre_256_avx_gen4 + }, +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + .multi_block = { + sha1_avx2, + sha224_avx2, + sha256_avx2, + sha384_avx2, + sha512_avx2 } +#endif } }, [RTE_AESNI_MB_AVX512] = { @@ -211,8 +280,20 @@ static const struct aesni_mb_op_fns job_ops[] = { aes_keyexp_256_avx512, aes_xcbc_expand_key_avx512, aes_cmac_subkey_gen_avx512, - aes_keyexp_128_enc_avx512 + aes_keyexp_128_enc_avx512, + aes_gcm_pre_128_avx_gen4, + aes_gcm_pre_192_avx_gen4, + aes_gcm_pre_256_avx_gen4 + }, +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + .multi_block = { + sha1_avx512, + sha224_avx512, + sha256_avx512, + sha384_avx512, + sha512_avx512 } +#endif } } }; diff --git a/drivers/crypto/aesni_mb/meson.build b/drivers/crypto/aesni_mb/meson.build new file mode 100644 index 00000000..aae0995e --- /dev/null +++ b/drivers/crypto/aesni_mb/meson.build @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Intel Corporation + +lib = cc.find_library('IPSec_MB', required: false) +if not lib.found() + build = false +else + ext_deps += lib +endif + +sources = files('rte_aesni_mb_pmd.c', 'rte_aesni_mb_pmd_ops.c') +deps += ['bus_vdev'] diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c index 93dc7a44..83250e32 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c @@ -14,6 +14,9 @@ #include "rte_aesni_mb_pmd_private.h" +#define AES_CCM_DIGEST_MIN_LEN 4 +#define AES_CCM_DIGEST_MAX_LEN 16 +#define HMAC_MAX_BLOCK_SIZE 128 static uint8_t cryptodev_driver_id; typedef void (*hash_one_block_t)(const void *data, void *digest); @@ -83,7 +86,8 @@ aesni_mb_get_chain_order(const struct rte_crypto_sym_xform *xform) } if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { - if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) { + if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM || + xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) { if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) return AESNI_MB_OP_AEAD_CIPHER_HASH; else @@ -101,6 +105,8 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops, const struct rte_crypto_sym_xform *xform) { hash_one_block_t hash_oneblock_fn; + unsigned int key_larger_block_size = 0; + uint8_t hashed_key[HMAC_MAX_BLOCK_SIZE] = { 0 }; if (xform == NULL) { sess->auth.algo = NULL_HASH; @@ -112,12 +118,23 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops, return -1; } + /* Set the request digest size */ + sess->auth.req_digest_len = xform->auth.digest_length; + /* Select auth generate/verify */ sess->auth.operation = xform->auth.op; /* Set Authentication Parameters */ if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_XCBC_MAC) { sess->auth.algo = AES_XCBC; + + uint16_t xcbc_mac_digest_len = + get_truncated_digest_byte_length(AES_XCBC); + if (sess->auth.req_digest_len != xcbc_mac_digest_len) { + AESNI_MB_LOG(ERR, "Invalid digest size\n"); + return -EINVAL; + } + sess->auth.gen_digest_len = sess->auth.req_digest_len; (*mb_ops->aux.keyexp.aes_xcbc)(xform->auth.key.data, sess->auth.xcbc.k1_expanded, sess->auth.xcbc.k2, sess->auth.xcbc.k3); @@ -126,6 +143,32 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops, if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_CMAC) { sess->auth.algo = AES_CMAC; + + uint16_t cmac_digest_len = get_digest_byte_length(AES_CMAC); + + if (sess->auth.req_digest_len > cmac_digest_len) { + AESNI_MB_LOG(ERR, "Invalid digest size\n"); + return -EINVAL; + } + /* + * Multi-buffer lib supports digest sizes from 4 to 16 bytes + * in version 0.50 and sizes of 12 and 16 bytes, + * in version 0.49. + * If size requested is different, generate the full digest + * (16 bytes) in a temporary location and then memcpy + * the requested number of bytes. + */ +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + if (sess->auth.req_digest_len < 4) +#else + uint16_t cmac_trunc_digest_len = + get_truncated_digest_byte_length(AES_CMAC); + if (sess->auth.req_digest_len != cmac_digest_len && + sess->auth.req_digest_len != cmac_trunc_digest_len) +#endif + sess->auth.gen_digest_len = cmac_digest_len; + else + sess->auth.gen_digest_len = sess->auth.req_digest_len; (*mb_ops->aux.keyexp.aes_cmac_expkey)(xform->auth.key.data, sess->auth.cmac.expkey); @@ -134,7 +177,6 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops, return 0; } - switch (xform->auth.algo) { case RTE_CRYPTO_AUTH_MD5_HMAC: sess->auth.algo = MD5; @@ -143,34 +185,107 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops, case RTE_CRYPTO_AUTH_SHA1_HMAC: sess->auth.algo = SHA1; hash_oneblock_fn = mb_ops->aux.one_block.sha1; +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + if (xform->auth.key.length > get_auth_algo_blocksize(SHA1)) { + mb_ops->aux.multi_block.sha1( + xform->auth.key.data, + xform->auth.key.length, + hashed_key); + key_larger_block_size = 1; + } +#endif break; case RTE_CRYPTO_AUTH_SHA224_HMAC: sess->auth.algo = SHA_224; hash_oneblock_fn = mb_ops->aux.one_block.sha224; +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + if (xform->auth.key.length > get_auth_algo_blocksize(SHA_224)) { + mb_ops->aux.multi_block.sha224( + xform->auth.key.data, + xform->auth.key.length, + hashed_key); + key_larger_block_size = 1; + } +#endif break; case RTE_CRYPTO_AUTH_SHA256_HMAC: sess->auth.algo = SHA_256; hash_oneblock_fn = mb_ops->aux.one_block.sha256; +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + if (xform->auth.key.length > get_auth_algo_blocksize(SHA_256)) { + mb_ops->aux.multi_block.sha256( + xform->auth.key.data, + xform->auth.key.length, + hashed_key); + key_larger_block_size = 1; + } +#endif break; case RTE_CRYPTO_AUTH_SHA384_HMAC: sess->auth.algo = SHA_384; hash_oneblock_fn = mb_ops->aux.one_block.sha384; +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + if (xform->auth.key.length > get_auth_algo_blocksize(SHA_384)) { + mb_ops->aux.multi_block.sha384( + xform->auth.key.data, + xform->auth.key.length, + hashed_key); + key_larger_block_size = 1; + } +#endif break; case RTE_CRYPTO_AUTH_SHA512_HMAC: sess->auth.algo = SHA_512; hash_oneblock_fn = mb_ops->aux.one_block.sha512; +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + if (xform->auth.key.length > get_auth_algo_blocksize(SHA_512)) { + mb_ops->aux.multi_block.sha512( + xform->auth.key.data, + xform->auth.key.length, + hashed_key); + key_larger_block_size = 1; + } +#endif break; default: AESNI_MB_LOG(ERR, "Unsupported authentication algorithm selection"); return -ENOTSUP; } + uint16_t trunc_digest_size = + get_truncated_digest_byte_length(sess->auth.algo); + uint16_t full_digest_size = + get_digest_byte_length(sess->auth.algo); + +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + if (sess->auth.req_digest_len > full_digest_size || + sess->auth.req_digest_len == 0) { +#else + if (sess->auth.req_digest_len != trunc_digest_size) { +#endif + AESNI_MB_LOG(ERR, "Invalid digest size\n"); + return -EINVAL; + } + + if (sess->auth.req_digest_len != trunc_digest_size && + sess->auth.req_digest_len != full_digest_size) + sess->auth.gen_digest_len = full_digest_size; + else + sess->auth.gen_digest_len = sess->auth.req_digest_len; /* Calculate Authentication precomputes */ - calculate_auth_precomputes(hash_oneblock_fn, + if (key_larger_block_size) { + calculate_auth_precomputes(hash_oneblock_fn, + sess->auth.pads.inner, sess->auth.pads.outer, + hashed_key, + xform->auth.key.length, + get_auth_algo_blocksize(sess->auth.algo)); + } else { + calculate_auth_precomputes(hash_oneblock_fn, sess->auth.pads.inner, sess->auth.pads.outer, xform->auth.key.data, xform->auth.key.length, get_auth_algo_blocksize(sess->auth.algo)); + } return 0; } @@ -330,7 +445,10 @@ aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops, struct aesni_mb_session *sess, const struct rte_crypto_sym_xform *xform) { - aes_keyexp_t aes_keyexp_fn; + union { + aes_keyexp_t aes_keyexp_fn; + aes_gcm_keyexp_t aes_gcm_keyexp_fn; + } keyexp; switch (xform->aead.op) { case RTE_CRYPTO_AEAD_OP_ENCRYPT: @@ -350,7 +468,53 @@ aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops, case RTE_CRYPTO_AEAD_AES_CCM: sess->cipher.mode = CCM; sess->auth.algo = AES_CCM; + + /* Check key length and choose key expansion function for AES */ + switch (xform->aead.key.length) { + case AES_128_BYTES: + sess->cipher.key_length_in_bytes = AES_128_BYTES; + keyexp.aes_keyexp_fn = mb_ops->aux.keyexp.aes128; + break; + default: + AESNI_MB_LOG(ERR, "Invalid cipher key length"); + return -EINVAL; + } + + /* Expanded cipher keys */ + (*keyexp.aes_keyexp_fn)(xform->aead.key.data, + sess->cipher.expanded_aes_keys.encode, + sess->cipher.expanded_aes_keys.decode); + break; + + case RTE_CRYPTO_AEAD_AES_GCM: + sess->cipher.mode = GCM; + sess->auth.algo = AES_GMAC; + + switch (xform->aead.key.length) { + case AES_128_BYTES: + sess->cipher.key_length_in_bytes = AES_128_BYTES; + keyexp.aes_gcm_keyexp_fn = + mb_ops->aux.keyexp.aes_gcm_128; + break; + case AES_192_BYTES: + sess->cipher.key_length_in_bytes = AES_192_BYTES; + keyexp.aes_gcm_keyexp_fn = + mb_ops->aux.keyexp.aes_gcm_192; + break; + case AES_256_BYTES: + sess->cipher.key_length_in_bytes = AES_256_BYTES; + keyexp.aes_gcm_keyexp_fn = + mb_ops->aux.keyexp.aes_gcm_256; + break; + default: + AESNI_MB_LOG(ERR, "Invalid cipher key length"); + return -EINVAL; + } + + (keyexp.aes_gcm_keyexp_fn)(xform->aead.key.data, + &sess->cipher.gcm_key); break; + default: AESNI_MB_LOG(ERR, "Unsupported aead mode parameter"); return -ENOTSUP; @@ -360,22 +524,15 @@ aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops, sess->iv.offset = xform->aead.iv.offset; sess->iv.length = xform->aead.iv.length; - /* Check key length and choose key expansion function for AES */ - - switch (xform->aead.key.length) { - case AES_128_BYTES: - sess->cipher.key_length_in_bytes = AES_128_BYTES; - aes_keyexp_fn = mb_ops->aux.keyexp.aes128; - break; - default: - AESNI_MB_LOG(ERR, "Invalid cipher key length"); + sess->auth.req_digest_len = xform->aead.digest_length; + /* CCM digests must be between 4 and 16 and an even number */ + if (sess->auth.req_digest_len < AES_CCM_DIGEST_MIN_LEN || + sess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN || + (sess->auth.req_digest_len & 1) == 1) { + AESNI_MB_LOG(ERR, "Invalid digest size\n"); return -EINVAL; } - - /* Expanded cipher keys */ - (*aes_keyexp_fn)(xform->aead.key.data, - sess->cipher.expanded_aes_keys.encode, - sess->cipher.expanded_aes_keys.decode); + sess->auth.gen_digest_len = sess->auth.req_digest_len; return 0; } @@ -397,19 +554,16 @@ aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops, sess->chain_order = HASH_CIPHER; auth_xform = xform; cipher_xform = xform->next; - sess->auth.digest_len = xform->auth.digest_length; break; case AESNI_MB_OP_CIPHER_HASH: sess->chain_order = CIPHER_HASH; auth_xform = xform->next; cipher_xform = xform; - sess->auth.digest_len = xform->auth.digest_length; break; case AESNI_MB_OP_HASH_ONLY: sess->chain_order = HASH_CIPHER; auth_xform = xform; cipher_xform = NULL; - sess->auth.digest_len = xform->auth.digest_length; break; case AESNI_MB_OP_CIPHER_ONLY: /* @@ -428,13 +582,11 @@ aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops, case AESNI_MB_OP_AEAD_CIPHER_HASH: sess->chain_order = CIPHER_HASH; sess->aead.aad_len = xform->aead.aad_length; - sess->auth.digest_len = xform->aead.digest_length; aead_xform = xform; break; case AESNI_MB_OP_AEAD_HASH_CIPHER: sess->chain_order = HASH_CIPHER; sess->aead.aad_len = xform->aead.aad_length; - sess->auth.digest_len = xform->aead.digest_length; aead_xform = xform; break; case AESNI_MB_OP_NOT_SUPPORTED: @@ -573,38 +725,62 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes; - if (job->cipher_mode == DES3) { - job->aes_enc_key_expanded = - session->cipher.exp_3des_keys.ks_ptr; - job->aes_dec_key_expanded = - session->cipher.exp_3des_keys.ks_ptr; - } else { - job->aes_enc_key_expanded = - session->cipher.expanded_aes_keys.encode; - job->aes_dec_key_expanded = - session->cipher.expanded_aes_keys.decode; - } - - - - /* Set authentication parameters */ job->hash_alg = session->auth.algo; - if (job->hash_alg == AES_XCBC) { + + switch (job->hash_alg) { + case AES_XCBC: job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded; job->u.XCBC._k2 = session->auth.xcbc.k2; job->u.XCBC._k3 = session->auth.xcbc.k3; - } else if (job->hash_alg == AES_CCM) { + + job->aes_enc_key_expanded = + session->cipher.expanded_aes_keys.encode; + job->aes_dec_key_expanded = + session->cipher.expanded_aes_keys.decode; + break; + + case AES_CCM: job->u.CCM.aad = op->sym->aead.aad.data + 18; job->u.CCM.aad_len_in_bytes = session->aead.aad_len; - } else if (job->hash_alg == AES_CMAC) { + job->aes_enc_key_expanded = + session->cipher.expanded_aes_keys.encode; + job->aes_dec_key_expanded = + session->cipher.expanded_aes_keys.decode; + break; + + case AES_CMAC: job->u.CMAC._key_expanded = session->auth.cmac.expkey; job->u.CMAC._skey1 = session->auth.cmac.skey1; job->u.CMAC._skey2 = session->auth.cmac.skey2; + job->aes_enc_key_expanded = + session->cipher.expanded_aes_keys.encode; + job->aes_dec_key_expanded = + session->cipher.expanded_aes_keys.decode; + break; - } else { + case AES_GMAC: + job->u.GCM.aad = op->sym->aead.aad.data; + job->u.GCM.aad_len_in_bytes = session->aead.aad_len; + job->aes_enc_key_expanded = &session->cipher.gcm_key; + job->aes_dec_key_expanded = &session->cipher.gcm_key; + break; + + default: job->u.HMAC._hashed_auth_key_xor_ipad = session->auth.pads.inner; job->u.HMAC._hashed_auth_key_xor_opad = session->auth.pads.outer; + + if (job->cipher_mode == DES3) { + job->aes_enc_key_expanded = + session->cipher.exp_3des_keys.ks_ptr; + job->aes_dec_key_expanded = + session->cipher.exp_3des_keys.ks_ptr; + } else { + job->aes_enc_key_expanded = + session->cipher.expanded_aes_keys.encode; + job->aes_dec_key_expanded = + session->cipher.expanded_aes_keys.decode; + } } /* Mutable crypto operation parameters */ @@ -625,7 +801,7 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, rte_pktmbuf_data_len(op->sym->m_src)); } else { m_dst = m_src; - if (job->hash_alg == AES_CCM) + if (job->hash_alg == AES_CCM || job->hash_alg == AES_GMAC) m_offset = op->sym->aead.data.offset; else m_offset = op->sym->cipher.data.offset; @@ -637,32 +813,33 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, job->auth_tag_output = qp->temp_digests[*digest_idx]; *digest_idx = (*digest_idx + 1) % MAX_JOBS; } else { - if (job->hash_alg == AES_CCM) + if (job->hash_alg == AES_CCM || job->hash_alg == AES_GMAC) job->auth_tag_output = op->sym->aead.digest.data; else job->auth_tag_output = op->sym->auth.digest.data; - } + if (session->auth.req_digest_len != session->auth.gen_digest_len) { + job->auth_tag_output = qp->temp_digests[*digest_idx]; + *digest_idx = (*digest_idx + 1) % MAX_JOBS; + } + } /* * Multi-buffer library current only support returning a truncated * digest length as specified in the relevant IPsec RFCs */ - if (job->hash_alg != AES_CCM && job->hash_alg != AES_CMAC) - job->auth_tag_output_len_in_bytes = - get_truncated_digest_byte_length(job->hash_alg); - else - job->auth_tag_output_len_in_bytes = session->auth.digest_len; + /* Set digest length */ + job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len; /* Set IV parameters */ - job->iv_len_in_bytes = session->iv.length; /* Data Parameter */ job->src = rte_pktmbuf_mtod(m_src, uint8_t *); job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset); - if (job->hash_alg == AES_CCM) { + switch (job->hash_alg) { + case AES_CCM: job->cipher_start_src_offset_in_bytes = op->sym->aead.data.offset; job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length; @@ -671,7 +848,19 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, job->iv = rte_crypto_op_ctod_offset(op, uint8_t *, session->iv.offset + 1); - } else { + break; + + case AES_GMAC: + job->cipher_start_src_offset_in_bytes = + op->sym->aead.data.offset; + job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset; + job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length; + job->msg_len_to_hash_in_bytes = job->msg_len_to_cipher_in_bytes; + job->iv = rte_crypto_op_ctod_offset(op, uint8_t *, + session->iv.offset); + break; + + default: job->cipher_start_src_offset_in_bytes = op->sym->cipher.data.offset; job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length; @@ -690,20 +879,37 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, } static inline void -verify_digest(struct aesni_mb_qp *qp __rte_unused, JOB_AES_HMAC *job, - struct rte_crypto_op *op) { +verify_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op, + struct aesni_mb_session *sess) +{ /* Verify digest if required */ - if (job->hash_alg == AES_CCM) { + if (job->hash_alg == AES_CCM || job->hash_alg == AES_GMAC) { if (memcmp(job->auth_tag_output, op->sym->aead.digest.data, - job->auth_tag_output_len_in_bytes) != 0) + sess->auth.req_digest_len) != 0) op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; } else { if (memcmp(job->auth_tag_output, op->sym->auth.digest.data, - job->auth_tag_output_len_in_bytes) != 0) + sess->auth.req_digest_len) != 0) op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; } } +static inline void +generate_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op, + struct aesni_mb_session *sess) +{ + /* No extra copy neeed */ + if (likely(sess->auth.req_digest_len == sess->auth.gen_digest_len)) + return; + + /* + * This can only happen for HMAC, so only digest + * for authentication algos is required + */ + memcpy(op->sym->auth.digest.data, job->auth_tag_output, + sess->auth.req_digest_len); +} + /** * Process a completed job and return rte_mbuf which job processed * @@ -730,7 +936,9 @@ post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job) if (job->hash_alg != NULL_HASH) { if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) - verify_digest(qp, job, op); + verify_digest(job, op, sess); + else + generate_digest(job, op, sess); } break; default: @@ -833,22 +1041,30 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, uint8_t digest_idx = qp->digest_idx; do { - /* Get next operation to process from ingress queue */ - retval = rte_ring_dequeue(qp->ingress_queue, (void **)&op); - if (retval < 0) - break; - /* Get next free mb job struct from mb manager */ job = (*qp->op_fns->job.get_next)(qp->mb_mgr); if (unlikely(job == NULL)) { /* if no free mb job structs we need to flush mb_mgr */ processed_jobs += flush_mb_mgr(qp, &ops[processed_jobs], - (nb_ops - processed_jobs) - 1); + nb_ops - processed_jobs); + + if (nb_ops == processed_jobs) + break; job = (*qp->op_fns->job.get_next)(qp->mb_mgr); } + /* + * Get next operation to process from ingress queue. + * There is no need to return the job to the MB_MGR + * if there are no more operations to process, since the MB_MGR + * can use that pointer again in next get_next calls. + */ + retval = rte_ring_dequeue(qp->ingress_queue, (void **)&op); + if (retval < 0) + break; + retval = set_mb_job_params(job, qp, op, &digest_idx); if (unlikely(retval != 0)) { qp->stats.dequeue_err_count++; diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c index ab26e5ae..43f6c26e 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c @@ -25,9 +25,15 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .increment = 1 }, .digest_size = { +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + .min = 1, + .max = 16, + .increment = 1 +#else .min = 12, .max = 12, .increment = 0 +#endif }, .iv_size = { 0 } }, } @@ -42,13 +48,23 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .block_size = 64, .key_size = { .min = 1, +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + .max = 65535, +#else .max = 64, +#endif .increment = 1 }, .digest_size = { +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + .min = 1, + .max = 20, + .increment = 1 +#else .min = 12, .max = 12, .increment = 0 +#endif }, .iv_size = { 0 } }, } @@ -63,13 +79,23 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .block_size = 64, .key_size = { .min = 1, +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + .max = 65535, +#else .max = 64, +#endif .increment = 1 }, .digest_size = { +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + .min = 1, + .max = 28, + .increment = 1 +#else .min = 14, .max = 14, .increment = 0 +#endif }, .iv_size = { 0 } }, } @@ -84,13 +110,23 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .block_size = 64, .key_size = { .min = 1, +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + .max = 65535, +#else .max = 64, +#endif .increment = 1 }, .digest_size = { +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + .min = 1, + .max = 32, + .increment = 1 +#else .min = 16, .max = 16, .increment = 0 +#endif }, .iv_size = { 0 } }, } @@ -105,13 +141,23 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .block_size = 128, .key_size = { .min = 1, +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + .max = 65535, +#else .max = 128, +#endif .increment = 1 }, .digest_size = { +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + .min = 1, + .max = 48, + .increment = 1 +#else .min = 24, .max = 24, .increment = 0 +#endif }, .iv_size = { 0 } }, } @@ -126,13 +172,23 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .block_size = 128, .key_size = { .min = 1, +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + .max = 65535, +#else .max = 128, +#endif .increment = 1 }, .digest_size = { +#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + .min = 1, + .max = 64, + .increment = 1 +#else .min = 32, .max = 32, .increment = 0 +#endif }, .iv_size = { 0 } }, } @@ -322,14 +378,44 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .increment = 0 }, .digest_size = { - .min = 12, + .min = 1, .max = 16, - .increment = 4 + .increment = 1 }, .iv_size = { 0 } }, } }, } }, + { /* AES GCM */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, + {.aead = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .digest_size = { + .min = 8, + .max = 16, + .increment = 4 + }, + .aad_size = { + .min = 0, + .max = 65535, + .increment = 1 + }, + .iv_size = { + .min = 12, + .max = 12, + .increment = 0 + } + }, } + }, } + }, RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() }; diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h index 70e9d18e..d8021cda 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h @@ -7,15 +7,6 @@ #include "aesni_mb_ops.h" -/* - * IMB_VERSION_NUM macro was introduced in version Multi-buffer 0.50, - * so if macro is not defined, it means that the version is 0.49. - */ -#if !defined(IMB_VERSION_NUM) -#define IMB_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + (c)) -#define IMB_VERSION_NUM IMB_VERSION(0, 49, 0) -#endif - #define CRYPTODEV_NAME_AESNI_MB_PMD crypto_aesni_mb /**< AES-NI Multi buffer PMD device name */ @@ -31,8 +22,8 @@ int aesni_mb_logtype_driver; #define HMAC_IPAD_VALUE (0x36) #define HMAC_OPAD_VALUE (0x5C) -/* Maximum length for digest (SHA-512 truncated needs 32 bytes) */ -#define DIGEST_LENGTH_MAX 32 +/* Maximum length for digest */ +#define DIGEST_LENGTH_MAX 64 static const unsigned auth_blocksize[] = { [MD5] = 64, [SHA1] = 64, @@ -64,7 +55,7 @@ static const unsigned auth_truncated_digest_byte_lengths[] = { [SHA_384] = 24, [SHA_512] = 32, [AES_XCBC] = 12, - [AES_CMAC] = 16, + [AES_CMAC] = 12, [AES_CCM] = 8, [NULL_HASH] = 0 }; @@ -91,11 +82,13 @@ static const unsigned auth_digest_byte_lengths[] = { [SHA_512] = 64, [AES_XCBC] = 16, [AES_CMAC] = 16, + [AES_GMAC] = 12, [NULL_HASH] = 0 }; /** - * Get the output digest size in bytes for a specified authentication algorithm + * Get the full digest size in bytes for a specified authentication algorithm + * (if available in the Multi-buffer library) * * @Note: this function will not return a valid value for a non-valid * authentication algorithm @@ -180,6 +173,8 @@ struct aesni_mb_session { const void *ks_ptr[3]; uint64_t key[3][16]; } exp_3des_keys; + + struct gcm_key_data gcm_key; }; /**< Expanded AES keys - Allocating space to * contain the maximum expanded key size which @@ -226,8 +221,10 @@ struct aesni_mb_session { } cmac; /**< Expanded XCBC authentication keys */ }; - /** digest size */ - uint16_t digest_len; + /** Generated digest size by the Multi-buffer library */ + uint16_t gen_digest_len; + /** Requested digest size from Cryptodev */ + uint16_t req_digest_len; } auth; struct { |