summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--build/external/packages/ipsec-mb.mk5
-rw-r--r--src/plugins/crypto_ipsecmb/ipsecmb.c143
2 files changed, 82 insertions, 66 deletions
diff --git a/build/external/packages/ipsec-mb.mk b/build/external/packages/ipsec-mb.mk
index 8750dc2d493..02c2e194456 100644
--- a/build/external/packages/ipsec-mb.mk
+++ b/build/external/packages/ipsec-mb.mk
@@ -11,17 +11,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-ipsec-mb_version := 1.2
+ipsec-mb_version := 1.3
ipsec-mb_tarball := v$(ipsec-mb_version).tar.gz
ipsec-mb_tarball_md5sum_0.54 := 258941f7ba90c275fcf9d19c622d2d21
ipsec-mb_tarball_md5sum_0.55 := deca674bca7ae2282890e1fa7f953609
ipsec-mb_tarball_md5sum_1.0 := 906e701937751e761671dc83a41cff65
ipsec-mb_tarball_md5sum_1.1 := 3916471d3713d27e42473cb6af9c65e5
ipsec-mb_tarball_md5sum_1.2 := f551d9c208893a436c1f5c146a615bd6
+ipsec-mb_tarball_md5sum_1.3 := d8692db9efe32a263b61f12ac0dca950
ipsec-mb_tarball_md5sum := $(ipsec-mb_tarball_md5sum_$(ipsec-mb_version))
ipsec-mb_tarball_strip_dirs := 1
-ipsec-mb_url := http://github.com/01org/intel-ipsec-mb/archive/$(ipsec-mb_tarball)
+ipsec-mb_url := http://github.com/intel/intel-ipsec-mb/archive/$(ipsec-mb_tarball)
define ipsec-mb_config_cmds
@true
diff --git a/src/plugins/crypto_ipsecmb/ipsecmb.c b/src/plugins/crypto_ipsecmb/ipsecmb.c
index 93654daee51..c75a2b8b6aa 100644
--- a/src/plugins/crypto_ipsecmb/ipsecmb.c
+++ b/src/plugins/crypto_ipsecmb/ipsecmb.c
@@ -31,8 +31,9 @@
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
- MB_MGR *mgr;
__m128i cbc_iv;
+ MB_MGR *mgr;
+ JOB_AES_HMAC burst_jobs[IMB_MAX_BURST_SIZE];
} ipsecmb_per_thread_data_t;
typedef struct
@@ -148,44 +149,50 @@ ipsecmb_ops_hmac_inline (vlib_main_t * vm, vnet_crypto_op_t * ops[],
ipsecmb_per_thread_data_t *ptd = vec_elt_at_index (imbm->per_thread_data,
vm->thread_index);
JOB_AES_HMAC *job;
- u32 i, n_fail = 0;
+ u32 i, n_fail = 0, ops_index = 0;
u8 scratch[n_ops][digest_size];
+ const u32 burst_sz =
+ (n_ops > IMB_MAX_BURST_SIZE) ? IMB_MAX_BURST_SIZE : n_ops;
- /*
- * queue all the jobs first ...
- */
- for (i = 0; i < n_ops; i++)
+ while (n_ops)
{
- vnet_crypto_op_t *op = ops[i];
- u8 *kd = (u8 *) imbm->key_data[op->key_index];
+ const u32 n = (n_ops > burst_sz) ? burst_sz : n_ops;
+ /*
+ * configure all the jobs first ...
+ */
+ for (i = 0; i < n; i++, ops_index++)
+ {
+ vnet_crypto_op_t *op = ops[ops_index];
+ const u8 *kd = (u8 *) imbm->key_data[op->key_index];
- job = IMB_GET_NEXT_JOB (ptd->mgr);
+ job = &ptd->burst_jobs[i];
- job->src = op->src;
- job->hash_start_src_offset_in_bytes = 0;
- job->msg_len_to_hash_in_bytes = op->len;
- job->hash_alg = alg;
- job->auth_tag_output_len_in_bytes = digest_size;
- job->auth_tag_output = scratch[i];
+ job->src = op->src;
+ job->hash_start_src_offset_in_bytes = 0;
+ job->msg_len_to_hash_in_bytes = op->len;
+ job->auth_tag_output_len_in_bytes = digest_size;
+ job->auth_tag_output = scratch[ops_index];
- job->cipher_mode = NULL_CIPHER;
- job->cipher_direction = DECRYPT;
- job->chain_order = HASH_CIPHER;
+ job->u.HMAC._hashed_auth_key_xor_ipad = kd;
+ job->u.HMAC._hashed_auth_key_xor_opad = kd + hash_size;
+ job->user_data = op;
+ }
- job->u.HMAC._hashed_auth_key_xor_ipad = kd;
- job->u.HMAC._hashed_auth_key_xor_opad = kd + hash_size;
- job->user_data = op;
+ /*
+ * submit all jobs to be processed and retire completed jobs
+ */
+ IMB_SUBMIT_HASH_BURST_NOCHECK (ptd->mgr, ptd->burst_jobs, n, alg);
- job = IMB_SUBMIT_JOB (ptd->mgr);
+ for (i = 0; i < n; i++)
+ {
+ job = &ptd->burst_jobs[i];
+ ipsecmb_retire_hmac_job (job, &n_fail, digest_size);
+ }
- if (job)
- ipsecmb_retire_hmac_job (job, &n_fail, digest_size);
+ n_ops -= n;
}
- while ((job = IMB_FLUSH_JOB (ptd->mgr)))
- ipsecmb_retire_hmac_job (job, &n_fail, digest_size);
-
- return n_ops - n_fail;
+ return ops_index - n_fail;
}
#define _(a, b, c, d, e, f) \
@@ -222,52 +229,57 @@ ipsecmb_ops_aes_cipher_inline (vlib_main_t *vm, vnet_crypto_op_t *ops[],
ipsecmb_per_thread_data_t *ptd = vec_elt_at_index (imbm->per_thread_data,
vm->thread_index);
JOB_AES_HMAC *job;
- u32 i, n_fail = 0;
+ u32 i, n_fail = 0, ops_index = 0;
+ const u32 burst_sz =
+ (n_ops > IMB_MAX_BURST_SIZE) ? IMB_MAX_BURST_SIZE : n_ops;
- for (i = 0; i < n_ops; i++)
+ while (n_ops)
{
- ipsecmb_aes_key_data_t *kd;
- vnet_crypto_op_t *op = ops[i];
- kd = (ipsecmb_aes_key_data_t *) imbm->key_data[op->key_index];
- __m128i iv;
+ const u32 n = (n_ops > burst_sz) ? burst_sz : n_ops;
- job = IMB_GET_NEXT_JOB (ptd->mgr);
+ for (i = 0; i < n; i++)
+ {
+ ipsecmb_aes_key_data_t *kd;
+ vnet_crypto_op_t *op = ops[ops_index++];
+ kd = (ipsecmb_aes_key_data_t *) imbm->key_data[op->key_index];
- job->src = op->src;
- job->dst = op->dst;
- job->msg_len_to_cipher_in_bytes = op->len;
- job->cipher_start_src_offset_in_bytes = 0;
+ job = &ptd->burst_jobs[i];
- job->hash_alg = NULL_HASH;
- job->cipher_mode = cipher_mode;
- job->cipher_direction = direction;
- job->chain_order = (direction == ENCRYPT ? CIPHER_HASH : HASH_CIPHER);
+ job->src = op->src;
+ job->dst = op->dst;
+ job->msg_len_to_cipher_in_bytes = op->len;
+ job->cipher_start_src_offset_in_bytes = 0;
- if ((direction == ENCRYPT) && (op->flags & VNET_CRYPTO_OP_FLAG_INIT_IV))
- {
- iv = ptd->cbc_iv;
- _mm_storeu_si128 ((__m128i *) op->iv, iv);
- ptd->cbc_iv = _mm_aesenc_si128 (iv, iv);
- }
+ job->hash_alg = NULL_HASH;
- job->aes_key_len_in_bytes = key_len / 8;
- job->aes_enc_key_expanded = kd->enc_key_exp;
- job->aes_dec_key_expanded = kd->dec_key_exp;
- job->iv = op->iv;
- job->iv_len_in_bytes = AES_BLOCK_SIZE;
+ if ((direction == ENCRYPT) &&
+ (op->flags & VNET_CRYPTO_OP_FLAG_INIT_IV))
+ {
+ const __m128i iv = ptd->cbc_iv;
+ _mm_storeu_si128 ((__m128i *) op->iv, iv);
+ ptd->cbc_iv = _mm_aesenc_si128 (iv, iv);
+ }
- job->user_data = op;
+ job->aes_enc_key_expanded = kd->enc_key_exp;
+ job->aes_dec_key_expanded = kd->dec_key_exp;
+ job->iv = op->iv;
+ job->iv_len_in_bytes = AES_BLOCK_SIZE;
- job = IMB_SUBMIT_JOB (ptd->mgr);
+ job->user_data = op;
+ }
- if (job)
- ipsecmb_retire_cipher_job (job, &n_fail);
- }
+ IMB_SUBMIT_CIPHER_BURST_NOCHECK (ptd->mgr, ptd->burst_jobs, n,
+ cipher_mode, direction, key_len / 8);
+ for (i = 0; i < n; i++)
+ {
+ job = &ptd->burst_jobs[i];
+ ipsecmb_retire_cipher_job (job, &n_fail);
+ }
- while ((job = IMB_FLUSH_JOB (ptd->mgr)))
- ipsecmb_retire_cipher_job (job, &n_fail);
+ n_ops -= n;
+ }
- return n_ops - n_fail;
+ return ops_index - n_fail;
}
#define _(a, b, c) \
@@ -462,7 +474,7 @@ ipsecmb_ops_chacha_poly (vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops,
ipsecmb_main_t *imbm = &ipsecmb_main;
ipsecmb_per_thread_data_t *ptd =
vec_elt_at_index (imbm->per_thread_data, vm->thread_index);
- struct IMB_JOB *job;
+ struct JOB_AES_HMAC *job;
MB_MGR *m = ptd->mgr;
u32 i, n_fail = 0, last_key_index = ~0;
u8 scratch[VLIB_FRAME_SIZE][16];
@@ -779,6 +791,7 @@ crypto_ipsecmb_init (vlib_main_t * vm)
MB_MGR *m = 0;
u32 eidx;
u8 *name;
+ const u32 burst_jobs_sz = sizeof (JOB_AES_HMAC) * IMB_MAX_BURST_SIZE;
if (!clib_cpu_supports_aes ())
return 0;
@@ -797,7 +810,9 @@ crypto_ipsecmb_init (vlib_main_t * vm)
vec_foreach (ptd, imbm->per_thread_data)
{
ptd->mgr = alloc_mb_mgr (0);
- if (clib_cpu_supports_avx512f ())
+ memset (ptd->burst_jobs, 0, burst_jobs_sz);
+
+ if (clib_cpu_supports_avx512f ())
init_mb_mgr_avx512 (ptd->mgr);
else if (clib_cpu_supports_avx2 ())
init_mb_mgr_avx2 (ptd->mgr);