diff options
Diffstat (limited to 'src/plugins/dpdk/cryptodev')
-rw-r--r-- | src/plugins/dpdk/cryptodev/cryptodev.c | 55 | ||||
-rw-r--r-- | src/plugins/dpdk/cryptodev/cryptodev.h | 166 | ||||
-rw-r--r-- | src/plugins/dpdk/cryptodev/cryptodev_op_data_path.c | 206 | ||||
-rw-r--r-- | src/plugins/dpdk/cryptodev/cryptodev_raw_data_path.c | 165 |
4 files changed, 353 insertions, 239 deletions
diff --git a/src/plugins/dpdk/cryptodev/cryptodev.c b/src/plugins/dpdk/cryptodev/cryptodev.c index fa54d2bf4fc..c66e9ed9427 100644 --- a/src/plugins/dpdk/cryptodev/cryptodev.c +++ b/src/plugins/dpdk/cryptodev/cryptodev.c @@ -667,37 +667,66 @@ VLIB_CLI_COMMAND (show_cryptodev_assignment, static) = { }; static clib_error_t * -cryptodev_show_sw_rings_fn (vlib_main_t *vm, unformat_input_t *input, - vlib_cli_command_t *cmd) +cryptodev_show_cache_rings_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) { cryptodev_main_t *cmt = &cryptodev_main; u32 thread_index = 0; vec_foreach_index (thread_index, cmt->per_thread_data) { cryptodev_engine_thread_t *cet = cmt->per_thread_data + thread_index; + cryptodev_cache_ring_t *ring = &cet->cache_ring; + u16 head = ring->head; + u16 tail = ring->tail; + u16 n_cached = ((head == tail) && (ring->frames[head].f == 0)) ? + 0 : + ((head == tail) && (ring->frames[head].f != 0)) ? + (CRYPTODEV_CACHE_QUEUE_MASK + 1) : + (head > tail) ? + (head - tail) : + (CRYPTODEV_CACHE_QUEUE_MASK - tail + head); + + u16 enq_head = ring->enq_head; + u16 deq_tail = ring->deq_tail; + u16 n_frames_inflight = + ((enq_head == deq_tail) && (ring->frames[enq_head].f == 0)) ? + 0 : + ((enq_head == deq_tail) && (ring->frames[enq_head].f != 0)) ? + CRYPTODEV_CACHE_QUEUE_MASK + 1 : + (enq_head > deq_tail) ? + (enq_head - deq_tail) : + (CRYPTODEV_CACHE_QUEUE_MASK - deq_tail + enq_head); + + u16 n_frames_processed = + ((tail == deq_tail) && (ring->frames[deq_tail].f == 0)) ? + 0 : + ((tail == deq_tail) && (ring->frames[deq_tail].f != 0)) ? + (CRYPTODEV_CACHE_QUEUE_MASK + 1) : + (deq_tail > tail) ? (deq_tail - tail) : + (CRYPTODEV_CACHE_QUEUE_MASK - tail + deq_tail); + if (vlib_num_workers () > 0 && thread_index == 0) continue; vlib_cli_output (vm, "\n\n"); - vlib_cli_output (vm, "Frames total: %d", cet->frames_on_ring); - vlib_cli_output (vm, "Frames pending in a ring: %d", - cet->frames_on_ring - cet->enqueued_not_dequeueq - - cet->deqeued_not_returned); + vlib_cli_output (vm, "Frames total: %d", n_cached); + vlib_cli_output (vm, "Frames pending in the ring: %d", + n_cached - n_frames_inflight - n_frames_processed); vlib_cli_output (vm, "Frames enqueued but not dequeued: %d", - cet->enqueued_not_dequeueq); + n_frames_inflight); vlib_cli_output (vm, "Frames dequed but not returned: %d", - cet->deqeued_not_returned); + n_frames_processed); vlib_cli_output (vm, "inflight: %d", cet->inflight); - vlib_cli_output (vm, "Head: %d", cet->frame_ring.head); - vlib_cli_output (vm, "Tail: %d", cet->frame_ring.tail); + vlib_cli_output (vm, "Head: %d", ring->head); + vlib_cli_output (vm, "Tail: %d", ring->tail); vlib_cli_output (vm, "\n\n"); } return 0; } VLIB_CLI_COMMAND (show_cryptodev_sw_rings, static) = { - .path = "show cryptodev sw-ring status", - .short_help = "show status of all cryptodev software rings", - .function = cryptodev_show_sw_rings_fn, + .path = "show cryptodev cache status", + .short_help = "show status of all cryptodev cache rings", + .function = cryptodev_show_cache_rings_fn, }; static clib_error_t * diff --git a/src/plugins/dpdk/cryptodev/cryptodev.h b/src/plugins/dpdk/cryptodev/cryptodev.h index 1dc5e03512a..e7bdfc1bddf 100644 --- a/src/plugins/dpdk/cryptodev/cryptodev.h +++ b/src/plugins/dpdk/cryptodev/cryptodev.h @@ -156,26 +156,51 @@ typedef struct typedef struct { vnet_crypto_async_frame_t *f; + union + { + struct + { + /* index of frame elt where enque to + * the crypto engine is happening */ + u8 enq_elts_head; + /* index of the frame elt where dequeue + * from the crypto engine is happening */ + u8 deq_elts_tail; + u8 elts_inflight; - u8 enqueued; - u8 dequeued; - u8 deq_state; - u8 frame_inflight; + u8 op_type; + u8 aad_len; + u8 n_elts; + u16 reserved; + }; + u64 raw; + }; - u8 op_type; - u8 aad_len; - u8 n_elts; - u8 reserved; -} cryptodev_async_ring_elt; + u64 frame_elts_errs_mask; +} cryptodev_cache_ring_elt_t; typedef struct { - cryptodev_async_ring_elt frames[VNET_CRYPTO_FRAME_POOL_SIZE]; - uint16_t head; - uint16_t tail; - uint16_t enq; /*record the frame currently being enqueued */ - uint16_t deq; /*record the frame currently being dequeued */ -} cryptodev_async_frame_sw_ring; + cryptodev_cache_ring_elt_t frames[VNET_CRYPTO_FRAME_POOL_SIZE]; + + union + { + struct + { + /* head of the cache ring */ + u16 head; + /* tail of the cache ring */ + u16 tail; + /* index of the frame where enqueue + * to the crypto engine is happening */ + u16 enq_head; + /* index of the frame where dequeue + * from the crypto engine is happening */ + u16 deq_tail; + }; + u64 raw; + }; +} cryptodev_cache_ring_t; typedef struct { @@ -194,13 +219,9 @@ typedef struct }; }; - cryptodev_async_frame_sw_ring frame_ring; + cryptodev_cache_ring_t cache_ring; u16 cryptodev_id; u16 cryptodev_q; - u16 frames_on_ring; - u16 enqueued_not_dequeueq; - u16 deqeued_not_returned; - u16 pending_to_qat; u16 inflight; } cryptodev_engine_thread_t; @@ -224,16 +245,107 @@ typedef struct extern cryptodev_main_t cryptodev_main; +#define CRYPTODEV_CACHE_RING_GET_FRAME(r, i) \ + ((r)->frames[(i) &CRYPTODEV_CACHE_QUEUE_MASK].f) + +#define CRYPTODEV_CACHE_RING_GET_ERR_MASK(r, i) \ + ((r)->frames[(i) &CRYPTODEV_CACHE_QUEUE_MASK].frame_elts_errs_mask) + +#define CRYPTODEV_CACHE_RING_GET_FRAME_ELTS_INFLIGHT(r, i) \ + (((r)->frames[(i) &CRYPTODEV_CACHE_QUEUE_MASK].enq_elts_head) - \ + ((r)->frames[(i) &CRYPTODEV_CACHE_QUEUE_MASK].deq_elts_tail)) + static_always_inline void -cryptodev_mark_frame_err_status (vnet_crypto_async_frame_t *f, - vnet_crypto_op_status_t s, - vnet_crypto_async_frame_state_t fs) +cryptodev_cache_ring_update_enq_head (cryptodev_cache_ring_t *r, + vnet_crypto_async_frame_t *f) +{ + if (r->frames[r->enq_head].enq_elts_head == f->n_elts) + { + r->enq_head++; + r->enq_head &= CRYPTODEV_CACHE_QUEUE_MASK; + f->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED; + } +} + +static_always_inline bool +cryptodev_cache_ring_update_deq_tail (cryptodev_cache_ring_t *r, + u16 *const deq) { - u32 n_elts = f->n_elts, i; + if (r->frames[*deq].deq_elts_tail == r->frames[*deq].n_elts) + { + *deq += 1; + *deq &= CRYPTODEV_CACHE_QUEUE_MASK; + return 1; + } + + return 0; +} +static_always_inline u64 +cryptodev_mark_frame_fill_err (vnet_crypto_async_frame_t *f, u64 current_err, + u16 index, u16 n, vnet_crypto_op_status_t op_s) +{ + u64 err = current_err; + u16 i; + + ERROR_ASSERT (index + n <= VNET_CRYPTO_FRAME_SIZE); + ERROR_ASSERT (op_s != VNET_CRYPTO_OP_STATUS_COMPLETED); + + for (i = index; i < (index + n); i++) + f->elts[i].status = op_s; + + err |= (~(~(0u) << n) << index); + + return err; +} + +static_always_inline cryptodev_cache_ring_elt_t * +cryptodev_cache_ring_push (cryptodev_cache_ring_t *r, + vnet_crypto_async_frame_t *f) +{ + u16 head = r->head; + cryptodev_cache_ring_elt_t *ring_elt = &r->frames[head]; + /** + * in debug mode we do the ring sanity test when a frame is enqueued to + * the ring. + **/ +#if CLIB_DEBUG > 0 + u16 tail = r->tail; + u16 n_cached = (head >= tail) ? (head - tail) : + (CRYPTODEV_CACHE_QUEUE_MASK - tail + head); + ERROR_ASSERT (n_cached < VNET_CRYPTO_FRAME_POOL_SIZE); + ERROR_ASSERT (r->raw == 0 && r->frames[head].raw == 0 && + r->frames[head].f == 0); +#endif + ring_elt->f = f; + ring_elt->n_elts = f->n_elts; + /* update head */ + r->head++; + r->head &= CRYPTODEV_CACHE_QUEUE_MASK; + return ring_elt; +} + +static_always_inline vnet_crypto_async_frame_t * +cryptodev_cache_ring_pop (cryptodev_cache_ring_t *r) +{ + vnet_crypto_async_frame_t *f; + u16 tail = r->tail; + cryptodev_cache_ring_elt_t *ring_elt = &r->frames[tail]; + + ERROR_ASSERT (r->frames[r->head].raw == 0 ? r->head != tail : 1); + ERROR_ASSERT (r->frames[tail].raw != 0); + ERROR_ASSERT (ring_elt->deq_elts_tail == ring_elt->enq_elts_head && + ring_elt->deq_elts_tail == ring_elt->n_elts); + + f = CRYPTODEV_CACHE_RING_GET_FRAME (r, tail); + f->state = CRYPTODEV_CACHE_RING_GET_ERR_MASK (r, r->tail) == 0 ? + VNET_CRYPTO_FRAME_STATE_SUCCESS : + VNET_CRYPTO_FRAME_STATE_ELT_ERROR; + + clib_memset (ring_elt, 0, sizeof (*ring_elt)); + r->tail++; + r->tail &= CRYPTODEV_CACHE_QUEUE_MASK; - for (i = 0; i < n_elts; i++) - f->elts[i].status = s; - f->state = fs; + return f; } int cryptodev_session_create (vlib_main_t *vm, vnet_crypto_key_index_t idx, diff --git a/src/plugins/dpdk/cryptodev/cryptodev_op_data_path.c b/src/plugins/dpdk/cryptodev/cryptodev_op_data_path.c index 56b91056e23..4e4295a9041 100644 --- a/src/plugins/dpdk/cryptodev/cryptodev_op_data_path.c +++ b/src/plugins/dpdk/cryptodev/cryptodev_op_data_path.c @@ -142,16 +142,14 @@ cryptodev_frame_linked_algs_enqueue (vlib_main_t *vm, { cryptodev_main_t *cmt = &cryptodev_main; cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index; - cryptodev_async_frame_sw_ring *ring = &cet->frame_ring; - cryptodev_async_ring_elt *ring_elt = &ring->frames[ring->head]; + cryptodev_cache_ring_t *ring = &cet->cache_ring; + ERROR_ASSERT (frame != 0); + ERROR_ASSERT (frame->n_elts > 0); + cryptodev_cache_ring_elt_t *ring_elt = + cryptodev_cache_ring_push (ring, frame); - cet->frames_on_ring++; - ring_elt->f = frame; - ring_elt->n_elts = frame->n_elts; ring_elt->aad_len = 1; ring_elt->op_type = (u8) op_type; - ring->head++; - ring->head &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1); return 0; } @@ -163,7 +161,8 @@ cryptodev_frame_linked_algs_enqueue_internal (vlib_main_t *vm, cryptodev_main_t *cmt = &cryptodev_main; clib_pmalloc_main_t *pm = vm->physmem_main.pmalloc_main; cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index; - cryptodev_async_frame_sw_ring *ring = &cet->frame_ring; + cryptodev_cache_ring_t *ring = &cet->cache_ring; + u16 *const enq = &ring->enq_head; vnet_crypto_async_frame_elt_t *fe; cryptodev_session_t *sess = 0; cryptodev_op_t *cops[CRYPTODE_ENQ_MAX] = {}; @@ -177,7 +176,7 @@ cryptodev_frame_linked_algs_enqueue_internal (vlib_main_t *vm, return; max_to_enq = clib_min (CRYPTODE_ENQ_MAX, - frame->n_elts - ring->frames[ring->enq].enqueued); + frame->n_elts - ring->frames[*enq].enq_elts_head); if (cet->inflight + max_to_enq > CRYPTODEV_MAX_INFLIGHT) return; @@ -187,14 +186,18 @@ cryptodev_frame_linked_algs_enqueue_internal (vlib_main_t *vm, if (PREDICT_FALSE ( rte_mempool_get_bulk (cet->cop_pool, (void **) cops, n_elts) < 0)) { - cryptodev_mark_frame_err_status (frame, - VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR, - VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED); + cryptodev_mark_frame_fill_err ( + frame, ring->frames[*enq].frame_elts_errs_mask, + ring->frames[*enq].enq_elts_head, max_to_enq, + VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); + ring->frames[*enq].enq_elts_head += max_to_enq; + ring->frames[*enq].deq_elts_tail += max_to_enq; + cryptodev_cache_ring_update_enq_head (ring, frame); return; } - fe = frame->elts + ring->frames[ring->enq].enqueued; - bi = frame->buffer_indices + ring->frames[ring->enq].enqueued; + fe = frame->elts + ring->frames[*enq].enq_elts_head; + bi = frame->buffer_indices + ring->frames[*enq].enq_elts_head; while (n_elts) { @@ -221,9 +224,10 @@ cryptodev_frame_linked_algs_enqueue_internal (vlib_main_t *vm, if (PREDICT_FALSE ( cryptodev_session_create (vm, last_key_index, 0) < 0)) { - cryptodev_mark_frame_err_status ( - frame, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR, - VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED); + cryptodev_mark_frame_fill_err ( + frame, ring->frames[*enq].frame_elts_errs_mask, + ring->frames[*enq].enq_elts_head, max_to_enq, + VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); goto error_exit; } } @@ -257,7 +261,7 @@ cryptodev_frame_linked_algs_enqueue_internal (vlib_main_t *vm, cryptodev_validate_mbuf (sop->m_src, b); clib_memcpy_fast (cop[0]->iv, fe->iv, 16); - ring->frames[ring->enq].enqueued++; + ring->frames[*enq].enq_elts_head++; cop++; bi++; fe++; @@ -267,21 +271,15 @@ cryptodev_frame_linked_algs_enqueue_internal (vlib_main_t *vm, n_enqueue = rte_cryptodev_enqueue_burst (cet->cryptodev_id, cet->cryptodev_q, (struct rte_crypto_op **) cops, max_to_enq); - ASSERT (n_enqueue == max_to_enq); + ERROR_ASSERT (n_enqueue == max_to_enq); cet->inflight += max_to_enq; - ring->frames[ring->enq].frame_inflight += max_to_enq; - if (ring->frames[ring->enq].enqueued == frame->n_elts) - { - cet->frame_ring.enq++; - cet->frame_ring.enq &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1); - frame->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED; - } - + cryptodev_cache_ring_update_enq_head (ring, frame); return; error_exit: - ring->enq++; - ring->enq &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1); + ring->frames[*enq].enq_elts_head += max_to_enq; + ring->frames[*enq].deq_elts_tail += max_to_enq; + cryptodev_cache_ring_update_enq_head (ring, frame); rte_mempool_put_bulk (cet->cop_pool, (void **) cops, max_to_enq); } @@ -292,16 +290,13 @@ cryptodev_frame_aead_enqueue (vlib_main_t *vm, { cryptodev_main_t *cmt = &cryptodev_main; cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index; - cryptodev_async_frame_sw_ring *ring = &cet->frame_ring; - cryptodev_async_ring_elt *ring_elt = &ring->frames[ring->head]; - cet->frames_on_ring++; - ring_elt->f = frame; - ring_elt->n_elts = frame->n_elts; + cryptodev_cache_ring_t *ring = &cet->cache_ring; + ERROR_ASSERT (frame != 0); + ERROR_ASSERT (frame->n_elts > 0); + cryptodev_cache_ring_elt_t *ring_elt = + cryptodev_cache_ring_push (ring, frame); ring_elt->aad_len = aad_len; ring_elt->op_type = (u8) op_type; - ring->head++; - ring->head &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1); - return 0; } @@ -312,7 +307,8 @@ cryptodev_aead_enqueue_internal (vlib_main_t *vm, { cryptodev_main_t *cmt = &cryptodev_main; cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index; - cryptodev_async_frame_sw_ring *ring = &cet->frame_ring; + cryptodev_cache_ring_t *ring = &cet->cache_ring; + u16 *const enq = &ring->enq_head; clib_pmalloc_main_t *pm = vm->physmem_main.pmalloc_main; vnet_crypto_async_frame_elt_t *fe; cryptodev_session_t *sess = 0; @@ -321,7 +317,7 @@ cryptodev_aead_enqueue_internal (vlib_main_t *vm, u32 *bi = 0; u32 n_enqueue = 0, n_elts; u32 last_key_index = ~0; - u16 left_to_enq = frame->n_elts - ring->frames[ring->enq].enqueued; + u16 left_to_enq = frame->n_elts - ring->frames[*enq].enq_elts_head; const u16 max_to_enq = clib_min (CRYPTODE_ENQ_MAX, left_to_enq); if (PREDICT_FALSE (frame == 0 || frame->n_elts == 0)) @@ -335,14 +331,18 @@ cryptodev_aead_enqueue_internal (vlib_main_t *vm, if (PREDICT_FALSE ( rte_mempool_get_bulk (cet->cop_pool, (void **) cops, n_elts) < 0)) { - cryptodev_mark_frame_err_status (frame, - VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR, - VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED); + cryptodev_mark_frame_fill_err ( + frame, ring->frames[*enq].frame_elts_errs_mask, + ring->frames[*enq].enq_elts_head, max_to_enq, + VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); + ring->frames[*enq].enq_elts_head += max_to_enq; + ring->frames[*enq].deq_elts_tail += max_to_enq; + cryptodev_cache_ring_update_enq_head (ring, frame); return -1; } - fe = frame->elts + ring->frames[ring->enq].enqueued; - bi = frame->buffer_indices + ring->frames[ring->enq].enqueued; + fe = frame->elts + ring->frames[*enq].enq_elts_head; + bi = frame->buffer_indices + ring->frames[*enq].enq_elts_head; while (n_elts) { @@ -367,9 +367,10 @@ cryptodev_aead_enqueue_internal (vlib_main_t *vm, if (PREDICT_FALSE (cryptodev_session_create (vm, last_key_index, aad_len) < 0)) { - cryptodev_mark_frame_err_status ( - frame, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR, - VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED); + cryptodev_mark_frame_fill_err ( + frame, ring->frames[*enq].frame_elts_errs_mask, + ring->frames[*enq].enq_elts_head, max_to_enq, + VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); goto error_exit; } } @@ -387,9 +388,10 @@ cryptodev_aead_enqueue_internal (vlib_main_t *vm, if (PREDICT_FALSE (cryptodev_session_create (vm, last_key_index, aad_len) < 0)) { - cryptodev_mark_frame_err_status ( - frame, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR, - VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED); + cryptodev_mark_frame_fill_err ( + frame, ring->frames[*enq].frame_elts_errs_mask, + ring->frames[*enq].enq_elts_head, max_to_enq, + VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); goto error_exit; } } @@ -426,7 +428,6 @@ cryptodev_aead_enqueue_internal (vlib_main_t *vm, clib_memcpy_fast (cop[0]->iv, fe->iv, 12); clib_memcpy_fast (cop[0]->aad, fe->aad, aad_len); - ring->frames[ring->enq].enqueued++; cop++; bi++; fe++; @@ -436,22 +437,17 @@ cryptodev_aead_enqueue_internal (vlib_main_t *vm, n_enqueue = rte_cryptodev_enqueue_burst (cet->cryptodev_id, cet->cryptodev_q, (struct rte_crypto_op **) cops, max_to_enq); - ASSERT (n_enqueue == max_to_enq); + ERROR_ASSERT (n_enqueue == max_to_enq); cet->inflight += max_to_enq; - ring->frames[ring->enq].frame_inflight += max_to_enq; - if (ring->frames[ring->enq].enqueued == frame->n_elts) - { - ring->enq++; - ring->enq &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1); - frame->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED; - cet->enqueued_not_dequeueq++; - } + ring->frames[*enq].enq_elts_head += max_to_enq; + cryptodev_cache_ring_update_enq_head (ring, frame); return 0; error_exit: - ring->enq++; - ring->enq &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1); + ring->frames[*enq].enq_elts_head += max_to_enq; + ring->frames[*enq].deq_elts_tail += max_to_enq; + cryptodev_cache_ring_update_enq_head (ring, frame); rte_mempool_put_bulk (cet->cop_pool, (void **) cops, max_to_enq); return -1; @@ -464,7 +460,8 @@ cryptodev_frame_dequeue_internal (vlib_main_t *vm, u32 *nb_elts_processed, cryptodev_main_t *cmt = &cryptodev_main; cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index; vnet_crypto_async_frame_t *frame = NULL; - cryptodev_async_frame_sw_ring *ring = &cet->frame_ring; + cryptodev_cache_ring_t *ring = &cet->cache_ring; + u16 *const deq = &ring->deq_tail; u16 n_deq, idx, left_to_deq, i; u16 max_to_deq = 0; u16 inflight = cet->inflight; @@ -472,24 +469,27 @@ cryptodev_frame_dequeue_internal (vlib_main_t *vm, u32 *nb_elts_processed, cryptodev_op_t *cops[CRYPTODE_DEQ_MAX] = {}; cryptodev_op_t **cop = cops; vnet_crypto_async_frame_elt_t *fe; - u32 n_elts; - u32 ss0 = 0, ss1 = 0, ss2 = 0, ss3 = 0; /* sum of status */ + u32 n_elts, n; + u64 err0 = 0, err1 = 0, err2 = 0, err3 = 0; /* partial errors mask */ - idx = ring->deq; + idx = ring->deq_tail; for (i = 0; i < VNET_CRYPTO_FRAME_POOL_SIZE; i++) { - if (PREDICT_TRUE (ring->frames[idx].frame_inflight > 0)) + u32 frame_inflight = + CRYPTODEV_CACHE_RING_GET_FRAME_ELTS_INFLIGHT (ring, idx); + + if (PREDICT_TRUE (frame_inflight > 0)) break; idx++; idx &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1); } - ASSERT (i != VNET_CRYPTO_FRAME_POOL_SIZE); - ring->deq = idx; + ERROR_ASSERT (i != VNET_CRYPTO_FRAME_POOL_SIZE); + ring->deq_tail = idx; left_to_deq = - ring->frames[ring->deq].f->n_elts - ring->frames[ring->deq].dequeued; + ring->frames[*deq].f->n_elts - ring->frames[*deq].deq_elts_tail; max_to_deq = clib_min (left_to_deq, CRYPTODE_DEQ_MAX); /* deq field can be used to track frame that is currently dequeued @@ -502,55 +502,50 @@ cryptodev_frame_dequeue_internal (vlib_main_t *vm, u32 *nb_elts_processed, if (n_deq == 0) return dequeue_more; - ss0 = ring->frames[ring->deq].deq_state; - ss1 = ring->frames[ring->deq].deq_state; - ss2 = ring->frames[ring->deq].deq_state; - ss3 = ring->frames[ring->deq].deq_state; - - frame = ring->frames[ring->deq].f; - fe = frame->elts + ring->frames[ring->deq].dequeued; + frame = ring->frames[*deq].f; + fe = frame->elts + ring->frames[*deq].deq_elts_tail; n_elts = n_deq; + n = ring->frames[*deq].deq_elts_tail; + while (n_elts > 4) { - ss0 |= fe[0].status = cryptodev_status_conversion[cop[0]->op.status]; - ss1 |= fe[1].status = cryptodev_status_conversion[cop[1]->op.status]; - ss2 |= fe[2].status = cryptodev_status_conversion[cop[2]->op.status]; - ss3 |= fe[3].status = cryptodev_status_conversion[cop[3]->op.status]; + fe[0].status = cryptodev_status_conversion[cop[0]->op.status]; + fe[1].status = cryptodev_status_conversion[cop[1]->op.status]; + fe[2].status = cryptodev_status_conversion[cop[2]->op.status]; + fe[3].status = cryptodev_status_conversion[cop[3]->op.status]; + + err0 |= (fe[0].status == VNET_CRYPTO_OP_STATUS_COMPLETED) << n; + err1 |= (fe[1].status == VNET_CRYPTO_OP_STATUS_COMPLETED) << (n + 1); + err2 |= (fe[2].status == VNET_CRYPTO_OP_STATUS_COMPLETED) << (n + 2); + err3 |= (fe[3].status == VNET_CRYPTO_OP_STATUS_COMPLETED) << (n + 3); cop += 4; fe += 4; n_elts -= 4; + n += 4; } while (n_elts) { - ss0 |= fe[0].status = cryptodev_status_conversion[cop[0]->op.status]; + fe[0].status = cryptodev_status_conversion[cop[0]->op.status]; + err0 |= (fe[0].status == VNET_CRYPTO_OP_STATUS_COMPLETED) << n; + n++; fe++; cop++; n_elts--; } - ring->frames[ring->deq].deq_state |= (u8) (ss0 | ss1 | ss2 | ss3); + ring->frames[*deq].frame_elts_errs_mask |= (err0 | err1 | err2 | err3); rte_mempool_put_bulk (cet->cop_pool, (void **) cops, n_deq); inflight -= n_deq; - ring->frames[ring->deq].dequeued += n_deq; - ring->frames[ring->deq].frame_inflight -= n_deq; - if (ring->frames[ring->deq].dequeued == ring->frames[ring->deq].n_elts) + ring->frames[*deq].deq_elts_tail += n_deq; + if (cryptodev_cache_ring_update_deq_tail (ring, deq)) { - frame->state = - (ss0 | ss1 | ss2 | ss3) == VNET_CRYPTO_OP_STATUS_COMPLETED ? - VNET_CRYPTO_FRAME_STATE_SUCCESS : - VNET_CRYPTO_FRAME_STATE_ELT_ERROR; - *nb_elts_processed = frame->n_elts; *enqueue_thread_idx = frame->enqueue_thread_index; - cet->deqeued_not_returned++; - cet->enqueued_not_dequeueq--; - ring->deq++; - ring->deq &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1); dequeue_more = (max_to_deq < CRYPTODE_DEQ_MAX); } @@ -559,7 +554,7 @@ cryptodev_frame_dequeue_internal (vlib_main_t *vm, u32 *nb_elts_processed, } static_always_inline void -cryptodev_enqueue_frame (vlib_main_t *vm, cryptodev_async_ring_elt *ring_elt) +cryptodev_enqueue_frame (vlib_main_t *vm, cryptodev_cache_ring_elt_t *ring_elt) { cryptodev_op_type_t op_type = (cryptodev_op_type_t) ring_elt->op_type; u8 linked_or_aad_len = ring_elt->aad_len; @@ -578,9 +573,9 @@ cryptodev_frame_dequeue (vlib_main_t *vm, u32 *nb_elts_processed, cryptodev_main_t *cmt = &cryptodev_main; vnet_crypto_main_t *cm = &crypto_main; cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index; + cryptodev_cache_ring_t *ring = &cet->cache_ring; + cryptodev_cache_ring_elt_t *ring_elt = &ring->frames[ring->tail]; - cryptodev_async_frame_sw_ring *ring = &cet->frame_ring; - cryptodev_async_ring_elt *ring_elt = &ring->frames[ring->tail]; vnet_crypto_async_frame_t *ret_frame = 0; u8 dequeue_more = 1; @@ -590,23 +585,16 @@ cryptodev_frame_dequeue (vlib_main_t *vm, u32 *nb_elts_processed, enqueue_thread_idx); } - if (PREDICT_TRUE (ring->frames[ring->enq].f != 0)) - cryptodev_enqueue_frame (vm, &ring->frames[ring->enq]); + if (PREDICT_TRUE (ring->frames[ring->enq_head].f != 0)) + cryptodev_enqueue_frame (vm, &ring->frames[ring->enq_head]); if (PREDICT_TRUE (ring_elt->f != 0)) { - if ((ring_elt->f->state == VNET_CRYPTO_FRAME_STATE_SUCCESS || - ring_elt->f->state == VNET_CRYPTO_FRAME_STATE_ELT_ERROR) && - ring_elt->enqueued == ring_elt->dequeued) + if (ring_elt->enq_elts_head == ring_elt->deq_elts_tail) { vlib_node_set_interrupt_pending ( vlib_get_main_by_index (vm->thread_index), cm->crypto_node_index); - ret_frame = ring_elt->f; - memset (ring_elt, 0, sizeof (*ring_elt)); - ring->tail += 1; - ring->tail &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1); - cet->frames_on_ring--; - cet->deqeued_not_returned--; + ret_frame = cryptodev_cache_ring_pop (ring); return ret_frame; } } diff --git a/src/plugins/dpdk/cryptodev/cryptodev_raw_data_path.c b/src/plugins/dpdk/cryptodev/cryptodev_raw_data_path.c index 9f0936a4081..19291eb3b59 100644 --- a/src/plugins/dpdk/cryptodev/cryptodev_raw_data_path.c +++ b/src/plugins/dpdk/cryptodev/cryptodev_raw_data_path.c @@ -96,7 +96,7 @@ cryptodev_reset_ctx (cryptodev_engine_thread_t *cet) { union rte_cryptodev_session_ctx sess_ctx; - ASSERT (cet->reset_sess != 0); + ERROR_ASSERT (cet->reset_sess != 0); sess_ctx.crypto_sess = cet->reset_sess; @@ -112,15 +112,14 @@ cryptodev_frame_linked_algs_enqueue (vlib_main_t *vm, { cryptodev_main_t *cmt = &cryptodev_main; cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index; - cryptodev_async_frame_sw_ring *ring = &cet->frame_ring; - cryptodev_async_ring_elt *ring_elt = &ring->frames[ring->head]; - cet->frames_on_ring++; - ring_elt->f = frame; - ring_elt->n_elts = frame->n_elts; + cryptodev_cache_ring_t *ring = &cet->cache_ring; + ERROR_ASSERT (frame != 0); + ERROR_ASSERT (frame->n_elts > 0); + cryptodev_cache_ring_elt_t *ring_elt = + cryptodev_cache_ring_push (ring, frame); + ring_elt->aad_len = 1; ring_elt->op_type = (u8) op_type; - ring->head++; - ring->head &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1); return 0; } @@ -135,13 +134,14 @@ cryptodev_frame_linked_algs_enqueue_internal (vlib_main_t *vm, vlib_buffer_t **b; struct rte_crypto_vec vec[CRYPTODEV_MAX_N_SGL]; struct rte_crypto_va_iova_ptr iv_vec, digest_vec; - cryptodev_async_frame_sw_ring *ring = &cet->frame_ring; + cryptodev_cache_ring_t *ring = &cet->cache_ring; + u16 *const enq = &ring->enq_head; u32 n_elts; u32 last_key_index = ~0; i16 min_ofs; u32 max_end; u32 max_to_enq = clib_min (CRYPTODE_ENQ_MAX, - frame->n_elts - ring->frames[ring->enq].enqueued); + frame->n_elts - ring->frames[*enq].enq_elts_head); u8 is_update = 0; int status; @@ -152,8 +152,8 @@ cryptodev_frame_linked_algs_enqueue_internal (vlib_main_t *vm, vlib_get_buffers (vm, frame->buffer_indices, cet->b, frame->n_elts); - b = cet->b + ring->frames[ring->enq].enqueued; - fe = frame->elts + ring->frames[ring->enq].enqueued; + b = cet->b + ring->frames[*enq].enq_elts_head; + fe = frame->elts + ring->frames[*enq].enq_elts_head; while (n_elts) { @@ -233,7 +233,7 @@ cryptodev_frame_linked_algs_enqueue_internal (vlib_main_t *vm, if (PREDICT_FALSE (status < 0)) goto error_exit; - ring->frames[ring->enq].enqueued += 1; + ring->frames[*enq].enq_elts_head += 1; b++; fe++; n_elts--; @@ -244,22 +244,19 @@ cryptodev_frame_linked_algs_enqueue_internal (vlib_main_t *vm, goto error_exit; cet->inflight += max_to_enq; - ring->frames[ring->enq].frame_inflight += max_to_enq; - if (ring->frames[ring->enq].enqueued == frame->n_elts) - { - cet->frame_ring.enq += 1; - cet->frame_ring.enq &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1); - frame->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED; - } + cryptodev_cache_ring_update_enq_head (ring, frame); return; error_exit: - cryptodev_mark_frame_err_status (frame, - VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR, - VNET_CRYPTO_FRAME_STATE_ELT_ERROR); + cryptodev_mark_frame_fill_err (frame, + ring->frames[*enq].frame_elts_errs_mask, + ring->frames[*enq].enq_elts_head, max_to_enq, + VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); + ring->frames[*enq].enq_elts_head += max_to_enq; + ring->frames[*enq].deq_elts_tail += max_to_enq; + cryptodev_cache_ring_update_enq_head (ring, frame); cryptodev_reset_ctx (cet); - cet->frame_ring.enq += 1; - cet->frame_ring.enq &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1); + return; } @@ -269,15 +266,14 @@ cryptodev_raw_aead_enqueue (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, { cryptodev_main_t *cmt = &cryptodev_main; cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index; - cryptodev_async_frame_sw_ring *ring = &cet->frame_ring; - cryptodev_async_ring_elt *ring_elt = &ring->frames[ring->head]; - cet->frames_on_ring++; - ring_elt->f = frame; - ring_elt->n_elts = frame->n_elts; + cryptodev_cache_ring_t *ring = &cet->cache_ring; + ERROR_ASSERT (frame != 0); + ERROR_ASSERT (frame->n_elts > 0); + cryptodev_cache_ring_elt_t *ring_elt = + cryptodev_cache_ring_push (ring, frame); + ring_elt->aad_len = aad_len; ring_elt->op_type = (u8) op_type; - ring->head++; - ring->head &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1); return 0; } @@ -288,7 +284,7 @@ cryptodev_raw_aead_enqueue_internal (vlib_main_t *vm, { cryptodev_main_t *cmt = &cryptodev_main; cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index; - cryptodev_async_frame_sw_ring *ring = &cet->frame_ring; + cryptodev_cache_ring_t *ring = &cet->cache_ring; vnet_crypto_async_frame_elt_t *fe; vlib_buffer_t **b; u32 n_elts; @@ -296,7 +292,8 @@ cryptodev_raw_aead_enqueue_internal (vlib_main_t *vm, struct rte_crypto_vec vec[CRYPTODEV_MAX_N_SGL]; struct rte_crypto_va_iova_ptr iv_vec, digest_vec, aad_vec; u32 last_key_index = ~0; - u16 left_to_enq = frame->n_elts - ring->frames[ring->enq].enqueued; + u16 *const enq = &ring->enq_head; + u16 left_to_enq = frame->n_elts - ring->frames[*enq].enq_elts_head; u16 max_to_enq = clib_min (CRYPTODE_ENQ_MAX, left_to_enq); u8 is_update = 0; int status; @@ -310,8 +307,8 @@ cryptodev_raw_aead_enqueue_internal (vlib_main_t *vm, vlib_get_buffers (vm, frame->buffer_indices, cet->b, frame->n_elts); - fe = frame->elts + ring->frames[ring->enq].enqueued; - b = cet->b + ring->frames[ring->enq].enqueued; + fe = frame->elts + ring->frames[*enq].enq_elts_head; + b = cet->b + ring->frames[*enq].enq_elts_head; cofs.raw = 0; while (n_elts) @@ -424,7 +421,7 @@ cryptodev_raw_aead_enqueue_internal (vlib_main_t *vm, if (PREDICT_FALSE (status < 0)) goto error_exit; - ring->frames[ring->enq].enqueued += 1; + ring->frames[*enq].enq_elts_head += 1; fe++; b++; n_elts--; @@ -435,24 +432,18 @@ cryptodev_raw_aead_enqueue_internal (vlib_main_t *vm, goto error_exit; cet->inflight += max_to_enq; - ring->frames[ring->enq].frame_inflight += max_to_enq; - if (ring->frames[ring->enq].enqueued == frame->n_elts) - { - ring->enq += 1; - ring->enq &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1); - frame->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED; - cet->enqueued_not_dequeueq++; - } - + cryptodev_cache_ring_update_enq_head (ring, frame); return; error_exit: - cryptodev_mark_frame_err_status (frame, - VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR, - VNET_CRYPTO_FRAME_STATE_ELT_ERROR); + cryptodev_mark_frame_fill_err (frame, + ring->frames[*enq].frame_elts_errs_mask, + ring->frames[*enq].enq_elts_head, max_to_enq, + VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); + ring->frames[*enq].enq_elts_head += max_to_enq; + ring->frames[*enq].deq_elts_tail += max_to_enq; + cryptodev_cache_ring_update_enq_head (ring, frame); cryptodev_reset_ctx (cet); - ring->enq += 1; - ring->enq &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1); return; } @@ -472,7 +463,8 @@ cryptodev_raw_dequeue_internal (vlib_main_t *vm, u32 *nb_elts_processed, cryptodev_main_t *cmt = &cryptodev_main; cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index; vnet_crypto_async_frame_t *frame; - cryptodev_async_frame_sw_ring *ring = &cet->frame_ring; + cryptodev_cache_ring_t *ring = &cet->cache_ring; + u16 *const deq = &ring->deq_tail; u32 n_success; u16 n_deq, indice, i, left_to_deq; u16 max_to_deq = 0; @@ -480,22 +472,22 @@ cryptodev_raw_dequeue_internal (vlib_main_t *vm, u32 *nb_elts_processed, u8 dequeue_more = 0; int dequeue_status; - indice = ring->deq; + indice = *deq; for (i = 0; i < VNET_CRYPTO_FRAME_POOL_SIZE; i++) { - if (PREDICT_TRUE (ring->frames[indice].frame_inflight > 0)) + if (PREDICT_TRUE ( + CRYPTODEV_CACHE_RING_GET_FRAME_ELTS_INFLIGHT (ring, indice) > 0)) break; indice += 1; - indice &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1); + indice &= CRYPTODEV_CACHE_QUEUE_MASK; } - ASSERT (i != VNET_CRYPTO_FRAME_POOL_SIZE); + ERROR_ASSERT (i != VNET_CRYPTO_FRAME_POOL_SIZE); - ring->deq = indice; + *deq = indice; - left_to_deq = - ring->frames[ring->deq].f->n_elts - ring->frames[ring->deq].dequeued; + left_to_deq = ring->frames[*deq].n_elts - ring->frames[*deq].deq_elts_tail; max_to_deq = clib_min (left_to_deq, CRYPTODE_DEQ_MAX); /* you can use deq field to track frame that is currently dequeued */ @@ -506,37 +498,39 @@ cryptodev_raw_dequeue_internal (vlib_main_t *vm, u32 *nb_elts_processed, cet->ctx, NULL, max_to_deq, cryptodev_post_dequeue, (void **) &frame, 0, &n_success, &dequeue_status); - if (!n_deq) + if (n_deq == 0) return dequeue_more; inflight -= n_deq; - ring->frames[ring->deq].dequeued += n_deq; - ring->frames[ring->deq].deq_state += n_success; - ring->frames[ring->deq].frame_inflight -= n_deq; + if (PREDICT_FALSE (n_success < n_deq)) + { + u16 idx = ring->frames[*deq].deq_elts_tail; - if (ring->frames[ring->deq].dequeued == ring->frames[ring->deq].n_elts) + for (i = 0; i < n_deq; i++) + { + if (frame->elts[idx + i].status != VNET_CRYPTO_OP_STATUS_COMPLETED) + ring->frames[*deq].frame_elts_errs_mask |= 1 << (idx + i); + } + } + ring->frames[*deq].deq_elts_tail += n_deq; + + if (cryptodev_cache_ring_update_deq_tail (ring, deq)) { - frame->state = ring->frames[ring->deq].deq_state == frame->n_elts ? - VNET_CRYPTO_FRAME_STATE_SUCCESS : - VNET_CRYPTO_FRAME_STATE_ELT_ERROR; *nb_elts_processed = frame->n_elts; *enqueue_thread_idx = frame->enqueue_thread_index; - cet->deqeued_not_returned++; - cet->enqueued_not_dequeueq--; - ring->deq += 1; - ring->deq &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1); dequeue_more = max_to_deq < CRYPTODE_DEQ_MAX; } int res = rte_cryptodev_raw_dequeue_done (cet->ctx, cet->inflight - inflight); - ASSERT (res == 0); + ERROR_ASSERT (res == 0); cet->inflight = inflight; return dequeue_more; } static_always_inline void -cryptodev_enqueue_frame (vlib_main_t *vm, cryptodev_async_ring_elt *ring_elt) +cryptodev_enqueue_frame_to_qat (vlib_main_t *vm, + cryptodev_cache_ring_elt_t *ring_elt) { cryptodev_op_type_t op_type = (cryptodev_op_type_t) ring_elt->op_type; u8 linked_or_aad_len = ring_elt->aad_len; @@ -555,8 +549,8 @@ cryptodev_raw_dequeue (vlib_main_t *vm, u32 *nb_elts_processed, cryptodev_main_t *cmt = &cryptodev_main; vnet_crypto_main_t *cm = &crypto_main; cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index; - cryptodev_async_frame_sw_ring *ring = &cet->frame_ring; - cryptodev_async_ring_elt *ring_elt = &ring->frames[ring->tail]; + cryptodev_cache_ring_t *ring = &cet->cache_ring; + cryptodev_cache_ring_elt_t *ring_elt = &ring->frames[ring->tail]; vnet_crypto_async_frame_t *ret_frame = 0; u8 dequeue_more = 1; @@ -566,26 +560,17 @@ cryptodev_raw_dequeue (vlib_main_t *vm, u32 *nb_elts_processed, enqueue_thread_idx); } - if (PREDICT_TRUE (ring->frames[ring->enq].f != 0)) - cryptodev_enqueue_frame (vm, &ring->frames[ring->enq]); + if (PREDICT_TRUE (ring->frames[ring->enq_head].f != 0)) + cryptodev_enqueue_frame_to_qat (vm, &ring->frames[ring->enq_head]); if (PREDICT_TRUE (ring_elt->f != 0)) { - if ((ring_elt->f->state == VNET_CRYPTO_FRAME_STATE_SUCCESS || - ring_elt->f->state == VNET_CRYPTO_FRAME_STATE_ELT_ERROR) && - ring_elt->enqueued == ring_elt->dequeued) + if (ring_elt->enq_elts_head == ring_elt->deq_elts_tail) { vlib_node_set_interrupt_pending ( vlib_get_main_by_index (vm->thread_index), cm->crypto_node_index); - ret_frame = ring_elt->f; - ring_elt->f = 0; - ring_elt->dequeued = 0; - ring_elt->enqueued = 0; - ring_elt->deq_state = 0; - ring->tail += 1; - ring->tail &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1); - cet->frames_on_ring--; - cet->deqeued_not_returned--; + ret_frame = cryptodev_cache_ring_pop (ring); + return ret_frame; } } @@ -684,7 +669,7 @@ cryptodev_register_raw_hdl (vlib_main_t *vm, u32 eidx) { u32 thread_id = cet - cmt->per_thread_data; u32 numa = vlib_get_main_by_index (thread_id)->numa_node; - u8 *name = format (0, "cache_frame_ring_%u_%u", numa, thread_id); + u8 *name = format (0, "cache_cache_ring_%u_%u", numa, thread_id); cet->aad_buf = rte_zmalloc_socket ( 0, CRYPTODEV_NB_CRYPTO_OPS * CRYPTODEV_MAX_AAD_SIZE, |