diff options
Diffstat (limited to 'src/plugins')
| -rw-r--r-- | src/plugins/crypto_sw_scheduler/CMakeLists.txt | 17 | ||||
| -rw-r--r-- | src/plugins/crypto_sw_scheduler/crypto_sw_scheduler.h | 61 | ||||
| -rw-r--r-- | src/plugins/crypto_sw_scheduler/main.c | 713 | ||||
| -rw-r--r-- | src/plugins/dpdk/cryptodev/cryptodev.c | 6 |
4 files changed, 795 insertions, 2 deletions
diff --git a/src/plugins/crypto_sw_scheduler/CMakeLists.txt b/src/plugins/crypto_sw_scheduler/CMakeLists.txt new file mode 100644 index 00000000000..b94b8f8804c --- /dev/null +++ b/src/plugins/crypto_sw_scheduler/CMakeLists.txt @@ -0,0 +1,17 @@ +# Copyright (c) 2020 Intel and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_vpp_plugin(crypto_sw_scheduler + SOURCES + main.c +) diff --git a/src/plugins/crypto_sw_scheduler/crypto_sw_scheduler.h b/src/plugins/crypto_sw_scheduler/crypto_sw_scheduler.h new file mode 100644 index 00000000000..9db42ba18ce --- /dev/null +++ b/src/plugins/crypto_sw_scheduler/crypto_sw_scheduler.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2020 Intel and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <vnet/crypto/crypto.h> + +#ifndef __crypto_sw_scheduler_h__ +#define __crypto_sw_scheduler_h__ + +#define CRYPTO_SW_SCHEDULER_QUEUE_SIZE 64 +#define CRYPTO_SW_SCHEDULER_QUEUE_MASK (CRYPTO_SW_SCHEDULER_QUEUE_SIZE - 1) + +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + u32 head; + u32 tail; + vnet_crypto_async_frame_t *jobs[0]; +} crypto_sw_scheduler_queue_t; + +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + crypto_sw_scheduler_queue_t *queues[VNET_CRYPTO_ASYNC_OP_N_IDS]; + vnet_crypto_op_t *crypto_ops; + vnet_crypto_op_t *integ_ops; + vnet_crypto_op_t *chained_crypto_ops; + vnet_crypto_op_t *chained_integ_ops; + vnet_crypto_op_chunk_t *chunks; + u8 self_crypto_enabled; +} crypto_sw_scheduler_per_thread_data_t; + +typedef struct +{ + u32 crypto_engine_index; + crypto_sw_scheduler_per_thread_data_t *per_thread_data; + vnet_crypto_key_t *keys; +} crypto_sw_scheduler_main_t; + +extern crypto_sw_scheduler_main_t crypto_sw_scheduler_main; + +#endif // __crypto_native_h__ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/crypto_sw_scheduler/main.c b/src/plugins/crypto_sw_scheduler/main.c new file mode 100644 index 00000000000..8f27fefe29a --- /dev/null +++ b/src/plugins/crypto_sw_scheduler/main.c @@ -0,0 +1,713 @@ +/* + * Copyright (c) 2020 Intel and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <vlib/vlib.h> +#include <vnet/plugin/plugin.h> +#include <vpp/app/version.h> + +#include "crypto_sw_scheduler.h" + +int +crypto_sw_scheduler_set_worker_crypto (u32 worker_idx, u8 enabled) +{ + crypto_sw_scheduler_main_t *cm = &crypto_sw_scheduler_main; + vlib_thread_main_t *tm = vlib_get_thread_main (); + crypto_sw_scheduler_per_thread_data_t *ptd = 0; + u32 count = 0, i = vlib_num_workers () > 0; + + if (worker_idx >= vlib_num_workers ()) + { + return VNET_API_ERROR_INVALID_VALUE; + } + + for (; i < tm->n_vlib_mains; i++) + { + ptd = cm->per_thread_data + i; + count += ptd->self_crypto_enabled; + } + + if (enabled || count > 1) + { + cm->per_thread_data[vlib_get_worker_thread_index + (worker_idx)].self_crypto_enabled = enabled; + } + else /* cannot disable all crypto workers */ + { + return VNET_API_ERROR_INVALID_VALUE_2; + } + return 0; +} + +static void +crypto_sw_scheduler_key_handler (vlib_main_t * vm, vnet_crypto_key_op_t kop, + vnet_crypto_key_index_t idx) +{ + crypto_sw_scheduler_main_t *cm = &crypto_sw_scheduler_main; + vnet_crypto_key_t *key = vnet_crypto_get_key (idx); + + vec_validate (cm->keys, idx); + + if (key->type == VNET_CRYPTO_KEY_TYPE_LINK) + { + if (kop == VNET_CRYPTO_KEY_OP_DEL) + { + cm->keys[idx].index_crypto = UINT32_MAX; + cm->keys[idx].index_integ = UINT32_MAX; + } + else + { + cm->keys[idx] = *key; + } + } +} + +static int +crypto_sw_scheduler_frame_enqueue (vlib_main_t * vm, + vnet_crypto_async_frame_t * frame) +{ + crypto_sw_scheduler_main_t *cm = &crypto_sw_scheduler_main; + crypto_sw_scheduler_per_thread_data_t *ptd + = vec_elt_at_index (cm->per_thread_data, vm->thread_index); + crypto_sw_scheduler_queue_t *q = ptd->queues[frame->op]; + u64 head = q->head; + + if (q->jobs[head & CRYPTO_SW_SCHEDULER_QUEUE_MASK]) + { + u32 n_elts = frame->n_elts, i; + for (i = 0; i < n_elts; i++) + frame->elts[i].status = VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR; + frame->state = VNET_CRYPTO_FRAME_STATE_ELT_ERROR; + return -1; + } + frame->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED; + q->jobs[head & CRYPTO_SW_SCHEDULER_QUEUE_MASK] = frame; + head += 1; + CLIB_MEMORY_STORE_BARRIER (); + q->head = head; + return 0; +} + +static_always_inline vnet_crypto_async_frame_t * +crypto_sw_scheduler_get_pending_frame (crypto_sw_scheduler_queue_t * q) +{ + vnet_crypto_async_frame_t *f; + u32 i; + u32 tail = q->tail; + u32 head = q->head; + + for (i = tail; i < head; i++) + { + f = q->jobs[i & CRYPTO_SW_SCHEDULER_QUEUE_MASK]; + if (!f) + continue; + if (clib_atomic_bool_cmp_and_swap + (&f->state, VNET_CRYPTO_FRAME_STATE_PENDING, + VNET_CRYPTO_FRAME_STATE_WORK_IN_PROGRESS)) + { + return f; + } + } + return NULL; +} + +static_always_inline vnet_crypto_async_frame_t * +crypto_sw_scheduler_get_completed_frame (crypto_sw_scheduler_queue_t * q) +{ + vnet_crypto_async_frame_t *f = 0; + if (q->jobs[q->tail & CRYPTO_SW_SCHEDULER_QUEUE_MASK] + && q->jobs[q->tail & CRYPTO_SW_SCHEDULER_QUEUE_MASK]->state + >= VNET_CRYPTO_FRAME_STATE_SUCCESS) + { + u32 tail = q->tail; + CLIB_MEMORY_STORE_BARRIER (); + q->tail++; + f = q->jobs[tail & CRYPTO_SW_SCHEDULER_QUEUE_MASK]; + q->jobs[tail & CRYPTO_SW_SCHEDULER_QUEUE_MASK] = 0; + } + return f; +} + +static_always_inline void +cryptodev_sw_scheduler_sgl (vlib_main_t * vm, + crypto_sw_scheduler_per_thread_data_t * ptd, + vlib_buffer_t * b, vnet_crypto_op_t * op, + i32 offset, i32 len) +{ + vnet_crypto_op_chunk_t *ch; + vlib_buffer_t *nb = b; + u32 n_chunks = 0; + u32 chunk_index = vec_len (ptd->chunks); + + op->flags |= VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS; + + while (len) + { + if (nb->current_data + nb->current_length > offset) + { + vec_add2 (ptd->chunks, ch, 1); + ch->src = ch->dst = nb->data + offset; + ch->len + = clib_min (nb->current_data + nb->current_length - offset, len); + len -= ch->len; + offset = 0; + n_chunks++; + if (!len) + break; + } + if (offset) + offset -= nb->current_data + nb->current_length; + if (nb->flags & VLIB_BUFFER_NEXT_PRESENT) + nb = vlib_get_buffer (vm, nb->next_buffer); + else + break; + } + + ASSERT (offset == 0 && len == 0); + op->chunk_index = chunk_index; + op->n_chunks = n_chunks; +} + +static_always_inline void +crypto_sw_scheduler_convert_aead (vlib_main_t * vm, + crypto_sw_scheduler_per_thread_data_t * ptd, + vnet_crypto_async_frame_elt_t * fe, + u32 index, u32 bi, + vnet_crypto_op_id_t op_id, u16 aad_len, + u8 tag_len) +{ + vlib_buffer_t *b = vlib_get_buffer (vm, bi); + vnet_crypto_op_t *op = 0; + + if (fe->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS) + { + vec_add2 (ptd->chained_crypto_ops, op, 1); + cryptodev_sw_scheduler_sgl (vm, ptd, b, op, fe->crypto_start_offset, + fe->crypto_total_length); + } + else + { + vec_add2 (ptd->crypto_ops, op, 1); + op->src = op->dst = b->data + fe->crypto_start_offset; + op->len = fe->crypto_total_length; + } + + op->op = op_id; + op->tag = fe->tag; + op->flags = fe->flags; + op->key_index = fe->key_index; + op->iv = fe->iv; + op->aad = fe->aad; + op->aad_len = aad_len; + op->tag_len = tag_len; + op->user_data = index; +} + +static_always_inline void +crypto_sw_scheduler_convert_link_crypto (vlib_main_t * vm, + crypto_sw_scheduler_per_thread_data_t + * ptd, vnet_crypto_key_t * key, + vnet_crypto_async_frame_elt_t * fe, + u32 index, u32 bi, + vnet_crypto_op_id_t crypto_op_id, + vnet_crypto_op_id_t integ_op_id, + u32 digest_len, u8 is_enc) +{ + vlib_buffer_t *b = vlib_get_buffer (vm, bi); + vnet_crypto_op_t *crypto_op = 0, *integ_op = 0; + + if (fe->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS) + { + vec_add2 (ptd->chained_crypto_ops, crypto_op, 1); + vec_add2 (ptd->chained_integ_ops, integ_op, 1); + cryptodev_sw_scheduler_sgl (vm, ptd, b, crypto_op, + fe->crypto_start_offset, + fe->crypto_total_length); + cryptodev_sw_scheduler_sgl (vm, ptd, b, integ_op, + fe->integ_start_offset, + fe->crypto_total_length + + fe->integ_length_adj); + } + else + { + vec_add2 (ptd->crypto_ops, crypto_op, 1); + vec_add2 (ptd->integ_ops, integ_op, 1); + crypto_op->src = crypto_op->dst = b->data + fe->crypto_start_offset; + crypto_op->len = fe->crypto_total_length; + integ_op->src = integ_op->dst = b->data + fe->integ_start_offset; + integ_op->len = fe->crypto_total_length + fe->integ_length_adj; + } + + crypto_op->op = crypto_op_id; + crypto_op->iv = fe->iv; + crypto_op->key_index = key->index_crypto; + crypto_op->user_data = 0; + integ_op->op = integ_op_id; + integ_op->digest = fe->digest; + integ_op->digest_len = digest_len; + integ_op->key_index = key->index_integ; + if (is_enc) + crypto_op->flags |= VNET_CRYPTO_OP_FLAG_INIT_IV; + else + integ_op->flags |= VNET_CRYPTO_OP_FLAG_HMAC_CHECK; + crypto_op->user_data = integ_op->user_data = index; +} + +static_always_inline void +process_ops (vlib_main_t * vm, vnet_crypto_async_frame_t * f, + vnet_crypto_op_t * ops, u8 * state) +{ + u32 n_fail, n_ops = vec_len (ops); + vnet_crypto_op_t *op = ops; + + if (n_ops == 0) + return; + + n_fail = n_ops - vnet_crypto_process_ops (vm, op, n_ops); + + while (n_fail) + { + ASSERT (op - ops < n_ops); + + if (op->status != VNET_CRYPTO_OP_STATUS_COMPLETED) + { + f->elts[op->user_data].status = op->status; + *state = VNET_CRYPTO_FRAME_STATE_ELT_ERROR; + n_fail--; + } + op++; + } +} + +static_always_inline void +process_chained_ops (vlib_main_t * vm, vnet_crypto_async_frame_t * f, + vnet_crypto_op_t * ops, vnet_crypto_op_chunk_t * chunks, + u8 * state) +{ + u32 n_fail, n_ops = vec_len (ops); + vnet_crypto_op_t *op = ops; + + if (n_ops == 0) + return; + + n_fail = n_ops - vnet_crypto_process_chained_ops (vm, op, chunks, n_ops); + + while (n_fail) + { + ASSERT (op - ops < n_ops); + + if (op->status != VNET_CRYPTO_OP_STATUS_COMPLETED) + { + f->elts[op->user_data].status = op->status; + *state = VNET_CRYPTO_FRAME_STATE_ELT_ERROR; + n_fail--; + } + op++; + } +} + +static_always_inline vnet_crypto_async_frame_t * +crypto_sw_scheduler_dequeue_aead (vlib_main_t * vm, + vnet_crypto_async_op_id_t async_op_id, + vnet_crypto_op_id_t sync_op_id, u8 tag_len, + u8 aad_len, u32 * nb_elts_processed, + u32 * enqueue_thread_idx) +{ + crypto_sw_scheduler_main_t *cm = &crypto_sw_scheduler_main; + crypto_sw_scheduler_per_thread_data_t *ptd = 0; + crypto_sw_scheduler_queue_t *q = 0; + vnet_crypto_async_frame_t *f = 0; + vnet_crypto_async_frame_elt_t *fe; + u32 *bi; + u32 n_elts; + int i = 0; + u8 state = VNET_CRYPTO_FRAME_STATE_SUCCESS; + + if (cm->per_thread_data[vm->thread_index].self_crypto_enabled) + { + /* *INDENT-OFF* */ + vec_foreach_index (i, cm->per_thread_data) + { + ptd = cm->per_thread_data + i; + q = ptd->queues[async_op_id]; + f = crypto_sw_scheduler_get_pending_frame (q); + if (f) + break; + } + /* *INDENT-ON* */ + } + + ptd = cm->per_thread_data + vm->thread_index; + + if (f) + { + *nb_elts_processed = n_elts = f->n_elts; + fe = f->elts; + bi = f->buffer_indices; + + vec_reset_length (ptd->crypto_ops); + vec_reset_length (ptd->chained_crypto_ops); + vec_reset_length (ptd->chunks); + + while (n_elts--) + { + if (n_elts > 1) + CLIB_PREFETCH (fe + 1, CLIB_CACHE_LINE_BYTES, LOAD); + + crypto_sw_scheduler_convert_aead (vm, ptd, fe, fe - f->elts, bi[0], + sync_op_id, aad_len, tag_len); + bi++; + fe++; + } + + process_ops (vm, f, ptd->crypto_ops, &state); + process_chained_ops (vm, f, ptd->chained_crypto_ops, ptd->chunks, + &state); + f->state = state; + *enqueue_thread_idx = f->enqueue_thread_index; + } + + return crypto_sw_scheduler_get_completed_frame (ptd->queues[async_op_id]); +} + +static_always_inline vnet_crypto_async_frame_t * +crypto_sw_scheduler_dequeue_link (vlib_main_t * vm, + vnet_crypto_async_op_id_t async_op_id, + vnet_crypto_op_id_t sync_crypto_op_id, + vnet_crypto_op_id_t sync_integ_op_id, + u16 digest_len, u8 is_enc, + u32 * nb_elts_processed, + u32 * enqueue_thread_idx) +{ + crypto_sw_scheduler_main_t *cm = &crypto_sw_scheduler_main; + crypto_sw_scheduler_per_thread_data_t *ptd = 0; + crypto_sw_scheduler_queue_t *q = 0; + vnet_crypto_async_frame_t *f = 0; + vnet_crypto_async_frame_elt_t *fe; + u32 *bi; + u32 n_elts; + int i = 0; + u8 state = VNET_CRYPTO_FRAME_STATE_SUCCESS; + + if (cm->per_thread_data[vm->thread_index].self_crypto_enabled) + { + /* *INDENT-OFF* */ + vec_foreach_index (i, cm->per_thread_data) + { + ptd = cm->per_thread_data + i; + q = ptd->queues[async_op_id]; + f = crypto_sw_scheduler_get_pending_frame (q); + if (f) + break; + } + /* *INDENT-ON* */ + } + + ptd = cm->per_thread_data + vm->thread_index; + + if (f) + { + vec_reset_length (ptd->crypto_ops); + vec_reset_length (ptd->integ_ops); + vec_reset_length (ptd->chained_crypto_ops); + vec_reset_length (ptd->chained_integ_ops); + vec_reset_length (ptd->chunks); + + *nb_elts_processed = n_elts = f->n_elts; + fe = f->elts; + bi = f->buffer_indices; + + while (n_elts--) + { + if (n_elts > 1) + CLIB_PREFETCH (fe + 1, CLIB_CACHE_LINE_BYTES, LOAD); + + crypto_sw_scheduler_convert_link_crypto (vm, ptd, + cm->keys + fe->key_index, + fe, fe - f->elts, bi[0], + sync_crypto_op_id, + sync_integ_op_id, + digest_len, is_enc); + bi++; + fe++; + } + + if (is_enc) + { + process_ops (vm, f, ptd->crypto_ops, &state); + process_chained_ops (vm, f, ptd->chained_crypto_ops, ptd->chunks, + &state); + process_ops (vm, f, ptd->integ_ops, &state); + process_chained_ops (vm, f, ptd->chained_integ_ops, ptd->chunks, + &state); + } + else + { + process_ops (vm, f, ptd->integ_ops, &state); + process_chained_ops (vm, f, ptd->chained_integ_ops, ptd->chunks, + &state); + process_ops (vm, f, ptd->crypto_ops, &state); + process_chained_ops (vm, f, ptd->chained_crypto_ops, ptd->chunks, + &state); + } + + f->state = state; + *enqueue_thread_idx = f->enqueue_thread_index; + } + + return crypto_sw_scheduler_get_completed_frame (ptd->queues[async_op_id]); +} + +static clib_error_t * +sw_scheduler_set_worker_crypto (vlib_main_t * vm, unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + unformat_input_t _line_input, *line_input = &_line_input; + u32 worker_index; + u8 crypto_enable; + int rv; + + /* Get a line of input. */ + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "worker %u", &worker_index)) + { + if (unformat (line_input, "crypto")) + { + if (unformat (line_input, "on")) + crypto_enable = 1; + else if (unformat (line_input, "off")) + crypto_enable = 0; + else + return (clib_error_return (0, "unknown input '%U'", + format_unformat_error, + line_input)); + } + else + return (clib_error_return (0, "unknown input '%U'", + format_unformat_error, line_input)); + } + else + return (clib_error_return (0, "unknown input '%U'", + format_unformat_error, line_input)); + } + + rv = crypto_sw_scheduler_set_worker_crypto (worker_index, crypto_enable); + if (rv == VNET_API_ERROR_INVALID_VALUE) + { + return (clib_error_return (0, "invalid worker idx: %d", worker_index)); + } + else if (rv == VNET_API_ERROR_INVALID_VALUE_2) + { + return (clib_error_return (0, "cannot disable all crypto workers")); + } + return 0; +} + +/*? + * This command sets if worker will do crypto processing. + * + * @cliexpar + * Example of how to set worker crypto processing off: + * @cliexstart{set sw_scheduler worker 0 crypto off} + * @cliexend + ?*/ +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (cmd_set_sw_scheduler_worker_crypto, static) = { + .path = "set sw_scheduler", + .short_help = "set sw_scheduler worker <idx> crypto <on|off>", + .function = sw_scheduler_set_worker_crypto, + .is_mp_safe = 1, +}; +/* *INDENT-ON* */ + +static clib_error_t * +sw_scheduler_show_workers (vlib_main_t * vm, unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + crypto_sw_scheduler_main_t *cm = &crypto_sw_scheduler_main; + u32 i; + + vlib_cli_output (vm, "%-7s%-20s%-8s", "ID", "Name", "Crypto"); + for (i = vlib_num_workers () >= 0; i < vlib_thread_main.n_vlib_mains; i++) + { + vlib_cli_output (vm, "%-7d%-20s%-8s", vlib_get_worker_index (i), + (vlib_worker_threads + i)->name, + cm-> + per_thread_data[i].self_crypto_enabled ? "on" : "off"); + } + + return 0; +} + +/*? + * This command displays sw_scheduler workers. + * + * @cliexpar + * Example of how to show workers: + * @cliexstart{show sw_scheduler workers} + * @cliexend + ?*/ +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (cmd_show_sw_scheduler_workers, static) = { + .path = "show sw_scheduler workers", + .short_help = "show sw_scheduler workers", + .function = sw_scheduler_show_workers, + .is_mp_safe = 1, +}; +/* *INDENT-ON* */ + +clib_error_t * +sw_scheduler_cli_init (vlib_main_t * vm) +{ + return 0; +} + +VLIB_INIT_FUNCTION (sw_scheduler_cli_init); + +/* *INDENT-OFF* */ +#define _(n, s, k, t, a) \ + static vnet_crypto_async_frame_t \ + *crypto_sw_scheduler_frame_dequeue_##n##_TAG_##t##_AAD_##a##_enc ( \ + vlib_main_t *vm, u32 *nb_elts_processed, u32 * thread_idx) \ + { \ + return crypto_sw_scheduler_dequeue_aead ( \ + vm, VNET_CRYPTO_OP_##n##_TAG##t##_AAD##a##_ENC, \ + VNET_CRYPTO_OP_##n##_ENC, t, a, nb_elts_processed, thread_idx); \ + } \ + static vnet_crypto_async_frame_t \ + *crypto_sw_scheduler_frame_dequeue_##n##_TAG_##t##_AAD_##a##_dec ( \ + vlib_main_t *vm, u32 *nb_elts_processed, u32 * thread_idx) \ + { \ + return crypto_sw_scheduler_dequeue_aead ( \ + vm, VNET_CRYPTO_OP_##n##_TAG##t##_AAD##a##_DEC, \ + VNET_CRYPTO_OP_##n##_DEC, t, a, nb_elts_processed, thread_idx); \ + } +foreach_crypto_aead_async_alg +#undef _ + +#define _(c, h, s, k, d) \ + static vnet_crypto_async_frame_t \ + *crypto_sw_scheduler_frame_dequeue_##c##_##h##_TAG##d##_enc ( \ + vlib_main_t *vm, u32 *nb_elts_processed, u32 * thread_idx) \ + { \ + return crypto_sw_scheduler_dequeue_link ( \ + vm, VNET_CRYPTO_OP_##c##_##h##_TAG##d##_ENC, \ + VNET_CRYPTO_OP_##c##_ENC, VNET_CRYPTO_OP_##h##_HMAC, d, 1, \ + nb_elts_processed, thread_idx); \ + } \ + static vnet_crypto_async_frame_t \ + *crypto_sw_scheduler_frame_dequeue_##c##_##h##_TAG##d##_dec ( \ + vlib_main_t *vm, u32 *nb_elts_processed, u32 * thread_idx) \ + { \ + return crypto_sw_scheduler_dequeue_link ( \ + vm, VNET_CRYPTO_OP_##c##_##h##_TAG##d##_DEC, \ + VNET_CRYPTO_OP_##c##_DEC, VNET_CRYPTO_OP_##h##_HMAC, d, 0, \ + nb_elts_processed, thread_idx); \ + } + foreach_crypto_link_async_alg +#undef _ + /* *INDENT-ON* */ + +crypto_sw_scheduler_main_t crypto_sw_scheduler_main; +clib_error_t * +crypto_sw_scheduler_init (vlib_main_t * vm) +{ + crypto_sw_scheduler_main_t *cm = &crypto_sw_scheduler_main; + vlib_thread_main_t *tm = vlib_get_thread_main (); + clib_error_t *error = 0; + crypto_sw_scheduler_per_thread_data_t *ptd; + + u32 queue_size = CRYPTO_SW_SCHEDULER_QUEUE_SIZE * sizeof (void *) + + sizeof (crypto_sw_scheduler_queue_t); + + vec_validate_aligned (cm->per_thread_data, tm->n_vlib_mains - 1, + CLIB_CACHE_LINE_BYTES); + + vec_foreach (ptd, cm->per_thread_data) + { + ptd->self_crypto_enabled = 1; + u32 i; + for (i = 0; i < VNET_CRYPTO_ASYNC_OP_N_IDS; i++) + { + crypto_sw_scheduler_queue_t *q + = clib_mem_alloc_aligned (queue_size, CLIB_CACHE_LINE_BYTES); + ASSERT (q != 0); + ptd->queues[i] = q; + clib_memset_u8 (q, 0, queue_size); + } + } + + cm->crypto_engine_index = + vnet_crypto_register_engine (vm, "sw_scheduler", 100, + "SW Scheduler Async Engine"); + + vnet_crypto_register_key_handler (vm, cm->crypto_engine_index, + crypto_sw_scheduler_key_handler); + + /* *INDENT-OFF* */ +#define _(n, s, k, t, a) \ + vnet_crypto_register_async_handler ( \ + vm, cm->crypto_engine_index, \ + VNET_CRYPTO_OP_##n##_TAG##t##_AAD##a##_ENC, \ + crypto_sw_scheduler_frame_enqueue, \ + crypto_sw_scheduler_frame_dequeue_##n##_TAG_##t##_AAD_##a##_enc); \ + vnet_crypto_register_async_handler ( \ + vm, cm->crypto_engine_index, \ + VNET_CRYPTO_OP_##n##_TAG##t##_AAD##a##_DEC, \ + crypto_sw_scheduler_frame_enqueue, \ + crypto_sw_scheduler_frame_dequeue_##n##_TAG_##t##_AAD_##a##_dec); + foreach_crypto_aead_async_alg +#undef _ + +#define _(c, h, s, k, d) \ + vnet_crypto_register_async_handler ( \ + vm, cm->crypto_engine_index, VNET_CRYPTO_OP_##c##_##h##_TAG##d##_ENC, \ + crypto_sw_scheduler_frame_enqueue, \ + crypto_sw_scheduler_frame_dequeue_##c##_##h##_TAG##d##_enc); \ + vnet_crypto_register_async_handler ( \ + vm, cm->crypto_engine_index, VNET_CRYPTO_OP_##c##_##h##_TAG##d##_DEC, \ + crypto_sw_scheduler_frame_enqueue, \ + crypto_sw_scheduler_frame_dequeue_##c##_##h##_TAG##d##_dec); + foreach_crypto_link_async_alg +#undef _ + /* *INDENT-ON* */ + + if (error) + vec_free (cm->per_thread_data); + + return error; +} + +/* *INDENT-OFF* */ +VLIB_INIT_FUNCTION (crypto_sw_scheduler_init) = { + .runs_after = VLIB_INITS ("vnet_crypto_init"), +}; + +VLIB_PLUGIN_REGISTER () = { + .version = VPP_BUILD_VER, + .description = "SW Scheduler Crypto Async Engine plugin", +}; +/* *INDENT-ON* */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/dpdk/cryptodev/cryptodev.c b/src/plugins/dpdk/cryptodev/cryptodev.c index 2ae09ce226e..6fc09c3813e 100644 --- a/src/plugins/dpdk/cryptodev/cryptodev.c +++ b/src/plugins/dpdk/cryptodev/cryptodev.c @@ -706,7 +706,8 @@ cryptodev_get_ring_head (struct rte_ring * ring) } static_always_inline vnet_crypto_async_frame_t * -cryptodev_frame_dequeue (vlib_main_t * vm) +cryptodev_frame_dequeue (vlib_main_t * vm, u32 * nb_elts_processed, + u32 * enqueue_thread_idx) { cryptodev_main_t *cmt = &cryptodev_main; cryptodev_numa_data_t *numa = cmt->per_numa_data + vm->numa_node; @@ -768,7 +769,8 @@ cryptodev_frame_dequeue (vlib_main_t * vm) VNET_CRYPTO_FRAME_STATE_SUCCESS : VNET_CRYPTO_FRAME_STATE_ELT_ERROR; rte_mempool_put_bulk (numa->cop_pool, (void **) cet->cops, frame->n_elts); - + *nb_elts_processed = frame->n_elts; + *enqueue_thread_idx = frame->enqueue_thread_index; return frame; } |