diff options
Diffstat (limited to 'src/vnet/crypto')
| -rw-r--r-- | src/vnet/crypto/cli.c | 98 | ||||
| -rw-r--r-- | src/vnet/crypto/crypto.api | 21 | ||||
| -rw-r--r-- | src/vnet/crypto/crypto.c | 236 | ||||
| -rw-r--r-- | src/vnet/crypto/crypto.h | 114 | ||||
| -rw-r--r-- | src/vnet/crypto/crypto_api.c | 14 | ||||
| -rw-r--r-- | src/vnet/crypto/node.c | 38 |
6 files changed, 261 insertions, 260 deletions
diff --git a/src/vnet/crypto/cli.c b/src/vnet/crypto/cli.c index a6098a18e11..2ca66f228c3 100644 --- a/src/vnet/crypto/cli.c +++ b/src/vnet/crypto/cli.c @@ -36,16 +36,13 @@ show_crypto_engines_command_fn (vlib_main_t * vm, } vlib_cli_output (vm, "%-20s%-8s%s", "Name", "Prio", "Description"); - /* *INDENT-OFF* */ vec_foreach (p, cm->engines) { vlib_cli_output (vm, "%-20s%-8u%s", p->name, p->priority, p->desc); } - /* *INDENT-ON* */ return 0; } -/* *INDENT-OFF* */ VLIB_CLI_COMMAND (show_crypto_engines_command, static) = { .path = "show crypto engines", @@ -67,7 +64,7 @@ format_vnet_crypto_engine_candidates (u8 * s, va_list * args) { vec_foreach (e, cm->engines) { - if (e->enqueue_handlers[id] && e->dequeue_handlers[id]) + if (e->enqueue_handlers[id] && e->dequeue_handler) { s = format (s, "%U", format_vnet_crypto_engine, e - cm->engines); if (ei == e - cm->engines) @@ -145,20 +142,18 @@ show_crypto_handlers_command_fn (vlib_main_t * vm, "Chained"); for (i = 0; i < VNET_CRYPTO_N_ALGS; i++) - vlib_cli_output (vm, "%-16U%U", format_vnet_crypto_alg, i, + vlib_cli_output (vm, "%-20U%U", format_vnet_crypto_alg, i, format_vnet_crypto_handlers, i); return 0; } -/* *INDENT-OFF* */ VLIB_CLI_COMMAND (show_crypto_handlers_command, static) = { .path = "show crypto handlers", .short_help = "show crypto handlers", .function = show_crypto_handlers_command_fn, }; -/* *INDENT-ON* */ static clib_error_t * set_crypto_handler_command_fn (vlib_main_t * vm, @@ -209,13 +204,11 @@ set_crypto_handler_command_fn (vlib_main_t * vm, char *key; u8 *value; - /* *INDENT-OFF* */ hash_foreach_mem (key, value, cm->alg_index_by_name, ({ (void) value; rc += vnet_crypto_set_handler2 (key, engine, oct); })); - /* *INDENT-ON* */ if (rc) vlib_cli_output (vm, "failed to set crypto engine!"); @@ -241,7 +234,6 @@ done: return error; } -/* *INDENT-OFF* */ VLIB_CLI_COMMAND (set_crypto_handler_command, static) = { .path = "set crypto handler", @@ -249,7 +241,6 @@ VLIB_CLI_COMMAND (set_crypto_handler_command, static) = " [simple|chained]", .function = set_crypto_handler_command_fn, }; -/* *INDENT-ON* */ static u8 * format_vnet_crypto_async_handlers (u8 * s, va_list * args) @@ -300,14 +291,12 @@ show_crypto_async_handlers_command_fn (vlib_main_t * vm, return 0; } -/* *INDENT-OFF* */ VLIB_CLI_COMMAND (show_crypto_async_handlers_command, static) = { .path = "show crypto async handlers", .short_help = "show crypto async handlers", .function = show_crypto_async_handlers_command_fn, }; -/* *INDENT-ON* */ static clib_error_t * @@ -316,7 +305,6 @@ show_crypto_async_status_command_fn (vlib_main_t * vm, vlib_cli_command_t * cmd) { vnet_crypto_main_t *cm = &crypto_main; - u32 skip_master = vlib_num_workers () > 0; vlib_thread_main_t *tm = vlib_get_thread_main (); unformat_input_t _line_input, *line_input = &_line_input; int i; @@ -324,12 +312,7 @@ show_crypto_async_status_command_fn (vlib_main_t * vm, if (unformat_user (input, unformat_line_input, line_input)) unformat_free (line_input); - vlib_cli_output (vm, "Crypto async dispatch mode: %s", - cm->dispatch_mode == - VNET_CRYPTO_ASYNC_DISPATCH_POLLING ? "POLLING" : - "INTERRUPT"); - - for (i = skip_master; i < tm->n_vlib_mains; i++) + for (i = 0; i < tm->n_vlib_mains; i++) { vlib_node_state_t state = vlib_node_get_state ( vlib_get_main_by_index (i), cm->crypto_node_index); @@ -343,14 +326,12 @@ show_crypto_async_status_command_fn (vlib_main_t * vm, return 0; } -/* *INDENT-OFF* */ VLIB_CLI_COMMAND (show_crypto_async_status_command, static) = { .path = "show crypto async status", .short_help = "show crypto async status", .function = show_crypto_async_status_command_fn, }; -/* *INDENT-ON* */ static clib_error_t * set_crypto_async_handler_command_fn (vlib_main_t * vm, @@ -394,13 +375,11 @@ set_crypto_async_handler_command_fn (vlib_main_t * vm, char *key; u8 *value; - /* *INDENT-OFF* */ hash_foreach_mem (key, value, cm->async_alg_index_by_name, ({ (void) value; rc += vnet_crypto_set_async_handler2 (key, engine); })); - /* *INDENT-ON* */ if (rc) vlib_cli_output (vm, "failed to set crypto engine!"); @@ -426,57 +405,52 @@ done: return error; } -/* *INDENT-OFF* */ VLIB_CLI_COMMAND (set_crypto_async_handler_command, static) = { .path = "set crypto async handler", .short_help = "set crypto async handler type [type2 type3 ...] engine", .function = set_crypto_async_handler_command_fn, }; -/* *INDENT-ON* */ - -static inline void -print_crypto_async_dispatch_warning () -{ - clib_warning ("Switching dispatch mode might not work is some situations."); - clib_warning - ("Use 'show crypto async status' to verify that the nodes' states were set"); - clib_warning ("and if not, set 'crypto async dispatch' mode again."); -} static clib_error_t * -set_crypto_async_dispatch_polling_command_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) +set_crypto_async_dispatch_command_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) { - print_crypto_async_dispatch_warning (); - vnet_crypto_set_async_dispatch_mode (VNET_CRYPTO_ASYNC_DISPATCH_POLLING); - return 0; -} + unformat_input_t _line_input, *line_input = &_line_input; + clib_error_t *error = 0; + u8 adaptive = 0; + u8 mode = VLIB_NODE_STATE_INTERRUPT; -static clib_error_t * -set_crypto_async_dispatch_interrupt_command_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - print_crypto_async_dispatch_warning (); - vnet_crypto_set_async_dispatch_mode (VNET_CRYPTO_ASYNC_DISPATCH_INTERRUPT); - return 0; + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "polling")) + mode = VLIB_NODE_STATE_POLLING; + else if (unformat (line_input, "interrupt")) + mode = VLIB_NODE_STATE_INTERRUPT; + else if (unformat (line_input, "adaptive")) + adaptive = 1; + else + { + error = clib_error_return (0, "invalid params"); + goto done; + } + } + + vnet_crypto_set_async_dispatch (mode, adaptive); +done: + unformat_free (line_input); + return error; } -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (set_crypto_async_dispatch_polling_command, static) = -{ - .path = "set crypto async dispatch polling", - .short_help = "set crypto async dispatch polling|interrupt", - .function = set_crypto_async_dispatch_polling_command_fn, -}; -VLIB_CLI_COMMAND (set_crypto_async_dispatch_interrupt_command, static) = -{ - .path = "set crypto async dispatch interrupt", - .short_help = "set crypto async dispatch polling|interrupt", - .function = set_crypto_async_dispatch_interrupt_command_fn, +VLIB_CLI_COMMAND (set_crypto_async_dispatch_mode_command, static) = { + .path = "set crypto async dispatch mode", + .short_help = "set crypto async dispatch mode <polling|interrupt|adaptive>", + .function = set_crypto_async_dispatch_command_fn, }; + /* * fd.io coding-style-patch-verification: ON * diff --git a/src/vnet/crypto/crypto.api b/src/vnet/crypto/crypto.api index 6eccd8524ba..8fec805dcfc 100644 --- a/src/vnet/crypto/crypto.api +++ b/src/vnet/crypto/crypto.api @@ -28,7 +28,8 @@ enum crypto_op_class_type:u8 CRYPTO_API_OP_BOTH, }; - /** \brief crypto: use polling or interrupt dispatch + /** \brief crypto: Use polling or interrupt dispatch. + Always unset the adaptive flag (that is why it is deprecated). @param client_index - opaque cookie to identify the sender @param context - sender context, to match reply w/ request @param mode - dispatch mode @@ -36,11 +37,29 @@ enum crypto_op_class_type:u8 autoreply define crypto_set_async_dispatch { + option deprecated; + option replaced_by="crypto_set_async_dispatch_v2"; u32 client_index; u32 context; vl_api_crypto_dispatch_mode_t mode; }; + /** \brief crypto: Change the way crypto operations are dispatched. + Use adaptive (or not) mode, starting in polling or interrupt state. + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param mode - dispatch initial state + @param adaptive - whether on not the state shall change depending on load +*/ + +autoreply define crypto_set_async_dispatch_v2 +{ + u32 client_index; + u32 context; + vl_api_crypto_dispatch_mode_t mode; + bool adaptive; +}; + /** \brief crypto: set crypto handler @param client_index - opaque cookie to identify the sender @param context - sender context, to match reply w/ request diff --git a/src/vnet/crypto/crypto.c b/src/vnet/crypto/crypto.c index 7903f88b7cb..c8e7ca90c9d 100644 --- a/src/vnet/crypto/crypto.c +++ b/src/vnet/crypto/crypto.c @@ -192,13 +192,16 @@ vnet_crypto_is_set_handler (vnet_crypto_alg_t alg) vnet_crypto_op_id_t opt = 0; int i; - if (alg > vec_len (cm->algs)) + if (alg >= vec_len (cm->algs)) return 0; for (i = 0; i < VNET_CRYPTO_OP_N_TYPES; i++) if ((opt = cm->algs[alg].op_by_type[i]) != 0) break; + if (opt >= vec_len (cm->ops_handlers)) + return 0; + return NULL != cm->ops_handlers[opt]; } @@ -275,30 +278,24 @@ vnet_crypto_register_ops_handlers (vlib_main_t * vm, u32 engine_index, } void -vnet_crypto_register_async_handler (vlib_main_t * vm, u32 engine_index, - vnet_crypto_async_op_id_t opt, - vnet_crypto_frame_enqueue_t * enqueue_hdl, - vnet_crypto_frame_dequeue_t * dequeue_hdl) +vnet_crypto_register_enqueue_handler (vlib_main_t *vm, u32 engine_index, + vnet_crypto_async_op_id_t opt, + vnet_crypto_frame_enqueue_t *enqueue_hdl) { vnet_crypto_main_t *cm = &crypto_main; vnet_crypto_engine_t *ae, *e = vec_elt_at_index (cm->engines, engine_index); vnet_crypto_async_op_data_t *otd = cm->async_opt_data + opt; vec_validate_aligned (cm->enqueue_handlers, VNET_CRYPTO_ASYNC_OP_N_IDS, CLIB_CACHE_LINE_BYTES); - vec_validate_aligned (cm->dequeue_handlers, VNET_CRYPTO_ASYNC_OP_N_IDS, - CLIB_CACHE_LINE_BYTES); - /* both enqueue hdl and dequeue hdl should present */ - if (!enqueue_hdl && !dequeue_hdl) + if (!enqueue_hdl) return; e->enqueue_handlers[opt] = enqueue_hdl; - e->dequeue_handlers[opt] = dequeue_hdl; if (otd->active_engine_index_async == ~0) { otd->active_engine_index_async = engine_index; cm->enqueue_handlers[opt] = enqueue_hdl; - cm->dequeue_handlers[opt] = dequeue_hdl; } ae = vec_elt_at_index (cm->engines, otd->active_engine_index_async); @@ -306,12 +303,79 @@ vnet_crypto_register_async_handler (vlib_main_t * vm, u32 engine_index, { otd->active_engine_index_async = engine_index; cm->enqueue_handlers[opt] = enqueue_hdl; - cm->dequeue_handlers[opt] = dequeue_hdl; } return; } +static int +engine_index_cmp (void *v1, void *v2) +{ + u32 *a1 = v1; + u32 *a2 = v2; + + if (*a1 > *a2) + return 1; + if (*a1 < *a2) + return -1; + return 0; +} + +static void +vnet_crypto_update_cm_dequeue_handlers (void) +{ + vnet_crypto_main_t *cm = &crypto_main; + vnet_crypto_async_op_data_t *otd; + vnet_crypto_engine_t *e; + u32 *active_engines = 0, *ei, last_ei = ~0, i; + + vec_reset_length (cm->dequeue_handlers); + + for (i = 0; i < VNET_CRYPTO_ASYNC_OP_N_IDS; i++) + { + otd = cm->async_opt_data + i; + if (otd->active_engine_index_async == ~0) + continue; + e = cm->engines + otd->active_engine_index_async; + if (!e->dequeue_handler) + continue; + vec_add1 (active_engines, otd->active_engine_index_async); + } + + vec_sort_with_function (active_engines, engine_index_cmp); + + vec_foreach (ei, active_engines) + { + if (ei[0] == last_ei) + continue; + if (ei[0] == ~0) + continue; + + e = cm->engines + ei[0]; + vec_add1 (cm->dequeue_handlers, e->dequeue_handler); + last_ei = ei[0]; + } + + vec_free (active_engines); +} + +void +vnet_crypto_register_dequeue_handler (vlib_main_t *vm, u32 engine_index, + vnet_crypto_frame_dequeue_t *deq_fn) +{ + vnet_crypto_main_t *cm = &crypto_main; + vnet_crypto_engine_t *e = vec_elt_at_index (cm->engines, engine_index); + + if (!deq_fn) + return; + + e->dequeue_handler = deq_fn; + + vnet_crypto_update_cm_dequeue_handlers (); + + return; +} + void vnet_crypto_register_key_handler (vlib_main_t * vm, u32 engine_index, vnet_crypto_key_handler_t * key_handler) @@ -370,8 +434,7 @@ vnet_crypto_key_add (vlib_main_t * vm, vnet_crypto_alg_t alg, u8 * data, if (!vnet_crypto_key_len_check (alg, length)) return ~0; - pool_get_aligned_will_expand (cm->keys, need_barrier_sync, - CLIB_CACHE_LINE_BYTES); + need_barrier_sync = pool_get_will_expand (cm->keys); /* If the cm->keys will expand, stop the parade. */ if (need_barrier_sync) vlib_worker_thread_barrier_sync (vm); @@ -386,11 +449,9 @@ vnet_crypto_key_add (vlib_main_t * vm, vnet_crypto_alg_t alg, u8 * data, key->alg = alg; vec_validate_aligned (key->data, length - 1, CLIB_CACHE_LINE_BYTES); clib_memcpy (key->data, data, length); - /* *INDENT-OFF* */ vec_foreach (engine, cm->engines) if (engine->key_op_handler) engine->key_op_handler (vm, VNET_CRYPTO_KEY_OP_ADD, index); - /* *INDENT-ON* */ return index; } @@ -401,25 +462,34 @@ vnet_crypto_key_del (vlib_main_t * vm, vnet_crypto_key_index_t index) vnet_crypto_engine_t *engine; vnet_crypto_key_t *key = pool_elt_at_index (cm->keys, index); - /* *INDENT-OFF* */ vec_foreach (engine, cm->engines) if (engine->key_op_handler) engine->key_op_handler (vm, VNET_CRYPTO_KEY_OP_DEL, index); - /* *INDENT-ON* */ if (key->type == VNET_CRYPTO_KEY_TYPE_DATA) { - clib_memset (key->data, 0, vec_len (key->data)); + clib_memset (key->data, 0xfe, vec_len (key->data)); vec_free (key->data); } else if (key->type == VNET_CRYPTO_KEY_TYPE_LINK) { - key->index_crypto = key->index_integ = 0; + key->index_crypto = key->index_integ = ~0; } pool_put (cm->keys, key); } +void +vnet_crypto_key_update (vlib_main_t *vm, vnet_crypto_key_index_t index) +{ + vnet_crypto_main_t *cm = &crypto_main; + vnet_crypto_engine_t *engine; + + vec_foreach (engine, cm->engines) + if (engine->key_op_handler) + engine->key_op_handler (vm, VNET_CRYPTO_KEY_OP_MODIFY, index); +} + vnet_crypto_async_alg_t vnet_crypto_link_algs (vnet_crypto_alg_t crypto_alg, vnet_crypto_alg_t integ_alg) @@ -458,50 +528,13 @@ vnet_crypto_key_add_linked (vlib_main_t * vm, key->index_integ = index_integ; key->async_alg = linked_alg; - /* *INDENT-OFF* */ vec_foreach (engine, cm->engines) if (engine->key_op_handler) engine->key_op_handler (vm, VNET_CRYPTO_KEY_OP_ADD, index); - /* *INDENT-ON* */ return index; } -clib_error_t * -crypto_dispatch_enable_disable (int is_enable) -{ - vnet_crypto_main_t *cm = &crypto_main; - vlib_thread_main_t *tm = vlib_get_thread_main (); - u32 skip_master = vlib_num_workers () > 0, i; - vlib_node_state_t state = VLIB_NODE_STATE_DISABLED; - u8 state_change = 0; - - CLIB_MEMORY_STORE_BARRIER (); - if (is_enable && cm->async_refcnt > 0) - { - state_change = 1; - state = - cm->dispatch_mode == - VNET_CRYPTO_ASYNC_DISPATCH_POLLING ? VLIB_NODE_STATE_POLLING : - VLIB_NODE_STATE_INTERRUPT; - } - - if (!is_enable && cm->async_refcnt == 0) - { - state_change = 1; - state = VLIB_NODE_STATE_DISABLED; - } - - if (state_change) - for (i = skip_master; i < tm->n_vlib_mains; i++) - { - vlib_main_t *ovm = vlib_get_main_by_index (i); - if (state != vlib_node_get_state (ovm, cm->crypto_node_index)) - vlib_node_set_state (ovm, cm->crypto_node_index, state); - } - return 0; -} - static_always_inline void crypto_set_active_async_engine (vnet_crypto_async_op_data_t * od, vnet_crypto_async_op_id_t id, u32 ei) @@ -509,11 +542,10 @@ crypto_set_active_async_engine (vnet_crypto_async_op_data_t * od, vnet_crypto_main_t *cm = &crypto_main; vnet_crypto_engine_t *ce = vec_elt_at_index (cm->engines, ei); - if (ce->enqueue_handlers[id] && ce->dequeue_handlers[id]) + if (ce->enqueue_handlers[id] && ce->dequeue_handler) { od->active_engine_index_async = ei; cm->enqueue_handlers[id] = ce->enqueue_handlers[id]; - cm->dequeue_handlers[id] = ce->dequeue_handlers[id]; } } @@ -546,6 +578,8 @@ vnet_crypto_set_async_handler2 (char *alg_name, char *engine) crypto_set_active_async_engine (od, id, p[0]); } + vnet_crypto_update_cm_dequeue_handlers (); + return 0; } @@ -561,13 +595,11 @@ vnet_crypto_register_post_node (vlib_main_t * vm, char *post_node_name) if (!pn) return ~0; - /* *INDENT-OFF* */ - vec_foreach (cm->next_nodes, nn) - { - if (nn->node_idx == pn->index) - return nn->next_idx; - } - /* *INDENT-ON* */ + vec_foreach (nn, cm->next_nodes) + { + if (nn->node_idx == pn->index) + return nn->next_idx; + } vec_validate (cm->next_nodes, index); nn = vec_elt_at_index (cm->next_nodes, index); @@ -580,70 +612,19 @@ vnet_crypto_register_post_node (vlib_main_t * vm, char *post_node_name) } void -vnet_crypto_request_async_mode (int is_enable) -{ - vnet_crypto_main_t *cm = &crypto_main; - vlib_thread_main_t *tm = vlib_get_thread_main (); - u32 skip_master = vlib_num_workers () > 0, i; - vlib_node_state_t state = VLIB_NODE_STATE_DISABLED; - u8 state_change = 0; - - CLIB_MEMORY_STORE_BARRIER (); - if (is_enable && cm->async_refcnt == 0) - { - state_change = 1; - state = - cm->dispatch_mode == VNET_CRYPTO_ASYNC_DISPATCH_POLLING ? - VLIB_NODE_STATE_POLLING : VLIB_NODE_STATE_INTERRUPT; - } - if (!is_enable && cm->async_refcnt == 1) - { - state_change = 1; - state = VLIB_NODE_STATE_DISABLED; - } - - if (state_change) - for (i = skip_master; i < tm->n_vlib_mains; i++) - { - vlib_main_t *ovm = vlib_get_main_by_index (i); - if (state != vlib_node_get_state (ovm, cm->crypto_node_index)) - vlib_node_set_state (ovm, cm->crypto_node_index, state); - } - - if (is_enable) - cm->async_refcnt += 1; - else if (cm->async_refcnt > 0) - cm->async_refcnt -= 1; -} - -void -vnet_crypto_set_async_dispatch_mode (u8 mode) +vnet_crypto_set_async_dispatch (u8 mode, u8 adaptive) { - vnet_crypto_main_t *cm = &crypto_main; - u32 skip_master = vlib_num_workers () > 0, i; vlib_thread_main_t *tm = vlib_get_thread_main (); - vlib_node_state_t state = VLIB_NODE_STATE_DISABLED; + u32 i, node_index = crypto_main.crypto_node_index; + vlib_node_state_t state = + mode ? VLIB_NODE_STATE_INTERRUPT : VLIB_NODE_STATE_POLLING; - CLIB_MEMORY_STORE_BARRIER (); - cm->dispatch_mode = mode; - if (mode == VNET_CRYPTO_ASYNC_DISPATCH_INTERRUPT) - { - state = - cm->async_refcnt == 0 ? - VLIB_NODE_STATE_DISABLED : VLIB_NODE_STATE_INTERRUPT; - } - else if (mode == VNET_CRYPTO_ASYNC_DISPATCH_POLLING) - { - state = - cm->async_refcnt == 0 ? - VLIB_NODE_STATE_DISABLED : VLIB_NODE_STATE_POLLING; - } - - for (i = skip_master; i < tm->n_vlib_mains; i++) + for (i = vlib_num_workers () > 0; i < tm->n_vlib_mains; i++) { vlib_main_t *ovm = vlib_get_main_by_index (i); - if (state != vlib_node_get_state (ovm, cm->crypto_node_index)) - vlib_node_set_state (ovm, cm->crypto_node_index, state); + vlib_node_set_state (ovm, node_index, state); + vlib_node_set_flag (ovm, node_index, VLIB_NODE_FLAG_ADAPTIVE_MODE, + adaptive); } } @@ -742,18 +723,15 @@ vnet_crypto_init (vlib_main_t * vm) vlib_thread_main_t *tm = vlib_get_thread_main (); vnet_crypto_thread_t *ct = 0; - cm->dispatch_mode = VNET_CRYPTO_ASYNC_DISPATCH_POLLING; cm->engine_index_by_name = hash_create_string ( /* size */ 0, sizeof (uword)); cm->alg_index_by_name = hash_create_string (0, sizeof (uword)); cm->async_alg_index_by_name = hash_create_string (0, sizeof (uword)); vec_validate_aligned (cm->threads, tm->n_vlib_mains, CLIB_CACHE_LINE_BYTES); vec_foreach (ct, cm->threads) - pool_alloc_aligned (ct->frame_pool, VNET_CRYPTO_FRAME_POOL_SIZE, - CLIB_CACHE_LINE_BYTES); + pool_init_fixed (ct->frame_pool, VNET_CRYPTO_FRAME_POOL_SIZE); vec_validate (cm->algs, VNET_CRYPTO_N_ALGS); vec_validate (cm->async_algs, VNET_CRYPTO_N_ASYNC_ALGS); - clib_bitmap_validate (cm->async_active_ids, VNET_CRYPTO_ASYNC_OP_N_IDS); #define _(n, s, l) \ vnet_crypto_init_cipher_data (VNET_CRYPTO_ALG_##n, \ diff --git a/src/vnet/crypto/crypto.h b/src/vnet/crypto/crypto.h index 71978b64835..89cf70d19e3 100644 --- a/src/vnet/crypto/crypto.h +++ b/src/vnet/crypto/crypto.h @@ -33,11 +33,14 @@ _(AES_256_CTR, "aes-256-ctr", 32) /* CRYPTO_ID, PRETTY_NAME, KEY_LENGTH_IN_BYTES */ -#define foreach_crypto_aead_alg \ - _(AES_128_GCM, "aes-128-gcm", 16) \ - _(AES_192_GCM, "aes-192-gcm", 24) \ - _(AES_256_GCM, "aes-256-gcm", 32) \ - _(CHACHA20_POLY1305, "chacha20-poly1305", 32) +#define foreach_crypto_aead_alg \ + _ (AES_128_GCM, "aes-128-gcm", 16) \ + _ (AES_192_GCM, "aes-192-gcm", 24) \ + _ (AES_256_GCM, "aes-256-gcm", 32) \ + _ (AES_128_NULL_GMAC, "aes-128-null-gmac", 16) \ + _ (AES_192_NULL_GMAC, "aes-192-null-gmac", 24) \ + _ (AES_256_NULL_GMAC, "aes-256-null-gmac", 32) \ + _ (CHACHA20_POLY1305, "chacha20-poly1305", 32) #define foreach_crypto_hash_alg \ _ (SHA1, "sha-1") \ @@ -82,15 +85,22 @@ typedef enum /** async crypto **/ /* CRYPTO_ID, PRETTY_NAME, KEY_LENGTH_IN_BYTES, TAG_LEN, AAD_LEN */ -#define foreach_crypto_aead_async_alg \ - _(AES_128_GCM, "aes-128-gcm-aad8", 16, 16, 8) \ - _(AES_128_GCM, "aes-128-gcm-aad12", 16, 16, 12) \ - _(AES_192_GCM, "aes-192-gcm-aad8", 24, 16, 8) \ - _(AES_192_GCM, "aes-192-gcm-aad12", 24, 16, 12) \ - _(AES_256_GCM, "aes-256-gcm-aad8", 32, 16, 8) \ - _(AES_256_GCM, "aes-256-gcm-aad12", 32, 16, 12) \ - _(CHACHA20_POLY1305, "chacha20-poly1305-aad8", 32, 16, 8) \ - _(CHACHA20_POLY1305, "chacha20-poly1305-aad12", 32, 16, 12) +#define foreach_crypto_aead_async_alg \ + _ (AES_128_GCM, "aes-128-gcm-aad8", 16, 16, 8) \ + _ (AES_128_GCM, "aes-128-gcm-aad12", 16, 16, 12) \ + _ (AES_192_GCM, "aes-192-gcm-aad8", 24, 16, 8) \ + _ (AES_192_GCM, "aes-192-gcm-aad12", 24, 16, 12) \ + _ (AES_256_GCM, "aes-256-gcm-aad8", 32, 16, 8) \ + _ (AES_256_GCM, "aes-256-gcm-aad12", 32, 16, 12) \ + _ (AES_128_NULL_GMAC, "aes-128-null-gmac-aad8", 16, 16, 8) \ + _ (AES_128_NULL_GMAC, "aes-128-null-gmac-aad12", 16, 16, 12) \ + _ (AES_192_NULL_GMAC, "aes-192-null-gmac-aad8", 24, 16, 8) \ + _ (AES_192_NULL_GMAC, "aes-192-null-gmac-aad12", 24, 16, 12) \ + _ (AES_256_NULL_GMAC, "aes-256-null-gmac-aad8", 32, 16, 8) \ + _ (AES_256_NULL_GMAC, "aes-256-null-gmac-aad12", 32, 16, 12) \ + _ (CHACHA20_POLY1305, "chacha20-poly1305-aad8", 32, 16, 8) \ + _ (CHACHA20_POLY1305, "chacha20-poly1305-aad12", 32, 16, 12) \ + _ (CHACHA20_POLY1305, "chacha20-poly1305", 32, 16, 0) /* CRYPTO_ID, INTEG_ID, PRETTY_NAME, KEY_LENGTH_IN_BYTES, DIGEST_LEN */ #define foreach_crypto_link_async_alg \ @@ -141,7 +151,6 @@ typedef enum VNET_CRYPTO_OP_N_STATUS, } vnet_crypto_op_status_t; -/* *INDENT-OFF* */ typedef enum { VNET_CRYPTO_ALG_NONE = 0, @@ -230,7 +239,6 @@ typedef enum #undef _ VNET_CRYPTO_N_OP_IDS, } vnet_crypto_op_id_t; -/* *INDENT-ON* */ typedef enum { @@ -259,9 +267,8 @@ typedef struct vnet_crypto_op_id_t op:16; vnet_crypto_op_status_t status:8; u8 flags; -#define VNET_CRYPTO_OP_FLAG_INIT_IV (1 << 0) -#define VNET_CRYPTO_OP_FLAG_HMAC_CHECK (1 << 1) -#define VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS (1 << 2) +#define VNET_CRYPTO_OP_FLAG_HMAC_CHECK (1 << 0) +#define VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS (1 << 1) union { @@ -337,7 +344,7 @@ typedef struct i16 crypto_start_offset; /* first buffer offset */ i16 integ_start_offset; /* adj total_length for integ, e.g.4 bytes for IPSec ESN */ - u16 integ_length_adj; + i16 integ_length_adj; vnet_crypto_op_status_t status : 8; u8 flags; /**< share same VNET_CRYPTO_OP_FLAG_* values */ } vnet_crypto_async_frame_elt_t; @@ -422,12 +429,15 @@ void vnet_crypto_register_key_handler (vlib_main_t * vm, u32 engine_index, /** async crypto register functions */ u32 vnet_crypto_register_post_node (vlib_main_t * vm, char *post_node_name); -void vnet_crypto_register_async_handler (vlib_main_t * vm, - u32 engine_index, - vnet_crypto_async_op_id_t opt, - vnet_crypto_frame_enqueue_t * enq_fn, - vnet_crypto_frame_dequeue_t * - deq_fn); + +void +vnet_crypto_register_enqueue_handler (vlib_main_t *vm, u32 engine_index, + vnet_crypto_async_op_id_t opt, + vnet_crypto_frame_enqueue_t *enq_fn); + +void +vnet_crypto_register_dequeue_handler (vlib_main_t *vm, u32 engine_index, + vnet_crypto_frame_dequeue_t *deq_fn); typedef struct { @@ -439,7 +449,7 @@ typedef struct vnet_crypto_chained_ops_handler_t * chained_ops_handlers[VNET_CRYPTO_N_OP_IDS]; vnet_crypto_frame_enqueue_t *enqueue_handlers[VNET_CRYPTO_ASYNC_OP_N_IDS]; - vnet_crypto_frame_dequeue_t *dequeue_handlers[VNET_CRYPTO_ASYNC_OP_N_IDS]; + vnet_crypto_frame_dequeue_t *dequeue_handler; } vnet_crypto_engine_t; typedef struct @@ -456,7 +466,6 @@ typedef struct vnet_crypto_chained_ops_handler_t **chained_ops_handlers; vnet_crypto_frame_enqueue_t **enqueue_handlers; vnet_crypto_frame_dequeue_t **dequeue_handlers; - clib_bitmap_t *async_active_ids; vnet_crypto_op_data_t opt_data[VNET_CRYPTO_N_OP_IDS]; vnet_crypto_async_op_data_t async_opt_data[VNET_CRYPTO_ASYNC_OP_N_IDS]; vnet_crypto_engine_t *engines; @@ -465,12 +474,8 @@ typedef struct uword *alg_index_by_name; uword *async_alg_index_by_name; vnet_crypto_async_alg_data_t *async_algs; - u32 async_refcnt; vnet_crypto_async_next_node_t *next_nodes; u32 crypto_node_index; -#define VNET_CRYPTO_ASYNC_DISPATCH_POLLING 0 -#define VNET_CRYPTO_ASYNC_DISPATCH_INTERRUPT 1 - u8 dispatch_mode; } vnet_crypto_main_t; extern vnet_crypto_main_t crypto_main; @@ -481,7 +486,7 @@ u32 vnet_crypto_process_chained_ops (vlib_main_t * vm, vnet_crypto_op_t ops[], u32 vnet_crypto_process_ops (vlib_main_t * vm, vnet_crypto_op_t ops[], u32 n_ops); - +void vnet_crypto_set_async_dispatch (u8 mode, u8 adaptive); int vnet_crypto_set_handler2 (char *ops_handler_name, char *engine, crypto_op_class_type_t oct); int vnet_crypto_is_set_handler (vnet_crypto_alg_t alg); @@ -489,6 +494,7 @@ int vnet_crypto_is_set_handler (vnet_crypto_alg_t alg); u32 vnet_crypto_key_add (vlib_main_t * vm, vnet_crypto_alg_t alg, u8 * data, u16 length); void vnet_crypto_key_del (vlib_main_t * vm, vnet_crypto_key_index_t index); +void vnet_crypto_key_update (vlib_main_t *vm, vnet_crypto_key_index_t index); /** * Use 2 created keys to generate new key for linked algs (cipher + integ) @@ -498,21 +504,13 @@ u32 vnet_crypto_key_add_linked (vlib_main_t * vm, vnet_crypto_key_index_t index_crypto, vnet_crypto_key_index_t index_integ); -clib_error_t *crypto_dispatch_enable_disable (int is_enable); - int vnet_crypto_set_async_handler2 (char *alg_name, char *engine); int vnet_crypto_is_set_async_handler (vnet_crypto_async_op_id_t opt); -void vnet_crypto_request_async_mode (int is_enable); - -void vnet_crypto_set_async_dispatch_mode (u8 mode); - vnet_crypto_async_alg_t vnet_crypto_link_algs (vnet_crypto_alg_t crypto_alg, vnet_crypto_alg_t integ_alg); -clib_error_t *crypto_dispatch_enable_disable (int is_enable); - format_function_t format_vnet_crypto_alg; format_function_t format_vnet_crypto_engine; format_function_t format_vnet_crypto_op; @@ -566,12 +564,16 @@ vnet_crypto_async_get_frame (vlib_main_t * vm, vnet_crypto_async_op_id_t opt) vnet_crypto_thread_t *ct = cm->threads + vm->thread_index; vnet_crypto_async_frame_t *f = NULL; - pool_get_aligned (ct->frame_pool, f, CLIB_CACHE_LINE_BYTES); - if (CLIB_DEBUG > 0) - clib_memset (f, 0xfe, sizeof (*f)); - f->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED; - f->op = opt; - f->n_elts = 0; + if (PREDICT_TRUE (pool_free_elts (ct->frame_pool))) + { + pool_get_aligned (ct->frame_pool, f, CLIB_CACHE_LINE_BYTES); +#if CLIB_DEBUG > 0 + clib_memset (f, 0xfe, sizeof (*f)); +#endif + f->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED; + f->op = opt; + f->n_elts = 0; + } return f; } @@ -591,20 +593,26 @@ vnet_crypto_async_submit_open_frame (vlib_main_t * vm, { vnet_crypto_main_t *cm = &crypto_main; vlib_thread_main_t *tm = vlib_get_thread_main (); - vnet_crypto_async_op_id_t opt = frame->op; - u32 i = vlib_num_workers () > 0; + u32 i; + vlib_node_t *n; frame->state = VNET_CRYPTO_FRAME_STATE_PENDING; frame->enqueue_thread_index = vm->thread_index; + if (PREDICT_FALSE (cm->enqueue_handlers == NULL)) + { + frame->state = VNET_CRYPTO_FRAME_STATE_ELT_ERROR; + return -1; + } + int ret = (cm->enqueue_handlers[frame->op]) (vm, frame); - clib_bitmap_set_no_check (cm->async_active_ids, opt, 1); if (PREDICT_TRUE (ret == 0)) { - if (cm->dispatch_mode == VNET_CRYPTO_ASYNC_DISPATCH_INTERRUPT) + n = vlib_get_node (vm, cm->crypto_node_index); + if (n->state == VLIB_NODE_STATE_INTERRUPT) { - for (; i < tm->n_vlib_mains; i++) + for (i = 0; i < tm->n_vlib_mains; i++) vlib_node_set_interrupt_pending (vlib_get_main_by_index (i), cm->crypto_node_index); } @@ -621,7 +629,7 @@ static_always_inline void vnet_crypto_async_add_to_frame (vlib_main_t *vm, vnet_crypto_async_frame_t *f, u32 key_index, u32 crypto_len, i16 integ_len_adj, i16 crypto_start_offset, - u16 integ_start_offset, u32 buffer_index, + i16 integ_start_offset, u32 buffer_index, u16 next_node, u8 *iv, u8 *tag, u8 *aad, u8 flags) { diff --git a/src/vnet/crypto/crypto_api.c b/src/vnet/crypto/crypto_api.c index 49b12a3d377..e701864a5ba 100644 --- a/src/vnet/crypto/crypto_api.c +++ b/src/vnet/crypto/crypto_api.c @@ -46,12 +46,24 @@ vl_api_crypto_set_async_dispatch_t_handler (vl_api_crypto_set_async_dispatch_t vl_api_crypto_set_async_dispatch_reply_t *rmp; int rv = 0; - vnet_crypto_set_async_dispatch_mode ((u8) mp->mode); + vnet_crypto_set_async_dispatch ((u8) mp->mode, 0); REPLY_MACRO (VL_API_CRYPTO_SET_ASYNC_DISPATCH_REPLY); } static void +vl_api_crypto_set_async_dispatch_v2_t_handler ( + vl_api_crypto_set_async_dispatch_v2_t *mp) +{ + vl_api_crypto_set_async_dispatch_v2_reply_t *rmp; + int rv = 0; + + vnet_crypto_set_async_dispatch ((u8) mp->mode, mp->adaptive ? 1 : 0); + + REPLY_MACRO (VL_API_CRYPTO_SET_ASYNC_DISPATCH_V2_REPLY); +} + +static void vl_api_crypto_set_handler_t_handler (vl_api_crypto_set_handler_t * mp) { vl_api_crypto_set_handler_reply_t *rmp; diff --git a/src/vnet/crypto/node.c b/src/vnet/crypto/node.c index e753f1ad1db..ee7f344ce68 100644 --- a/src/vnet/crypto/node.c +++ b/src/vnet/crypto/node.c @@ -135,8 +135,11 @@ crypto_dequeue_frame (vlib_main_t * vm, vlib_node_runtime_t * node, vnet_crypto_async_free_frame (vm, cf); } /* signal enqueue-thread to dequeue the processed frame (n_elts>0) */ - if (cm->dispatch_mode == VNET_CRYPTO_ASYNC_DISPATCH_INTERRUPT - && n_elts > 0) + if (n_elts > 0 && + ((node->state == VLIB_NODE_STATE_POLLING && + (node->flags & + VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE)) || + node->state == VLIB_NODE_STATE_INTERRUPT)) { vlib_node_set_interrupt_pending ( vlib_get_main_by_index (enqueue_thread_idx), @@ -158,27 +161,35 @@ VLIB_NODE_FN (crypto_dispatch_node) (vlib_main_t * vm, { vnet_crypto_main_t *cm = &crypto_main; vnet_crypto_thread_t *ct = cm->threads + vm->thread_index; - u32 n_dispatched = 0, n_cache = 0; - u32 index; - - /* *INDENT-OFF* */ - clib_bitmap_foreach (index, cm->async_active_ids) { - n_cache = crypto_dequeue_frame (vm, node, ct, cm->dequeue_handlers[index], - n_cache, &n_dispatched); - } - /* *INDENT-ON* */ + u32 n_dispatched = 0, n_cache = 0, index; + vec_foreach_index (index, cm->dequeue_handlers) + { + n_cache = crypto_dequeue_frame ( + vm, node, ct, cm->dequeue_handlers[index], n_cache, &n_dispatched); + } if (n_cache) vlib_buffer_enqueue_to_next_vec (vm, node, &ct->buffer_indices, &ct->nexts, n_cache); + /* if there are still pending tasks and node in interrupt mode, + sending current thread signal to dequeue next loop */ + if (pool_elts (ct->frame_pool) > 0 && + ((node->state == VLIB_NODE_STATE_POLLING && + (node->flags & + VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE)) || + node->state == VLIB_NODE_STATE_INTERRUPT)) + { + vlib_node_set_interrupt_pending (vm, node->node_index); + } + return n_dispatched; } -/* *INDENT-OFF* */ VLIB_REGISTER_NODE (crypto_dispatch_node) = { .name = "crypto-dispatch", .type = VLIB_NODE_TYPE_INPUT, - .state = VLIB_NODE_STATE_DISABLED, + .flags = VLIB_NODE_FLAG_ADAPTIVE_MODE, + .state = VLIB_NODE_STATE_INTERRUPT, .format_trace = format_crypto_dispatch_trace, .n_errors = ARRAY_LEN(vnet_crypto_async_error_strings), @@ -192,7 +203,6 @@ VLIB_REGISTER_NODE (crypto_dispatch_node) = { #undef _ }, }; -/* *INDENT-ON* */ /* * fd.io coding-style-patch-verification: ON |