From efcad1a9d22c4a664f3004cafe09d9c3a68e1620 Mon Sep 17 00:00:00 2001 From: Filip Tehlar Date: Tue, 4 Feb 2020 09:36:04 +0000 Subject: ipsec: add support for chained buffers Type: feature Change-Id: Ie072a7c2bbb1e4a77f7001754f01897efd30fc53 Signed-off-by: Filip Tehlar --- src/vnet/crypto/cli.c | 30 ++++++-- src/vnet/crypto/crypto.c | 183 +++++++++++++++++++++++++++++++++++++---------- src/vnet/crypto/crypto.h | 95 +++++++++++++++++++++--- src/vnet/crypto/format.c | 5 +- 4 files changed, 256 insertions(+), 57 deletions(-) (limited to 'src/vnet/crypto') diff --git a/src/vnet/crypto/cli.c b/src/vnet/crypto/cli.c index 8d523aec484..ef6371ad3dc 100644 --- a/src/vnet/crypto/cli.c +++ b/src/vnet/crypto/cli.c @@ -74,14 +74,24 @@ format_vnet_crypto_handlers (u8 * s, va_list * args) od = cm->opt_data + id; if (first == 0) s = format (s, "\n%U", format_white_space, indent); - s = format (s, "%-20U%-20U", format_vnet_crypto_op_type, od->type, - format_vnet_crypto_engine, od->active_engine_index,s); + s = format (s, "%-22U%-20U", format_vnet_crypto_op_type, od->type, 0, + format_vnet_crypto_engine, od->active_engine_index_simple,s); vec_foreach (e, cm->engines) { if (e->ops_handlers[id] != 0) s = format (s, "%U ", format_vnet_crypto_engine, e - cm->engines); } + + s = format (s, "\n%U", format_white_space, indent); + s = format (s, "%-22U%-20U", format_vnet_crypto_op_type, od->type, 1, + format_vnet_crypto_engine, + od->active_engine_index_chained); + vec_foreach (e, cm->engines) + { + if (e->chained_ops_handlers[id] != 0) + s = format (s, "%U ", format_vnet_crypto_engine, e - cm->engines); + } first = 0; } return s; @@ -98,7 +108,7 @@ show_crypto_handlers_command_fn (vlib_main_t * vm, if (unformat_user (input, unformat_line_input, line_input)) unformat_free (line_input); - vlib_cli_output (vm, "%-20s%-20s%-20s%s", "Algo", "Type", "Active", + vlib_cli_output (vm, "%-20s%-22s%-20s%s", "Algo", "Type", "Active", "Candidates"); for (i = 0; i < VNET_CRYPTO_N_ALGS; i++) @@ -128,6 +138,7 @@ set_crypto_handler_command_fn (vlib_main_t * vm, char **args = 0, *s, **arg, *engine = 0; int all = 0; clib_error_t *error = 0; + crypto_op_class_type_t oct = CRYPTO_OP_BOTH; if (!unformat_user (input, unformat_line_input, line_input)) return 0; @@ -136,6 +147,12 @@ set_crypto_handler_command_fn (vlib_main_t * vm, { if (unformat (line_input, "all")) all = 1; + else if (unformat (line_input, "simple")) + oct = CRYPTO_OP_SIMPLE; + else if (unformat (line_input, "chained")) + oct = CRYPTO_OP_CHAINED; + else if (unformat (line_input, "both")) + oct = CRYPTO_OP_BOTH; else if (unformat (line_input, "%s", &s)) vec_add1 (args, s); else @@ -163,7 +180,7 @@ set_crypto_handler_command_fn (vlib_main_t * vm, hash_foreach_mem (key, value, cm->alg_index_by_name, ({ (void) value; - rc += vnet_crypto_set_handler (key, engine); + rc += vnet_crypto_set_handler2 (key, engine, oct); })); /* *INDENT-ON* */ @@ -174,7 +191,7 @@ set_crypto_handler_command_fn (vlib_main_t * vm, { vec_foreach (arg, args) { - rc = vnet_crypto_set_handler (arg[0], engine); + rc = vnet_crypto_set_handler2 (arg[0], engine, oct); if (rc) { vlib_cli_output (vm, "failed to set engine %s for %s!", @@ -195,7 +212,8 @@ done: VLIB_CLI_COMMAND (set_crypto_handler_command, static) = { .path = "set crypto handler", - .short_help = "set crypto handler cipher [cipher2 cipher3 ...] engine", + .short_help = "set crypto handler cipher [cipher2 cipher3 ...] engine" + " [simple|chained]", .function = set_crypto_handler_command_fn, }; /* *INDENT-ON* */ diff --git a/src/vnet/crypto/crypto.c b/src/vnet/crypto/crypto.c index 6cd12108f6c..4458bfcff4d 100644 --- a/src/vnet/crypto/crypto.c +++ b/src/vnet/crypto/crypto.c @@ -19,31 +19,52 @@ vnet_crypto_main_t crypto_main; +static_always_inline void +crypto_set_op_status (vnet_crypto_op_t * ops[], u32 n_ops, int status) +{ + while (n_ops--) + { + ops[0]->status = status; + ops++; + } +} + static_always_inline u32 vnet_crypto_process_ops_call_handler (vlib_main_t * vm, vnet_crypto_main_t * cm, vnet_crypto_op_id_t opt, - vnet_crypto_op_t * ops[], u32 n_ops) + vnet_crypto_op_t * ops[], + vnet_crypto_op_chunk_t * chunks, + u32 n_ops) { + u32 rv = 0; if (n_ops == 0) return 0; - if (cm->ops_handlers[opt] == 0) + if (chunks) { - while (n_ops--) - { - ops[0]->status = VNET_CRYPTO_OP_STATUS_FAIL_NO_HANDLER; - ops++; - } - return 0; - } - return (cm->ops_handlers[opt]) (vm, ops, n_ops); + if (cm->chained_ops_handlers[opt] == 0) + crypto_set_op_status (ops, n_ops, + VNET_CRYPTO_OP_STATUS_FAIL_NO_HANDLER); + else + rv = (cm->chained_ops_handlers[opt]) (vm, ops, chunks, n_ops); + } + else + { + if (cm->ops_handlers[opt] == 0) + crypto_set_op_status (ops, n_ops, + VNET_CRYPTO_OP_STATUS_FAIL_NO_HANDLER); + else + rv = (cm->ops_handlers[opt]) (vm, ops, n_ops); + } + return rv; } -u32 -vnet_crypto_process_ops (vlib_main_t * vm, vnet_crypto_op_t ops[], u32 n_ops) +static_always_inline u32 +vnet_crypto_process_ops_inline (vlib_main_t * vm, vnet_crypto_op_t ops[], + vnet_crypto_op_chunk_t * chunks, u32 n_ops) { vnet_crypto_main_t *cm = &crypto_main; const int op_q_size = VLIB_FRAME_SIZE; @@ -61,7 +82,8 @@ vnet_crypto_process_ops (vlib_main_t * vm, vnet_crypto_op_t ops[], u32 n_ops) if (current_op_type != opt || n_op_queue >= op_q_size) { rv += vnet_crypto_process_ops_call_handler (vm, cm, current_op_type, - op_queue, n_op_queue); + op_queue, chunks, + n_op_queue); n_op_queue = 0; current_op_type = opt; } @@ -70,10 +92,23 @@ vnet_crypto_process_ops (vlib_main_t * vm, vnet_crypto_op_t ops[], u32 n_ops) } rv += vnet_crypto_process_ops_call_handler (vm, cm, current_op_type, - op_queue, n_op_queue); + op_queue, chunks, n_op_queue); return rv; } +u32 +vnet_crypto_process_ops (vlib_main_t * vm, vnet_crypto_op_t ops[], u32 n_ops) +{ + return vnet_crypto_process_ops_inline (vm, ops, 0, n_ops); +} + +u32 +vnet_crypto_process_chained_ops (vlib_main_t * vm, vnet_crypto_op_t ops[], + vnet_crypto_op_chunk_t * chunks, u32 n_ops) +{ + return vnet_crypto_process_ops_inline (vm, ops, chunks, n_ops); +} + u32 vnet_crypto_register_engine (vlib_main_t * vm, char *name, int prio, char *desc) @@ -91,13 +126,40 @@ vnet_crypto_register_engine (vlib_main_t * vm, char *name, int prio, return p - cm->engines; } +static_always_inline void +crypto_set_active_engine (vnet_crypto_op_data_t * od, + vnet_crypto_op_id_t id, u32 ei, + crypto_op_class_type_t oct) +{ + vnet_crypto_main_t *cm = &crypto_main; + vnet_crypto_engine_t *ce = vec_elt_at_index (cm->engines, ei); + + if (oct == CRYPTO_OP_BOTH || oct == CRYPTO_OP_CHAINED) + { + if (ce->chained_ops_handlers[id]) + { + od->active_engine_index_chained = ei; + cm->chained_ops_handlers[id] = ce->chained_ops_handlers[id]; + } + } + + if (oct == CRYPTO_OP_BOTH || oct == CRYPTO_OP_SIMPLE) + { + if (ce->ops_handlers[id]) + { + od->active_engine_index_simple = ei; + cm->ops_handlers[id] = ce->ops_handlers[id]; + } + } +} + int -vnet_crypto_set_handler (char *alg_name, char *engine) +vnet_crypto_set_handler2 (char *alg_name, char *engine, + crypto_op_class_type_t oct) { uword *p; vnet_crypto_main_t *cm = &crypto_main; vnet_crypto_alg_data_t *ad; - vnet_crypto_engine_t *ce; int i; p = hash_get_mem (cm->alg_index_by_name, alg_name); @@ -110,20 +172,15 @@ vnet_crypto_set_handler (char *alg_name, char *engine) if (!p) return -1; - ce = vec_elt_at_index (cm->engines, p[0]); - - for (i = 0; i < VNET_CRYPTO_OP_N_TYPES; i++) + for (i = 0; i < VNET_CRYPTO_OP_N_TYPES; i += 2) { vnet_crypto_op_data_t *od; vnet_crypto_op_id_t id = ad->op_by_type[i]; if (id == 0) continue; + od = cm->opt_data + id; - if (ce->ops_handlers[id]) - { - od->active_engine_index = p[0]; - cm->ops_handlers[id] = ce->ops_handlers[id]; - } + crypto_set_active_engine (od, id, p[0], oct); } return 0; @@ -138,33 +195,77 @@ vnet_crypto_is_set_handler (vnet_crypto_alg_t alg) } void -vnet_crypto_register_ops_handler (vlib_main_t * vm, u32 engine_index, - vnet_crypto_op_id_t opt, - vnet_crypto_ops_handler_t * fn) +vnet_crypto_register_ops_handler_inline (vlib_main_t * vm, u32 engine_index, + vnet_crypto_op_id_t opt, + vnet_crypto_ops_handler_t * fn, + vnet_crypto_chained_ops_handler_t * + cfn) { vnet_crypto_main_t *cm = &crypto_main; vnet_crypto_engine_t *ae, *e = vec_elt_at_index (cm->engines, engine_index); vnet_crypto_op_data_t *otd = cm->opt_data + opt; vec_validate_aligned (cm->ops_handlers, VNET_CRYPTO_N_OP_IDS - 1, CLIB_CACHE_LINE_BYTES); - e->ops_handlers[opt] = fn; + vec_validate_aligned (cm->chained_ops_handlers, VNET_CRYPTO_N_OP_IDS - 1, + CLIB_CACHE_LINE_BYTES); - if (otd->active_engine_index == ~0) + if (fn) { - otd->active_engine_index = engine_index; - cm->ops_handlers[opt] = fn; - return; + e->ops_handlers[opt] = fn; + if (otd->active_engine_index_simple == ~0) + { + otd->active_engine_index_simple = engine_index; + cm->ops_handlers[opt] = fn; + } + + ae = vec_elt_at_index (cm->engines, otd->active_engine_index_simple); + if (ae->priority < e->priority) + crypto_set_active_engine (otd, opt, engine_index, CRYPTO_OP_SIMPLE); } - ae = vec_elt_at_index (cm->engines, otd->active_engine_index); - if (ae->priority < e->priority) + + if (cfn) { - otd->active_engine_index = engine_index; - cm->ops_handlers[opt] = fn; + e->chained_ops_handlers[opt] = cfn; + if (otd->active_engine_index_chained == ~0) + { + otd->active_engine_index_chained = engine_index; + cm->chained_ops_handlers[opt] = cfn; + } + + ae = vec_elt_at_index (cm->engines, otd->active_engine_index_chained); + if (ae->priority < e->priority) + crypto_set_active_engine (otd, opt, engine_index, CRYPTO_OP_CHAINED); } return; } +void +vnet_crypto_register_ops_handler (vlib_main_t * vm, u32 engine_index, + vnet_crypto_op_id_t opt, + vnet_crypto_ops_handler_t * fn) +{ + vnet_crypto_register_ops_handler_inline (vm, engine_index, opt, fn, 0); +} + +void +vnet_crypto_register_chained_ops_handler (vlib_main_t * vm, u32 engine_index, + vnet_crypto_op_id_t opt, + vnet_crypto_chained_ops_handler_t * + fn) +{ + vnet_crypto_register_ops_handler_inline (vm, engine_index, opt, 0, fn); +} + +void +vnet_crypto_register_ops_handlers (vlib_main_t * vm, u32 engine_index, + vnet_crypto_op_id_t opt, + vnet_crypto_ops_handler_t * fn, + vnet_crypto_chained_ops_handler_t * cfn) +{ + vnet_crypto_register_ops_handler_inline (vm, engine_index, opt, fn, cfn); +} + void vnet_crypto_register_key_handler (vlib_main_t * vm, u32 engine_index, vnet_crypto_key_handler_t * key_handler) @@ -253,10 +354,13 @@ vnet_crypto_init_cipher_data (vnet_crypto_alg_t alg, vnet_crypto_op_id_t eid, { vnet_crypto_op_type_t eopt, dopt; vnet_crypto_main_t *cm = &crypto_main; + cm->algs[alg].name = name; cm->opt_data[eid].alg = cm->opt_data[did].alg = alg; - cm->opt_data[eid].active_engine_index = ~0; - cm->opt_data[did].active_engine_index = ~0; + cm->opt_data[eid].active_engine_index_simple = ~0; + cm->opt_data[did].active_engine_index_simple = ~0; + cm->opt_data[eid].active_engine_index_chained = ~0; + cm->opt_data[did].active_engine_index_chained = ~0; if (is_aead) { eopt = VNET_CRYPTO_OP_TYPE_AEAD_ENCRYPT; @@ -282,7 +386,8 @@ vnet_crypto_init_hmac_data (vnet_crypto_alg_t alg, cm->algs[alg].name = name; cm->algs[alg].op_by_type[VNET_CRYPTO_OP_TYPE_HMAC] = id; cm->opt_data[id].alg = alg; - cm->opt_data[id].active_engine_index = ~0; + cm->opt_data[id].active_engine_index_simple = ~0; + cm->opt_data[id].active_engine_index_chained = ~0; cm->opt_data[id].type = VNET_CRYPTO_OP_TYPE_HMAC; hash_set_mem (cm->alg_index_by_name, name, alg); } diff --git a/src/vnet/crypto/crypto.h b/src/vnet/crypto/crypto.h index 626e71d5030..f89ecf9f3d5 100644 --- a/src/vnet/crypto/crypto.h +++ b/src/vnet/crypto/crypto.h @@ -116,38 +116,82 @@ typedef enum } vnet_crypto_op_id_t; /* *INDENT-ON* */ +typedef enum +{ + CRYPTO_OP_SIMPLE, + CRYPTO_OP_CHAINED, + CRYPTO_OP_BOTH, +} crypto_op_class_type_t; + typedef struct { char *name; vnet_crypto_op_id_t op_by_type[VNET_CRYPTO_OP_N_TYPES]; } vnet_crypto_alg_data_t; +typedef struct +{ + u8 *src; + u8 *dst; + u32 len; +} vnet_crypto_op_chunk_t; + typedef struct { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + uword user_data; vnet_crypto_op_id_t op:16; vnet_crypto_op_status_t status:8; u8 flags; #define VNET_CRYPTO_OP_FLAG_INIT_IV (1 << 0) #define VNET_CRYPTO_OP_FLAG_HMAC_CHECK (1 << 1) - u32 key_index; - u32 len; +#define VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS (1 << 2) + + union + { + u8 digest_len; + u8 tag_len; + }; u16 aad_len; - u8 digest_len, tag_len; + + union + { + struct + { + u8 *src; + u8 *dst; + }; + + /* valid if VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS is set */ + u16 n_chunks; + }; + + union + { + u32 len; + /* valid if VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS is set */ + u32 chunk_index; + }; + + u32 key_index; u8 *iv; - u8 *src; - u8 *dst; u8 *aad; - u8 *tag; - u8 *digest; - uword user_data; + + union + { + u8 *tag; + u8 *digest; + }; } vnet_crypto_op_t; +STATIC_ASSERT_SIZEOF (vnet_crypto_op_t, CLIB_CACHE_LINE_BYTES); + typedef struct { vnet_crypto_op_type_t type; vnet_crypto_alg_t alg; - u32 active_engine_index; + u32 active_engine_index_simple; + u32 active_engine_index_chained; } vnet_crypto_op_data_t; typedef struct @@ -158,6 +202,11 @@ typedef struct typedef u32 vnet_crypto_key_index_t; +typedef u32 (vnet_crypto_chained_ops_handler_t) (vlib_main_t * vm, + vnet_crypto_op_t * ops[], + vnet_crypto_op_chunk_t * + chunks, u32 n_ops); + typedef u32 (vnet_crypto_ops_handler_t) (vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops); @@ -171,6 +220,18 @@ u32 vnet_crypto_register_engine (vlib_main_t * vm, char *name, int prio, void vnet_crypto_register_ops_handler (vlib_main_t * vm, u32 engine_index, vnet_crypto_op_id_t opt, vnet_crypto_ops_handler_t * oph); + +void vnet_crypto_register_chained_ops_handler (vlib_main_t * vm, + u32 engine_index, + vnet_crypto_op_id_t opt, + vnet_crypto_chained_ops_handler_t + * oph); +void vnet_crypto_register_ops_handlers (vlib_main_t * vm, u32 engine_index, + vnet_crypto_op_id_t opt, + vnet_crypto_ops_handler_t * fn, + vnet_crypto_chained_ops_handler_t * + cfn); + void vnet_crypto_register_key_handler (vlib_main_t * vm, u32 engine_index, vnet_crypto_key_handler_t * keyh); @@ -181,6 +242,8 @@ typedef struct int priority; vnet_crypto_key_handler_t *key_op_handler; vnet_crypto_ops_handler_t *ops_handlers[VNET_CRYPTO_N_OP_IDS]; + vnet_crypto_chained_ops_handler_t + * chained_ops_handlers[VNET_CRYPTO_N_OP_IDS]; } vnet_crypto_engine_t; typedef struct @@ -188,6 +251,7 @@ typedef struct vnet_crypto_alg_data_t *algs; vnet_crypto_thread_t *threads; vnet_crypto_ops_handler_t **ops_handlers; + vnet_crypto_chained_ops_handler_t **chained_ops_handlers; vnet_crypto_op_data_t opt_data[VNET_CRYPTO_N_OP_IDS]; vnet_crypto_engine_t *engines; vnet_crypto_key_t *keys; @@ -200,10 +264,14 @@ extern vnet_crypto_main_t crypto_main; u32 vnet_crypto_submit_ops (vlib_main_t * vm, vnet_crypto_op_t ** jobs, u32 n_jobs); +u32 vnet_crypto_process_chained_ops (vlib_main_t * vm, vnet_crypto_op_t ops[], + vnet_crypto_op_chunk_t * chunks, + u32 n_ops); u32 vnet_crypto_process_ops (vlib_main_t * vm, vnet_crypto_op_t ops[], u32 n_ops); -int vnet_crypto_set_handler (char *ops_handler_name, char *engine); +int vnet_crypto_set_handler2 (char *ops_handler_name, char *engine, + crypto_op_class_type_t oct); int vnet_crypto_is_set_handler (vnet_crypto_alg_t alg); u32 vnet_crypto_key_add (vlib_main_t * vm, vnet_crypto_alg_t alg, @@ -225,6 +293,7 @@ vnet_crypto_op_init (vnet_crypto_op_t * op, vnet_crypto_op_id_t type) op->op = type; op->flags = 0; op->key_index = ~0; + op->n_chunks = 0; } static_always_inline vnet_crypto_op_type_t @@ -243,6 +312,12 @@ vnet_crypto_get_key (vnet_crypto_key_index_t index) return vec_elt_at_index (cm->keys, index); } +static_always_inline int +vnet_crypto_set_handler (char *alg_name, char *engine) +{ + return vnet_crypto_set_handler2 (alg_name, engine, CRYPTO_OP_BOTH); +} + #endif /* included_vnet_crypto_crypto_h */ /* diff --git a/src/vnet/crypto/format.c b/src/vnet/crypto/format.c index 715941e0ee7..3210ab9eac0 100644 --- a/src/vnet/crypto/format.c +++ b/src/vnet/crypto/format.c @@ -54,7 +54,7 @@ format_vnet_crypto_op (u8 * s, va_list * args) vnet_crypto_op_id_t op = va_arg (*args, int); // vnet_crypto_op_id_t); vnet_crypto_op_data_t *otd = cm->opt_data + op; - return format (s, "%U-%U", format_vnet_crypto_op_type, otd->type, + return format (s, "%U-%U", format_vnet_crypto_op_type, otd->type, 0, format_vnet_crypto_alg, otd->alg); } @@ -62,6 +62,7 @@ u8 * format_vnet_crypto_op_type (u8 * s, va_list * args) { vnet_crypto_op_type_t opt = va_arg (*args, vnet_crypto_op_type_t); + int is_chained = va_arg (*args, int); char *strings[] = { #define _(n, s) [VNET_CRYPTO_OP_TYPE_##n] = s, foreach_crypto_op_type @@ -71,7 +72,7 @@ format_vnet_crypto_op_type (u8 * s, va_list * args) if (opt >= VNET_CRYPTO_OP_N_TYPES) return format (s, "unknown"); - return format (s, "%s", strings[opt]); + return format (s, "%s%s", strings[opt], is_chained ? "-chained" : ""); } u8 * -- cgit 1.2.3-korg