diff options
Diffstat (limited to 'src')
45 files changed, 1385 insertions, 526 deletions
diff --git a/src/plugins/acl/acl_test.c b/src/plugins/acl/acl_test.c index fddb3d532ff..8404689dc06 100644 --- a/src/plugins/acl/acl_test.c +++ b/src/plugins/acl/acl_test.c @@ -18,6 +18,8 @@ *------------------------------------------------------------------ */ +#include <byteswap.h> + #include <vat/vat.h> #include <vlibapi/api.h> #include <vlibmemory/api.h> diff --git a/src/plugins/acl/sess_mgmt_node.c b/src/plugins/acl/sess_mgmt_node.c index e049a3ffa85..418baef9b6b 100644 --- a/src/plugins/acl/sess_mgmt_node.c +++ b/src/plugins/acl/sess_mgmt_node.c @@ -371,8 +371,9 @@ send_one_worker_interrupt (vlib_main_t * vm, acl_main_t * am, } void -aclp_post_session_change_request (acl_main_t * am, u32 target_thread, - u32 target_session, u32 request_type) +aclp_post_session_change_request (acl_main_t *am, u32 target_thread, + u32 target_session, + acl_fa_sess_req_t request_type) { acl_fa_per_worker_data_t *pw_me = &am->per_worker_data[os_get_thread_index ()]; diff --git a/src/plugins/crypto_native/CMakeLists.txt b/src/plugins/crypto_native/CMakeLists.txt index 9b6091610d9..5499ed4608a 100644 --- a/src/plugins/crypto_native/CMakeLists.txt +++ b/src/plugins/crypto_native/CMakeLists.txt @@ -12,8 +12,8 @@ # limitations under the License. if(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|x86_64.*|AMD64.*") - list(APPEND VARIANTS "slm\;-march=silvermont") - list(APPEND VARIANTS "hsw\;-march=haswell") + list(APPEND VARIANTS "slm\;-march=silvermont -maes") + list(APPEND VARIANTS "hsw\;-march=haswell -maes") if(compiler_flag_march_skylake_avx512 AND compiler_flag_mprefer_vector_width_256) list(APPEND VARIANTS "skx\;-march=skylake-avx512 -mprefer-vector-width=256") endif() @@ -23,16 +23,15 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|x86_64.*|AMD64.*") if(compiler_flag_march_alderlake) list(APPEND VARIANTS "adl\;-march=alderlake -mprefer-vector-width=256") endif() - set (COMPILE_FILES aes_cbc.c aes_gcm.c aes_ctr.c) - set (COMPILE_OPTS -Wall -fno-common -maes) endif() if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*)") list(APPEND VARIANTS "armv8\;-march=armv8.1-a+crc+crypto") - set (COMPILE_FILES aes_cbc.c aes_gcm.c aes_ctr.c) - set (COMPILE_OPTS -Wall -fno-common) endif() +set (COMPILE_FILES aes_cbc.c aes_gcm.c aes_ctr.c sha2.c) +set (COMPILE_OPTS -Wall -fno-common) + if (NOT VARIANTS) return() endif() diff --git a/src/plugins/crypto_native/aes_cbc.c b/src/plugins/crypto_native/aes_cbc.c index c84390c3108..dd7ca3f1cf1 100644 --- a/src/plugins/crypto_native/aes_cbc.c +++ b/src/plugins/crypto_native/aes_cbc.c @@ -249,18 +249,30 @@ decrypt: return n_ops; } -#define foreach_aes_cbc_handler_type _(128) _(192) _(256) - -#define _(x) \ -static u32 aes_ops_dec_aes_cbc_##x \ -(vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops) \ -{ return aes_ops_dec_aes_cbc (vm, ops, n_ops, AES_KEY_##x); } \ -static u32 aes_ops_enc_aes_cbc_##x \ -(vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops) \ -{ return aes_ops_enc_aes_cbc (vm, ops, n_ops, AES_KEY_##x); } \ - -foreach_aes_cbc_handler_type; -#undef _ +static int +aes_cbc_cpu_probe () +{ +#if defined(__VAES__) && defined(__AVX512F__) + if (clib_cpu_supports_vaes () && clib_cpu_supports_avx512f ()) + return 50; +#elif defined(__VAES__) + if (clib_cpu_supports_vaes ()) + return 40; +#elif defined(__AVX512F__) + if (clib_cpu_supports_avx512f ()) + return 30; +#elif defined(__AVX2__) + if (clib_cpu_supports_avx2 ()) + return 20; +#elif __AES__ + if (clib_cpu_supports_aes ()) + return 10; +#elif __aarch64__ + if (clib_cpu_supports_aarch64_aes ()) + return 10; +#endif + return -1; +} static void * aes_cbc_key_exp_128 (vnet_crypto_key_t *key) @@ -289,43 +301,39 @@ aes_cbc_key_exp_256 (vnet_crypto_key_t *key) return kd; } -#include <fcntl.h> - -clib_error_t * -#if defined(__VAES__) && defined(__AVX512F__) -crypto_native_aes_cbc_init_icl (vlib_main_t *vm) -#elif defined(__VAES__) -crypto_native_aes_cbc_init_adl (vlib_main_t *vm) -#elif __AVX512F__ -crypto_native_aes_cbc_init_skx (vlib_main_t * vm) -#elif __aarch64__ -crypto_native_aes_cbc_init_neon (vlib_main_t * vm) -#elif __AVX2__ -crypto_native_aes_cbc_init_hsw (vlib_main_t * vm) -#else -crypto_native_aes_cbc_init_slm (vlib_main_t * vm) -#endif -{ - crypto_native_main_t *cm = &crypto_native_main; +#define foreach_aes_cbc_handler_type _ (128) _ (192) _ (256) + +#define _(x) \ + static u32 aes_ops_enc_aes_cbc_##x (vlib_main_t *vm, \ + vnet_crypto_op_t *ops[], u32 n_ops) \ + { \ + return aes_ops_enc_aes_cbc (vm, ops, n_ops, AES_KEY_##x); \ + } \ + \ + CRYPTO_NATIVE_OP_HANDLER (aes_##x##_cbc_enc) = { \ + .op_id = VNET_CRYPTO_OP_AES_##x##_CBC_ENC, \ + .fn = aes_ops_enc_aes_cbc_##x, \ + .probe = aes_cbc_cpu_probe, \ + }; \ + \ + static u32 aes_ops_dec_aes_cbc_##x (vlib_main_t *vm, \ + vnet_crypto_op_t *ops[], u32 n_ops) \ + { \ + return aes_ops_dec_aes_cbc (vm, ops, n_ops, AES_KEY_##x); \ + } \ + \ + CRYPTO_NATIVE_OP_HANDLER (aes_##x##_cbc_dec) = { \ + .op_id = VNET_CRYPTO_OP_AES_##x##_CBC_DEC, \ + .fn = aes_ops_dec_aes_cbc_##x, \ + .probe = aes_cbc_cpu_probe, \ + }; \ + \ + CRYPTO_NATIVE_KEY_HANDLER (aes_##x##_cbc) = { \ + .alg_id = VNET_CRYPTO_ALG_AES_##x##_CBC, \ + .key_fn = aes_cbc_key_exp_##x, \ + .probe = aes_cbc_cpu_probe, \ + }; -#define _(x) \ - vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \ - VNET_CRYPTO_OP_AES_##x##_CBC_ENC, \ - aes_ops_enc_aes_cbc_##x); \ - vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \ - VNET_CRYPTO_OP_AES_##x##_CBC_DEC, \ - aes_ops_dec_aes_cbc_##x); \ - cm->key_fn[VNET_CRYPTO_ALG_AES_##x##_CBC] = aes_cbc_key_exp_##x; - foreach_aes_cbc_handler_type; +foreach_aes_cbc_handler_type; #undef _ - return 0; -} - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/plugins/crypto_native/aes_ctr.c b/src/plugins/crypto_native/aes_ctr.c index 3a219510419..d02a7b69b9d 100644 --- a/src/plugins/crypto_native/aes_ctr.c +++ b/src/plugins/crypto_native/aes_ctr.c @@ -81,32 +81,50 @@ aes_ctr_key_exp (vnet_crypto_key_t *key, aes_key_size_t ks) foreach_aes_ctr_handler_type; #undef _ -clib_error_t * +static int +probe () +{ #if defined(__VAES__) && defined(__AVX512F__) -crypto_native_aes_ctr_init_icl (vlib_main_t *vm) + if (clib_cpu_supports_vaes () && clib_cpu_supports_avx512f ()) + return 50; #elif defined(__VAES__) -crypto_native_aes_ctr_init_adl (vlib_main_t *vm) -#elif __AVX512F__ -crypto_native_aes_ctr_init_skx (vlib_main_t *vm) -#elif __AVX2__ -crypto_native_aes_ctr_init_hsw (vlib_main_t *vm) + if (clib_cpu_supports_vaes ()) + return 40; +#elif defined(__AVX512F__) + if (clib_cpu_supports_avx512f ()) + return 30; +#elif defined(__AVX2__) + if (clib_cpu_supports_avx2 ()) + return 20; +#elif __AES__ + if (clib_cpu_supports_aes ()) + return 10; #elif __aarch64__ -crypto_native_aes_ctr_init_neon (vlib_main_t *vm) -#else -crypto_native_aes_ctr_init_slm (vlib_main_t *vm) + if (clib_cpu_supports_aarch64_aes ()) + return 10; #endif -{ - crypto_native_main_t *cm = &crypto_native_main; + return -1; +} -#define _(x) \ - vnet_crypto_register_ops_handlers ( \ - vm, cm->crypto_engine_index, VNET_CRYPTO_OP_AES_##x##_CTR_ENC, \ - aes_ops_aes_ctr_##x, aes_ops_aes_ctr_##x##_chained); \ - vnet_crypto_register_ops_handlers ( \ - vm, cm->crypto_engine_index, VNET_CRYPTO_OP_AES_##x##_CTR_DEC, \ - aes_ops_aes_ctr_##x, aes_ops_aes_ctr_##x##_chained); \ - cm->key_fn[VNET_CRYPTO_ALG_AES_##x##_CTR] = aes_ctr_key_exp_##x; - foreach_aes_ctr_handler_type; +#define _(b) \ + CRYPTO_NATIVE_OP_HANDLER (aes_##b##_ctr_enc) = { \ + .op_id = VNET_CRYPTO_OP_AES_##b##_CTR_ENC, \ + .fn = aes_ops_aes_ctr_##b, \ + .cfn = aes_ops_aes_ctr_##b##_chained, \ + .probe = probe, \ + }; \ + \ + CRYPTO_NATIVE_OP_HANDLER (aes_##b##_ctr_dec) = { \ + .op_id = VNET_CRYPTO_OP_AES_##b##_CTR_DEC, \ + .fn = aes_ops_aes_ctr_##b, \ + .cfn = aes_ops_aes_ctr_##b##_chained, \ + .probe = probe, \ + }; \ + CRYPTO_NATIVE_KEY_HANDLER (aes_##b##_ctr) = { \ + .alg_id = VNET_CRYPTO_ALG_AES_##b##_CTR, \ + .key_fn = aes_ctr_key_exp_##b, \ + .probe = probe, \ + }; + +_ (128) _ (192) _ (256) #undef _ - return 0; -} diff --git a/src/plugins/crypto_native/aes_gcm.c b/src/plugins/crypto_native/aes_gcm.c index 6589d411975..220788d4e97 100644 --- a/src/plugins/crypto_native/aes_gcm.c +++ b/src/plugins/crypto_native/aes_gcm.c @@ -118,40 +118,49 @@ aes_gcm_key_exp (vnet_crypto_key_t *key, aes_key_size_t ks) foreach_aes_gcm_handler_type; #undef _ -clib_error_t * +static int +probe () +{ #if defined(__VAES__) && defined(__AVX512F__) -crypto_native_aes_gcm_init_icl (vlib_main_t *vm) + if (clib_cpu_supports_vpclmulqdq () && clib_cpu_supports_vaes () && + clib_cpu_supports_avx512f ()) + return 50; #elif defined(__VAES__) -crypto_native_aes_gcm_init_adl (vlib_main_t *vm) -#elif __AVX512F__ -crypto_native_aes_gcm_init_skx (vlib_main_t *vm) -#elif __AVX2__ -crypto_native_aes_gcm_init_hsw (vlib_main_t *vm) + if (clib_cpu_supports_vpclmulqdq () && clib_cpu_supports_vaes ()) + return 40; +#elif defined(__AVX512F__) + if (clib_cpu_supports_pclmulqdq () && clib_cpu_supports_avx512f ()) + return 30; +#elif defined(__AVX2__) + if (clib_cpu_supports_pclmulqdq () && clib_cpu_supports_avx2 ()) + return 20; +#elif __AES__ + if (clib_cpu_supports_pclmulqdq () && clib_cpu_supports_aes ()) + return 10; #elif __aarch64__ -crypto_native_aes_gcm_init_neon (vlib_main_t *vm) -#else -crypto_native_aes_gcm_init_slm (vlib_main_t *vm) + if (clib_cpu_supports_aarch64_aes ()) + return 10; #endif -{ - crypto_native_main_t *cm = &crypto_native_main; - -#define _(x) \ - vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \ - VNET_CRYPTO_OP_AES_##x##_GCM_ENC, \ - aes_ops_enc_aes_gcm_##x); \ - vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \ - VNET_CRYPTO_OP_AES_##x##_GCM_DEC, \ - aes_ops_dec_aes_gcm_##x); \ - cm->key_fn[VNET_CRYPTO_ALG_AES_##x##_GCM] = aes_gcm_key_exp_##x; - foreach_aes_gcm_handler_type; -#undef _ - return 0; + return -1; } -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ +#define _(b) \ + CRYPTO_NATIVE_OP_HANDLER (aes_##b##_gcm_enc) = { \ + .op_id = VNET_CRYPTO_OP_AES_##b##_GCM_ENC, \ + .fn = aes_ops_enc_aes_gcm_##b, \ + .probe = probe, \ + }; \ + \ + CRYPTO_NATIVE_OP_HANDLER (aes_##b##_gcm_dec) = { \ + .op_id = VNET_CRYPTO_OP_AES_##b##_GCM_DEC, \ + .fn = aes_ops_dec_aes_gcm_##b, \ + .probe = probe, \ + }; \ + CRYPTO_NATIVE_KEY_HANDLER (aes_##b##_gcm) = { \ + .alg_id = VNET_CRYPTO_ALG_AES_##b##_GCM, \ + .key_fn = aes_gcm_key_exp_##b, \ + .probe = probe, \ + }; + +_ (128) _ (192) _ (256) +#undef _ diff --git a/src/plugins/crypto_native/crypto_native.h b/src/plugins/crypto_native/crypto_native.h index c15b8cbd1da..3d18e8cabd0 100644 --- a/src/plugins/crypto_native/crypto_native.h +++ b/src/plugins/crypto_native/crypto_native.h @@ -19,33 +19,66 @@ #define __crypto_native_h__ typedef void *(crypto_native_key_fn_t) (vnet_crypto_key_t * key); +typedef int (crypto_native_variant_probe_t) (); + +typedef struct crypto_native_op_handler +{ + struct crypto_native_op_handler *next; + vnet_crypto_op_id_t op_id; + vnet_crypto_ops_handler_t *fn; + vnet_crypto_chained_ops_handler_t *cfn; + crypto_native_variant_probe_t *probe; + int priority; +} crypto_native_op_handler_t; + +typedef struct crypto_native_key_handler +{ + struct crypto_native_key_handler *next; + vnet_crypto_alg_t alg_id; + crypto_native_key_fn_t *key_fn; + crypto_native_variant_probe_t *probe; + int priority; +} crypto_native_key_handler_t; typedef struct { u32 crypto_engine_index; crypto_native_key_fn_t *key_fn[VNET_CRYPTO_N_ALGS]; void **key_data; + crypto_native_op_handler_t *op_handlers; + crypto_native_key_handler_t *key_handlers; } crypto_native_main_t; extern crypto_native_main_t crypto_native_main; -#define foreach_crypto_native_march_variant \ - _ (slm) _ (hsw) _ (skx) _ (icl) _ (adl) _ (neon) - -#define _(v) \ - clib_error_t __clib_weak *crypto_native_aes_cbc_init_##v (vlib_main_t *vm); \ - clib_error_t __clib_weak *crypto_native_aes_ctr_init_##v (vlib_main_t *vm); \ - clib_error_t __clib_weak *crypto_native_aes_gcm_init_##v (vlib_main_t *vm); - -foreach_crypto_native_march_variant; -#undef _ +#define CRYPTO_NATIVE_OP_HANDLER(x) \ + static crypto_native_op_handler_t __crypto_native_op_handler_##x; \ + static void __clib_constructor __crypto_native_op_handler_cb_##x (void) \ + { \ + crypto_native_main_t *cm = &crypto_native_main; \ + int priority = __crypto_native_op_handler_##x.probe (); \ + if (priority >= 0) \ + { \ + __crypto_native_op_handler_##x.priority = priority; \ + __crypto_native_op_handler_##x.next = cm->op_handlers; \ + cm->op_handlers = &__crypto_native_op_handler_##x; \ + } \ + } \ + static crypto_native_op_handler_t __crypto_native_op_handler_##x +#define CRYPTO_NATIVE_KEY_HANDLER(x) \ + static crypto_native_key_handler_t __crypto_native_key_handler_##x; \ + static void __clib_constructor __crypto_native_key_handler_cb_##x (void) \ + { \ + crypto_native_main_t *cm = &crypto_native_main; \ + int priority = __crypto_native_key_handler_##x.probe (); \ + if (priority >= 0) \ + { \ + __crypto_native_key_handler_##x.priority = priority; \ + __crypto_native_key_handler_##x.next = cm->key_handlers; \ + cm->key_handlers = &__crypto_native_key_handler_##x; \ + } \ + } \ + static crypto_native_key_handler_t __crypto_native_key_handler_##x #endif /* __crypto_native_h__ */ -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/plugins/crypto_native/main.c b/src/plugins/crypto_native/main.c index 8a59be319b9..2bc0d98f196 100644 --- a/src/plugins/crypto_native/main.c +++ b/src/plugins/crypto_native/main.c @@ -63,95 +63,52 @@ clib_error_t * crypto_native_init (vlib_main_t * vm) { crypto_native_main_t *cm = &crypto_native_main; - clib_error_t *error = 0; - if (clib_cpu_supports_x86_aes () == 0 && - clib_cpu_supports_aarch64_aes () == 0) + if (cm->op_handlers == 0) return 0; cm->crypto_engine_index = vnet_crypto_register_engine (vm, "native", 100, "Native ISA Optimized Crypto"); - if (0); -#if __x86_64__ - else if (crypto_native_aes_cbc_init_icl && clib_cpu_supports_vaes () && - clib_cpu_supports_avx512f ()) - error = crypto_native_aes_cbc_init_icl (vm); - else if (crypto_native_aes_cbc_init_adl && clib_cpu_supports_vaes ()) - error = crypto_native_aes_cbc_init_adl (vm); - else if (crypto_native_aes_cbc_init_skx && clib_cpu_supports_avx512f ()) - error = crypto_native_aes_cbc_init_skx (vm); - else if (crypto_native_aes_cbc_init_hsw && clib_cpu_supports_avx2 ()) - error = crypto_native_aes_cbc_init_hsw (vm); - else if (crypto_native_aes_cbc_init_slm) - error = crypto_native_aes_cbc_init_slm (vm); -#endif -#if __aarch64__ - else if (crypto_native_aes_cbc_init_neon) - error = crypto_native_aes_cbc_init_neon (vm); -#endif - else - error = clib_error_return (0, "No AES CBC implemenation available"); - - if (error) - return error; - - if (0) - ; -#if __x86_64__ - else if (crypto_native_aes_ctr_init_icl && clib_cpu_supports_vaes () && - clib_cpu_supports_avx512f ()) - error = crypto_native_aes_ctr_init_icl (vm); - else if (crypto_native_aes_ctr_init_adl && clib_cpu_supports_vaes ()) - error = crypto_native_aes_ctr_init_adl (vm); - else if (crypto_native_aes_ctr_init_skx && clib_cpu_supports_avx512f ()) - error = crypto_native_aes_ctr_init_skx (vm); - else if (crypto_native_aes_ctr_init_hsw && clib_cpu_supports_avx2 ()) - error = crypto_native_aes_ctr_init_hsw (vm); - else if (crypto_native_aes_ctr_init_slm) - error = crypto_native_aes_ctr_init_slm (vm); -#endif -#if __aarch64__ - else if (crypto_native_aes_ctr_init_neon) - error = crypto_native_aes_ctr_init_neon (vm); -#endif - else - error = clib_error_return (0, "No AES CTR implemenation available"); - - if (error) - return error; - -#if __x86_64__ - if (clib_cpu_supports_pclmulqdq ()) + crypto_native_op_handler_t *oh = cm->op_handlers; + crypto_native_key_handler_t *kh = cm->key_handlers; + crypto_native_op_handler_t **best_by_op_id = 0; + crypto_native_key_handler_t **best_by_alg_id = 0; + + while (oh) { - if (crypto_native_aes_gcm_init_icl && clib_cpu_supports_vaes () && - clib_cpu_supports_avx512f ()) - error = crypto_native_aes_gcm_init_icl (vm); - else if (crypto_native_aes_gcm_init_adl && clib_cpu_supports_vaes ()) - error = crypto_native_aes_gcm_init_adl (vm); - else if (crypto_native_aes_gcm_init_skx && clib_cpu_supports_avx512f ()) - error = crypto_native_aes_gcm_init_skx (vm); - else if (crypto_native_aes_gcm_init_hsw && clib_cpu_supports_avx2 ()) - error = crypto_native_aes_gcm_init_hsw (vm); - else if (crypto_native_aes_gcm_init_slm) - error = crypto_native_aes_gcm_init_slm (vm); - else - error = clib_error_return (0, "No AES GCM implemenation available"); - - if (error) - return error; + vec_validate (best_by_op_id, oh->op_id); + + if (best_by_op_id[oh->op_id] == 0 || + best_by_op_id[oh->op_id]->priority < oh->priority) + best_by_op_id[oh->op_id] = oh; + + oh = oh->next; } -#endif -#if __aarch64__ - if (crypto_native_aes_gcm_init_neon) - error = crypto_native_aes_gcm_init_neon (vm); - else - error = clib_error_return (0, "No AES GCM implemenation available"); - - if (error) - return error; -#endif + + while (kh) + { + vec_validate (best_by_alg_id, kh->alg_id); + + if (best_by_alg_id[kh->alg_id] == 0 || + best_by_alg_id[kh->alg_id]->priority < kh->priority) + best_by_alg_id[kh->alg_id] = kh; + + kh = kh->next; + } + + vec_foreach_pointer (oh, best_by_op_id) + if (oh) + vnet_crypto_register_ops_handlers (vm, cm->crypto_engine_index, + oh->op_id, oh->fn, oh->cfn); + + vec_foreach_pointer (kh, best_by_alg_id) + if (kh) + cm->key_fn[kh->alg_id] = kh->key_fn; + + vec_free (best_by_op_id); + vec_free (best_by_alg_id); vnet_crypto_register_key_handler (vm, cm->crypto_engine_index, crypto_native_key_handler); diff --git a/src/plugins/crypto_native/sha2.c b/src/plugins/crypto_native/sha2.c new file mode 100644 index 00000000000..459ce6d8e79 --- /dev/null +++ b/src/plugins/crypto_native/sha2.c @@ -0,0 +1,186 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2024 Cisco Systems, Inc. + */ + +#include <vlib/vlib.h> +#include <vnet/plugin/plugin.h> +#include <vnet/crypto/crypto.h> +#include <crypto_native/crypto_native.h> +#include <vppinfra/crypto/sha2.h> + +static_always_inline u32 +crypto_native_ops_hash_sha2 (vlib_main_t *vm, vnet_crypto_op_t *ops[], + u32 n_ops, vnet_crypto_op_chunk_t *chunks, + clib_sha2_type_t type, int maybe_chained) +{ + vnet_crypto_op_t *op = ops[0]; + clib_sha2_ctx_t ctx; + u32 n_left = n_ops; + +next: + if (op->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS) + { + vnet_crypto_op_chunk_t *chp = chunks + op->chunk_index; + clib_sha2_init (&ctx, type); + for (int j = 0; j < op->n_chunks; j++, chp++) + clib_sha2_update (&ctx, chp->src, chp->len); + clib_sha2_final (&ctx, op->digest); + } + else + clib_sha2 (type, op->src, op->len, op->digest); + + op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; + + if (--n_left) + { + op += 1; + goto next; + } + + return n_ops; +} + +static_always_inline u32 +crypto_native_ops_hmac_sha2 (vlib_main_t *vm, vnet_crypto_op_t *ops[], + u32 n_ops, vnet_crypto_op_chunk_t *chunks, + clib_sha2_type_t type) +{ + crypto_native_main_t *cm = &crypto_native_main; + vnet_crypto_op_t *op = ops[0]; + u32 n_left = n_ops; + clib_sha2_hmac_ctx_t ctx; + u8 buffer[64]; + u32 sz, n_fail = 0; + + for (; n_left; n_left--, op++) + { + clib_sha2_hmac_init ( + &ctx, type, (clib_sha2_hmac_key_data_t *) cm->key_data[op->key_index]); + if (op->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS) + { + vnet_crypto_op_chunk_t *chp = chunks + op->chunk_index; + for (int j = 0; j < op->n_chunks; j++, chp++) + clib_sha2_hmac_update (&ctx, chp->src, chp->len); + } + else + clib_sha2_hmac_update (&ctx, op->src, op->len); + + clib_sha2_hmac_final (&ctx, buffer); + + if (op->digest_len) + { + sz = op->digest_len; + if (op->flags & VNET_CRYPTO_OP_FLAG_HMAC_CHECK) + { + if ((memcmp (op->digest, buffer, sz))) + { + n_fail++; + op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC; + continue; + } + } + else + clib_memcpy_fast (op->digest, buffer, sz); + } + else + { + sz = clib_sha2_variants[type].digest_size; + if (op->flags & VNET_CRYPTO_OP_FLAG_HMAC_CHECK) + { + if ((memcmp (op->digest, buffer, sz))) + { + n_fail++; + op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC; + continue; + } + } + else + clib_memcpy_fast (op->digest, buffer, sz); + } + + op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; + } + + return n_ops - n_fail; +} + +static void * +sha2_key_add (vnet_crypto_key_t *key, clib_sha2_type_t type) +{ + clib_sha2_hmac_key_data_t *kd; + + kd = clib_mem_alloc_aligned (sizeof (*kd), CLIB_CACHE_LINE_BYTES); + clib_sha2_hmac_key_data (type, key->data, vec_len (key->data), kd); + + return kd; +} + +static int +probe () +{ +#if defined(__SHA__) && defined(__x86_64__) + if (clib_cpu_supports_sha ()) + return 50; +#elif defined(__ARM_FEATURE_SHA2) + if (clib_cpu_supports_sha2 ()) + return 10; +#endif + return -1; +} + +#define _(b) \ + static u32 crypto_native_ops_hash_sha##b ( \ + vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops) \ + { \ + return crypto_native_ops_hash_sha2 (vm, ops, n_ops, 0, CLIB_SHA2_##b, 0); \ + } \ + \ + static u32 crypto_native_ops_chained_hash_sha##b ( \ + vlib_main_t *vm, vnet_crypto_op_t *ops[], vnet_crypto_op_chunk_t *chunks, \ + u32 n_ops) \ + { \ + return crypto_native_ops_hash_sha2 (vm, ops, n_ops, chunks, \ + CLIB_SHA2_##b, 1); \ + } \ + \ + static u32 crypto_native_ops_hmac_sha##b ( \ + vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops) \ + { \ + return crypto_native_ops_hmac_sha2 (vm, ops, n_ops, 0, CLIB_SHA2_##b); \ + } \ + \ + static u32 crypto_native_ops_chained_hmac_sha##b ( \ + vlib_main_t *vm, vnet_crypto_op_t *ops[], vnet_crypto_op_chunk_t *chunks, \ + u32 n_ops) \ + { \ + return crypto_native_ops_hmac_sha2 (vm, ops, n_ops, chunks, \ + CLIB_SHA2_##b); \ + } \ + \ + static void *sha2_##b##_key_add (vnet_crypto_key_t *k) \ + { \ + return sha2_key_add (k, CLIB_SHA2_##b); \ + } \ + \ + CRYPTO_NATIVE_OP_HANDLER (crypto_native_hash_sha##b) = { \ + .op_id = VNET_CRYPTO_OP_SHA##b##_HASH, \ + .fn = crypto_native_ops_hash_sha##b, \ + .cfn = crypto_native_ops_chained_hash_sha##b, \ + .probe = probe, \ + }; \ + CRYPTO_NATIVE_OP_HANDLER (crypto_native_hmac_sha##b) = { \ + .op_id = VNET_CRYPTO_OP_SHA##b##_HMAC, \ + .fn = crypto_native_ops_hmac_sha##b, \ + .cfn = crypto_native_ops_chained_hmac_sha##b, \ + .probe = probe, \ + }; \ + CRYPTO_NATIVE_KEY_HANDLER (crypto_native_hmac_sha##b) = { \ + .alg_id = VNET_CRYPTO_ALG_HMAC_SHA##b, \ + .key_fn = sha2_##b##_key_add, \ + .probe = probe, \ + }; + +_ (224) +_ (256) + +#undef _ diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 03a1d59f7cc..8c5ed95b062 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -114,7 +114,7 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) if ((rrv = roc_nix_dev_init (cd->nix))) return cnx_return_roc_err (dev, rrv, "roc_nix_dev_init"); - if (roc_nix_npc_mac_addr_get (cd->nix, mac_addr)) + if ((rrv = roc_nix_npc_mac_addr_get (cd->nix, mac_addr))) return cnx_return_roc_err (dev, rrv, "roc_nix_npc_mac_addr_get"); vnet_dev_port_add_args_t port_add_args = { diff --git a/src/plugins/dpdk/CMakeLists.txt b/src/plugins/dpdk/CMakeLists.txt index 48b1548f9c2..48c56f35282 100644 --- a/src/plugins/dpdk/CMakeLists.txt +++ b/src/plugins/dpdk/CMakeLists.txt @@ -90,8 +90,10 @@ else() ############################################################################## # libnuma ############################################################################## - vpp_plugin_find_library(dpdk NUMA_LIB "numa") - list(APPEND DPDK_LINK_LIBRARIES ${NUMA_LIB}) + if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") + vpp_plugin_find_library(dpdk NUMA_LIB "numa") + list(APPEND DPDK_LINK_LIBRARIES ${NUMA_LIB}) + endif() ############################################################################## # Mellanox libraries diff --git a/src/plugins/dpdk/device/common.c b/src/plugins/dpdk/device/common.c index dc6b0c1c952..7a49c5aaef2 100644 --- a/src/plugins/dpdk/device/common.c +++ b/src/plugins/dpdk/device/common.c @@ -491,6 +491,7 @@ dpdk_get_pci_device (const struct rte_eth_dev_info *info) return NULL; } +#ifdef __linux__ /* If this device is VMBUS return pointer to info, otherwise NULL */ struct rte_vmbus_device * dpdk_get_vmbus_device (const struct rte_eth_dev_info *info) @@ -507,6 +508,7 @@ dpdk_get_vmbus_device (const struct rte_eth_dev_info *info) else return NULL; } +#endif /* __linux__ */ /* * fd.io coding-style-patch-verification: ON diff --git a/src/plugins/dpdk/device/dpdk.h b/src/plugins/dpdk/device/dpdk.h index c22a67a07e7..88a4d9ff618 100644 --- a/src/plugins/dpdk/device/dpdk.h +++ b/src/plugins/dpdk/device/dpdk.h @@ -24,7 +24,9 @@ #include <rte_eal.h> #include <rte_bus_pci.h> +#ifdef __linux__ #include <rte_bus_vmbus.h> +#endif /* __linux__ */ #include <rte_ethdev.h> #include <rte_version.h> #include <rte_net.h> @@ -35,7 +37,9 @@ #include <bus_driver.h> #include <bus_pci_driver.h> +#ifdef __linux__ #include <bus_vmbus_driver.h> +#endif /* __linux__ */ #endif #include <vnet/devices/devices.h> diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c index d15cfd7233a..2d038b907bf 100644 --- a/src/plugins/dpdk/device/init.c +++ b/src/plugins/dpdk/device/init.c @@ -187,9 +187,11 @@ dpdk_find_startup_config (struct rte_eth_dev_info *di) { dpdk_main_t *dm = &dpdk_main; struct rte_pci_device *pci_dev; - struct rte_vmbus_device *vmbus_dev; vlib_pci_addr_t pci_addr; +#ifdef __linux__ + struct rte_vmbus_device *vmbus_dev; vlib_vmbus_addr_t vmbus_addr; +#endif /* __linux__ */ uword *p = 0; if ((pci_dev = dpdk_get_pci_device (di))) @@ -202,6 +204,7 @@ dpdk_find_startup_config (struct rte_eth_dev_info *di) hash_get (dm->conf->device_config_index_by_pci_addr, pci_addr.as_u32); } +#ifdef __linux__ if ((vmbus_dev = dpdk_get_vmbus_device (di))) { unformat_input_t input_vmbus; @@ -216,6 +219,7 @@ dpdk_find_startup_config (struct rte_eth_dev_info *di) &vmbus_addr); unformat_free (&input_vmbus); } +#endif /* __linux__ */ if (p) return pool_elt_at_index (dm->conf->dev_confs, p[0]); @@ -566,8 +570,18 @@ dpdk_bind_devices_to_uio (dpdk_config_main_t * conf) continue; } +#ifdef __FreeBSD__ + /* + * The defines for the PCI_CLASS_* types are platform specific and differ + * on FreeBSD. + */ + if (d->device_class != PCI_CLASS_NETWORK && + d->device_class != PCI_CLASS_PROCESSOR_CO) + continue; +#else if (d->device_class != PCI_CLASS_NETWORK_ETHERNET && d->device_class != PCI_CLASS_PROCESSOR_CO) continue; +#endif /* __FreeBSD__ */ if (num_whitelisted) { @@ -1045,15 +1059,11 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) u8 no_vmbus = 0; u8 file_prefix = 0; u8 *socket_mem = 0; - u8 *huge_dir_path = 0; u32 vendor, device, domain, bus, func; void *fmt_func; void *fmt_addr; f64 poll_interval; - huge_dir_path = - format (0, "%s/hugepages%c", vlib_unix_get_runtime_dir (), 0); - conf->device_config_index_by_pci_addr = hash_create (0, sizeof (uword)); mhash_init (&conf->device_config_index_by_vmbus_addr, sizeof (uword), sizeof (vlib_vmbus_addr_t)); @@ -1396,11 +1406,6 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) RTE_VECT_SIMD_256 : RTE_VECT_SIMD_512); - /* lazy umount hugepages */ - umount2 ((char *) huge_dir_path, MNT_DETACH); - rmdir ((char *) huge_dir_path); - vec_free (huge_dir_path); - /* main thread 1st */ if ((error = dpdk_buffer_pools_create (vm))) return error; diff --git a/src/plugins/dpdk/main.c b/src/plugins/dpdk/main.c index 47007219482..9781d0ed7f0 100644 --- a/src/plugins/dpdk/main.c +++ b/src/plugins/dpdk/main.c @@ -13,13 +13,6 @@ * limitations under the License. */ -#include <unistd.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> -#include <linux/vfio.h> -#include <sys/ioctl.h> - #include <vnet/vnet.h> #include <vnet/plugin/plugin.h> #include <dpdk/device/dpdk.h> diff --git a/src/plugins/fateshare/fateshare.c b/src/plugins/fateshare/fateshare.c index 33ee167bce3..971d32303db 100644 --- a/src/plugins/fateshare/fateshare.c +++ b/src/plugins/fateshare/fateshare.c @@ -17,6 +17,7 @@ #include <vnet/vnet.h> #include <vnet/plugin/plugin.h> +#include <vppinfra/unix.h> #include <fateshare/fateshare.h> #include <vlibapi/api.h> @@ -26,7 +27,11 @@ #include <sys/types.h> #include <sys/wait.h> +#ifdef __linux__ #include <sys/prctl.h> // prctl(), PR_SET_PDEATHSIG +#else +#include <sys/procctl.h> +#endif /* __linux__ */ #include <limits.h> fateshare_main_t fateshare_main; @@ -86,12 +91,23 @@ launch_monitor (fateshare_main_t *kmp) { dup2 (logfd, 1); dup2 (logfd, 2); +#ifdef __linux__ int r = prctl (PR_SET_PDEATHSIG, SIGTERM); if (r == -1) { perror (0); exit (1); } +#else + int r, s = SIGTERM; + + r = procctl (P_PID, 0, PROC_PDEATHSIG_CTL, &s); + if (r == -1) + { + perror (0); + exit (1); + } +#endif /* __linux__ */ pid_t current_ppid = getppid (); if (current_ppid != ppid_before_fork) { @@ -197,24 +213,30 @@ fateshare_config (vlib_main_t *vm, unformat_input_t *input) if (fmp->monitor_cmd == 0) { - char *p, path[PATH_MAX]; - int rv; + char *p; + u8 *path; /* find executable path */ - if ((rv = readlink ("/proc/self/exe", path, PATH_MAX - 1)) == -1) + path = os_get_exec_path (); + + if (path == 0) return clib_error_return ( - 0, "could not stat /proc/self/exe - set monitor manually"); + 0, "could not get exec path - set monitor manually"); - /* readlink doesn't provide null termination */ - path[rv] = 0; + /* add null termination */ + vec_add1 (path, 0); /* strip filename */ - if ((p = strrchr (path, '/')) == 0) - return clib_error_return ( - 0, "could not determine vpp directory - set monitor manually"); + if ((p = strrchr ((char *) path, '/')) == 0) + { + vec_free (path); + return clib_error_return ( + 0, "could not determine vpp directory - set monitor manually"); + } *p = 0; fmp->monitor_cmd = format (0, "%s/vpp_fateshare_monitor\0", path); + vec_free (path); } if (fmp->monitor_logfile == 0) { diff --git a/src/plugins/fateshare/vpp_fateshare_monitor.c b/src/plugins/fateshare/vpp_fateshare_monitor.c index 7b203884c4e..7af451ccffe 100644 --- a/src/plugins/fateshare/vpp_fateshare_monitor.c +++ b/src/plugins/fateshare/vpp_fateshare_monitor.c @@ -4,7 +4,12 @@ #include <sys/types.h> #include <sys/wait.h> +#ifdef __linux__ #include <sys/prctl.h> // prctl(), PR_SET_PDEATHSIG +#else +#include <signal.h> +#include <sys/procctl.h> +#endif /* __linux__ */ #include <sys/stat.h> #include <fcntl.h> @@ -82,6 +87,7 @@ launch_command (char *scmd, char *logname_base) } /* child */ +#ifdef __linux__ int r = prctl (PR_SET_PDEATHSIG, SIGTERM); if (r == -1) { @@ -89,6 +95,17 @@ launch_command (char *scmd, char *logname_base) sleep (5); exit (1); } +#else + int r, s = SIGTERM; + + r = procctl (P_PID, 0, PROC_PDEATHSIG_CTL, &s); + if (r == -1) + { + perror ("procctl"); + exit (1); + } +#endif /* __linux__ */ + if (getppid () != ppid_before_fork) { sleep (5); @@ -180,12 +197,23 @@ main (int argc, char **argv) exit (2); } +#ifdef __linux__ int r = prctl (PR_SET_PDEATHSIG, SIGTERM); if (r == -1) { perror (0); exit (1); } +#else + int r, s = SIGTERM; + + r = procctl (P_PID, 0, PROC_PDEATHSIG_CTL, &s); + if (r == -1) + { + perror ("procctl"); + exit (1); + } +#endif /* __linux__ */ /* Establish handler. */ struct sigaction sa; diff --git a/src/plugins/hs_apps/CMakeLists.txt b/src/plugins/hs_apps/CMakeLists.txt index bd43eb34afa..179c9c7a4c4 100644 --- a/src/plugins/hs_apps/CMakeLists.txt +++ b/src/plugins/hs_apps/CMakeLists.txt @@ -55,7 +55,7 @@ if(VPP_BUILD_VCL_TESTS) ) add_vpp_executable(${test} SOURCES "vcl/${test}.c" - LINK_LIBRARIES vppcom pthread + LINK_LIBRARIES vppcom pthread ${EPOLL_LIB} NO_INSTALL ) endforeach() @@ -68,7 +68,7 @@ if(VPP_BUILD_VCL_TESTS) SOURCES "vcl/${test}.c" vcl/vcl_test_protos.c - LINK_LIBRARIES vppcom pthread + LINK_LIBRARIES vppcom pthread ${EPOLL_LIB} NO_INSTALL ) endforeach() diff --git a/src/plugins/hs_apps/http_client_cli.c b/src/plugins/hs_apps/http_client_cli.c index f44d4e1bcd1..085a2b69bf7 100644 --- a/src/plugins/hs_apps/http_client_cli.c +++ b/src/plugins/hs_apps/http_client_cli.c @@ -370,7 +370,7 @@ hcc_connect () } static clib_error_t * -hcc_run (vlib_main_t *vm) +hcc_run (vlib_main_t *vm, int print_output) { vlib_thread_main_t *vtm = vlib_get_thread_main (); hcc_main_t *hcm = &hcc_main; @@ -407,11 +407,12 @@ hcc_run (vlib_main_t *vm) goto cleanup; case HCC_REPLY_RECEIVED: - vlib_cli_output (vm, "%v", hcm->http_response); + if (print_output) + vlib_cli_output (vm, "%v", hcm->http_response); vec_free (hcm->http_response); break; default: - clib_error_return (0, "unexpected event %d", event_type); + err = clib_error_return (0, "unexpected event %d", event_type); break; } @@ -448,7 +449,7 @@ hcc_command_fn (vlib_main_t *vm, unformat_input_t *input, u64 seg_size; u8 *appns_id = 0; clib_error_t *err = 0; - int rv; + int rv, print_output = 1; hcm->prealloc_fifos = 0; hcm->private_segment_size = 0; @@ -472,6 +473,8 @@ hcc_command_fn (vlib_main_t *vm, unformat_input_t *input, hcm->fifo_size <<= 10; else if (unformat (line_input, "uri %s", &hcm->uri)) ; + else if (unformat (line_input, "no-output")) + print_output = 0; else if (unformat (line_input, "appns %_%v%_", &appns_id)) ; else if (unformat (line_input, "secret %lu", &hcm->appns_secret)) @@ -506,7 +509,7 @@ hcc_command_fn (vlib_main_t *vm, unformat_input_t *input, vnet_session_enable_disable (vm, 1 /* turn on TCP, etc. */); vlib_worker_thread_barrier_release (vm); - err = hcc_run (vm); + err = hcc_run (vm, print_output); if (hcc_detach ()) { @@ -526,7 +529,7 @@ done: VLIB_CLI_COMMAND (hcc_command, static) = { .path = "http cli client", .short_help = "[appns <app-ns> secret <appns-secret>] uri http://<ip-addr> " - "query <query-string>", + "query <query-string> [no-output]", .function = hcc_command_fn, .is_mp_safe = 1, }; diff --git a/src/plugins/http/http.c b/src/plugins/http/http.c index 036e6929987..37a6de71bc7 100644 --- a/src/plugins/http/http.c +++ b/src/plugins/http/http.c @@ -74,14 +74,14 @@ format_http_state (u8 *s, va_list *va) return format (s, "unknown"); } -static inline void -http_state_change (http_conn_t *hc, http_state_t state) -{ - HTTP_DBG (1, "changing http state %U -> %U", format_http_state, - hc->http_state, format_http_state, state); - ASSERT (hc->http_state != state); - hc->http_state = state; -} +#define http_state_change(_hc, _state) \ + do \ + { \ + HTTP_DBG (1, "changing http state %U -> %U", format_http_state, \ + (_hc)->http_state, format_http_state, _state); \ + (_hc)->http_state = _state; \ + } \ + while (0) static inline http_worker_t * http_worker_get (u32 thread_index) @@ -140,6 +140,7 @@ http_listener_free (http_conn_t *lhc) { http_main_t *hm = &http_main; + vec_free (lhc->app_name); if (CLIB_DEBUG) memset (lhc, 0xfc, sizeof (*lhc)); pool_put (hm->listener_pool, lhc); @@ -372,7 +373,7 @@ static const char *http_redirect_template = "HTTP/1.1 %s\r\n"; static const char *http_response_template = "HTTP/1.1 %s\r\n" "Date: %U GMT\r\n" "Expires: %U GMT\r\n" - "Server: VPP Static\r\n" + "Server: %s\r\n" "Content-Type: %s\r\n" "Content-Length: %lu\r\n\r\n"; @@ -577,7 +578,7 @@ http_state_wait_server_reply (http_conn_t *hc, transport_send_params_t *sp) { hc->rx_buf_offset = 0; vec_reset_length (hc->rx_buf); - http_state_change (hc, HTTP_STATE_WAIT_CLIENT_METHOD); + http_state_change (hc, HTTP_STATE_WAIT_APP_METHOD); } else { @@ -585,7 +586,8 @@ http_state_wait_server_reply (http_conn_t *hc, transport_send_params_t *sp) } app_wrk = app_worker_get_if_valid (as->app_wrk_index); - app_worker_rx_notify (app_wrk, as); + if (app_wrk) + app_worker_rx_notify (app_wrk, as); return HTTP_SM_STOP; } else @@ -594,7 +596,6 @@ http_state_wait_server_reply (http_conn_t *hc, transport_send_params_t *sp) ec = HTTP_STATUS_METHOD_NOT_ALLOWED; goto error; } - return HTTP_SM_STOP; error: @@ -734,6 +735,7 @@ http_state_wait_app_reply (http_conn_t *hc, transport_send_params_t *sp) * Add headers. For now: * - current time * - expiration time + * - server name * - content type * - data length */ @@ -748,6 +750,8 @@ http_state_wait_app_reply (http_conn_t *hc, transport_send_params_t *sp) format_clib_timebase_time, now, /* Expires */ format_clib_timebase_time, now + 600.0, + /* Server */ + hc->app_name, /* Content type */ http_content_type_str[msg.content_type], /* Length */ @@ -791,7 +795,6 @@ error: static http_sm_result_t http_state_wait_app_method (http_conn_t *hc, transport_send_params_t *sp) { - http_status_code_t sc; http_msg_t msg; session_t *as; u8 *buf = 0, *request; @@ -806,19 +809,15 @@ http_state_wait_app_method (http_conn_t *hc, transport_send_params_t *sp) if (msg.data.type > HTTP_MSG_DATA_PTR) { clib_warning ("no data"); - sc = HTTP_STATUS_INTERNAL_ERROR; goto error; } if (msg.type != HTTP_MSG_REQUEST) { clib_warning ("unexpected message type %d", msg.type); - sc = HTTP_STATUS_INTERNAL_ERROR; goto error; } - sc = msg.code; - vec_validate (buf, msg.data.len - 1); rv = svm_fifo_dequeue (as->tx_fifo, msg.data.len, buf); ASSERT (rv == msg.data.len); @@ -828,7 +827,6 @@ http_state_wait_app_method (http_conn_t *hc, transport_send_params_t *sp) if (offset != vec_len (request)) { clib_warning ("sending request failed!"); - sc = HTTP_STATUS_INTERNAL_ERROR; goto error; } @@ -837,83 +835,76 @@ http_state_wait_app_method (http_conn_t *hc, transport_send_params_t *sp) vec_free (buf); vec_free (request); - return HTTP_SM_CONTINUE; + return HTTP_SM_STOP; error: - clib_warning ("unexpected msg type from app %u", msg.type); - http_send_error (hc, sc); session_transport_closing_notify (&hc->connection); http_disconnect_transport (hc); - return HTTP_SM_STOP; -} - -static void -http_app_enqueue (http_conn_t *hc, session_t *as) -{ - app_worker_t *app_wrk; - u32 dlen, max_enq, n_enq; - int rv; - - dlen = vec_len (hc->rx_buf) - hc->rx_buf_offset; - if (!dlen) - return; - - max_enq = svm_fifo_max_enqueue (as->rx_fifo); - n_enq = clib_min (max_enq, dlen); - rv = svm_fifo_enqueue (as->rx_fifo, n_enq, &hc->rx_buf[hc->rx_buf_offset]); - if (rv < 0) - return; - - hc->rx_buf_offset += rv; - if (hc->rx_buf_offset >= vec_len (hc->rx_buf)) - { - vec_reset_length (hc->rx_buf); - hc->rx_buf_offset = 0; - } - - app_wrk = app_worker_get_if_valid (as->app_wrk_index); - ASSERT (app_wrk); - app_worker_rx_notify (app_wrk, as); + return HTTP_SM_ERROR; } static http_sm_result_t http_state_client_io_more_data (http_conn_t *hc, transport_send_params_t *sp) { session_t *as, *ts; - u32 max_deq; - int n_read; + app_worker_t *app_wrk; + svm_fifo_seg_t _seg, *seg = &_seg; + u32 max_len, max_deq, max_enq, n_segs = 1; + int rv, len; as = session_get_from_handle (hc->h_pa_session_handle); ts = session_get_from_handle (hc->h_tc_session_handle); - http_app_enqueue (hc, as); - - if (hc->to_recv == 0) + max_deq = svm_fifo_max_dequeue (ts->rx_fifo); + if (max_deq == 0) { - http_state_change (hc, HTTP_STATE_WAIT_CLIENT_METHOD); + HTTP_DBG (1, "no data to deq"); return HTTP_SM_STOP; } - max_deq = svm_fifo_max_dequeue (ts->rx_fifo); - if (max_deq > 0) + max_enq = svm_fifo_max_enqueue (as->rx_fifo); + if (max_enq == 0) { - vec_validate (hc->rx_buf, max_deq - 1); - n_read = svm_fifo_dequeue (ts->rx_fifo, max_deq, hc->rx_buf); - ASSERT (n_read == max_deq); + HTTP_DBG (1, "app's rx fifo full"); + svm_fifo_add_want_deq_ntf (as->rx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); + return HTTP_SM_STOP; + } - if (svm_fifo_is_empty (ts->rx_fifo)) - svm_fifo_unset_event (ts->rx_fifo); + max_len = clib_min (max_enq, max_deq); + len = svm_fifo_segments (ts->rx_fifo, 0, seg, &n_segs, max_len); + if (len < 0) + { + HTTP_DBG (1, "svm_fifo_segments() len %d", len); + return HTTP_SM_STOP; + } - hc->to_recv -= n_read; - vec_set_len (hc->rx_buf, n_read); + rv = svm_fifo_enqueue_segments (as->rx_fifo, seg, 1, 0 /* allow partial */); + if (rv < 0) + { + clib_warning ("data enqueue failed, rv: %d", rv); + return HTTP_SM_ERROR; } - if (hc->rx_buf_offset < vec_len (hc->rx_buf) || - svm_fifo_max_dequeue_cons (ts->rx_fifo)) + svm_fifo_dequeue_drop (ts->rx_fifo, rv); + if (rv > hc->to_recv) { - session_enqueue_notify (ts); + clib_warning ("http protocol error: received more data than expected"); + session_transport_closing_notify (&hc->connection); + http_disconnect_transport (hc); + http_state_change (hc, HTTP_STATE_WAIT_APP_METHOD); + return HTTP_SM_ERROR; } - return HTTP_SM_CONTINUE; + hc->to_recv -= rv; + HTTP_DBG (1, "drained %d from ts; remains %d", rv, hc->to_recv); + + app_wrk = app_worker_get_if_valid (as->app_wrk_index); + if (app_wrk) + app_worker_rx_notify (app_wrk, as); + + if (svm_fifo_max_dequeue_cons (ts->rx_fifo)) + session_enqueue_notify (ts); + + return HTTP_SM_STOP; } static http_sm_result_t @@ -983,6 +974,7 @@ static void http_req_run_state_machine (http_conn_t *hc, transport_send_params_t *sp) { http_sm_result_t res; + do { res = state_funcs[hc->http_state](hc, sp); @@ -1010,6 +1002,12 @@ http_ts_rx_callback (session_t *ts) return -1; } + if (hc->state == HTTP_CONN_STATE_CLOSED) + { + svm_fifo_dequeue_drop_all (ts->tx_fifo); + return 0; + } + http_req_run_state_machine (hc, 0); if (hc->state == HTTP_CONN_STATE_TRANSPORT_CLOSED) @@ -1205,6 +1203,11 @@ http_start_listen (u32 app_listener_index, transport_endpoint_cfg_t *tep) lhc->c_s_index = app_listener_index; lhc->c_flags |= TRANSPORT_CONNECTION_F_NO_LOOKUP; + if (vec_len (app->name)) + lhc->app_name = vec_dup (app->name); + else + lhc->app_name = format (0, "VPP server app"); + return lhc_index; } diff --git a/src/plugins/http/http.h b/src/plugins/http/http.h index dbae5ac4611..c9912dd6db8 100644 --- a/src/plugins/http/http.h +++ b/src/plugins/http/http.h @@ -227,6 +227,7 @@ typedef struct http_tc_ http_conn_state_t state; u32 timer_handle; + u8 *app_name; /* * Current request diff --git a/src/plugins/lisp/lisp-cp/lisp_types.h b/src/plugins/lisp/lisp-cp/lisp_types.h index 21bd72178d7..e92f8f80c70 100644 --- a/src/plugins/lisp/lisp-cp/lisp_types.h +++ b/src/plugins/lisp/lisp-cp/lisp_types.h @@ -198,7 +198,8 @@ u8 gid_address_len (gid_address_t * a); void *gid_address_cast (gid_address_t * gid, gid_address_type_t type); void gid_address_copy (gid_address_t * dst, gid_address_t * src); u32 gid_address_parse (u8 * offset, gid_address_t * a); -void gid_address_ip_set (gid_address_t * dst, void *src, u8 version); +void gid_address_ip_set (gid_address_t *dst, void *src, + ip_address_family_t version); #define gid_address_type(_a) (_a)->type #define gid_address_ippref(_a) (_a)->ippref diff --git a/src/plugins/tlsopenssl/tls_openssl.c b/src/plugins/tlsopenssl/tls_openssl.c index 98982439b96..5d172a0adcf 100644 --- a/src/plugins/tlsopenssl/tls_openssl.c +++ b/src/plugins/tlsopenssl/tls_openssl.c @@ -1179,18 +1179,13 @@ int tls_openssl_set_ciphers (char *ciphers) { openssl_main_t *om = &openssl_main; - int i; if (!ciphers) { return -1; } - vec_validate (om->ciphers, strlen (ciphers)); - for (i = 0; i < vec_len (om->ciphers) - 1; i++) - { - om->ciphers[i] = toupper (ciphers[i]); - } + vec_validate_init_c_string (om->ciphers, ciphers, strlen (ciphers)); return 0; diff --git a/src/plugins/unittest/gso_test.c b/src/plugins/unittest/gso_test.c index 54eb7422c87..43c614341d2 100644 --- a/src/plugins/unittest/gso_test.c +++ b/src/plugins/unittest/gso_test.c @@ -96,12 +96,94 @@ GSO_TEST_REGISTER_DATA (gso_ipv6_tcp, static) = { .is_ip6 = 1, }; +/* + * this does not support tunnel packets + */ +static void +set_hdr_offsets (vlib_buffer_t *b0, u8 is_l2) +{ + u16 ethertype = 0, l2hdr_sz = 0; + vnet_buffer_oflags_t oflags = 0; + u8 l4_proto = 0; + + if (!is_l2) + { + switch (b0->data[0] & 0xf0) + { + case 0x40: + ethertype = ETHERNET_TYPE_IP4; + break; + case 0x60: + ethertype = ETHERNET_TYPE_IP6; + break; + } + } + else + { + ethernet_header_t *eh = (ethernet_header_t *) b0->data; + ethertype = clib_net_to_host_u16 (eh->type); + l2hdr_sz = sizeof (ethernet_header_t); + + if (ethernet_frame_is_tagged (ethertype)) + { + ethernet_vlan_header_t *vlan = (ethernet_vlan_header_t *) (eh + 1); + + ethertype = clib_net_to_host_u16 (vlan->type); + l2hdr_sz += sizeof (*vlan); + if (ethertype == ETHERNET_TYPE_VLAN) + { + vlan++; + ethertype = clib_net_to_host_u16 (vlan->type); + l2hdr_sz += sizeof (*vlan); + } + } + } + + vnet_buffer (b0)->l2_hdr_offset = 0; + vnet_buffer (b0)->l3_hdr_offset = l2hdr_sz; + + if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP4)) + { + ip4_header_t *ip4 = (ip4_header_t *) (b0->data + l2hdr_sz); + vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + ip4_header_bytes (ip4); + l4_proto = ip4->protocol; + oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM; + b0->flags |= (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID | + VNET_BUFFER_F_L3_HDR_OFFSET_VALID | + VNET_BUFFER_F_L4_HDR_OFFSET_VALID); + } + else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6)) + { + ip6_header_t *ip6 = (ip6_header_t *) (b0->data + l2hdr_sz); + vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + sizeof (ip6_header_t); + /* FIXME IPv6 EH traversal */ + l4_proto = ip6->protocol; + b0->flags |= (VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID | + VNET_BUFFER_F_L3_HDR_OFFSET_VALID | + VNET_BUFFER_F_L4_HDR_OFFSET_VALID); + } + if (l4_proto == IP_PROTOCOL_TCP) + { + oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; + } + else if (l4_proto == IP_PROTOCOL_UDP) + { + oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; + } + if (oflags) + vnet_buffer_offload_flags_set (b0, oflags); +} + static u32 -fill_buffers (vlib_main_t *vm, u32 *buffer_indices, u8 *data, u32 data_size, - u32 n_buffers, u32 buffer_size, u32 packet_size, u32 gso_size, - u32 l4_hdr_len) +fill_buffers (vlib_main_t *vm, u32 *buffer_indices, + gso_test_data_t *gso_test_data, u32 n_buffers, u32 buffer_size, + u32 packet_size, u32 gso_size) { u32 i; + u8 *data = gso_test_data->data; + u32 data_size = gso_test_data->data_size; + u32 l4_hdr_len = gso_test_data->l4_hdr_len; + u8 is_l2 = gso_test_data->is_l2; for (i = 0; i < n_buffers; i++) { @@ -153,6 +235,8 @@ fill_buffers (vlib_main_t *vm, u32 *buffer_indices, u8 *data, u32 data_size, len += fill_data_size; } while (k < n_bufs); + + set_hdr_offsets (b, is_l2); b->flags |= VNET_BUFFER_F_GSO; vnet_buffer2 (b)->gso_size = gso_size; vnet_buffer2 (b)->gso_l4_hdr_sz = l4_hdr_len; @@ -165,17 +249,14 @@ fill_buffers (vlib_main_t *vm, u32 *buffer_indices, u8 *data, u32 data_size, static_always_inline u32 gso_segment_buffer_test (vlib_main_t *vm, u32 bi, - vnet_interface_per_thread_data_t *ptd, u8 is_l2, - u8 is_ip6) + vnet_interface_per_thread_data_t *ptd, u8 is_l2) { vlib_buffer_t *b = vlib_get_buffer (vm, bi); - generic_header_offset_t gho = { 0 }; u32 n_tx_bytes = 0; if (PREDICT_TRUE (b->flags & VNET_BUFFER_F_GSO)) { - vnet_generic_header_offset_parser (b, &gho, is_l2, !is_ip6, is_ip6); - n_tx_bytes = gso_segment_buffer_inline (vm, ptd, b, &gho, is_l2, is_ip6); + n_tx_bytes = gso_segment_buffer_inline (vm, ptd, b, is_l2); } return n_tx_bytes; @@ -237,19 +318,16 @@ test_gso_perf (vlib_main_t *vm, gso_test_main_t *gtm) vlib_buffer_free (vm, buffer_indices, n_alloc); goto done; } - n_filled = - fill_buffers (vm, buffer_indices, gso_test_data->data, - gso_test_data->data_size, n_buffers, buffer_size, - packet_size, gso_size, gso_test_data->l4_hdr_len); + n_filled = fill_buffers (vm, buffer_indices, gso_test_data, n_buffers, + buffer_size, packet_size, gso_size); u8 is_l2 = gso_test_data->is_l2; - u8 is_ip6 = gso_test_data->is_ip6; for (k = 0; k < warmup_rounds; k++) { for (j = 0; j < n_filled; j++) - gso_segment_buffer_test (vm, buffer_indices[j], &ptd[j], is_l2, - is_ip6); + gso_segment_buffer_test (vm, buffer_indices[j], &ptd[j], is_l2); + for (j = 0; j < n_filled; j++) { vlib_buffer_free (vm, ptd[j].split_buffers, @@ -264,8 +342,9 @@ test_gso_perf (vlib_main_t *vm, gso_test_main_t *gtm) { t0 = clib_cpu_time_now (); for (j = 0; j < n_filled; j++) - gso_segment_buffer_test (vm, buffer_indices[j], &ptd[j], is_l2, - is_ip6); + gso_segment_buffer_test (vm, buffer_indices[j], &ptd[j], + is_l2); + t1 = clib_cpu_time_now (); t2[i] += (t1 - t0); for (j = 0; j < n_filled; j++) diff --git a/src/vat/ip_types.c b/src/vat/ip_types.c index abcd2eaf648..248205287a4 100644 --- a/src/vat/ip_types.c +++ b/src/vat/ip_types.c @@ -205,9 +205,9 @@ ip_address_family_to_link_type (ip_address_family_t af) return (VNET_LINK_IP4); } - void -ip_address_set (ip_address_t * dst, const void *src, u8 version) +ip_address_set (ip_address_t *dst, const void *src, + ip_address_family_t version) { ip_addr_version (dst) = version; diff --git a/src/vcl/CMakeLists.txt b/src/vcl/CMakeLists.txt index 2f738f39d1a..c8835e771c1 100644 --- a/src/vcl/CMakeLists.txt +++ b/src/vcl/CMakeLists.txt @@ -11,11 +11,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -if(NOT "${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") - message(WARNING "-- vppcom is currently only support on Linux - disabled") - return() -endif() - ############################################################################## # vppcom shared library ############################################################################## @@ -40,6 +35,9 @@ if (LDP_HAS_GNU_SOURCE) add_compile_definitions(HAVE_GNU_SOURCE) endif(LDP_HAS_GNU_SOURCE) +if("${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD") + message("WARNING: vcl_ldpreload isn't supported on FreeBSD - disabled") +else() add_vpp_library(vcl_ldpreload SOURCES ldp_socket_wrapper.c @@ -48,6 +46,7 @@ add_vpp_library(vcl_ldpreload LINK_LIBRARIES vppinfra svm vlibmemoryclient rt pthread vppcom dl ) +endif() add_vpp_headers(vcl ldp.h diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c index 58f6ac0a134..a557093e897 100644 --- a/src/vcl/vppcom.c +++ b/src/vcl/vppcom.c @@ -2087,7 +2087,16 @@ read_again: ASSERT (rv >= 0); if (peek) - return rv; + { + /* Request new notifications if more data enqueued */ + if (rv < n || rv == svm_fifo_max_dequeue_cons (rx_fifo)) + { + if (is_ct) + svm_fifo_unset_event (s->rx_fifo); + svm_fifo_unset_event (rx_fifo); + } + return rv; + } n_read += rv; diff --git a/src/vlib/freebsd/pci.c b/src/vlib/freebsd/pci.c new file mode 100644 index 00000000000..a4e9eb2dda6 --- /dev/null +++ b/src/vlib/freebsd/pci.c @@ -0,0 +1,380 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Tom Jones <thj@freebsd.org> + * + * This software was developed by Tom Jones <thj@freebsd.org> under sponsorship + * from the FreeBSD Foundation. + * + */ + +#include <vlib/vlib.h> +#include <vlib/pci/pci.h> +#include <vlib/unix/unix.h> + +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/ioctl.h> +#include <sys/eventfd.h> + +#include <sys/pciio.h> + +#include <fcntl.h> +#include <dirent.h> +#include <net/if.h> + +extern vlib_pci_main_t freebsd_pci_main; + +uword +vlib_pci_get_private_data (vlib_main_t *vm, vlib_pci_dev_handle_t h) +{ + return 0; +} + +void +vlib_pci_set_private_data (vlib_main_t *vm, vlib_pci_dev_handle_t h, + uword private_data) +{ +} + +vlib_pci_addr_t * +vlib_pci_get_addr (vlib_main_t *vm, vlib_pci_dev_handle_t h) +{ + return NULL; +} + +u32 +vlib_pci_get_numa_node (vlib_main_t *vm, vlib_pci_dev_handle_t h) +{ + return 0; +} + +u32 +vlib_pci_get_num_msix_interrupts (vlib_main_t *vm, vlib_pci_dev_handle_t h) +{ + return 0; +} + +/* Call to allocate/initialize the pci subsystem. + This is not an init function so that users can explicitly enable + pci only when it's needed. */ +clib_error_t *pci_bus_init (vlib_main_t *vm); + +vlib_pci_device_info_t * +vlib_pci_get_device_info (vlib_main_t *vm, vlib_pci_addr_t *addr, + clib_error_t **error) +{ + /* Populate a vlib_pci_device_info_t from the given address */ + clib_error_t *err = NULL; + vlib_pci_device_info_t *di = NULL; + + int fd = -1; + struct pci_conf_io pci; + struct pci_conf match; + struct pci_match_conf pattern; + bzero (&match, sizeof (match)); + bzero (&pattern, sizeof (pattern)); + + pattern.pc_sel.pc_domain = addr->domain; + pattern.pc_sel.pc_bus = addr->bus; + pattern.pc_sel.pc_dev = addr->slot; + pattern.pc_sel.pc_func = addr->function; + pattern.flags = PCI_GETCONF_MATCH_DOMAIN | PCI_GETCONF_MATCH_BUS | + PCI_GETCONF_MATCH_DEV | PCI_GETCONF_MATCH_FUNC; + + pci.pat_buf_len = sizeof (pattern); + pci.num_patterns = 1; + pci.patterns = &pattern; + pci.match_buf_len = sizeof (match); + pci.num_matches = 1; + pci.matches = &match; + pci.offset = 0; + pci.generation = 0; + pci.status = 0; + + fd = open ("/dev/pci", 0); + if (fd == -1) + { + err = clib_error_return_unix (0, "open '/dev/pci'"); + goto error; + } + + if (ioctl (fd, PCIOCGETCONF, &pci) == -1) + { + err = clib_error_return_unix (0, "reading PCIOCGETCONF"); + goto error; + } + + di = clib_mem_alloc (sizeof (vlib_pci_device_info_t)); + clib_memset (di, 0, sizeof (vlib_pci_device_info_t)); + + di->addr.as_u32 = addr->as_u32; + di->numa_node = 0; /* TODO: Place holder until we have NUMA on FreeBSD */ + + di->device_class = match.pc_class; + di->vendor_id = match.pc_vendor; + di->device_id = match.pc_device; + di->revision = match.pc_revid; + + di->product_name = NULL; + di->vpd_r = 0; + di->vpd_w = 0; + di->driver_name = format (0, "%s", &match.pd_name); + di->iommu_group = -1; + + goto done; + +error: + vlib_pci_free_device_info (di); + di = NULL; +done: + if (error) + *error = err; + close (fd); + return di; +} + +clib_error_t *__attribute__ ((weak)) +vlib_pci_get_device_root_bus (vlib_pci_addr_t *addr, vlib_pci_addr_t *root_bus) +{ + return NULL; +} + +clib_error_t * +vlib_pci_bind_to_uio (vlib_main_t *vm, vlib_pci_addr_t *addr, + char *uio_drv_name, int force) +{ + clib_error_t *error = 0; + + if (error) + { + return error; + } + + if (strncmp ("auto", uio_drv_name, 5) == 0) + { + /* TODO: We should confirm that nic_uio is loaded and return an error. */ + uio_drv_name = "nic_uio"; + } + return error; +} + +clib_error_t * +vlib_pci_register_intx_handler (vlib_main_t *vm, vlib_pci_dev_handle_t h, + pci_intx_handler_function_t *intx_handler) +{ + return NULL; +} + +clib_error_t * +vlib_pci_unregister_intx_handler (vlib_main_t *vm, vlib_pci_dev_handle_t h) +{ + return NULL; +} + +clib_error_t * +vlib_pci_register_msix_handler (vlib_main_t *vm, vlib_pci_dev_handle_t h, + u32 start, u32 count, + pci_msix_handler_function_t *msix_handler) +{ + return NULL; +} + +clib_error_t * +vlib_pci_unregister_msix_handler (vlib_main_t *vm, vlib_pci_dev_handle_t h, + u32 start, u32 count) +{ + return NULL; +} + +clib_error_t * +vlib_pci_enable_msix_irq (vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 start, + u16 count) +{ + return NULL; +} + +uword +vlib_pci_get_msix_file_index (vlib_main_t *vm, vlib_pci_dev_handle_t h, + u16 index) +{ + return 0; +} + +clib_error_t * +vlib_pci_disable_msix_irq (vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 start, + u16 count) +{ + return NULL; +} + +/* Configuration space read/write. */ +clib_error_t * +vlib_pci_read_write_config (vlib_main_t *vm, vlib_pci_dev_handle_t h, + vlib_read_or_write_t read_or_write, uword address, + void *data, u32 n_bytes) +{ + return NULL; +} + +clib_error_t * +vlib_pci_map_region (vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource, + void **result) +{ + return NULL; +} + +clib_error_t * +vlib_pci_map_region_fixed (vlib_main_t *vm, vlib_pci_dev_handle_t h, + u32 resource, u8 *addr, void **result) +{ + return NULL; +} + +clib_error_t * +vlib_pci_io_region (vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource) +{ + return NULL; +} + +clib_error_t * +vlib_pci_read_write_io (vlib_main_t *vm, vlib_pci_dev_handle_t h, + vlib_read_or_write_t read_or_write, uword offset, + void *data, u32 length) +{ + return NULL; +} + +clib_error_t * +vlib_pci_map_dma (vlib_main_t *vm, vlib_pci_dev_handle_t h, void *ptr) +{ + return NULL; +} + +int +vlib_pci_supports_virtual_addr_dma (vlib_main_t *vm, vlib_pci_dev_handle_t h) +{ + return 0; +} + +clib_error_t * +vlib_pci_device_open (vlib_main_t *vm, vlib_pci_addr_t *addr, + pci_device_id_t ids[], vlib_pci_dev_handle_t *handle) +{ + return NULL; +} + +void +vlib_pci_device_close (vlib_main_t *vm, vlib_pci_dev_handle_t h) +{ +} + +void +init_device_from_registered (vlib_main_t *vm, vlib_pci_device_info_t *di) +{ +} + +static int +pci_addr_cmp (void *v1, void *v2) +{ + vlib_pci_addr_t *a1 = v1; + vlib_pci_addr_t *a2 = v2; + + if (a1->domain > a2->domain) + return 1; + if (a1->domain < a2->domain) + return -1; + if (a1->bus > a2->bus) + return 1; + if (a1->bus < a2->bus) + return -1; + if (a1->slot > a2->slot) + return 1; + if (a1->slot < a2->slot) + return -1; + if (a1->function > a2->function) + return 1; + if (a1->function < a2->function) + return -1; + return 0; +} + +vlib_pci_addr_t * +vlib_pci_get_all_dev_addrs () +{ + vlib_pci_addr_t *addrs = 0; + + int fd = -1; + struct pci_conf_io pci; + struct pci_conf matches[32]; + bzero (matches, sizeof (matches)); + + pci.pat_buf_len = 0; + pci.num_patterns = 0; + pci.patterns = NULL; + pci.match_buf_len = sizeof (matches); + pci.num_matches = 32; + pci.matches = (struct pci_conf *) &matches; + pci.offset = 0; + pci.generation = 0; + pci.status = 0; + + fd = open ("/dev/pci", 0); + if (fd == -1) + { + clib_error_return_unix (0, "opening /dev/pci"); + return (NULL); + } + + if (ioctl (fd, PCIOCGETCONF, &pci) == -1) + { + clib_error_return_unix (0, "reading pci config"); + close (fd); + return (NULL); + } + + for (int i = 0; i < pci.num_matches; i++) + { + struct pci_conf *m = &pci.matches[i]; + vlib_pci_addr_t addr; + + addr.domain = m->pc_sel.pc_domain; + addr.bus = m->pc_sel.pc_bus; + addr.slot = m->pc_sel.pc_dev; + addr.function = m->pc_sel.pc_func; + + vec_add1 (addrs, addr); + } + + vec_sort_with_function (addrs, pci_addr_cmp); + close (fd); + + return addrs; +} + +clib_error_t * +freebsd_pci_init (vlib_main_t *vm) +{ + vlib_pci_main_t *pm = &pci_main; + vlib_pci_addr_t *addr = 0, *addrs; + + pm->vlib_main = vm; + + ASSERT (sizeof (vlib_pci_addr_t) == sizeof (u32)); + + addrs = vlib_pci_get_all_dev_addrs (); + vec_foreach (addr, addrs) + { + vlib_pci_device_info_t *d; + if ((d = vlib_pci_get_device_info (vm, addr, 0))) + { + init_device_from_registered (vm, d); + vlib_pci_free_device_info (d); + } + } + + return 0; +} + +VLIB_INIT_FUNCTION (freebsd_pci_init) = { + .runs_after = VLIB_INITS ("unix_input_init"), +}; diff --git a/src/vlib/linux/pci.c b/src/vlib/linux/pci.c index 08923bebcdf..f7c63bc3607 100644 --- a/src/vlib/linux/pci.c +++ b/src/vlib/linux/pci.c @@ -55,6 +55,7 @@ #include <linux/ethtool.h> #include <linux/sockios.h> #include <linux/vfio.h> +#include <limits.h> #include <sys/eventfd.h> #define SYSFS_DEVICES_PCI "/sys/devices/pci" diff --git a/src/vlib/threads.c b/src/vlib/threads.c index 3994afc2cea..bbcb4ec2979 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -16,6 +16,9 @@ #include <signal.h> #include <math.h> +#ifdef __FreeBSD__ +#include <pthread_np.h> +#endif /* __FreeBSD__ */ #include <vppinfra/format.h> #include <vppinfra/time_range.h> #include <vppinfra/interrupt.h> @@ -272,7 +275,11 @@ vlib_thread_init (vlib_main_t * vm) w->thread_mheap = clib_mem_get_heap (); w->thread_stack = vlib_thread_stacks[0]; w->cpu_id = tm->main_lcore; +#ifdef __FreeBSD__ + w->lwp = pthread_getthreadid_np (); +#else w->lwp = syscall (SYS_gettid); +#endif /* __FreeBSD__ */ w->thread_id = pthread_self (); tm->n_vlib_mains = 1; @@ -448,7 +455,11 @@ vlib_worker_thread_bootstrap_fn (void *arg) { vlib_worker_thread_t *w = arg; +#ifdef __FreeBSD__ + w->lwp = pthread_getthreadid_np (); +#else w->lwp = syscall (SYS_gettid); +#endif /* __FreeBSD__ */ w->thread_id = pthread_self (); __os_thread_index = w - vlib_worker_threads; diff --git a/src/vlib/unix/main.c b/src/vlib/unix/main.c index 38eef4125a9..ee28ca8f1aa 100644 --- a/src/vlib/unix/main.c +++ b/src/vlib/unix/main.c @@ -39,6 +39,7 @@ #include <vlib/vlib.h> #include <vlib/unix/unix.h> #include <vlib/unix/plugin.h> +#include <vppinfra/unix.h> #include <limits.h> #include <signal.h> @@ -612,22 +613,23 @@ vlib_unix_main (int argc, char *argv[]) vlib_main_t *vm = vlib_get_first_main (); /* one and only time for this! */ unformat_input_t input; clib_error_t *e; - char buffer[PATH_MAX]; int i; vec_validate_aligned (vgm->vlib_mains, 0, CLIB_CACHE_LINE_BYTES); - if ((i = readlink ("/proc/self/exe", buffer, sizeof (buffer) - 1)) > 0) + vgm->exec_path = (char *) os_get_exec_path (); + + if (vgm->exec_path) { - int j; - buffer[i] = 0; - vgm->exec_path = vec_new (char, i + 1); - clib_memcpy_fast (vgm->exec_path, buffer, i + 1); - for (j = i - 1; j > 0; j--) - if (buffer[j - 1] == '/') + for (i = vec_len (vgm->exec_path) - 1; i > 0; i--) + if (vgm->exec_path[i - 1] == '/') break; - vgm->name = vec_new (char, i - j + 1); - clib_memcpy_fast (vgm->name, buffer + j, i - j + 1); + + vgm->name = 0; + + vec_add (vgm->name, vgm->exec_path + i, vec_len (vgm->exec_path) - i); + vec_add1 (vgm->exec_path, 0); + vec_add1 (vgm->name, 0); } else vgm->exec_path = vgm->name = argv[0]; diff --git a/src/vlibmemory/socket_client.c b/src/vlibmemory/socket_client.c index 5fa19c6a9c0..ad28136dc07 100644 --- a/src/vlibmemory/socket_client.c +++ b/src/vlibmemory/socket_client.c @@ -22,6 +22,14 @@ #define _GNU_SOURCE #include <sys/socket.h> +#ifdef __FreeBSD__ +#define _WANT_UCRED +#include <sys/types.h> +#include <sys/param.h> +#include <sys/ucred.h> +#include <sys/un.h> +#endif /* __FreeBSD__ */ + #include <svm/ssvm.h> #include <vlibmemory/socket_client.h> #include <vlibmemory/memory_client.h> @@ -278,7 +286,11 @@ vl_sock_api_recv_fd_msg_internal (socket_client_main_t * scm, int fds[], struct msghdr mh = { 0 }; struct iovec iov[1]; ssize_t size = 0; +#ifdef __linux__ struct ucred *cr = 0; +#elif __FreeBSD__ + struct cmsgcred *cr = 0; +#endif /* __linux__ */ struct cmsghdr *cmsg; pid_t pid __attribute__ ((unused)); uid_t uid __attribute__ ((unused)); @@ -318,6 +330,7 @@ vl_sock_api_recv_fd_msg_internal (socket_client_main_t * scm, int fds[], { if (cmsg->cmsg_level == SOL_SOCKET) { +#ifdef __linux__ if (cmsg->cmsg_type == SCM_CREDENTIALS) { cr = (struct ucred *) CMSG_DATA (cmsg); @@ -325,6 +338,15 @@ vl_sock_api_recv_fd_msg_internal (socket_client_main_t * scm, int fds[], gid = cr->gid; pid = cr->pid; } +#elif __FreeBSD__ + if (cmsg->cmsg_type == SCM_CREDS) + { + cr = (struct cmsgcred *) CMSG_DATA (cmsg); + uid = cr->cmcred_uid; + gid = cr->cmcred_gid; + pid = cr->cmcred_pid; + } +#endif /* __linux__ */ else if (cmsg->cmsg_type == SCM_RIGHTS) { clib_memcpy_fast (fds, CMSG_DATA (cmsg), sizeof (int) * n_fds); diff --git a/src/vnet/gso/gso.h b/src/vnet/gso/gso.h index 883a4941ee2..dee5da5c70b 100644 --- a/src/vnet/gso/gso.h +++ b/src/vnet/gso/gso.h @@ -39,13 +39,13 @@ gso_init_bufs_from_template_base (vlib_buffer_t **bufs, vlib_buffer_t *b0, u32 flags, u16 n_bufs, u16 hdr_sz) { u32 i = n_bufs; - while (i >= 4) + while (i >= 6) { /* prefetches */ CLIB_PREFETCH (bufs[2], 2 * CLIB_CACHE_LINE_BYTES, LOAD); CLIB_PREFETCH (bufs[3], 2 * CLIB_CACHE_LINE_BYTES, LOAD); - vlib_prefetch_buffer_data (bufs[2], LOAD); - vlib_prefetch_buffer_data (bufs[3], LOAD); + vlib_prefetch_buffer_data (bufs[4], LOAD); + vlib_prefetch_buffer_data (bufs[5], LOAD); /* copying objects from cacheline 0 */ bufs[0]->current_data = 0; @@ -70,10 +70,26 @@ gso_init_bufs_from_template_base (vlib_buffer_t **bufs, vlib_buffer_t *b0, bufs[0]->total_length_not_including_first_buffer = 0; bufs[1]->total_length_not_including_first_buffer = 0; + clib_memcpy_fast (&bufs[0]->opaque2, &b0->opaque2, sizeof (b0->opaque2)); + clib_memcpy_fast (&bufs[1]->opaque2, &b0->opaque2, sizeof (b0->opaque2)); + /* copying data */ clib_memcpy_fast (bufs[0]->data, vlib_buffer_get_current (b0), hdr_sz); clib_memcpy_fast (bufs[1]->data, vlib_buffer_get_current (b0), hdr_sz); + /* header offset fixup */ + vnet_buffer (bufs[0])->l2_hdr_offset -= b0->current_data; + vnet_buffer (bufs[0])->l3_hdr_offset -= b0->current_data; + vnet_buffer (bufs[0])->l4_hdr_offset -= b0->current_data; + vnet_buffer2 (bufs[0])->outer_l3_hdr_offset -= b0->current_data; + vnet_buffer2 (bufs[0])->outer_l4_hdr_offset -= b0->current_data; + + vnet_buffer (bufs[1])->l2_hdr_offset -= b0->current_data; + vnet_buffer (bufs[1])->l3_hdr_offset -= b0->current_data; + vnet_buffer (bufs[1])->l4_hdr_offset -= b0->current_data; + vnet_buffer2 (bufs[1])->outer_l3_hdr_offset -= b0->current_data; + vnet_buffer2 (bufs[1])->outer_l4_hdr_offset -= b0->current_data; + bufs += 2; i -= 2; } @@ -92,10 +108,18 @@ gso_init_bufs_from_template_base (vlib_buffer_t **bufs, vlib_buffer_t *b0, /* copying objects from cacheline 1 */ bufs[0]->trace_handle = b0->trace_handle; bufs[0]->total_length_not_including_first_buffer = 0; + clib_memcpy_fast (&bufs[0]->opaque2, &b0->opaque2, sizeof (b0->opaque2)); /* copying data */ clib_memcpy_fast (bufs[0]->data, vlib_buffer_get_current (b0), hdr_sz); + /* header offset fixup */ + vnet_buffer (bufs[0])->l2_hdr_offset -= b0->current_data; + vnet_buffer (bufs[0])->l3_hdr_offset -= b0->current_data; + vnet_buffer (bufs[0])->l4_hdr_offset -= b0->current_data; + vnet_buffer2 (bufs[0])->outer_l3_hdr_offset -= b0->current_data; + vnet_buffer2 (bufs[0])->outer_l4_hdr_offset -= b0->current_data; + bufs++; i--; } @@ -103,28 +127,41 @@ gso_init_bufs_from_template_base (vlib_buffer_t **bufs, vlib_buffer_t *b0, static_always_inline void gso_fixup_segmented_buf (vlib_main_t *vm, vlib_buffer_t *b0, u32 next_tcp_seq, - int is_l2, int is_ip6, generic_header_offset_t *gho, - clib_ip_csum_t *c, u8 tcp_flags) + int is_l2, u8 oflags, u16 hdr_sz, u16 l4_hdr_sz, + clib_ip_csum_t *c, u8 tcp_flags, u8 is_prefetch, + vlib_buffer_t *b1) { - ip4_header_t *ip4 = - (ip4_header_t *) (vlib_buffer_get_current (b0) + gho->l3_hdr_offset + - gho->outer_hdr_sz); - ip6_header_t *ip6 = - (ip6_header_t *) (vlib_buffer_get_current (b0) + gho->l3_hdr_offset + - gho->outer_hdr_sz); - tcp_header_t *tcp = - (tcp_header_t *) (vlib_buffer_get_current (b0) + gho->l4_hdr_offset + - gho->outer_hdr_sz); + i16 l3_hdr_offset = vnet_buffer (b0)->l3_hdr_offset; + i16 l4_hdr_offset = vnet_buffer (b0)->l4_hdr_offset; + + ip4_header_t *ip4 = (ip4_header_t *) (b0->data + l3_hdr_offset); + ip6_header_t *ip6 = (ip6_header_t *) (b0->data + l3_hdr_offset); + tcp_header_t *tcp = (tcp_header_t *) (b0->data + l4_hdr_offset); tcp->flags = tcp_flags; tcp->seq_number = clib_host_to_net_u32 (next_tcp_seq); c->odd = 0; - if (is_ip6) + if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM) + { + ip4->length = + clib_host_to_net_u16 (b0->current_length - hdr_sz + + (l4_hdr_offset - l3_hdr_offset) + l4_hdr_sz); + ip4->checksum = 0; + ip4->checksum = ip4_header_checksum (ip4); + vnet_buffer_offload_flags_clear (b0, (VNET_BUFFER_OFFLOAD_F_IP_CKSUM | + VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)); + c->sum += clib_mem_unaligned (&ip4->src_address, u32); + c->sum += clib_mem_unaligned (&ip4->dst_address, u32); + c->sum += clib_host_to_net_u32 ( + (clib_net_to_host_u16 (ip4->length) - ip4_header_bytes (ip4)) + + (ip4->protocol << 16)); + } + else { - ip6->payload_length = clib_host_to_net_u16 ( - b0->current_length - gho->l4_hdr_offset - gho->outer_hdr_sz); + ip6->payload_length = + clib_host_to_net_u16 (b0->current_length - hdr_sz + l4_hdr_sz); vnet_buffer_offload_flags_clear (b0, VNET_BUFFER_OFFLOAD_F_TCP_CKSUM); ip6_psh_t psh = { 0 }; u32 *p = (u32 *) &psh; @@ -135,24 +172,15 @@ gso_fixup_segmented_buf (vlib_main_t *vm, vlib_buffer_t *b0, u32 next_tcp_seq, for (int i = 0; i < 10; i++) c->sum += p[i]; } - else - { - ip4->length = clib_host_to_net_u16 ( - b0->current_length - gho->l3_hdr_offset - gho->outer_hdr_sz); - if (gho->gho_flags & GHO_F_IP4) - ip4->checksum = ip4_header_checksum (ip4); - vnet_buffer_offload_flags_clear (b0, (VNET_BUFFER_OFFLOAD_F_IP_CKSUM | - VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)); - c->sum += clib_mem_unaligned (&ip4->src_address, u32); - c->sum += clib_mem_unaligned (&ip4->dst_address, u32); - c->sum += clib_host_to_net_u32 ( - (clib_net_to_host_u16 (ip4->length) - ip4_header_bytes (ip4)) + - (ip4->protocol << 16)); - } - clib_ip_csum_chunk (c, (u8 *) tcp, gho->l4_hdr_sz); + + if (is_prefetch) + CLIB_PREFETCH (vlib_buffer_get_current (b1) + hdr_sz, + CLIB_CACHE_LINE_BYTES, LOAD); + + clib_ip_csum_chunk (c, (u8 *) tcp, l4_hdr_sz); tcp->checksum = clib_ip_csum_fold (c); - if (!is_l2 && ((gho->gho_flags & GHO_F_TUNNEL) == 0)) + if (!is_l2 && ((oflags & VNET_BUFFER_OFFLOAD_F_TNL_MASK) == 0)) { u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX]; @@ -169,16 +197,20 @@ gso_fixup_segmented_buf (vlib_main_t *vm, vlib_buffer_t *b0, u32 next_tcp_seq, static_always_inline u32 gso_segment_buffer_inline (vlib_main_t *vm, vnet_interface_per_thread_data_t *ptd, - vlib_buffer_t *b, generic_header_offset_t *gho, - int is_l2, int is_ip6) + vlib_buffer_t *b, int is_l2) { vlib_buffer_t **bufs = 0; u32 n_tx_bytes = 0; + + u8 oflags = vnet_buffer (b)->oflags; + i16 l4_hdr_offset = vnet_buffer (b)->l4_hdr_offset; u16 gso_size = vnet_buffer2 (b)->gso_size; + u16 l4_hdr_sz = vnet_buffer2 (b)->gso_l4_hdr_sz; + u8 tcp_flags = 0, tcp_flags_no_fin_psh = 0; u32 default_bflags = b->flags & ~(VNET_BUFFER_F_GSO | VLIB_BUFFER_NEXT_PRESENT); - u16 hdr_sz = gho->hdr_sz + gho->outer_hdr_sz; + u16 hdr_sz = (l4_hdr_offset - b->current_data) + l4_hdr_sz; u32 next_tcp_seq = 0, tcp_seq = 0; u32 data_size = vlib_buffer_length_in_chain (vm, b) - hdr_sz; u16 size = @@ -200,9 +232,8 @@ gso_segment_buffer_inline (vlib_main_t *vm, vec_validate (bufs, n_bufs - 1); vlib_get_buffers (vm, ptd->split_buffers, bufs, n_bufs); - tcp_header_t *tcp = - (tcp_header_t *) (vlib_buffer_get_current (b) + gho->l4_hdr_offset + - gho->outer_hdr_sz); + tcp_header_t *tcp = (tcp_header_t *) (b->data + l4_hdr_offset); + tcp_seq = next_tcp_seq = clib_net_to_host_u32 (tcp->seq_number); /* store original flags for last packet and reset FIN and PSH */ tcp_flags = tcp->flags; @@ -247,11 +278,11 @@ gso_segment_buffer_inline (vlib_main_t *vm, if (0 == dst_left && data_size) { vlib_prefetch_buffer_header (bufs[i + 1], LOAD); - vlib_prefetch_buffer_data (bufs[i + 1], LOAD); n_tx_bytes += bufs[i]->current_length; - gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, is_ip6, gho, - &c, tcp_flags_no_fin_psh); + gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, oflags, hdr_sz, + l4_hdr_sz, &c, tcp_flags_no_fin_psh, 1, + bufs[i + 1]); i++; dst_left = size; dst_ptr = vlib_buffer_get_current (bufs[i]) + hdr_sz; @@ -264,8 +295,8 @@ gso_segment_buffer_inline (vlib_main_t *vm, ASSERT ((i + 1) == n_alloc); n_tx_bytes += bufs[i]->current_length; - gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, is_ip6, gho, &c, - tcp_flags); + gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, oflags, hdr_sz, + l4_hdr_sz, &c, tcp_flags, 0, NULL); vec_free (bufs); return n_tx_bytes; diff --git a/src/vnet/gso/node.c b/src/vnet/gso/node.c index 910f1585e7f..c1d4459476e 100644 --- a/src/vnet/gso/node.c +++ b/src/vnet/gso/node.c @@ -80,113 +80,108 @@ format_gso_trace (u8 * s, va_list * args) return s; } -static_always_inline u16 -tso_segment_ipip_tunnel_fixup (vlib_main_t * vm, - vnet_interface_per_thread_data_t * ptd, - vlib_buffer_t * sb0, - generic_header_offset_t * gho) +static_always_inline void +tso_segment_ipip_tunnel_fixup (vlib_main_t *vm, + vnet_interface_per_thread_data_t *ptd, + vlib_buffer_t *sb0) { u16 n_tx_bufs = vec_len (ptd->split_buffers); - u16 i = 0, n_tx_bytes = 0; + u16 i = 0; while (i < n_tx_bufs) { vlib_buffer_t *b0 = vlib_get_buffer (vm, ptd->split_buffers[i]); + i16 outer_l3_hdr_offset = vnet_buffer2 (b0)->outer_l3_hdr_offset; + i16 l3_hdr_offset = vnet_buffer (b0)->l3_hdr_offset; - ip4_header_t *ip4 = - (ip4_header_t *) (vlib_buffer_get_current (b0) + - gho->outer_l3_hdr_offset); - ip6_header_t *ip6 = - (ip6_header_t *) (vlib_buffer_get_current (b0) + - gho->outer_l3_hdr_offset); + ip4_header_t *ip4 = (ip4_header_t *) (b0->data + outer_l3_hdr_offset); + ip6_header_t *ip6 = (ip6_header_t *) (b0->data + outer_l3_hdr_offset); - if (gho->gho_flags & GHO_F_OUTER_IP4) + if (vnet_buffer (b0)->oflags & VNET_BUFFER_OFFLOAD_F_OUTER_IP_CKSUM) { - ip4->length = - clib_host_to_net_u16 (b0->current_length - - gho->outer_l3_hdr_offset); + ip4->length = clib_host_to_net_u16 ( + b0->current_length - (outer_l3_hdr_offset - b0->current_data)); ip4->checksum = ip4_header_checksum (ip4); + vnet_buffer_offload_flags_clear ( + b0, VNET_BUFFER_OFFLOAD_F_OUTER_IP_CKSUM | + VNET_BUFFER_OFFLOAD_F_TNL_IPIP); } - else if (gho->gho_flags & GHO_F_OUTER_IP6) + else { - ip6->payload_length = - clib_host_to_net_u16 (b0->current_length - - gho->outer_l4_hdr_offset); + ip6->payload_length = clib_host_to_net_u16 ( + b0->current_length - (l3_hdr_offset - b0->current_data)); + vnet_buffer_offload_flags_clear (b0, VNET_BUFFER_OFFLOAD_F_TNL_IPIP); } - n_tx_bytes += gho->outer_hdr_sz; i++; } - return n_tx_bytes; } static_always_inline void -tso_segment_vxlan_tunnel_headers_fixup (vlib_main_t * vm, vlib_buffer_t * b, - generic_header_offset_t * gho) +tso_segment_vxlan_tunnel_headers_fixup (vlib_main_t *vm, vlib_buffer_t *b) { - u8 proto = 0; ip4_header_t *ip4 = 0; ip6_header_t *ip6 = 0; udp_header_t *udp = 0; + i16 outer_l3_hdr_offset = vnet_buffer2 (b)->outer_l3_hdr_offset; + i16 outer_l4_hdr_offset = vnet_buffer2 (b)->outer_l4_hdr_offset; - ip4 = - (ip4_header_t *) (vlib_buffer_get_current (b) + gho->outer_l3_hdr_offset); - ip6 = - (ip6_header_t *) (vlib_buffer_get_current (b) + gho->outer_l3_hdr_offset); - udp = - (udp_header_t *) (vlib_buffer_get_current (b) + gho->outer_l4_hdr_offset); + ip4 = (ip4_header_t *) (b->data + outer_l3_hdr_offset); + ip6 = (ip6_header_t *) (b->data + outer_l3_hdr_offset); + udp = (udp_header_t *) (b->data + outer_l4_hdr_offset); - if (gho->gho_flags & GHO_F_OUTER_IP4) + if (vnet_buffer (b)->oflags & VNET_BUFFER_OFFLOAD_F_OUTER_IP_CKSUM) { - proto = ip4->protocol; - ip4->length = - clib_host_to_net_u16 (b->current_length - gho->outer_l3_hdr_offset); + ip4->length = clib_host_to_net_u16 ( + b->current_length - (outer_l3_hdr_offset - b->current_data)); ip4->checksum = ip4_header_checksum (ip4); + if (vnet_buffer (b)->oflags & VNET_BUFFER_OFFLOAD_F_OUTER_UDP_CKSUM) + { + udp->length = clib_host_to_net_u16 ( + b->current_length - (outer_l4_hdr_offset - b->current_data)); + // udp checksum is 0, in udp tunnel + udp->checksum = 0; + } + vnet_buffer_offload_flags_clear ( + b, VNET_BUFFER_OFFLOAD_F_OUTER_IP_CKSUM | + VNET_BUFFER_OFFLOAD_F_OUTER_UDP_CKSUM | + VNET_BUFFER_OFFLOAD_F_TNL_VXLAN); } - else if (gho->gho_flags & GHO_F_OUTER_IP6) - { - proto = ip6->protocol; - ip6->payload_length = - clib_host_to_net_u16 (b->current_length - gho->outer_l4_hdr_offset); - } - if (proto == IP_PROTOCOL_UDP) + else { - int bogus; - udp->length = - clib_host_to_net_u16 (b->current_length - gho->outer_l4_hdr_offset); - udp->checksum = 0; - if (gho->gho_flags & GHO_F_OUTER_IP6) + ip6->payload_length = clib_host_to_net_u16 ( + b->current_length - (outer_l4_hdr_offset - b->current_data)); + + if (vnet_buffer (b)->oflags & VNET_BUFFER_OFFLOAD_F_OUTER_UDP_CKSUM) { + int bogus; + udp->length = ip6->payload_length; + // udp checksum is 0, in udp tunnel + udp->checksum = 0; udp->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ip6, &bogus); + vnet_buffer_offload_flags_clear ( + b, VNET_BUFFER_OFFLOAD_F_OUTER_UDP_CKSUM | + VNET_BUFFER_OFFLOAD_F_TNL_VXLAN); } - else if (gho->gho_flags & GHO_F_OUTER_IP4) - { - udp->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4); - } - /* FIXME: it should be OUTER_UDP_CKSUM */ - vnet_buffer_offload_flags_clear (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM); } } -static_always_inline u16 -tso_segment_vxlan_tunnel_fixup (vlib_main_t * vm, - vnet_interface_per_thread_data_t * ptd, - vlib_buffer_t * sb0, - generic_header_offset_t * gho) +static_always_inline void +tso_segment_vxlan_tunnel_fixup (vlib_main_t *vm, + vnet_interface_per_thread_data_t *ptd, + vlib_buffer_t *sb0) { u16 n_tx_bufs = vec_len (ptd->split_buffers); - u16 i = 0, n_tx_bytes = 0; + u16 i = 0; while (i < n_tx_bufs) { vlib_buffer_t *b0 = vlib_get_buffer (vm, ptd->split_buffers[i]); - tso_segment_vxlan_tunnel_headers_fixup (vm, b0, gho); - n_tx_bytes += gho->outer_hdr_sz; + tso_segment_vxlan_tunnel_headers_fixup (vm, b0); i++; } - return n_tx_bytes; } static_always_inline u16 @@ -682,32 +677,10 @@ vnet_gso_node_inline (vlib_main_t * vm, to_next -= 1; n_left_to_next += 1; /* undo the counting. */ - generic_header_offset_t gho = { 0 }; u32 n_tx_bytes = 0; - u32 inner_is_ip6 = is_ip6; - - vnet_generic_header_offset_parser (b[0], &gho, is_l2, - is_ip4, is_ip6); - - if (PREDICT_FALSE (gho.gho_flags & GHO_F_TUNNEL)) - { - if (PREDICT_FALSE - (gho.gho_flags & (GHO_F_GRE_TUNNEL | - GHO_F_GENEVE_TUNNEL))) - { - /* not supported yet */ - drop_one_buffer_and_count (vm, vnm, node, from - 1, - hi->sw_if_index, - GSO_ERROR_UNHANDLED_TYPE); - b += 1; - continue; - } - inner_is_ip6 = (gho.gho_flags & GHO_F_IP6) != 0; - } - - n_tx_bytes = gso_segment_buffer_inline (vm, ptd, b[0], &gho, - is_l2, inner_is_ip6); + n_tx_bytes = + gso_segment_buffer_inline (vm, ptd, b[0], is_l2); if (PREDICT_FALSE (n_tx_bytes == 0)) { @@ -718,19 +691,15 @@ vnet_gso_node_inline (vlib_main_t * vm, continue; } - - if (PREDICT_FALSE (gho.gho_flags & GHO_F_VXLAN_TUNNEL)) + if (PREDICT_FALSE (vnet_buffer (b[0])->oflags & + VNET_BUFFER_OFFLOAD_F_TNL_VXLAN)) { - n_tx_bytes += - tso_segment_vxlan_tunnel_fixup (vm, ptd, b[0], &gho); + tso_segment_vxlan_tunnel_fixup (vm, ptd, b[0]); } - else - if (PREDICT_FALSE - (gho.gho_flags & (GHO_F_IPIP_TUNNEL | - GHO_F_IPIP6_TUNNEL))) + else if (PREDICT_FALSE (vnet_buffer (b[0])->oflags & + VNET_BUFFER_OFFLOAD_F_TNL_IPIP)) { - n_tx_bytes += - tso_segment_ipip_tunnel_fixup (vm, ptd, b[0], &gho); + tso_segment_ipip_tunnel_fixup (vm, ptd, b[0]); } u16 n_tx_bufs = vec_len (ptd->split_buffers); @@ -744,7 +713,6 @@ vnet_gso_node_inline (vlib_main_t * vm, { sbi0 = to_next[0] = from_seg[0]; sb0 = vlib_get_buffer (vm, sbi0); - vnet_buffer_offload_flags_clear (sb0, 0x7F); ASSERT (sb0->current_length > 0); to_next += 1; from_seg += 1; diff --git a/src/vnet/interface_api.c b/src/vnet/interface_api.c index 2995836672d..c727e519138 100644 --- a/src/vnet/interface_api.c +++ b/src/vnet/interface_api.c @@ -1020,21 +1020,19 @@ vl_api_sw_interface_set_interface_name_t_handler ( { vl_api_sw_interface_set_interface_name_reply_t *rmp; vnet_main_t *vnm = vnet_get_main (); - u32 sw_if_index = ntohl (mp->sw_if_index); - vnet_sw_interface_t *si = vnet_get_sw_interface (vnm, sw_if_index); clib_error_t *error; int rv = 0; + VALIDATE_SW_IF_INDEX (mp); + + u32 sw_if_index = ntohl (mp->sw_if_index); + vnet_sw_interface_t *si = vnet_get_sw_interface (vnm, sw_if_index); + if (mp->name[0] == 0) { rv = VNET_API_ERROR_INVALID_VALUE; goto out; } - if (si == 0) - { - rv = VNET_API_ERROR_INVALID_SW_IF_INDEX; - goto out; - } error = vnet_rename_interface (vnm, si->hw_if_index, (char *) mp->name); if (error) @@ -1044,6 +1042,7 @@ vl_api_sw_interface_set_interface_name_t_handler ( } out: + BAD_SW_IF_INDEX_LABEL; REPLY_MACRO (VL_API_SW_INTERFACE_SET_INTERFACE_NAME_REPLY); } diff --git a/src/vnet/ip/ip.c b/src/vnet/ip/ip.c index d045c2f37c1..586f7dfbc85 100644 --- a/src/vnet/ip/ip.c +++ b/src/vnet/ip/ip.c @@ -201,7 +201,8 @@ ip_feature_enable_disable (ip_address_family_t af, } int -ip_flow_hash_set (ip_address_family_t af, u32 table_id, u32 flow_hash_config) +ip_flow_hash_set (ip_address_family_t af, u32 table_id, + flow_hash_config_t flow_hash_config) { fib_protocol_t fproto; u32 fib_index; diff --git a/src/vnet/ipsec/ipsec_itf.c b/src/vnet/ipsec/ipsec_itf.c index 6e66d10660b..b86bf6a110c 100644 --- a/src/vnet/ipsec/ipsec_itf.c +++ b/src/vnet/ipsec/ipsec_itf.c @@ -381,6 +381,7 @@ ipsec_itf_create_cli (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { unformat_input_t _line_input, *line_input = &_line_input; + tunnel_mode_t mode = TUNNEL_MODE_P2P; u32 instance, sw_if_index; clib_error_t *error; mac_address_t mac; @@ -396,6 +397,8 @@ ipsec_itf_create_cli (vlib_main_t * vm, { if (unformat (line_input, "instance %d", &instance)) ; + else if (unformat (line_input, "p2mp")) + mode = TUNNEL_MODE_MP; else { error = clib_error_return (0, "unknown input: %U", @@ -410,7 +413,7 @@ ipsec_itf_create_cli (vlib_main_t * vm, return error; } - rv = ipsec_itf_create (instance, TUNNEL_MODE_P2P, &sw_if_index); + rv = ipsec_itf_create (instance, mode, &sw_if_index); if (rv) return clib_error_return (0, "iPSec interface create failed"); @@ -425,13 +428,13 @@ ipsec_itf_create_cli (vlib_main_t * vm, * * @cliexpar * The following two command syntaxes are equivalent: - * @cliexcmd{ipsec itf create [instance <instance>]} + * @cliexcmd{ipsec itf create [instance <instance>] [p2mp]} * Example of how to create a ipsec interface: * @cliexcmd{ipsec itf create} ?*/ VLIB_CLI_COMMAND (ipsec_itf_create_command, static) = { .path = "ipsec itf create", - .short_help = "ipsec itf create [instance <instance>]", + .short_help = "ipsec itf create [instance <instance>] [p2mp]", .function = ipsec_itf_create_cli, }; diff --git a/src/vnet/pg/input.c b/src/vnet/pg/input.c index f81485de65f..321472c4d85 100644 --- a/src/vnet/pg/input.c +++ b/src/vnet/pg/input.c @@ -1578,7 +1578,7 @@ fill_buffer_offload_flags (vlib_main_t *vm, u32 *buffers, u32 n_buffers, (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID | VNET_BUFFER_F_L3_HDR_OFFSET_VALID | VNET_BUFFER_F_L4_HDR_OFFSET_VALID); - if (buffer_oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM) + if (buffer_oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM || gso_enabled) oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM; } else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6)) @@ -1596,7 +1596,7 @@ fill_buffer_offload_flags (vlib_main_t *vm, u32 *buffers, u32 n_buffers, if (l4_proto == IP_PROTOCOL_TCP) { - if (buffer_oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM) + if (buffer_oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM || gso_enabled) oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; /* only set GSO flag for chained buffers */ diff --git a/src/vnet/session/application.h b/src/vnet/session/application.h index 5fea61bdab4..c68a911230f 100644 --- a/src/vnet/session/application.h +++ b/src/vnet/session/application.h @@ -342,7 +342,7 @@ session_error_t app_worker_start_listen (app_worker_t *app_wrk, int app_worker_stop_listen (app_worker_t * app_wrk, app_listener_t * al); int app_worker_init_accepted (session_t * s); int app_worker_listened_notify (app_worker_t *app_wrk, session_handle_t alsh, - u32 opaque, int err); + u32 opaque, session_error_t err); int app_worker_unlisten_reply (app_worker_t *app_wrk, session_handle_t sh, u32 opaque, session_error_t err); int app_worker_accept_notify (app_worker_t * app_wrk, session_t * s); diff --git a/src/vnet/session/application_interface.c b/src/vnet/session/application_interface.c index 86f3dcdece6..a62f914d43a 100644 --- a/src/vnet/session/application_interface.c +++ b/src/vnet/session/application_interface.c @@ -106,8 +106,8 @@ parse_uri (char *uri, session_endpoint_cfg_t *sep) return 0; } -int -vnet_bind_uri (vnet_listen_args_t * a) +session_error_t +vnet_bind_uri (vnet_listen_args_t *a) { session_endpoint_cfg_t sep = SESSION_ENDPOINT_CFG_NULL; int rv; diff --git a/src/vpp/vnet/main.c b/src/vpp/vnet/main.c index 71434a9c065..c57efd59a62 100644 --- a/src/vpp/vnet/main.c +++ b/src/vpp/vnet/main.c @@ -22,6 +22,8 @@ #include <vppinfra/clib.h> #include <vppinfra/cpu.h> +#include <vppinfra/bitmap.h> +#include <vppinfra/unix.h> #include <vlib/vlib.h> #include <vlib/unix/unix.h> #include <vlib/threads.h> @@ -43,25 +45,26 @@ static void vpp_find_plugin_path () { extern char *vat_plugin_path; - char *p, path[PATH_MAX]; - int rv; - u8 *s; + char *p; + u8 *s, *path; /* find executable path */ - if ((rv = readlink ("/proc/self/exe", path, PATH_MAX - 1)) == -1) + path = os_get_exec_path (); + + if (!path) return; - /* readlink doesn't provide null termination */ - path[rv] = 0; + /* add null termination */ + vec_add1 (path, 0); /* strip filename */ - if ((p = strrchr (path, '/')) == 0) - return; + if ((p = strrchr ((char *) path, '/')) == 0) + goto done; *p = 0; /* strip bin/ */ - if ((p = strrchr (path, '/')) == 0) - return; + if ((p = strrchr ((char *) path, '/')) == 0) + goto done; *p = 0; s = format (0, "%s/" CLIB_LIB_DIR "/vpp_plugins", path, path); @@ -71,6 +74,9 @@ vpp_find_plugin_path () s = format (0, "%s/" CLIB_LIB_DIR "/vpp_api_test_plugins", path, path); vec_add1 (s, 0); vat_plugin_path = (char *) s; + +done: + vec_free (path); } static void diff --git a/src/vppinfra/CMakeLists.txt b/src/vppinfra/CMakeLists.txt index f34ceed9d15..5878f0612f0 100644 --- a/src/vppinfra/CMakeLists.txt +++ b/src/vppinfra/CMakeLists.txt @@ -229,7 +229,12 @@ elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD") ) endif() -option(VPP_USE_EXTERNAL_LIBEXECINFO "Use external libexecinfo (useful for non-glibc targets)." OFF) +if("${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD") + option(VPP_USE_EXTERNAL_LIBEXECINFO "Use external libexecinfo (useful for non-glibc targets)." ON) +else() + option(VPP_USE_EXTERNAL_LIBEXECINFO "Use external libexecinfo (useful for non-glibc targets)." OFF) +endif() + if(VPP_USE_EXTERNAL_LIBEXECINFO) set(EXECINFO_LIB execinfo) endif() diff --git a/src/vppinfra/pmalloc.c b/src/vppinfra/pmalloc.c index 2a27379b573..85b9db9d56c 100644 --- a/src/vppinfra/pmalloc.c +++ b/src/vppinfra/pmalloc.c @@ -17,6 +17,9 @@ #include <stdlib.h> #include <sys/types.h> #include <sys/stat.h> +#ifdef __FreeBSD__ +#include <sys/memrange.h> +#endif /* __FreeBSD__ */ #include <fcntl.h> #include <unistd.h> #include <sched.h> @@ -184,8 +187,9 @@ next_chunk: } static void -pmalloc_update_lookup_table (clib_pmalloc_main_t * pm, u32 first, u32 count) +pmalloc_update_lookup_table (clib_pmalloc_main_t *pm, u32 first, u32 count) { +#ifdef __linux uword seek, va, pa, p; int fd; u32 elts_per_page = 1U << (pm->def_log2_page_sz - pm->lookup_log2_page_sz); @@ -223,6 +227,45 @@ pmalloc_update_lookup_table (clib_pmalloc_main_t * pm, u32 first, u32 count) if (fd != -1) close (fd); +#elif defined(__FreeBSD__) + struct mem_extract meme; + uword p; + int fd; + u32 elts_per_page = 1U << (pm->def_log2_page_sz - pm->lookup_log2_page_sz); + + vec_validate_aligned (pm->lookup_table, + vec_len (pm->pages) * elts_per_page - 1, + CLIB_CACHE_LINE_BYTES); + + p = (uword) first * elts_per_page; + if (pm->flags & CLIB_PMALLOC_F_NO_PAGEMAP) + { + while (p < (uword) elts_per_page * count) + { + pm->lookup_table[p] = + pointer_to_uword (pm->base) + (p << pm->lookup_log2_page_sz); + p++; + } + return; + } + + fd = open ((char *) "/dev/mem", O_RDONLY); + if (fd == -1) + return; + + while (p < (uword) elts_per_page * count) + { + meme.me_vaddr = + pointer_to_uword (pm->base) + (p << pm->lookup_log2_page_sz); + if (ioctl (fd, MEM_EXTRACT_PADDR, &meme) == -1) + continue; + pm->lookup_table[p] = meme.me_vaddr - meme.me_paddr; + p++; + } + return; +#else +#error "Unsupported OS" +#endif } static inline clib_pmalloc_page_t * diff --git a/src/vppinfra/unix-misc.c b/src/vppinfra/unix-misc.c index 4dbc5ce98ce..29cbe0a557d 100644 --- a/src/vppinfra/unix-misc.c +++ b/src/vppinfra/unix-misc.c @@ -42,6 +42,8 @@ #include <vppinfra/format.h> #ifdef __linux__ #include <vppinfra/linux/sysfs.h> +#else +#include <sys/sysctl.h> #endif #include <sys/stat.h> @@ -358,6 +360,28 @@ os_get_cpu_phys_core_id (int cpu_id) #endif } +__clib_export u8 * +os_get_exec_path () +{ + u8 *rv = 0; +#ifdef __linux__ + char tmp[PATH_MAX]; + ssize_t sz = readlink ("/proc/self/exe", tmp, sizeof (tmp)); + + if (sz <= 0) + return 0; +#else + char tmp[MAXPATHLEN]; + int mib[4] = { CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1 }; + size_t sz = MAXPATHLEN; + + if (sysctl (mib, 4, tmp, &sz, NULL, 0) == -1) + return 0; +#endif + vec_add (rv, tmp, sz); + return rv; +} + /* * fd.io coding-style-patch-verification: ON * diff --git a/src/vppinfra/unix.h b/src/vppinfra/unix.h index 3ad57b05e72..abda21879f9 100644 --- a/src/vppinfra/unix.h +++ b/src/vppinfra/unix.h @@ -71,6 +71,10 @@ clib_bitmap_t *os_get_cpu_on_node_bitmap (int node); /* Retrieve physical core id of specific cpu, -1 if not available */ int os_get_cpu_phys_core_id (int cpu); +/* Retrieve the path of the current executable as a vector (not + * null-terminated). */ +u8 *os_get_exec_path (); + #endif /* included_clib_unix_h */ /* |