diff options
Diffstat (limited to 'src')
177 files changed, 6348 insertions, 1975 deletions
diff --git a/src/cmake/api.cmake b/src/cmake/api.cmake index 0e273a7d4dc..10e89d77594 100644 --- a/src/cmake/api.cmake +++ b/src/cmake/api.cmake @@ -70,7 +70,7 @@ function(vpp_generate_api_json_header file dir component) ) install( FILES ${output_name} - DESTINATION share/vpp/api/${dir}/ + DESTINATION ${CMAKE_INSTALL_DATADIR}/vpp/api/${dir}/ COMPONENT ${component} ) endfunction() @@ -101,7 +101,7 @@ function(vpp_generate_vapi_c_header f) ) install( FILES ${output_name} - DESTINATION include/vapi + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/vapi COMPONENT vpp-dev ) endfunction () @@ -128,7 +128,7 @@ function (vpp_generate_vapi_cpp_header f) ) install( FILES ${output_name} - DESTINATION include/vapi + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/vapi COMPONENT vpp-dev ) endfunction () diff --git a/src/cmake/library.cmake b/src/cmake/library.cmake index 45b3944eb34..a06a795c69f 100644 --- a/src/cmake/library.cmake +++ b/src/cmake/library.cmake @@ -72,7 +72,7 @@ macro(add_vpp_library lib) ${CMAKE_CURRENT_BINARY_DIR}/${file}_types.h ${CMAKE_CURRENT_BINARY_DIR}/${file}_tojson.h ${CMAKE_CURRENT_BINARY_DIR}/${file}_fromjson.h - DESTINATION include/${lib}/${dir} + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${lib}/${dir} COMPONENT vpp-dev ) endforeach() @@ -96,7 +96,7 @@ macro(add_vpp_library lib) get_filename_component(dir ${file} DIRECTORY) install( FILES ${file} - DESTINATION include/${lib}/${dir} + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${lib}/${dir} COMPONENT ${ARG_COMPONENT}-dev ) endforeach() @@ -111,7 +111,7 @@ function (add_vpp_headers path) get_filename_component(dir ${file} DIRECTORY) install( FILES ${file} - DESTINATION include/${path}/${dir} + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${path}/${dir} COMPONENT vpp-dev ) endforeach() diff --git a/src/cmake/plugin.cmake b/src/cmake/plugin.cmake index df96aa8765f..f971e5f1401 100644 --- a/src/cmake/plugin.cmake +++ b/src/cmake/plugin.cmake @@ -44,7 +44,7 @@ macro(add_vpp_plugin name) ${CMAKE_CURRENT_BINARY_DIR}/${f}.h ${CMAKE_CURRENT_BINARY_DIR}/${f}_enum.h ${CMAKE_CURRENT_BINARY_DIR}/${f}_types.h - DESTINATION include/vpp_plugins/${name}/${dir} + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/vpp_plugins/${name}/${dir} COMPONENT ${PLUGIN_DEV_COMPONENT} ) endforeach() @@ -86,7 +86,7 @@ macro(add_vpp_plugin name) get_filename_component(dir ${file} DIRECTORY) install( FILES ${file} - DESTINATION include/vpp_plugins/${name}/${dir} + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/vpp_plugins/${name}/${dir} COMPONENT vpp-dev ) endforeach() diff --git a/src/plugins/acl/acl_test.c b/src/plugins/acl/acl_test.c index 0c218687bdf..98803a916cb 100644 --- a/src/plugins/acl/acl_test.c +++ b/src/plugins/acl/acl_test.c @@ -18,6 +18,8 @@ *------------------------------------------------------------------ */ +#include <byteswap.h> + #include <vat/vat.h> #include <vlibapi/api.h> #include <vlibmemory/api.h> @@ -112,7 +114,7 @@ static void vl_api_acl_interface_list_details_t_handler int i; vat_main_t * vam = acl_test_main.vat_main; u8 *out = 0; - vl_api_acl_interface_list_details_t_endian(mp); + vl_api_acl_interface_list_details_t_endian (mp, 0 /* from network */); out = format(out, "sw_if_index: %d, count: %d, n_input: %d\n", mp->sw_if_index, mp->count, mp->n_input); out = format(out, " input "); for(i=0; i<mp->count; i++) { @@ -139,7 +141,8 @@ static void vl_api_acl_interface_etype_whitelist_details_t_handler int i; vat_main_t * vam = acl_test_main.vat_main; u8 *out = 0; - vl_api_acl_interface_etype_whitelist_details_t_endian(mp); + vl_api_acl_interface_etype_whitelist_details_t_endian ( + mp, 0 /* from network */); out = format(out, "sw_if_index: %d, count: %d, n_input: %d\n", mp->sw_if_index, mp->count, mp->n_input); out = format(out, " input "); for(i=0; i<mp->count; i++) { @@ -157,9 +160,9 @@ static void vl_api_acl_plugin_get_conn_table_max_entries_reply_t_handler (vl_api_acl_plugin_get_conn_table_max_entries_reply_t * mp) { vat_main_t * vam = acl_test_main.vat_main; - clib_warning("\nConn table max entries: %d", - __bswap_64(mp->conn_table_max_entries) ); - vam->result_ready = 1; + clib_warning ("\nConn table max entries: %d", + clib_net_to_host_u64 (mp->conn_table_max_entries)); + vam->result_ready = 1; } static inline u8 * @@ -171,15 +174,15 @@ vl_api_acl_rule_t_pretty_format (u8 *out, vl_api_acl_rule_t * a) inet_ntop(af, &a->src_prefix.address.un, (void *)src, sizeof(src)); inet_ntop(af, &a->dst_prefix.address.un, (void *)dst, sizeof(dst)); - out = format(out, "%s action %d src %s/%d dst %s/%d proto %d sport %d-%d dport %d-%d tcpflags %d mask %d", - a->src_prefix.address.af ? "ipv6" : "ipv4", a->is_permit, - src, a->src_prefix.len, - dst, a->dst_prefix.len, - a->proto, - a->srcport_or_icmptype_first, a->srcport_or_icmptype_last, - a->dstport_or_icmpcode_first, a->dstport_or_icmpcode_last, - a->tcp_flags_value, a->tcp_flags_mask); - return(out); + out = format (out, + "%s action %d src %s/%d dst %s/%d proto %d sport %d-%d dport " + "%d-%d tcpflags %d mask %d", + a->src_prefix.address.af ? "ipv6" : "ipv4", a->is_permit, src, + a->src_prefix.len, dst, a->dst_prefix.len, a->proto, + a->srcport_or_icmptype_first, a->srcport_or_icmptype_last, + a->dstport_or_icmpcode_first, a->dstport_or_icmpcode_last, + a->tcp_flags_value, a->tcp_flags_mask); + return (out); } @@ -189,9 +192,10 @@ static void vl_api_acl_details_t_handler { int i; vat_main_t * vam = acl_test_main.vat_main; - vl_api_acl_details_t_endian(mp); - u8 *out = 0; - out = format(0, "acl_index: %d, count: %d\n tag {%s}\n", mp->acl_index, mp->count, mp->tag); + vl_api_acl_details_t_endian (mp, 0 /* from network */); + u8 *out = 0; + out = format (0, "acl_index: %d, count: %d\n tag {%s}\n", + mp->acl_index, mp->count, mp->tag); for(i=0; i<mp->count; i++) { out = format(out, " "); out = vl_api_acl_rule_t_pretty_format(out, &mp->r[i]); @@ -223,8 +227,9 @@ static void vl_api_macip_acl_details_t_handler { int i; vat_main_t * vam = acl_test_main.vat_main; - vl_api_macip_acl_details_t_endian(mp); - u8 *out = format(0,"MACIP acl_index: %d, count: %d\n tag {%s}\n", mp->acl_index, mp->count, mp->tag); + vl_api_macip_acl_details_t_endian (mp, 0 /* from network */); + u8 *out = format (0, "MACIP acl_index: %d, count: %d\n tag {%s}\n", + mp->acl_index, mp->count, mp->tag); for(i=0; i<mp->count; i++) { out = format(out, " "); out = vl_api_macip_acl_rule_t_pretty_format(out, &mp->r[i]); diff --git a/src/plugins/acl/sess_mgmt_node.c b/src/plugins/acl/sess_mgmt_node.c index e049a3ffa85..418baef9b6b 100644 --- a/src/plugins/acl/sess_mgmt_node.c +++ b/src/plugins/acl/sess_mgmt_node.c @@ -371,8 +371,9 @@ send_one_worker_interrupt (vlib_main_t * vm, acl_main_t * am, } void -aclp_post_session_change_request (acl_main_t * am, u32 target_thread, - u32 target_session, u32 request_type) +aclp_post_session_change_request (acl_main_t *am, u32 target_thread, + u32 target_session, + acl_fa_sess_req_t request_type) { acl_fa_per_worker_data_t *pw_me = &am->per_worker_data[os_get_thread_index ()]; diff --git a/src/plugins/crypto_native/CMakeLists.txt b/src/plugins/crypto_native/CMakeLists.txt index 9b6091610d9..5499ed4608a 100644 --- a/src/plugins/crypto_native/CMakeLists.txt +++ b/src/plugins/crypto_native/CMakeLists.txt @@ -12,8 +12,8 @@ # limitations under the License. if(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|x86_64.*|AMD64.*") - list(APPEND VARIANTS "slm\;-march=silvermont") - list(APPEND VARIANTS "hsw\;-march=haswell") + list(APPEND VARIANTS "slm\;-march=silvermont -maes") + list(APPEND VARIANTS "hsw\;-march=haswell -maes") if(compiler_flag_march_skylake_avx512 AND compiler_flag_mprefer_vector_width_256) list(APPEND VARIANTS "skx\;-march=skylake-avx512 -mprefer-vector-width=256") endif() @@ -23,16 +23,15 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|x86_64.*|AMD64.*") if(compiler_flag_march_alderlake) list(APPEND VARIANTS "adl\;-march=alderlake -mprefer-vector-width=256") endif() - set (COMPILE_FILES aes_cbc.c aes_gcm.c aes_ctr.c) - set (COMPILE_OPTS -Wall -fno-common -maes) endif() if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*)") list(APPEND VARIANTS "armv8\;-march=armv8.1-a+crc+crypto") - set (COMPILE_FILES aes_cbc.c aes_gcm.c aes_ctr.c) - set (COMPILE_OPTS -Wall -fno-common) endif() +set (COMPILE_FILES aes_cbc.c aes_gcm.c aes_ctr.c sha2.c) +set (COMPILE_OPTS -Wall -fno-common) + if (NOT VARIANTS) return() endif() diff --git a/src/plugins/crypto_native/aes_cbc.c b/src/plugins/crypto_native/aes_cbc.c index c84390c3108..dd7ca3f1cf1 100644 --- a/src/plugins/crypto_native/aes_cbc.c +++ b/src/plugins/crypto_native/aes_cbc.c @@ -249,18 +249,30 @@ decrypt: return n_ops; } -#define foreach_aes_cbc_handler_type _(128) _(192) _(256) - -#define _(x) \ -static u32 aes_ops_dec_aes_cbc_##x \ -(vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops) \ -{ return aes_ops_dec_aes_cbc (vm, ops, n_ops, AES_KEY_##x); } \ -static u32 aes_ops_enc_aes_cbc_##x \ -(vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops) \ -{ return aes_ops_enc_aes_cbc (vm, ops, n_ops, AES_KEY_##x); } \ - -foreach_aes_cbc_handler_type; -#undef _ +static int +aes_cbc_cpu_probe () +{ +#if defined(__VAES__) && defined(__AVX512F__) + if (clib_cpu_supports_vaes () && clib_cpu_supports_avx512f ()) + return 50; +#elif defined(__VAES__) + if (clib_cpu_supports_vaes ()) + return 40; +#elif defined(__AVX512F__) + if (clib_cpu_supports_avx512f ()) + return 30; +#elif defined(__AVX2__) + if (clib_cpu_supports_avx2 ()) + return 20; +#elif __AES__ + if (clib_cpu_supports_aes ()) + return 10; +#elif __aarch64__ + if (clib_cpu_supports_aarch64_aes ()) + return 10; +#endif + return -1; +} static void * aes_cbc_key_exp_128 (vnet_crypto_key_t *key) @@ -289,43 +301,39 @@ aes_cbc_key_exp_256 (vnet_crypto_key_t *key) return kd; } -#include <fcntl.h> - -clib_error_t * -#if defined(__VAES__) && defined(__AVX512F__) -crypto_native_aes_cbc_init_icl (vlib_main_t *vm) -#elif defined(__VAES__) -crypto_native_aes_cbc_init_adl (vlib_main_t *vm) -#elif __AVX512F__ -crypto_native_aes_cbc_init_skx (vlib_main_t * vm) -#elif __aarch64__ -crypto_native_aes_cbc_init_neon (vlib_main_t * vm) -#elif __AVX2__ -crypto_native_aes_cbc_init_hsw (vlib_main_t * vm) -#else -crypto_native_aes_cbc_init_slm (vlib_main_t * vm) -#endif -{ - crypto_native_main_t *cm = &crypto_native_main; +#define foreach_aes_cbc_handler_type _ (128) _ (192) _ (256) + +#define _(x) \ + static u32 aes_ops_enc_aes_cbc_##x (vlib_main_t *vm, \ + vnet_crypto_op_t *ops[], u32 n_ops) \ + { \ + return aes_ops_enc_aes_cbc (vm, ops, n_ops, AES_KEY_##x); \ + } \ + \ + CRYPTO_NATIVE_OP_HANDLER (aes_##x##_cbc_enc) = { \ + .op_id = VNET_CRYPTO_OP_AES_##x##_CBC_ENC, \ + .fn = aes_ops_enc_aes_cbc_##x, \ + .probe = aes_cbc_cpu_probe, \ + }; \ + \ + static u32 aes_ops_dec_aes_cbc_##x (vlib_main_t *vm, \ + vnet_crypto_op_t *ops[], u32 n_ops) \ + { \ + return aes_ops_dec_aes_cbc (vm, ops, n_ops, AES_KEY_##x); \ + } \ + \ + CRYPTO_NATIVE_OP_HANDLER (aes_##x##_cbc_dec) = { \ + .op_id = VNET_CRYPTO_OP_AES_##x##_CBC_DEC, \ + .fn = aes_ops_dec_aes_cbc_##x, \ + .probe = aes_cbc_cpu_probe, \ + }; \ + \ + CRYPTO_NATIVE_KEY_HANDLER (aes_##x##_cbc) = { \ + .alg_id = VNET_CRYPTO_ALG_AES_##x##_CBC, \ + .key_fn = aes_cbc_key_exp_##x, \ + .probe = aes_cbc_cpu_probe, \ + }; -#define _(x) \ - vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \ - VNET_CRYPTO_OP_AES_##x##_CBC_ENC, \ - aes_ops_enc_aes_cbc_##x); \ - vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \ - VNET_CRYPTO_OP_AES_##x##_CBC_DEC, \ - aes_ops_dec_aes_cbc_##x); \ - cm->key_fn[VNET_CRYPTO_ALG_AES_##x##_CBC] = aes_cbc_key_exp_##x; - foreach_aes_cbc_handler_type; +foreach_aes_cbc_handler_type; #undef _ - return 0; -} - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/plugins/crypto_native/aes_ctr.c b/src/plugins/crypto_native/aes_ctr.c index 3a219510419..d02a7b69b9d 100644 --- a/src/plugins/crypto_native/aes_ctr.c +++ b/src/plugins/crypto_native/aes_ctr.c @@ -81,32 +81,50 @@ aes_ctr_key_exp (vnet_crypto_key_t *key, aes_key_size_t ks) foreach_aes_ctr_handler_type; #undef _ -clib_error_t * +static int +probe () +{ #if defined(__VAES__) && defined(__AVX512F__) -crypto_native_aes_ctr_init_icl (vlib_main_t *vm) + if (clib_cpu_supports_vaes () && clib_cpu_supports_avx512f ()) + return 50; #elif defined(__VAES__) -crypto_native_aes_ctr_init_adl (vlib_main_t *vm) -#elif __AVX512F__ -crypto_native_aes_ctr_init_skx (vlib_main_t *vm) -#elif __AVX2__ -crypto_native_aes_ctr_init_hsw (vlib_main_t *vm) + if (clib_cpu_supports_vaes ()) + return 40; +#elif defined(__AVX512F__) + if (clib_cpu_supports_avx512f ()) + return 30; +#elif defined(__AVX2__) + if (clib_cpu_supports_avx2 ()) + return 20; +#elif __AES__ + if (clib_cpu_supports_aes ()) + return 10; #elif __aarch64__ -crypto_native_aes_ctr_init_neon (vlib_main_t *vm) -#else -crypto_native_aes_ctr_init_slm (vlib_main_t *vm) + if (clib_cpu_supports_aarch64_aes ()) + return 10; #endif -{ - crypto_native_main_t *cm = &crypto_native_main; + return -1; +} -#define _(x) \ - vnet_crypto_register_ops_handlers ( \ - vm, cm->crypto_engine_index, VNET_CRYPTO_OP_AES_##x##_CTR_ENC, \ - aes_ops_aes_ctr_##x, aes_ops_aes_ctr_##x##_chained); \ - vnet_crypto_register_ops_handlers ( \ - vm, cm->crypto_engine_index, VNET_CRYPTO_OP_AES_##x##_CTR_DEC, \ - aes_ops_aes_ctr_##x, aes_ops_aes_ctr_##x##_chained); \ - cm->key_fn[VNET_CRYPTO_ALG_AES_##x##_CTR] = aes_ctr_key_exp_##x; - foreach_aes_ctr_handler_type; +#define _(b) \ + CRYPTO_NATIVE_OP_HANDLER (aes_##b##_ctr_enc) = { \ + .op_id = VNET_CRYPTO_OP_AES_##b##_CTR_ENC, \ + .fn = aes_ops_aes_ctr_##b, \ + .cfn = aes_ops_aes_ctr_##b##_chained, \ + .probe = probe, \ + }; \ + \ + CRYPTO_NATIVE_OP_HANDLER (aes_##b##_ctr_dec) = { \ + .op_id = VNET_CRYPTO_OP_AES_##b##_CTR_DEC, \ + .fn = aes_ops_aes_ctr_##b, \ + .cfn = aes_ops_aes_ctr_##b##_chained, \ + .probe = probe, \ + }; \ + CRYPTO_NATIVE_KEY_HANDLER (aes_##b##_ctr) = { \ + .alg_id = VNET_CRYPTO_ALG_AES_##b##_CTR, \ + .key_fn = aes_ctr_key_exp_##b, \ + .probe = probe, \ + }; + +_ (128) _ (192) _ (256) #undef _ - return 0; -} diff --git a/src/plugins/crypto_native/aes_gcm.c b/src/plugins/crypto_native/aes_gcm.c index 6589d411975..220788d4e97 100644 --- a/src/plugins/crypto_native/aes_gcm.c +++ b/src/plugins/crypto_native/aes_gcm.c @@ -118,40 +118,49 @@ aes_gcm_key_exp (vnet_crypto_key_t *key, aes_key_size_t ks) foreach_aes_gcm_handler_type; #undef _ -clib_error_t * +static int +probe () +{ #if defined(__VAES__) && defined(__AVX512F__) -crypto_native_aes_gcm_init_icl (vlib_main_t *vm) + if (clib_cpu_supports_vpclmulqdq () && clib_cpu_supports_vaes () && + clib_cpu_supports_avx512f ()) + return 50; #elif defined(__VAES__) -crypto_native_aes_gcm_init_adl (vlib_main_t *vm) -#elif __AVX512F__ -crypto_native_aes_gcm_init_skx (vlib_main_t *vm) -#elif __AVX2__ -crypto_native_aes_gcm_init_hsw (vlib_main_t *vm) + if (clib_cpu_supports_vpclmulqdq () && clib_cpu_supports_vaes ()) + return 40; +#elif defined(__AVX512F__) + if (clib_cpu_supports_pclmulqdq () && clib_cpu_supports_avx512f ()) + return 30; +#elif defined(__AVX2__) + if (clib_cpu_supports_pclmulqdq () && clib_cpu_supports_avx2 ()) + return 20; +#elif __AES__ + if (clib_cpu_supports_pclmulqdq () && clib_cpu_supports_aes ()) + return 10; #elif __aarch64__ -crypto_native_aes_gcm_init_neon (vlib_main_t *vm) -#else -crypto_native_aes_gcm_init_slm (vlib_main_t *vm) + if (clib_cpu_supports_aarch64_aes ()) + return 10; #endif -{ - crypto_native_main_t *cm = &crypto_native_main; - -#define _(x) \ - vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \ - VNET_CRYPTO_OP_AES_##x##_GCM_ENC, \ - aes_ops_enc_aes_gcm_##x); \ - vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \ - VNET_CRYPTO_OP_AES_##x##_GCM_DEC, \ - aes_ops_dec_aes_gcm_##x); \ - cm->key_fn[VNET_CRYPTO_ALG_AES_##x##_GCM] = aes_gcm_key_exp_##x; - foreach_aes_gcm_handler_type; -#undef _ - return 0; + return -1; } -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ +#define _(b) \ + CRYPTO_NATIVE_OP_HANDLER (aes_##b##_gcm_enc) = { \ + .op_id = VNET_CRYPTO_OP_AES_##b##_GCM_ENC, \ + .fn = aes_ops_enc_aes_gcm_##b, \ + .probe = probe, \ + }; \ + \ + CRYPTO_NATIVE_OP_HANDLER (aes_##b##_gcm_dec) = { \ + .op_id = VNET_CRYPTO_OP_AES_##b##_GCM_DEC, \ + .fn = aes_ops_dec_aes_gcm_##b, \ + .probe = probe, \ + }; \ + CRYPTO_NATIVE_KEY_HANDLER (aes_##b##_gcm) = { \ + .alg_id = VNET_CRYPTO_ALG_AES_##b##_GCM, \ + .key_fn = aes_gcm_key_exp_##b, \ + .probe = probe, \ + }; + +_ (128) _ (192) _ (256) +#undef _ diff --git a/src/plugins/crypto_native/crypto_native.h b/src/plugins/crypto_native/crypto_native.h index c15b8cbd1da..3d18e8cabd0 100644 --- a/src/plugins/crypto_native/crypto_native.h +++ b/src/plugins/crypto_native/crypto_native.h @@ -19,33 +19,66 @@ #define __crypto_native_h__ typedef void *(crypto_native_key_fn_t) (vnet_crypto_key_t * key); +typedef int (crypto_native_variant_probe_t) (); + +typedef struct crypto_native_op_handler +{ + struct crypto_native_op_handler *next; + vnet_crypto_op_id_t op_id; + vnet_crypto_ops_handler_t *fn; + vnet_crypto_chained_ops_handler_t *cfn; + crypto_native_variant_probe_t *probe; + int priority; +} crypto_native_op_handler_t; + +typedef struct crypto_native_key_handler +{ + struct crypto_native_key_handler *next; + vnet_crypto_alg_t alg_id; + crypto_native_key_fn_t *key_fn; + crypto_native_variant_probe_t *probe; + int priority; +} crypto_native_key_handler_t; typedef struct { u32 crypto_engine_index; crypto_native_key_fn_t *key_fn[VNET_CRYPTO_N_ALGS]; void **key_data; + crypto_native_op_handler_t *op_handlers; + crypto_native_key_handler_t *key_handlers; } crypto_native_main_t; extern crypto_native_main_t crypto_native_main; -#define foreach_crypto_native_march_variant \ - _ (slm) _ (hsw) _ (skx) _ (icl) _ (adl) _ (neon) - -#define _(v) \ - clib_error_t __clib_weak *crypto_native_aes_cbc_init_##v (vlib_main_t *vm); \ - clib_error_t __clib_weak *crypto_native_aes_ctr_init_##v (vlib_main_t *vm); \ - clib_error_t __clib_weak *crypto_native_aes_gcm_init_##v (vlib_main_t *vm); - -foreach_crypto_native_march_variant; -#undef _ +#define CRYPTO_NATIVE_OP_HANDLER(x) \ + static crypto_native_op_handler_t __crypto_native_op_handler_##x; \ + static void __clib_constructor __crypto_native_op_handler_cb_##x (void) \ + { \ + crypto_native_main_t *cm = &crypto_native_main; \ + int priority = __crypto_native_op_handler_##x.probe (); \ + if (priority >= 0) \ + { \ + __crypto_native_op_handler_##x.priority = priority; \ + __crypto_native_op_handler_##x.next = cm->op_handlers; \ + cm->op_handlers = &__crypto_native_op_handler_##x; \ + } \ + } \ + static crypto_native_op_handler_t __crypto_native_op_handler_##x +#define CRYPTO_NATIVE_KEY_HANDLER(x) \ + static crypto_native_key_handler_t __crypto_native_key_handler_##x; \ + static void __clib_constructor __crypto_native_key_handler_cb_##x (void) \ + { \ + crypto_native_main_t *cm = &crypto_native_main; \ + int priority = __crypto_native_key_handler_##x.probe (); \ + if (priority >= 0) \ + { \ + __crypto_native_key_handler_##x.priority = priority; \ + __crypto_native_key_handler_##x.next = cm->key_handlers; \ + cm->key_handlers = &__crypto_native_key_handler_##x; \ + } \ + } \ + static crypto_native_key_handler_t __crypto_native_key_handler_##x #endif /* __crypto_native_h__ */ -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/plugins/crypto_native/main.c b/src/plugins/crypto_native/main.c index 8a59be319b9..2bc0d98f196 100644 --- a/src/plugins/crypto_native/main.c +++ b/src/plugins/crypto_native/main.c @@ -63,95 +63,52 @@ clib_error_t * crypto_native_init (vlib_main_t * vm) { crypto_native_main_t *cm = &crypto_native_main; - clib_error_t *error = 0; - if (clib_cpu_supports_x86_aes () == 0 && - clib_cpu_supports_aarch64_aes () == 0) + if (cm->op_handlers == 0) return 0; cm->crypto_engine_index = vnet_crypto_register_engine (vm, "native", 100, "Native ISA Optimized Crypto"); - if (0); -#if __x86_64__ - else if (crypto_native_aes_cbc_init_icl && clib_cpu_supports_vaes () && - clib_cpu_supports_avx512f ()) - error = crypto_native_aes_cbc_init_icl (vm); - else if (crypto_native_aes_cbc_init_adl && clib_cpu_supports_vaes ()) - error = crypto_native_aes_cbc_init_adl (vm); - else if (crypto_native_aes_cbc_init_skx && clib_cpu_supports_avx512f ()) - error = crypto_native_aes_cbc_init_skx (vm); - else if (crypto_native_aes_cbc_init_hsw && clib_cpu_supports_avx2 ()) - error = crypto_native_aes_cbc_init_hsw (vm); - else if (crypto_native_aes_cbc_init_slm) - error = crypto_native_aes_cbc_init_slm (vm); -#endif -#if __aarch64__ - else if (crypto_native_aes_cbc_init_neon) - error = crypto_native_aes_cbc_init_neon (vm); -#endif - else - error = clib_error_return (0, "No AES CBC implemenation available"); - - if (error) - return error; - - if (0) - ; -#if __x86_64__ - else if (crypto_native_aes_ctr_init_icl && clib_cpu_supports_vaes () && - clib_cpu_supports_avx512f ()) - error = crypto_native_aes_ctr_init_icl (vm); - else if (crypto_native_aes_ctr_init_adl && clib_cpu_supports_vaes ()) - error = crypto_native_aes_ctr_init_adl (vm); - else if (crypto_native_aes_ctr_init_skx && clib_cpu_supports_avx512f ()) - error = crypto_native_aes_ctr_init_skx (vm); - else if (crypto_native_aes_ctr_init_hsw && clib_cpu_supports_avx2 ()) - error = crypto_native_aes_ctr_init_hsw (vm); - else if (crypto_native_aes_ctr_init_slm) - error = crypto_native_aes_ctr_init_slm (vm); -#endif -#if __aarch64__ - else if (crypto_native_aes_ctr_init_neon) - error = crypto_native_aes_ctr_init_neon (vm); -#endif - else - error = clib_error_return (0, "No AES CTR implemenation available"); - - if (error) - return error; - -#if __x86_64__ - if (clib_cpu_supports_pclmulqdq ()) + crypto_native_op_handler_t *oh = cm->op_handlers; + crypto_native_key_handler_t *kh = cm->key_handlers; + crypto_native_op_handler_t **best_by_op_id = 0; + crypto_native_key_handler_t **best_by_alg_id = 0; + + while (oh) { - if (crypto_native_aes_gcm_init_icl && clib_cpu_supports_vaes () && - clib_cpu_supports_avx512f ()) - error = crypto_native_aes_gcm_init_icl (vm); - else if (crypto_native_aes_gcm_init_adl && clib_cpu_supports_vaes ()) - error = crypto_native_aes_gcm_init_adl (vm); - else if (crypto_native_aes_gcm_init_skx && clib_cpu_supports_avx512f ()) - error = crypto_native_aes_gcm_init_skx (vm); - else if (crypto_native_aes_gcm_init_hsw && clib_cpu_supports_avx2 ()) - error = crypto_native_aes_gcm_init_hsw (vm); - else if (crypto_native_aes_gcm_init_slm) - error = crypto_native_aes_gcm_init_slm (vm); - else - error = clib_error_return (0, "No AES GCM implemenation available"); - - if (error) - return error; + vec_validate (best_by_op_id, oh->op_id); + + if (best_by_op_id[oh->op_id] == 0 || + best_by_op_id[oh->op_id]->priority < oh->priority) + best_by_op_id[oh->op_id] = oh; + + oh = oh->next; } -#endif -#if __aarch64__ - if (crypto_native_aes_gcm_init_neon) - error = crypto_native_aes_gcm_init_neon (vm); - else - error = clib_error_return (0, "No AES GCM implemenation available"); - - if (error) - return error; -#endif + + while (kh) + { + vec_validate (best_by_alg_id, kh->alg_id); + + if (best_by_alg_id[kh->alg_id] == 0 || + best_by_alg_id[kh->alg_id]->priority < kh->priority) + best_by_alg_id[kh->alg_id] = kh; + + kh = kh->next; + } + + vec_foreach_pointer (oh, best_by_op_id) + if (oh) + vnet_crypto_register_ops_handlers (vm, cm->crypto_engine_index, + oh->op_id, oh->fn, oh->cfn); + + vec_foreach_pointer (kh, best_by_alg_id) + if (kh) + cm->key_fn[kh->alg_id] = kh->key_fn; + + vec_free (best_by_op_id); + vec_free (best_by_alg_id); vnet_crypto_register_key_handler (vm, cm->crypto_engine_index, crypto_native_key_handler); diff --git a/src/plugins/crypto_native/sha2.c b/src/plugins/crypto_native/sha2.c new file mode 100644 index 00000000000..459ce6d8e79 --- /dev/null +++ b/src/plugins/crypto_native/sha2.c @@ -0,0 +1,186 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2024 Cisco Systems, Inc. + */ + +#include <vlib/vlib.h> +#include <vnet/plugin/plugin.h> +#include <vnet/crypto/crypto.h> +#include <crypto_native/crypto_native.h> +#include <vppinfra/crypto/sha2.h> + +static_always_inline u32 +crypto_native_ops_hash_sha2 (vlib_main_t *vm, vnet_crypto_op_t *ops[], + u32 n_ops, vnet_crypto_op_chunk_t *chunks, + clib_sha2_type_t type, int maybe_chained) +{ + vnet_crypto_op_t *op = ops[0]; + clib_sha2_ctx_t ctx; + u32 n_left = n_ops; + +next: + if (op->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS) + { + vnet_crypto_op_chunk_t *chp = chunks + op->chunk_index; + clib_sha2_init (&ctx, type); + for (int j = 0; j < op->n_chunks; j++, chp++) + clib_sha2_update (&ctx, chp->src, chp->len); + clib_sha2_final (&ctx, op->digest); + } + else + clib_sha2 (type, op->src, op->len, op->digest); + + op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; + + if (--n_left) + { + op += 1; + goto next; + } + + return n_ops; +} + +static_always_inline u32 +crypto_native_ops_hmac_sha2 (vlib_main_t *vm, vnet_crypto_op_t *ops[], + u32 n_ops, vnet_crypto_op_chunk_t *chunks, + clib_sha2_type_t type) +{ + crypto_native_main_t *cm = &crypto_native_main; + vnet_crypto_op_t *op = ops[0]; + u32 n_left = n_ops; + clib_sha2_hmac_ctx_t ctx; + u8 buffer[64]; + u32 sz, n_fail = 0; + + for (; n_left; n_left--, op++) + { + clib_sha2_hmac_init ( + &ctx, type, (clib_sha2_hmac_key_data_t *) cm->key_data[op->key_index]); + if (op->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS) + { + vnet_crypto_op_chunk_t *chp = chunks + op->chunk_index; + for (int j = 0; j < op->n_chunks; j++, chp++) + clib_sha2_hmac_update (&ctx, chp->src, chp->len); + } + else + clib_sha2_hmac_update (&ctx, op->src, op->len); + + clib_sha2_hmac_final (&ctx, buffer); + + if (op->digest_len) + { + sz = op->digest_len; + if (op->flags & VNET_CRYPTO_OP_FLAG_HMAC_CHECK) + { + if ((memcmp (op->digest, buffer, sz))) + { + n_fail++; + op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC; + continue; + } + } + else + clib_memcpy_fast (op->digest, buffer, sz); + } + else + { + sz = clib_sha2_variants[type].digest_size; + if (op->flags & VNET_CRYPTO_OP_FLAG_HMAC_CHECK) + { + if ((memcmp (op->digest, buffer, sz))) + { + n_fail++; + op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC; + continue; + } + } + else + clib_memcpy_fast (op->digest, buffer, sz); + } + + op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; + } + + return n_ops - n_fail; +} + +static void * +sha2_key_add (vnet_crypto_key_t *key, clib_sha2_type_t type) +{ + clib_sha2_hmac_key_data_t *kd; + + kd = clib_mem_alloc_aligned (sizeof (*kd), CLIB_CACHE_LINE_BYTES); + clib_sha2_hmac_key_data (type, key->data, vec_len (key->data), kd); + + return kd; +} + +static int +probe () +{ +#if defined(__SHA__) && defined(__x86_64__) + if (clib_cpu_supports_sha ()) + return 50; +#elif defined(__ARM_FEATURE_SHA2) + if (clib_cpu_supports_sha2 ()) + return 10; +#endif + return -1; +} + +#define _(b) \ + static u32 crypto_native_ops_hash_sha##b ( \ + vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops) \ + { \ + return crypto_native_ops_hash_sha2 (vm, ops, n_ops, 0, CLIB_SHA2_##b, 0); \ + } \ + \ + static u32 crypto_native_ops_chained_hash_sha##b ( \ + vlib_main_t *vm, vnet_crypto_op_t *ops[], vnet_crypto_op_chunk_t *chunks, \ + u32 n_ops) \ + { \ + return crypto_native_ops_hash_sha2 (vm, ops, n_ops, chunks, \ + CLIB_SHA2_##b, 1); \ + } \ + \ + static u32 crypto_native_ops_hmac_sha##b ( \ + vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops) \ + { \ + return crypto_native_ops_hmac_sha2 (vm, ops, n_ops, 0, CLIB_SHA2_##b); \ + } \ + \ + static u32 crypto_native_ops_chained_hmac_sha##b ( \ + vlib_main_t *vm, vnet_crypto_op_t *ops[], vnet_crypto_op_chunk_t *chunks, \ + u32 n_ops) \ + { \ + return crypto_native_ops_hmac_sha2 (vm, ops, n_ops, chunks, \ + CLIB_SHA2_##b); \ + } \ + \ + static void *sha2_##b##_key_add (vnet_crypto_key_t *k) \ + { \ + return sha2_key_add (k, CLIB_SHA2_##b); \ + } \ + \ + CRYPTO_NATIVE_OP_HANDLER (crypto_native_hash_sha##b) = { \ + .op_id = VNET_CRYPTO_OP_SHA##b##_HASH, \ + .fn = crypto_native_ops_hash_sha##b, \ + .cfn = crypto_native_ops_chained_hash_sha##b, \ + .probe = probe, \ + }; \ + CRYPTO_NATIVE_OP_HANDLER (crypto_native_hmac_sha##b) = { \ + .op_id = VNET_CRYPTO_OP_SHA##b##_HMAC, \ + .fn = crypto_native_ops_hmac_sha##b, \ + .cfn = crypto_native_ops_chained_hmac_sha##b, \ + .probe = probe, \ + }; \ + CRYPTO_NATIVE_KEY_HANDLER (crypto_native_hmac_sha##b) = { \ + .alg_id = VNET_CRYPTO_ALG_HMAC_SHA##b, \ + .key_fn = sha2_##b##_key_add, \ + .probe = probe, \ + }; + +_ (224) +_ (256) + +#undef _ diff --git a/src/plugins/crypto_sw_scheduler/main.c b/src/plugins/crypto_sw_scheduler/main.c index b32c8ae0594..73a158e86b2 100644 --- a/src/plugins/crypto_sw_scheduler/main.c +++ b/src/plugins/crypto_sw_scheduler/main.c @@ -455,7 +455,9 @@ crypto_sw_scheduler_dequeue (vlib_main_t *vm, u32 *nb_elts_processed, crypto_sw_scheduler_queue_t *current_queue = 0; u32 tail, head; u8 found = 0; + u8 recheck_queues = 1; +run_next_queues: /* get a pending frame to process */ if (ptd->self_crypto_enabled) { @@ -565,6 +567,11 @@ crypto_sw_scheduler_dequeue (vlib_main_t *vm, u32 *nb_elts_processed, return f; } + if (!found && recheck_queues) + { + recheck_queues = 0; + goto run_next_queues; + } return 0; } diff --git a/src/plugins/dev_iavf/port.c b/src/plugins/dev_iavf/port.c index 982436d9b45..90e81e960c4 100644 --- a/src/plugins/dev_iavf/port.c +++ b/src/plugins/dev_iavf/port.c @@ -42,29 +42,35 @@ iavf_port_vlan_strip_disable (vlib_main_t *vm, vnet_dev_port_t *port) vnet_dev_t *dev = port->dev; iavf_port_t *ap = vnet_dev_get_port_data (port); virtchnl_vlan_caps_t vc; - vnet_dev_rv_t rv; + vnet_dev_rv_t rv = VNET_DEV_ERR_NOT_SUPPORTED; u32 outer, inner; const u32 mask = VIRTCHNL_VLAN_ETHERTYPE_8100; - if ((ap->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN_V2) == 0) - return iavf_vc_op_disable_vlan_stripping (vm, dev); + if (ap->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN_V2) + { + if ((rv = iavf_vc_op_get_offload_vlan_v2_caps (vm, dev, &vc))) + return rv; - if ((rv = iavf_vc_op_get_offload_vlan_v2_caps (vm, dev, &vc))) - return rv; + outer = vc.offloads.stripping_support.outer; + inner = vc.offloads.stripping_support.inner; - outer = vc.offloads.stripping_support.outer; - inner = vc.offloads.stripping_support.inner; + outer = outer & VIRTCHNL_VLAN_TOGGLE ? outer & mask : 0; + inner = inner & VIRTCHNL_VLAN_TOGGLE ? inner & mask : 0; - outer = outer & VIRTCHNL_VLAN_TOGGLE ? outer & mask : 0; - inner = inner & VIRTCHNL_VLAN_TOGGLE ? inner & mask : 0; + virtchnl_vlan_setting_t vs = { + .vport_id = ap->vsi_id, + .outer_ethertype_setting = outer, + .inner_ethertype_setting = inner, + }; - virtchnl_vlan_setting_t vs = { - .vport_id = ap->vsi_id, - .outer_ethertype_setting = outer, - .inner_ethertype_setting = inner, - }; + if ((rv = iavf_vc_op_disable_vlan_stripping_v2 (vm, dev, &vs))) + return rv; + } - return iavf_vc_op_disable_vlan_stripping_v2 (vm, dev, &vs); + if (ap->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN) + return iavf_vc_op_disable_vlan_stripping (vm, dev); + + return rv; } vnet_dev_rv_t @@ -275,7 +281,12 @@ iavf_port_init (vlib_main_t *vm, vnet_dev_port_t *port) u64_bit_set (&ap->intr_mode_per_rxq_bitmap, q->queue_id, 1); if ((rv = iavf_port_vlan_strip_disable (vm, port))) - return rv; + { + if (rv == VNET_DEV_ERR_NOT_SUPPORTED) + log_warn (port->dev, "device doesn't support vlan stripping"); + else + return rv; + } if ((rv = iavf_port_init_rss (vm, port))) return rv; diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 47be8a8f9a4..97a11e0d0d7 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -51,7 +51,9 @@ static struct } _ (0xa063, RVU_PF, "Marvell Octeon Resource Virtualization Unit PF"), - _ (0xa0f8, RVU_VF, "Marvell Octeon Resource Virtualization Unit VF"), + _ (0xa064, RVU_VF, "Marvell Octeon Resource Virtualization Unit VF"), + _ (0xa0f8, LBK_VF, "Marvell Octeon Loopback Unit VF"), + _ (0xa0f7, SDP_VF, "Marvell Octeon System DPI Packet Interface Unit VF"), _ (0xa0f3, CPT_VF, "Marvell Octeon Cryptographic Accelerator Unit VF"), #undef _ }; @@ -113,7 +115,7 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) if ((rrv = roc_nix_dev_init (cd->nix))) return cnx_return_roc_err (dev, rrv, "roc_nix_dev_init"); - if (roc_nix_npc_mac_addr_get (cd->nix, mac_addr)) + if ((rrv = roc_nix_npc_mac_addr_get (cd->nix, mac_addr))) return cnx_return_roc_err (dev, rrv, "roc_nix_npc_mac_addr_get"); vnet_dev_port_add_args_t port_add_args = { @@ -239,12 +241,19 @@ oct_init (vlib_main_t *vm, vnet_dev_t *dev) strncpy ((char *) cd->plt_pci_dev.name, dev->device_id, sizeof (cd->plt_pci_dev.name) - 1); - if (cd->type == OCT_DEVICE_TYPE_RVU_PF || cd->type == OCT_DEVICE_TYPE_RVU_VF) - return oct_init_nix (vm, dev); - else if (cd->type == OCT_DEVICE_TYPE_CPT_VF) - return oct_init_cpt (vm, dev); - else - return VNET_DEV_ERR_UNSUPPORTED_DEVICE; + switch (cd->type) + { + case OCT_DEVICE_TYPE_RVU_PF: + case OCT_DEVICE_TYPE_RVU_VF: + case OCT_DEVICE_TYPE_SDP_VF: + return oct_init_nix (vm, dev); + + case OCT_DEVICE_TYPE_CPT_VF: + return oct_init_cpt (vm, dev); + + default: + return VNET_DEV_ERR_UNSUPPORTED_DEVICE; + } return 0; } diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index fd8a92c7b3d..e43cde0a35f 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -15,11 +15,15 @@ #include <base/roc_api.h> #include <dev_octeon/hw_defs.h> +#define OCT_BATCH_ALLOC_IOVA0_MASK 0xFFFFFFFFFFFFFF80 + typedef enum { OCT_DEVICE_TYPE_UNKNOWN = 0, OCT_DEVICE_TYPE_RVU_PF, OCT_DEVICE_TYPE_RVU_VF, + OCT_DEVICE_TYPE_LBK_VF, + OCT_DEVICE_TYPE_SDP_VF, OCT_DEVICE_TYPE_CPT_VF, } __clib_packed oct_device_type_t; @@ -72,13 +76,15 @@ typedef struct typedef struct { CLIB_ALIGN_MARK (cl, 128); - union - { - struct npa_batch_alloc_status_s status; - u64 iova[16]; - }; + u64 iova[16]; } oct_npa_batch_alloc_cl128_t; +typedef union +{ + struct npa_batch_alloc_status_s status; + u64 as_u64; +} oct_npa_batch_alloc_status_t; + STATIC_ASSERT_SIZEOF (oct_npa_batch_alloc_cl128_t, 128); typedef struct @@ -157,7 +163,8 @@ vnet_dev_rv_t oct_flow_query (vlib_main_t *, vnet_dev_port_t *, u32, uword, _ (AURA_BATCH_ALLOC_ISSUE_FAIL, aura_batch_alloc_issue_fail, ERROR, \ "aura batch alloc issue failed") \ _ (AURA_BATCH_ALLOC_NOT_READY, aura_batch_alloc_not_ready, ERROR, \ - "aura batch alloc not ready") + "aura batch alloc not ready") \ + _ (MTU_EXCEEDED, mtu_exceeded, ERROR, "mtu exceeded") typedef enum { diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 00ad8b9c477..98a4c28b37d 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -284,8 +284,11 @@ oct_txq_stop (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) for (n = ctq->ba_num_cl, cl = ctq->ba_buffer + ctq->ba_first_cl; n; cl++, n--) { - if (cl->status.ccode != 0) - for (u32 i = 0; i < cl->status.count; i++) + oct_npa_batch_alloc_status_t st; + + st.as_u64 = __atomic_load_n (cl->iova, __ATOMIC_ACQUIRE); + if (st.status.ccode != ALLOC_CCODE_INVAL) + for (u32 i = 0; i < st.status.count; i++) { vlib_buffer_t *b = (vlib_buffer_t *) (cl->iova[i] + off); vlib_buffer_free_one (vm, vlib_get_buffer_index (vm, b)); @@ -376,6 +379,82 @@ oct_port_stop (vlib_main_t *vm, vnet_dev_port_t *port) } vnet_dev_rv_t +oct_validate_config_promisc_mode (vnet_dev_port_t *port, int enable) +{ + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + + if (roc_nix_is_vf_or_sdp (nix)) + return VNET_DEV_ERR_UNSUPPORTED_DEVICE; + + return VNET_DEV_OK; +} + +vnet_dev_rv_t +oct_op_config_promisc_mode (vlib_main_t *vm, vnet_dev_port_t *port, int enable) +{ + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + int rv; + + rv = roc_nix_npc_promisc_ena_dis (nix, enable); + if (rv) + { + return oct_roc_err (dev, rv, "roc_nix_npc_promisc_ena_dis failed"); + } + + rv = roc_nix_mac_promisc_mode_enable (nix, enable); + if (rv) + { + return oct_roc_err (dev, rv, + "roc_nix_mac_promisc_mode_enable(%s) failed", + enable ? "true" : "false"); + } + + return VNET_DEV_OK; +} + +static vnet_dev_rv_t +oct_port_add_del_eth_addr (vlib_main_t *vm, vnet_dev_port_t *port, + vnet_dev_hw_addr_t *addr, int is_add, + int is_primary) +{ + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + vnet_dev_rv_t rv = VNET_DEV_OK; + + i32 rrv; + + if (is_primary) + { + if (is_add) + { + /* Update mac address at NPC */ + rrv = roc_nix_npc_mac_addr_set (nix, (u8 *) addr); + if (rrv) + rv = oct_roc_err (dev, rrv, "roc_nix_npc_mac_addr_set() failed"); + + /* Update mac address at CGX for PFs only */ + if (!roc_nix_is_vf_or_sdp (nix)) + { + rrv = roc_nix_mac_addr_set (nix, (u8 *) addr); + if (rrv) + { + /* Rollback to previous mac address */ + roc_nix_npc_mac_addr_set (nix, + (u8 *) &port->primary_hw_addr); + rv = oct_roc_err (dev, rrv, "roc_nix_mac_addr_set() failed"); + } + } + } + } + return rv; +} + +vnet_dev_rv_t oct_port_cfg_change_validate (vlib_main_t *vm, vnet_dev_port_t *port, vnet_dev_port_cfg_change_req_t *req) { @@ -389,6 +468,8 @@ oct_port_cfg_change_validate (vlib_main_t *vm, vnet_dev_port_t *port, break; case VNET_DEV_PORT_CFG_PROMISC_MODE: + rv = oct_validate_config_promisc_mode (port, req->promisc); + break; case VNET_DEV_PORT_CFG_CHANGE_PRIMARY_HW_ADDR: case VNET_DEV_PORT_CFG_ADD_SECONDARY_HW_ADDR: case VNET_DEV_PORT_CFG_REMOVE_SECONDARY_HW_ADDR: @@ -418,9 +499,13 @@ oct_port_cfg_change (vlib_main_t *vm, vnet_dev_port_t *port, switch (req->type) { case VNET_DEV_PORT_CFG_PROMISC_MODE: + rv = oct_op_config_promisc_mode (vm, port, req->promisc); break; case VNET_DEV_PORT_CFG_CHANGE_PRIMARY_HW_ADDR: + rv = oct_port_add_del_eth_addr (vm, port, &req->addr, + /* is_add */ 1, + /* is_primary */ 1); break; case VNET_DEV_PORT_CFG_ADD_SECONDARY_HW_ADDR: diff --git a/src/plugins/dev_octeon/queue.c b/src/plugins/dev_octeon/queue.c index 9378fc3b7c7..d6ae794fb8d 100644 --- a/src/plugins/dev_octeon/queue.c +++ b/src/plugins/dev_octeon/queue.c @@ -57,12 +57,20 @@ oct_tx_queue_alloc (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); vnet_dev_port_t *port = txq->port; vnet_dev_t *dev = port->dev; + u32 sz = sizeof (void *) * ROC_CN10K_NPA_BATCH_ALLOC_MAX_PTRS; + vnet_dev_rv_t rv; log_debug (dev, "tx_queue_alloc: queue %u alocated", txq->queue_id); - return vnet_dev_dma_mem_alloc ( - vm, dev, sizeof (void *) * ROC_CN10K_NPA_BATCH_ALLOC_MAX_PTRS, 128, - (void **) &ctq->ba_buffer); + rv = vnet_dev_dma_mem_alloc (vm, dev, sz, 128, (void **) &ctq->ba_buffer); + + if (rv != VNET_DEV_OK) + return rv; + + clib_memset_u64 (ctq->ba_buffer, OCT_BATCH_ALLOC_IOVA0_MASK, + ROC_CN10K_NPA_BATCH_ALLOC_MAX_PTRS); + + return rv; } void diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index 5f7e5a8469b..997f1356199 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -217,7 +217,8 @@ oct_rxq_refill (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u16 n_refill) while (n_lines >= batch_max_lines) { - n_alloc = vlib_buffer_alloc (vm, buffer_indices, batch_max_bufs); + n_alloc = + vlib_buffer_alloc_from_pool (vm, buffer_indices, batch_max_bufs, bpi); if (PREDICT_FALSE (n_alloc < batch_max_bufs)) goto alloc_fail; oct_rxq_refill_batch (vm, lmt_id, addr, lines, buffer_indices, w0, @@ -313,7 +314,7 @@ oct_rx_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, n_desc = (status.tail - head) & cq_mask; if (n_desc == 0) - return 0; + goto refill; vlib_get_new_next_frame (vm, node, ctx->next_index, ctx->to_next, ctx->n_left_to_next); @@ -365,14 +366,15 @@ oct_rx_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_put_next_frame (vm, node, ctx->next_index, ctx->n_left_to_next); - n_enq = crq->n_enq - ctx->n_segs; - n_enq += oct_rxq_refill (vm, rxq, rxq->size - n_enq); - crq->n_enq = n_enq; - vlib_increment_combined_counter ( vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, thr_idx, ctx->hw_if_index, ctx->n_rx_pkts, ctx->n_rx_bytes); +refill: + n_enq = crq->n_enq - ctx->n_segs; + n_enq += oct_rxq_refill (vm, rxq, rxq->size - n_enq); + crq->n_enq = n_enq; + return ctx->n_rx_pkts; } diff --git a/src/plugins/dev_octeon/tx_node.c b/src/plugins/dev_octeon/tx_node.c index 5deaa82a0c1..a2e4b07de8a 100644 --- a/src/plugins/dev_octeon/tx_node.c +++ b/src/plugins/dev_octeon/tx_node.c @@ -22,8 +22,11 @@ typedef struct u32 n_tx_bytes; u32 n_drop; vlib_buffer_t *drop[VLIB_FRAME_SIZE]; + u32 n_exd_mtu; + vlib_buffer_t *exd_mtu[VLIB_FRAME_SIZE]; u32 batch_alloc_not_ready; u32 batch_alloc_issue_fail; + int max_pkt_len; u16 lmt_id; u64 lmt_ioaddr; lmt_line_t *lmt_lines; @@ -46,9 +49,12 @@ oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq) for (cl = ctq->ba_buffer + ctq->ba_first_cl; num_cl > 0; num_cl--, cl++) { - u8 count; - if (cl->status.ccode == ALLOC_CCODE_INVAL) + oct_npa_batch_alloc_status_t st; + + if ((st.as_u64 = __atomic_load_n (cl->iova, __ATOMIC_RELAXED)) == + OCT_BATCH_ALLOC_IOVA0_MASK + ALLOC_CCODE_INVAL) { + cl_not_ready: ctx->batch_alloc_not_ready++; n_freed = bi - (u32 *) ctq->ba_buffer; if (n_freed > 0) @@ -63,11 +69,15 @@ oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq) return 0; } - count = cl->status.count; + if (st.status.count > 8 && + __atomic_load_n (cl->iova + 8, __ATOMIC_RELAXED) == + OCT_BATCH_ALLOC_IOVA0_MASK) + goto cl_not_ready; + #if (CLIB_DEBUG > 0) - cl->status.count = cl->status.ccode = 0; + cl->iova[0] &= OCT_BATCH_ALLOC_IOVA0_MASK; #endif - if (PREDICT_TRUE (count == 16)) + if (PREDICT_TRUE (st.status.count == 16)) { /* optimize for likely case where cacheline is full */ vlib_get_buffer_indices_with_offset (vm, (void **) cl, bi, 16, @@ -76,9 +86,9 @@ oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq) } else { - vlib_get_buffer_indices_with_offset (vm, (void **) cl, bi, count, - off); - bi += count; + vlib_get_buffer_indices_with_offset (vm, (void **) cl, bi, + st.status.count, off); + bi += st.status.count; } } @@ -89,7 +99,8 @@ oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq) /* clear status bits in each cacheline */ n = cl - ctq->ba_buffer; for (u32 i = 0; i < n; i++) - ctq->ba_buffer[i].iova[0] = 0; + ctq->ba_buffer[i].iova[0] = ctq->ba_buffer[i].iova[8] = + OCT_BATCH_ALLOC_IOVA0_MASK; ctq->ba_num_cl = ctq->ba_first_cl = 0; } @@ -125,7 +136,8 @@ oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq) static_always_inline u8 oct_tx_enq1 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vlib_buffer_t *b, - lmt_line_t *line, u32 flags, int simple, int trace) + lmt_line_t *line, u32 flags, int simple, int trace, u32 *n, + u8 *dpl) { u8 n_dwords = 2; u32 total_len = 0; @@ -140,6 +152,12 @@ oct_tx_enq1 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vlib_buffer_t *b, }, }; + if (PREDICT_FALSE (vlib_buffer_length_in_chain (vm, b) > ctx->max_pkt_len)) + { + ctx->exd_mtu[ctx->n_exd_mtu++] = b; + return 0; + } + if (!simple && flags & VLIB_BUFFER_NEXT_PRESENT) { u8 n_tail_segs = 0; @@ -151,7 +169,7 @@ oct_tx_enq1 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vlib_buffer_t *b, tail_segs[n_tail_segs++] = t; if (n_tail_segs > 5) { - ctx->drop[ctx->n_drop++] = t; + ctx->drop[ctx->n_drop++] = b; return 0; } } @@ -223,6 +241,9 @@ oct_tx_enq1 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vlib_buffer_t *b, for (u32 i = 0; i < n_dwords; i++) line->dwords[i] = d.as_u128[i]; + *dpl = n_dwords; + *n = *n + 1; + return n_dwords; } @@ -232,7 +253,7 @@ oct_tx_enq16 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq, { u8 dwords_per_line[16], *dpl = dwords_per_line; u64 lmt_arg, ioaddr, n_lines; - u32 n_left, or_flags_16 = 0; + u32 n_left, or_flags_16 = 0, n = 0; const u32 not_simple_flags = VLIB_BUFFER_NEXT_PRESENT | VNET_BUFFER_F_OFFLOAD; lmt_line_t *l = ctx->lmt_lines; @@ -240,7 +261,7 @@ oct_tx_enq16 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq, /* Data Store Memory Barrier - outer shareable domain */ asm volatile("dmb oshst" ::: "memory"); - for (n_left = n_pkts; n_left >= 8; n_left -= 8, b += 8, l += 8) + for (n_left = n_pkts; n_left >= 8; n_left -= 8, b += 8) { u32 f0, f1, f2, f3, f4, f5, f6, f7, or_f = 0; vlib_prefetch_buffer_header (b[8], LOAD); @@ -261,48 +282,54 @@ oct_tx_enq16 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq, if ((or_f & not_simple_flags) == 0) { int simple = 1; - oct_tx_enq1 (vm, ctx, b[0], l, f0, simple, trace); - oct_tx_enq1 (vm, ctx, b[1], l + 1, f1, simple, trace); + oct_tx_enq1 (vm, ctx, b[0], l, f0, simple, trace, &n, &dpl[n]); + oct_tx_enq1 (vm, ctx, b[1], l + n, f1, simple, trace, &n, &dpl[n]); vlib_prefetch_buffer_header (b[13], LOAD); - oct_tx_enq1 (vm, ctx, b[2], l + 2, f2, simple, trace); - oct_tx_enq1 (vm, ctx, b[3], l + 3, f3, simple, trace); + oct_tx_enq1 (vm, ctx, b[2], l + n, f2, simple, trace, &n, &dpl[n]); + oct_tx_enq1 (vm, ctx, b[3], l + n, f3, simple, trace, &n, &dpl[n]); vlib_prefetch_buffer_header (b[14], LOAD); - oct_tx_enq1 (vm, ctx, b[4], l + 4, f4, simple, trace); - oct_tx_enq1 (vm, ctx, b[5], l + 5, f5, simple, trace); + oct_tx_enq1 (vm, ctx, b[4], l + n, f4, simple, trace, &n, &dpl[n]); + oct_tx_enq1 (vm, ctx, b[5], l + n, f5, simple, trace, &n, &dpl[n]); vlib_prefetch_buffer_header (b[15], LOAD); - oct_tx_enq1 (vm, ctx, b[6], l + 6, f6, simple, trace); - oct_tx_enq1 (vm, ctx, b[7], l + 7, f7, simple, trace); - dpl[0] = dpl[1] = dpl[2] = dpl[3] = 2; - dpl[4] = dpl[5] = dpl[6] = dpl[7] = 2; + oct_tx_enq1 (vm, ctx, b[6], l + n, f6, simple, trace, &n, &dpl[n]); + oct_tx_enq1 (vm, ctx, b[7], l + n, f7, simple, trace, &n, &dpl[n]); } else { int simple = 0; - dpl[0] = oct_tx_enq1 (vm, ctx, b[0], l, f0, simple, trace); - dpl[1] = oct_tx_enq1 (vm, ctx, b[1], l + 1, f1, simple, trace); + oct_tx_enq1 (vm, ctx, b[0], l, f0, simple, trace, &n, &dpl[n]); + oct_tx_enq1 (vm, ctx, b[1], l + n, f1, simple, trace, &n, &dpl[n]); vlib_prefetch_buffer_header (b[13], LOAD); - dpl[2] = oct_tx_enq1 (vm, ctx, b[2], l + 2, f2, simple, trace); - dpl[3] = oct_tx_enq1 (vm, ctx, b[3], l + 3, f3, simple, trace); + oct_tx_enq1 (vm, ctx, b[2], l + n, f2, simple, trace, &n, &dpl[n]); + oct_tx_enq1 (vm, ctx, b[3], l + n, f3, simple, trace, &n, &dpl[n]); vlib_prefetch_buffer_header (b[14], LOAD); - dpl[4] = oct_tx_enq1 (vm, ctx, b[4], l + 4, f4, simple, trace); - dpl[5] = oct_tx_enq1 (vm, ctx, b[5], l + 5, f5, simple, trace); + oct_tx_enq1 (vm, ctx, b[4], l + n, f4, simple, trace, &n, &dpl[n]); + oct_tx_enq1 (vm, ctx, b[5], l + n, f5, simple, trace, &n, &dpl[n]); vlib_prefetch_buffer_header (b[15], LOAD); - dpl[6] = oct_tx_enq1 (vm, ctx, b[6], l + 6, f6, simple, trace); - dpl[7] = oct_tx_enq1 (vm, ctx, b[7], l + 7, f7, simple, trace); + oct_tx_enq1 (vm, ctx, b[6], l + n, f6, simple, trace, &n, &dpl[n]); + oct_tx_enq1 (vm, ctx, b[7], l + n, f7, simple, trace, &n, &dpl[n]); } - dpl += 8; + dpl += n; + l += n; + n = 0; } - for (; n_left > 0; n_left -= 1, b += 1, l += 1) + for (; n_left > 0; n_left -= 1, b += 1) { u32 f0 = b[0]->flags; - dpl++[0] = oct_tx_enq1 (vm, ctx, b[0], l, f0, 0, trace); + oct_tx_enq1 (vm, ctx, b[0], l, f0, 0, trace, &n, &dpl[n]); or_flags_16 |= f0; + dpl += n; + l += n; + n = 0; } lmt_arg = ctx->lmt_id; ioaddr = ctx->lmt_ioaddr; - n_lines = n_pkts; + n_lines = dpl - dwords_per_line; + + if (PREDICT_FALSE (!n_lines)) + return n_pkts; if (PREDICT_FALSE (or_flags_16 & VLIB_BUFFER_NEXT_PRESENT)) { @@ -342,6 +369,8 @@ VNET_DEV_NODE_FN (oct_tx_node) vnet_dev_tx_node_runtime_t *rt = vnet_dev_get_tx_node_runtime (node); vnet_dev_tx_queue_t *txq = rt->tx_queue; oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + vnet_dev_t *dev = txq->port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); u32 node_index = node->node_index; u32 *from = vlib_frame_vector_args (frame); u32 n, n_enq, n_left, n_pkts = frame->n_vectors; @@ -355,6 +384,7 @@ VNET_DEV_NODE_FN (oct_tx_node) .sq = ctq->sq.qid, .sizem1 = 1, }, + .max_pkt_len = roc_nix_max_pkt_len (cd->nix), .lmt_id = lmt_id, .lmt_ioaddr = ctq->io_addr, .lmt_lines = ctq->lmt_addr + (lmt_id << ROC_LMT_LINE_SIZE_LOG2), @@ -388,21 +418,25 @@ VNET_DEV_NODE_FN (oct_tx_node) n += oct_tx_enq16 (vm, &ctx, txq, b, n_left, /* trace */ 0); } - ctq->n_enq = n_enq + n; + ctq->n_enq = n_enq + n - ctx.n_drop - ctx.n_exd_mtu; if (n < n_pkts) { - n = n_pkts - n; - vlib_buffer_free (vm, from + n, n); + u32 n_free = n_pkts - n; + vlib_buffer_free (vm, from + n, n_free); vlib_error_count (vm, node->node_index, OCT_TX_NODE_CTR_NO_FREE_SLOTS, - n); - n_pkts -= ctx.n_drop; + n_free); + n_pkts -= n_free; } if (ctx.n_drop) vlib_error_count (vm, node->node_index, OCT_TX_NODE_CTR_CHAIN_TOO_LONG, ctx.n_drop); + if (PREDICT_FALSE (ctx.n_exd_mtu)) + vlib_error_count (vm, node->node_index, OCT_TX_NODE_CTR_MTU_EXCEEDED, + ctx.n_exd_mtu); + if (ctx.batch_alloc_not_ready) vlib_error_count (vm, node_index, OCT_TX_NODE_CTR_AURA_BATCH_ALLOC_NOT_READY, @@ -423,5 +457,13 @@ VNET_DEV_NODE_FN (oct_tx_node) n_pkts -= ctx.n_drop; } + if (PREDICT_FALSE (ctx.n_exd_mtu)) + { + u32 bi[VLIB_FRAME_SIZE]; + vlib_get_buffer_indices (vm, ctx.exd_mtu, bi, ctx.n_exd_mtu); + vlib_buffer_free (vm, bi, ctx.n_exd_mtu); + n_pkts -= ctx.n_exd_mtu; + } + return n_pkts; } diff --git a/src/plugins/dpdk/CMakeLists.txt b/src/plugins/dpdk/CMakeLists.txt index 48b1548f9c2..48c56f35282 100644 --- a/src/plugins/dpdk/CMakeLists.txt +++ b/src/plugins/dpdk/CMakeLists.txt @@ -90,8 +90,10 @@ else() ############################################################################## # libnuma ############################################################################## - vpp_plugin_find_library(dpdk NUMA_LIB "numa") - list(APPEND DPDK_LINK_LIBRARIES ${NUMA_LIB}) + if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") + vpp_plugin_find_library(dpdk NUMA_LIB "numa") + list(APPEND DPDK_LINK_LIBRARIES ${NUMA_LIB}) + endif() ############################################################################## # Mellanox libraries diff --git a/src/plugins/dpdk/device/cli.c b/src/plugins/dpdk/device/cli.c index c838800deb4..77f9a27f97b 100644 --- a/src/plugins/dpdk/device/cli.c +++ b/src/plugins/dpdk/device/cli.c @@ -89,12 +89,18 @@ show_dpdk_physmem (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { clib_error_t *err = 0; - u32 pipe_max_size; int fds[2]; u8 *s = 0; int n, n_try; FILE *f; + /* + * XXX: Pipes on FreeBSD grow dynamically up to 64KB (FreeBSD 15), don't + * manually tweak this value on FreeBSD at the moment. + */ +#ifdef __linux__ + u32 pipe_max_size; + err = clib_sysfs_read ("/proc/sys/fs/pipe-max-size", "%u", &pipe_max_size); if (err) @@ -112,6 +118,7 @@ show_dpdk_physmem (vlib_main_t * vm, unformat_input_t * input, err = clib_error_return_unix (0, "fcntl(F_SETPIPE_SZ)"); goto error; } +#endif /* __linux__ */ if (fcntl (fds[0], F_SETFL, O_NONBLOCK) == -1) { diff --git a/src/plugins/dpdk/device/common.c b/src/plugins/dpdk/device/common.c index dc6b0c1c952..7a49c5aaef2 100644 --- a/src/plugins/dpdk/device/common.c +++ b/src/plugins/dpdk/device/common.c @@ -491,6 +491,7 @@ dpdk_get_pci_device (const struct rte_eth_dev_info *info) return NULL; } +#ifdef __linux__ /* If this device is VMBUS return pointer to info, otherwise NULL */ struct rte_vmbus_device * dpdk_get_vmbus_device (const struct rte_eth_dev_info *info) @@ -507,6 +508,7 @@ dpdk_get_vmbus_device (const struct rte_eth_dev_info *info) else return NULL; } +#endif /* __linux__ */ /* * fd.io coding-style-patch-verification: ON diff --git a/src/plugins/dpdk/device/dpdk.h b/src/plugins/dpdk/device/dpdk.h index c22a67a07e7..88a4d9ff618 100644 --- a/src/plugins/dpdk/device/dpdk.h +++ b/src/plugins/dpdk/device/dpdk.h @@ -24,7 +24,9 @@ #include <rte_eal.h> #include <rte_bus_pci.h> +#ifdef __linux__ #include <rte_bus_vmbus.h> +#endif /* __linux__ */ #include <rte_ethdev.h> #include <rte_version.h> #include <rte_net.h> @@ -35,7 +37,9 @@ #include <bus_driver.h> #include <bus_pci_driver.h> +#ifdef __linux__ #include <bus_vmbus_driver.h> +#endif /* __linux__ */ #endif #include <vnet/devices/devices.h> diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c index d15cfd7233a..e416efe2e4d 100644 --- a/src/plugins/dpdk/device/init.c +++ b/src/plugins/dpdk/device/init.c @@ -187,9 +187,11 @@ dpdk_find_startup_config (struct rte_eth_dev_info *di) { dpdk_main_t *dm = &dpdk_main; struct rte_pci_device *pci_dev; - struct rte_vmbus_device *vmbus_dev; vlib_pci_addr_t pci_addr; +#ifdef __linux__ + struct rte_vmbus_device *vmbus_dev; vlib_vmbus_addr_t vmbus_addr; +#endif /* __linux__ */ uword *p = 0; if ((pci_dev = dpdk_get_pci_device (di))) @@ -202,6 +204,7 @@ dpdk_find_startup_config (struct rte_eth_dev_info *di) hash_get (dm->conf->device_config_index_by_pci_addr, pci_addr.as_u32); } +#ifdef __linux__ if ((vmbus_dev = dpdk_get_vmbus_device (di))) { unformat_input_t input_vmbus; @@ -216,6 +219,7 @@ dpdk_find_startup_config (struct rte_eth_dev_info *di) &vmbus_addr); unformat_free (&input_vmbus); } +#endif /* __linux__ */ if (p) return pool_elt_at_index (dm->conf->dev_confs, p[0]); @@ -566,8 +570,18 @@ dpdk_bind_devices_to_uio (dpdk_config_main_t * conf) continue; } +#ifdef __FreeBSD__ + /* + * The defines for the PCI_CLASS_* types are platform specific and differ + * on FreeBSD. + */ + if (d->device_class != PCI_CLASS_NETWORK && + d->device_class != PCI_CLASS_PROCESSOR_CO) + continue; +#else if (d->device_class != PCI_CLASS_NETWORK_ETHERNET && d->device_class != PCI_CLASS_PROCESSOR_CO) continue; +#endif /* __FreeBSD__ */ if (num_whitelisted) { @@ -1031,12 +1045,14 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) dpdk_main_t *dm = &dpdk_main; clib_error_t *error = 0; dpdk_config_main_t *conf = &dpdk_config_main; - vlib_thread_main_t *tm = vlib_get_thread_main (); dpdk_device_config_t *devconf; vlib_pci_addr_t pci_addr = { 0 }; vlib_vmbus_addr_t vmbus_addr = { 0 }; unformat_input_t sub_input; +#ifdef __linux + vlib_thread_main_t *tm = vlib_get_thread_main (); uword default_hugepage_sz, x; +#endif /* __linux__ */ u8 *s, *tmp = 0; int ret, i; int num_whitelisted = 0; @@ -1045,15 +1061,11 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) u8 no_vmbus = 0; u8 file_prefix = 0; u8 *socket_mem = 0; - u8 *huge_dir_path = 0; u32 vendor, device, domain, bus, func; void *fmt_func; void *fmt_addr; f64 poll_interval; - huge_dir_path = - format (0, "%s/hugepages%c", vlib_unix_get_runtime_dir (), 0); - conf->device_config_index_by_pci_addr = hash_create (0, sizeof (uword)); mhash_init (&conf->device_config_index_by_vmbus_addr, sizeof (uword), sizeof (vlib_vmbus_addr_t)); @@ -1248,6 +1260,11 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) { vec_add1 (conf->eal_init_args, (u8 *) "--in-memory"); +#ifdef __linux__ + /* + * FreeBSD performs huge page prealloc through a dedicated kernel mode + * this process is only required on Linux. + */ default_hugepage_sz = clib_mem_get_default_hugepage_size (); clib_bitmap_foreach (x, tm->cpu_socket_bitmap) @@ -1262,6 +1279,7 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) if ((e = clib_sysfs_prealloc_hugepages(x, 0, n_pages))) clib_error_report (e); } +#endif /* __linux__ */ } /* on/off dpdk's telemetry thread */ @@ -1270,6 +1288,7 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) vec_add1 (conf->eal_init_args, (u8 *) "--no-telemetry"); } +#ifdef __linux__ if (!file_prefix) { tmp = format (0, "--file-prefix%c", 0); @@ -1277,6 +1296,7 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) tmp = format (0, "vpp%c", 0); vec_add1 (conf->eal_init_args, tmp); } +#endif if (no_pci == 0 && geteuid () == 0) dpdk_bind_devices_to_uio (conf); @@ -1396,11 +1416,6 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) RTE_VECT_SIMD_256 : RTE_VECT_SIMD_512); - /* lazy umount hugepages */ - umount2 ((char *) huge_dir_path, MNT_DETACH); - rmdir ((char *) huge_dir_path); - vec_free (huge_dir_path); - /* main thread 1st */ if ((error = dpdk_buffer_pools_create (vm))) return error; diff --git a/src/plugins/dpdk/main.c b/src/plugins/dpdk/main.c index 47007219482..9781d0ed7f0 100644 --- a/src/plugins/dpdk/main.c +++ b/src/plugins/dpdk/main.c @@ -13,13 +13,6 @@ * limitations under the License. */ -#include <unistd.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> -#include <linux/vfio.h> -#include <sys/ioctl.h> - #include <vnet/vnet.h> #include <vnet/plugin/plugin.h> #include <dpdk/device/dpdk.h> diff --git a/src/plugins/fateshare/fateshare.c b/src/plugins/fateshare/fateshare.c index 33ee167bce3..971d32303db 100644 --- a/src/plugins/fateshare/fateshare.c +++ b/src/plugins/fateshare/fateshare.c @@ -17,6 +17,7 @@ #include <vnet/vnet.h> #include <vnet/plugin/plugin.h> +#include <vppinfra/unix.h> #include <fateshare/fateshare.h> #include <vlibapi/api.h> @@ -26,7 +27,11 @@ #include <sys/types.h> #include <sys/wait.h> +#ifdef __linux__ #include <sys/prctl.h> // prctl(), PR_SET_PDEATHSIG +#else +#include <sys/procctl.h> +#endif /* __linux__ */ #include <limits.h> fateshare_main_t fateshare_main; @@ -86,12 +91,23 @@ launch_monitor (fateshare_main_t *kmp) { dup2 (logfd, 1); dup2 (logfd, 2); +#ifdef __linux__ int r = prctl (PR_SET_PDEATHSIG, SIGTERM); if (r == -1) { perror (0); exit (1); } +#else + int r, s = SIGTERM; + + r = procctl (P_PID, 0, PROC_PDEATHSIG_CTL, &s); + if (r == -1) + { + perror (0); + exit (1); + } +#endif /* __linux__ */ pid_t current_ppid = getppid (); if (current_ppid != ppid_before_fork) { @@ -197,24 +213,30 @@ fateshare_config (vlib_main_t *vm, unformat_input_t *input) if (fmp->monitor_cmd == 0) { - char *p, path[PATH_MAX]; - int rv; + char *p; + u8 *path; /* find executable path */ - if ((rv = readlink ("/proc/self/exe", path, PATH_MAX - 1)) == -1) + path = os_get_exec_path (); + + if (path == 0) return clib_error_return ( - 0, "could not stat /proc/self/exe - set monitor manually"); + 0, "could not get exec path - set monitor manually"); - /* readlink doesn't provide null termination */ - path[rv] = 0; + /* add null termination */ + vec_add1 (path, 0); /* strip filename */ - if ((p = strrchr (path, '/')) == 0) - return clib_error_return ( - 0, "could not determine vpp directory - set monitor manually"); + if ((p = strrchr ((char *) path, '/')) == 0) + { + vec_free (path); + return clib_error_return ( + 0, "could not determine vpp directory - set monitor manually"); + } *p = 0; fmp->monitor_cmd = format (0, "%s/vpp_fateshare_monitor\0", path); + vec_free (path); } if (fmp->monitor_logfile == 0) { diff --git a/src/plugins/fateshare/vpp_fateshare_monitor.c b/src/plugins/fateshare/vpp_fateshare_monitor.c index 7b203884c4e..7af451ccffe 100644 --- a/src/plugins/fateshare/vpp_fateshare_monitor.c +++ b/src/plugins/fateshare/vpp_fateshare_monitor.c @@ -4,7 +4,12 @@ #include <sys/types.h> #include <sys/wait.h> +#ifdef __linux__ #include <sys/prctl.h> // prctl(), PR_SET_PDEATHSIG +#else +#include <signal.h> +#include <sys/procctl.h> +#endif /* __linux__ */ #include <sys/stat.h> #include <fcntl.h> @@ -82,6 +87,7 @@ launch_command (char *scmd, char *logname_base) } /* child */ +#ifdef __linux__ int r = prctl (PR_SET_PDEATHSIG, SIGTERM); if (r == -1) { @@ -89,6 +95,17 @@ launch_command (char *scmd, char *logname_base) sleep (5); exit (1); } +#else + int r, s = SIGTERM; + + r = procctl (P_PID, 0, PROC_PDEATHSIG_CTL, &s); + if (r == -1) + { + perror ("procctl"); + exit (1); + } +#endif /* __linux__ */ + if (getppid () != ppid_before_fork) { sleep (5); @@ -180,12 +197,23 @@ main (int argc, char **argv) exit (2); } +#ifdef __linux__ int r = prctl (PR_SET_PDEATHSIG, SIGTERM); if (r == -1) { perror (0); exit (1); } +#else + int r, s = SIGTERM; + + r = procctl (P_PID, 0, PROC_PDEATHSIG_CTL, &s); + if (r == -1) + { + perror ("procctl"); + exit (1); + } +#endif /* __linux__ */ /* Establish handler. */ struct sigaction sa; diff --git a/src/plugins/hs_apps/CMakeLists.txt b/src/plugins/hs_apps/CMakeLists.txt index bd43eb34afa..179c9c7a4c4 100644 --- a/src/plugins/hs_apps/CMakeLists.txt +++ b/src/plugins/hs_apps/CMakeLists.txt @@ -55,7 +55,7 @@ if(VPP_BUILD_VCL_TESTS) ) add_vpp_executable(${test} SOURCES "vcl/${test}.c" - LINK_LIBRARIES vppcom pthread + LINK_LIBRARIES vppcom pthread ${EPOLL_LIB} NO_INSTALL ) endforeach() @@ -68,7 +68,7 @@ if(VPP_BUILD_VCL_TESTS) SOURCES "vcl/${test}.c" vcl/vcl_test_protos.c - LINK_LIBRARIES vppcom pthread + LINK_LIBRARIES vppcom pthread ${EPOLL_LIB} NO_INSTALL ) endforeach() diff --git a/src/plugins/hs_apps/http_client_cli.c b/src/plugins/hs_apps/http_client_cli.c index f44d4e1bcd1..1a321bf44a8 100644 --- a/src/plugins/hs_apps/http_client_cli.c +++ b/src/plugins/hs_apps/http_client_cli.c @@ -13,11 +13,9 @@ * limitations under the License. */ -#include <vnet/session/application.h> #include <vnet/session/application_interface.h> #include <vnet/session/session.h> #include <http/http.h> -#include <hs_apps/http_cli.h> #define HCC_DEBUG 0 @@ -68,6 +66,7 @@ typedef struct typedef enum { HCC_REPLY_RECEIVED = 100, + HCC_TRANSPORT_CLOSED, } hcc_cli_signal_t; static hcc_main_t hcc_main; @@ -273,6 +272,17 @@ hcc_ts_cleanup_callback (session_t *s, session_cleanup_ntf_t ntf) hcc_session_free (s->thread_index, hs); } +static void +hcc_ts_transport_closed (session_t *s) +{ + hcc_main_t *hcm = &hcc_main; + + HCC_DBG ("transport closed"); + + vlib_process_signal_event_mt (hcm->vlib_main, hcm->cli_node_index, + HCC_TRANSPORT_CLOSED, 0); +} + static session_cb_vft_t hcc_session_cb_vft = { .session_accept_callback = hcc_ts_accept_callback, .session_disconnect_callback = hcc_ts_disconnect_callback, @@ -281,6 +291,7 @@ static session_cb_vft_t hcc_session_cb_vft = { .builtin_app_tx_callback = hcc_ts_tx_callback, .session_reset_callback = hcc_ts_reset_callback, .session_cleanup_callback = hcc_ts_cleanup_callback, + .session_transport_closed_callback = hcc_ts_transport_closed, }; static clib_error_t * @@ -370,7 +381,7 @@ hcc_connect () } static clib_error_t * -hcc_run (vlib_main_t *vm) +hcc_run (vlib_main_t *vm, int print_output) { vlib_thread_main_t *vtm = vlib_get_thread_main (); hcc_main_t *hcm = &hcc_main; @@ -407,11 +418,15 @@ hcc_run (vlib_main_t *vm) goto cleanup; case HCC_REPLY_RECEIVED: - vlib_cli_output (vm, "%v", hcm->http_response); + if (print_output) + vlib_cli_output (vm, "%v", hcm->http_response); vec_free (hcm->http_response); break; + case HCC_TRANSPORT_CLOSED: + err = clib_error_return (0, "error, transport closed"); + break; default: - clib_error_return (0, "unexpected event %d", event_type); + err = clib_error_return (0, "unexpected event %d", event_type); break; } @@ -448,7 +463,7 @@ hcc_command_fn (vlib_main_t *vm, unformat_input_t *input, u64 seg_size; u8 *appns_id = 0; clib_error_t *err = 0; - int rv; + int rv, print_output = 1; hcm->prealloc_fifos = 0; hcm->private_segment_size = 0; @@ -472,6 +487,8 @@ hcc_command_fn (vlib_main_t *vm, unformat_input_t *input, hcm->fifo_size <<= 10; else if (unformat (line_input, "uri %s", &hcm->uri)) ; + else if (unformat (line_input, "no-output")) + print_output = 0; else if (unformat (line_input, "appns %_%v%_", &appns_id)) ; else if (unformat (line_input, "secret %lu", &hcm->appns_secret)) @@ -506,7 +523,7 @@ hcc_command_fn (vlib_main_t *vm, unformat_input_t *input, vnet_session_enable_disable (vm, 1 /* turn on TCP, etc. */); vlib_worker_thread_barrier_release (vm); - err = hcc_run (vm); + err = hcc_run (vm, print_output); if (hcc_detach ()) { @@ -526,7 +543,7 @@ done: VLIB_CLI_COMMAND (hcc_command, static) = { .path = "http cli client", .short_help = "[appns <app-ns> secret <appns-secret>] uri http://<ip-addr> " - "query <query-string>", + "query <query-string> [no-output]", .function = hcc_command_fn, .is_mp_safe = 1, }; diff --git a/src/plugins/http/http.c b/src/plugins/http/http.c index 036e6929987..526e665903a 100644 --- a/src/plugins/http/http.c +++ b/src/plugins/http/http.c @@ -74,14 +74,14 @@ format_http_state (u8 *s, va_list *va) return format (s, "unknown"); } -static inline void -http_state_change (http_conn_t *hc, http_state_t state) -{ - HTTP_DBG (1, "changing http state %U -> %U", format_http_state, - hc->http_state, format_http_state, state); - ASSERT (hc->http_state != state); - hc->http_state = state; -} +#define http_state_change(_hc, _state) \ + do \ + { \ + HTTP_DBG (1, "changing http state %U -> %U", format_http_state, \ + (_hc)->http_state, format_http_state, _state); \ + (_hc)->http_state = _state; \ + } \ + while (0) static inline http_worker_t * http_worker_get (u32 thread_index) @@ -140,6 +140,7 @@ http_listener_free (http_conn_t *lhc) { http_main_t *hm = &http_main; + vec_free (lhc->app_name); if (CLIB_DEBUG) memset (lhc, 0xfc, sizeof (*lhc)); pool_put (hm->listener_pool, lhc); @@ -372,12 +373,12 @@ static const char *http_redirect_template = "HTTP/1.1 %s\r\n"; static const char *http_response_template = "HTTP/1.1 %s\r\n" "Date: %U GMT\r\n" "Expires: %U GMT\r\n" - "Server: VPP Static\r\n" + "Server: %s\r\n" "Content-Type: %s\r\n" "Content-Length: %lu\r\n\r\n"; static const char *http_request_template = "GET %s HTTP/1.1\r\n" - "User-Agent: VPP HTTP client\r\n" + "User-Agent: %s\r\n" "Accept: */*\r\n"; static u32 @@ -520,17 +521,19 @@ http_state_wait_server_reply (http_conn_t *hc, transport_send_params_t *sp) http_msg_t msg = {}; app_worker_t *app_wrk; session_t *as; - http_status_code_t ec; rv = http_read_message (hc); /* Nothing yet, wait for data or timer expire */ if (rv) - return HTTP_SM_STOP; + { + HTTP_DBG (1, "no data to deq"); + return HTTP_SM_STOP; + } if (vec_len (hc->rx_buf) < 8) { - ec = HTTP_STATUS_BAD_REQUEST; + clib_warning ("response buffer too short"); goto error; } @@ -546,9 +549,7 @@ http_state_wait_server_reply (http_conn_t *hc, transport_send_params_t *sp) if (rv) { clib_warning ("failed to parse http reply"); - session_transport_closing_notify (&hc->connection); - http_disconnect_transport (hc); - return -1; + goto error; } msg.data.len = content_length; u32 dlen = vec_len (hc->rx_buf) - hc->rx_buf_offset; @@ -577,7 +578,7 @@ http_state_wait_server_reply (http_conn_t *hc, transport_send_params_t *sp) { hc->rx_buf_offset = 0; vec_reset_length (hc->rx_buf); - http_state_change (hc, HTTP_STATE_WAIT_CLIENT_METHOD); + http_state_change (hc, HTTP_STATE_WAIT_APP_METHOD); } else { @@ -585,23 +586,20 @@ http_state_wait_server_reply (http_conn_t *hc, transport_send_params_t *sp) } app_wrk = app_worker_get_if_valid (as->app_wrk_index); - app_worker_rx_notify (app_wrk, as); + if (app_wrk) + app_worker_rx_notify (app_wrk, as); return HTTP_SM_STOP; } else { - HTTP_DBG (0, "Unknown http method %v", hc->rx_buf); - ec = HTTP_STATUS_METHOD_NOT_ALLOWED; + clib_warning ("Unknown http method %v", hc->rx_buf); goto error; } - return HTTP_SM_STOP; error: - - http_send_error (hc, ec); session_transport_closing_notify (&hc->connection); + session_transport_closed_notify (&hc->connection); http_disconnect_transport (hc); - return HTTP_SM_ERROR; } @@ -734,6 +732,7 @@ http_state_wait_app_reply (http_conn_t *hc, transport_send_params_t *sp) * Add headers. For now: * - current time * - expiration time + * - server name * - content type * - data length */ @@ -748,6 +747,8 @@ http_state_wait_app_reply (http_conn_t *hc, transport_send_params_t *sp) format_clib_timebase_time, now, /* Expires */ format_clib_timebase_time, now + 600.0, + /* Server */ + hc->app_name, /* Content type */ http_content_type_str[msg.content_type], /* Length */ @@ -791,7 +792,6 @@ error: static http_sm_result_t http_state_wait_app_method (http_conn_t *hc, transport_send_params_t *sp) { - http_status_code_t sc; http_msg_t msg; session_t *as; u8 *buf = 0, *request; @@ -806,29 +806,24 @@ http_state_wait_app_method (http_conn_t *hc, transport_send_params_t *sp) if (msg.data.type > HTTP_MSG_DATA_PTR) { clib_warning ("no data"); - sc = HTTP_STATUS_INTERNAL_ERROR; goto error; } if (msg.type != HTTP_MSG_REQUEST) { clib_warning ("unexpected message type %d", msg.type); - sc = HTTP_STATUS_INTERNAL_ERROR; goto error; } - sc = msg.code; - vec_validate (buf, msg.data.len - 1); rv = svm_fifo_dequeue (as->tx_fifo, msg.data.len, buf); ASSERT (rv == msg.data.len); - request = format (0, http_request_template, buf); + request = format (0, http_request_template, buf, hc->app_name); offset = http_send_data (hc, request, vec_len (request), 0); if (offset != vec_len (request)) { clib_warning ("sending request failed!"); - sc = HTTP_STATUS_INTERNAL_ERROR; goto error; } @@ -837,83 +832,85 @@ http_state_wait_app_method (http_conn_t *hc, transport_send_params_t *sp) vec_free (buf); vec_free (request); - return HTTP_SM_CONTINUE; + return HTTP_SM_STOP; error: - clib_warning ("unexpected msg type from app %u", msg.type); - http_send_error (hc, sc); + svm_fifo_dequeue_drop_all (as->tx_fifo); session_transport_closing_notify (&hc->connection); + session_transport_closed_notify (&hc->connection); http_disconnect_transport (hc); - return HTTP_SM_STOP; -} - -static void -http_app_enqueue (http_conn_t *hc, session_t *as) -{ - app_worker_t *app_wrk; - u32 dlen, max_enq, n_enq; - int rv; - - dlen = vec_len (hc->rx_buf) - hc->rx_buf_offset; - if (!dlen) - return; - - max_enq = svm_fifo_max_enqueue (as->rx_fifo); - n_enq = clib_min (max_enq, dlen); - rv = svm_fifo_enqueue (as->rx_fifo, n_enq, &hc->rx_buf[hc->rx_buf_offset]); - if (rv < 0) - return; - - hc->rx_buf_offset += rv; - if (hc->rx_buf_offset >= vec_len (hc->rx_buf)) - { - vec_reset_length (hc->rx_buf); - hc->rx_buf_offset = 0; - } - - app_wrk = app_worker_get_if_valid (as->app_wrk_index); - ASSERT (app_wrk); - app_worker_rx_notify (app_wrk, as); + return HTTP_SM_ERROR; } static http_sm_result_t http_state_client_io_more_data (http_conn_t *hc, transport_send_params_t *sp) { session_t *as, *ts; - u32 max_deq; - int n_read; + app_worker_t *app_wrk; + svm_fifo_seg_t _seg, *seg = &_seg; + u32 max_len, max_deq, max_enq, n_segs = 1; + int rv, len; as = session_get_from_handle (hc->h_pa_session_handle); ts = session_get_from_handle (hc->h_tc_session_handle); - http_app_enqueue (hc, as); + max_deq = svm_fifo_max_dequeue (ts->rx_fifo); + if (max_deq == 0) + { + HTTP_DBG (1, "no data to deq"); + return HTTP_SM_STOP; + } - if (hc->to_recv == 0) + max_enq = svm_fifo_max_enqueue (as->rx_fifo); + if (max_enq == 0) { - http_state_change (hc, HTTP_STATE_WAIT_CLIENT_METHOD); + HTTP_DBG (1, "app's rx fifo full"); + svm_fifo_add_want_deq_ntf (as->rx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); return HTTP_SM_STOP; } - max_deq = svm_fifo_max_dequeue (ts->rx_fifo); - if (max_deq > 0) + max_len = clib_min (max_enq, max_deq); + len = svm_fifo_segments (ts->rx_fifo, 0, seg, &n_segs, max_len); + if (len < 0) { - vec_validate (hc->rx_buf, max_deq - 1); - n_read = svm_fifo_dequeue (ts->rx_fifo, max_deq, hc->rx_buf); - ASSERT (n_read == max_deq); + HTTP_DBG (1, "svm_fifo_segments() len %d", len); + return HTTP_SM_STOP; + } - if (svm_fifo_is_empty (ts->rx_fifo)) - svm_fifo_unset_event (ts->rx_fifo); + rv = svm_fifo_enqueue_segments (as->rx_fifo, seg, 1, 0 /* allow partial */); + if (rv < 0) + { + clib_warning ("data enqueue failed, rv: %d", rv); + return HTTP_SM_ERROR; + } - hc->to_recv -= n_read; - vec_set_len (hc->rx_buf, n_read); + svm_fifo_dequeue_drop (ts->rx_fifo, rv); + if (rv > hc->to_recv) + { + clib_warning ("http protocol error: received more data than expected"); + session_transport_closing_notify (&hc->connection); + http_disconnect_transport (hc); + http_state_change (hc, HTTP_STATE_WAIT_APP_METHOD); + return HTTP_SM_ERROR; } + hc->to_recv -= rv; + HTTP_DBG (1, "drained %d from ts; remains %d", rv, hc->to_recv); - if (hc->rx_buf_offset < vec_len (hc->rx_buf) || - svm_fifo_max_dequeue_cons (ts->rx_fifo)) + if (hc->to_recv == 0) { - session_enqueue_notify (ts); + hc->rx_buf_offset = 0; + vec_reset_length (hc->rx_buf); + http_state_change (hc, HTTP_STATE_WAIT_APP_METHOD); } - return HTTP_SM_CONTINUE; + + app_wrk = app_worker_get_if_valid (as->app_wrk_index); + if (app_wrk) + app_worker_rx_notify (app_wrk, as); + + if (svm_fifo_max_dequeue_cons (ts->rx_fifo)) + session_enqueue_notify (ts); + + return HTTP_SM_STOP; } static http_sm_result_t @@ -983,6 +980,7 @@ static void http_req_run_state_machine (http_conn_t *hc, transport_send_params_t *sp) { http_sm_result_t res; + do { res = state_funcs[hc->http_state](hc, sp); @@ -1010,6 +1008,12 @@ http_ts_rx_callback (session_t *ts) return -1; } + if (hc->state == HTTP_CONN_STATE_CLOSED) + { + svm_fifo_dequeue_drop_all (ts->tx_fifo); + return 0; + } + http_req_run_state_machine (hc, 0); if (hc->state == HTTP_CONN_STATE_TRANSPORT_CLOSED) @@ -1153,6 +1157,11 @@ http_transport_connect (transport_endpoint_cfg_t *tep) hc->state = HTTP_CONN_STATE_CONNECTING; cargs->api_context = hc_index; + if (vec_len (app->name)) + hc->app_name = vec_dup (app->name); + else + hc->app_name = format (0, "VPP HTTP client"); + HTTP_DBG (1, "hc ho_index %x", hc_index); if ((error = vnet_connect (cargs))) @@ -1205,6 +1214,11 @@ http_start_listen (u32 app_listener_index, transport_endpoint_cfg_t *tep) lhc->c_s_index = app_listener_index; lhc->c_flags |= TRANSPORT_CONNECTION_F_NO_LOOKUP; + if (vec_len (app->name)) + lhc->app_name = vec_dup (app->name); + else + lhc->app_name = format (0, "VPP server app"); + return lhc_index; } @@ -1245,7 +1259,11 @@ http_transport_close (u32 hc_index, u32 thread_index) http_disconnect_transport (hc); return; } - + else if (hc->state == HTTP_CONN_STATE_CLOSED) + { + HTTP_DBG (1, "nothing to do, already closed"); + return; + } as = session_get_from_handle (hc->h_pa_session_handle); /* Nothing more to send, confirm close */ diff --git a/src/plugins/http/http.h b/src/plugins/http/http.h index dbae5ac4611..c9912dd6db8 100644 --- a/src/plugins/http/http.h +++ b/src/plugins/http/http.h @@ -227,6 +227,7 @@ typedef struct http_tc_ http_conn_state_t state; u32 timer_handle; + u8 *app_name; /* * Current request diff --git a/src/plugins/ikev2/ikev2.api b/src/plugins/ikev2/ikev2.api index 58b7fc05d9e..de276e7f3ea 100644 --- a/src/plugins/ikev2/ikev2.api +++ b/src/plugins/ikev2/ikev2.api @@ -72,8 +72,6 @@ define ikev2_sa_dump { u32 client_index; u32 context; - - option status = "in_progress"; }; /** \brief Dump all SAs @@ -86,6 +84,17 @@ define ikev2_sa_v2_dump u32 context; }; +/** \brief Dump all SAs + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request +*/ +define ikev2_sa_v3_dump +{ + u32 client_index; + u32 context; + option status = "in_progress"; +}; + /** \brief Details about IKE SA @param context - sender context, to match reply w/ request @param retval - return code @@ -97,7 +106,6 @@ define ikev2_sa_details i32 retval; vl_api_ikev2_sa_t sa; - option status = "in_progress"; }; /** \brief Details about IKE SA @@ -113,6 +121,20 @@ define ikev2_sa_v2_details vl_api_ikev2_sa_v2_t sa; }; +/** \brief Details about IKE SA + @param context - sender context, to match reply w/ request + @param retval - return code + @param sa - SA data +*/ +define ikev2_sa_v3_details +{ + u32 context; + i32 retval; + + vl_api_ikev2_sa_v3_t sa; + option status = "in_progress"; +}; + /** \brief Dump child SA of specific SA @param client_index - opaque cookie to identify the sender @param context - sender context, to match reply w/ request @@ -125,7 +147,6 @@ define ikev2_child_sa_dump u32 sa_index; option vat_help = "sa_index <index>"; - option status = "in_progress"; }; /** \brief Child SA details @@ -139,6 +160,34 @@ define ikev2_child_sa_details i32 retval; vl_api_ikev2_child_sa_t child_sa; +}; + +/** \brief Dump child SA of specific SA + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param sa_index - index of specific sa +*/ +define ikev2_child_sa_v2_dump +{ + u32 client_index; + u32 context; + + u32 sa_index; + option vat_help = "sa_index <index>"; + option status = "in_progress"; +}; + +/** \brief Child SA details + @param context - sender context, to match reply w/ request + @param retval - return code + @param child_sa - child SA data +*/ +define ikev2_child_sa_v2_details +{ + u32 context; + i32 retval; + + vl_api_ikev2_child_sa_v2_t child_sa; option status = "in_progress"; }; diff --git a/src/plugins/ikev2/ikev2.c b/src/plugins/ikev2/ikev2.c index d559565487e..9bea2c96d12 100644 --- a/src/plugins/ikev2/ikev2.c +++ b/src/plugins/ikev2/ikev2.c @@ -1888,7 +1888,7 @@ ikev2_sa_match_ts (ikev2_sa_t * sa) } static ikev2_profile_t * -ikev2_select_profile (ikev2_main_t *km, ikev2_sa_t *sa, +ikev2_select_profile (vlib_main_t *vm, ikev2_main_t *km, ikev2_sa_t *sa, ikev2_sa_transform_t *tr_prf, u8 *key_pad) { ikev2_profile_t *ret = 0, *p; @@ -1928,6 +1928,7 @@ ikev2_select_profile (ikev2_main_t *km, ikev2_sa_t *sa, if (!clib_memcmp (auth, sa_auth->data, vec_len (sa_auth->data))) { ikev2_set_state (sa, IKEV2_STATE_AUTHENTICATED); + sa->auth_timestamp = vlib_time_now (vm); vec_free (auth); ret = p; break; @@ -1946,6 +1947,7 @@ ikev2_select_profile (ikev2_main_t *km, ikev2_sa_t *sa, if (ikev2_verify_sign (p->auth.key, sa_auth->data, authmsg) == 1) { ikev2_set_state (sa, IKEV2_STATE_AUTHENTICATED); + sa->auth_timestamp = vlib_time_now (vm); ret = p; break; } @@ -1961,7 +1963,7 @@ ikev2_select_profile (ikev2_main_t *km, ikev2_sa_t *sa, } static void -ikev2_sa_auth (ikev2_sa_t *sa) +ikev2_sa_auth (ikev2_sa_t *sa, vlib_main_t *vm) { ikev2_main_t *km = &ikev2_main; ikev2_profile_t *sel_p = 0; @@ -1982,7 +1984,7 @@ ikev2_sa_auth (ikev2_sa_t *sa) } key_pad = format (0, "%s", IKEV2_KEY_PAD); - sel_p = ikev2_select_profile (km, sa, tr_prf, key_pad); + sel_p = ikev2_select_profile (vm, km, sa, tr_prf, key_pad); if (sel_p) { @@ -2230,6 +2232,8 @@ ikev2_create_tunnel_interface (vlib_main_t *vm, ikev2_sa_t *sa, clib_memset (&a, 0, sizeof (a)); + child->timestamp = vlib_time_now (vm); + if (!child->r_proposals) { ikev2_set_state (sa, IKEV2_STATE_NO_PROPOSAL_CHOSEN); @@ -3424,7 +3428,7 @@ ikev2_node_internal (vlib_main_t *vm, vlib_node_runtime_t *node, sa0->dst_port = clib_net_to_host_u16 (udp0->src_port); res = ikev2_process_auth_req (vm, sa0, ike0, rlen); if (res) - ikev2_sa_auth (sa0); + ikev2_sa_auth (sa0, vm); else vlib_node_increment_counter (vm, node->node_index, IKEV2_ERROR_MALFORMED_PACKET, 1); diff --git a/src/plugins/ikev2/ikev2_api.c b/src/plugins/ikev2/ikev2_api.c index bb44e9f364f..c9608aa660b 100644 --- a/src/plugins/ikev2/ikev2_api.c +++ b/src/plugins/ikev2/ikev2_api.c @@ -173,7 +173,7 @@ send_profile (ikev2_profile_t * profile, vl_api_registration_t * reg, rmp->profile.lifetime_jitter = profile->lifetime_jitter; rmp->profile.handover = profile->handover; - vl_api_ikev2_profile_t_endian (&rmp->profile); + vl_api_ikev2_profile_t_endian (&rmp->profile, 1 /* to network */); vl_api_send_msg (reg, (u8 *) rmp); } @@ -291,7 +291,7 @@ send_sa (ikev2_sa_t * sa, vl_api_ikev2_sa_dump_t * mp, u32 api_sa_index) ikev2_copy_stats (&rsa->stats, &sa->stats); - vl_api_ikev2_sa_t_endian(rsa); + vl_api_ikev2_sa_t_endian (rsa, 1 /* to network */); }); } @@ -382,7 +382,7 @@ send_sa_v2 (ikev2_sa_t *sa, vl_api_ikev2_sa_v2_dump_t *mp, u32 api_sa_index) ikev2_copy_stats (&rsa->stats, &sa->stats); - vl_api_ikev2_sa_v2_t_endian (rsa); + vl_api_ikev2_sa_v2_t_endian (rsa, 1 /* to network */); }); } @@ -405,6 +405,100 @@ vl_api_ikev2_sa_v2_dump_t_handler (vl_api_ikev2_sa_v2_dump_t *mp) } static void +send_sa_v3 (ikev2_sa_t *sa, vl_api_ikev2_sa_v3_dump_t *mp, u32 api_sa_index) +{ + ikev2_main_t *km = &ikev2_main; + vl_api_ikev2_sa_v3_details_t *rmp = 0; + int rv = 0; + ikev2_sa_transform_t *tr; + ikev2_profile_t *p; + p = pool_elt_at_index (km->profiles, sa->profile_index); + vlib_main_t *vm = vlib_get_main (); + + REPLY_MACRO2_ZERO (VL_API_IKEV2_SA_V3_DETAILS, { + vl_api_ikev2_sa_v3_t *rsa = &rmp->sa; + vl_api_ikev2_keys_t *k = &rsa->keys; + + int size_data = sizeof (rsa->profile_name) - 1; + if (vec_len (p->name) < size_data) + size_data = vec_len (p->name); + clib_memcpy (rsa->profile_name, p->name, size_data); + + rsa->state = ikev2_state_encode (sa->state); + + rsa->uptime = vlib_time_now (vm) - sa->auth_timestamp; + + rsa->sa_index = api_sa_index; + ip_address_encode2 (&sa->iaddr, &rsa->iaddr); + ip_address_encode2 (&sa->raddr, &rsa->raddr); + rsa->ispi = sa->ispi; + rsa->rspi = sa->rspi; + cp_id (&rsa->i_id, &sa->i_id); + cp_id (&rsa->r_id, &sa->r_id); + + tr = ikev2_sa_get_td_for_type (sa->r_proposals, IKEV2_TRANSFORM_TYPE_ENCR); + if (tr) + cp_sa_transform (&rsa->encryption, tr); + + tr = ikev2_sa_get_td_for_type (sa->r_proposals, IKEV2_TRANSFORM_TYPE_PRF); + if (tr) + cp_sa_transform (&rsa->prf, tr); + + tr = + ikev2_sa_get_td_for_type (sa->r_proposals, IKEV2_TRANSFORM_TYPE_INTEG); + if (tr) + cp_sa_transform (&rsa->integrity, tr); + + tr = ikev2_sa_get_td_for_type (sa->r_proposals, IKEV2_TRANSFORM_TYPE_DH); + if (tr) + cp_sa_transform (&rsa->dh, tr); + + k->sk_d_len = vec_len (sa->sk_d); + clib_memcpy (&k->sk_d, sa->sk_d, k->sk_d_len); + + k->sk_ai_len = vec_len (sa->sk_ai); + clib_memcpy (&k->sk_ai, sa->sk_ai, k->sk_ai_len); + + k->sk_ar_len = vec_len (sa->sk_ar); + clib_memcpy (&k->sk_ar, sa->sk_ar, k->sk_ar_len); + + k->sk_ei_len = vec_len (sa->sk_ei); + clib_memcpy (&k->sk_ei, sa->sk_ei, k->sk_ei_len); + + k->sk_er_len = vec_len (sa->sk_er); + clib_memcpy (&k->sk_er, sa->sk_er, k->sk_er_len); + + k->sk_pi_len = vec_len (sa->sk_pi); + clib_memcpy (&k->sk_pi, sa->sk_pi, k->sk_pi_len); + + k->sk_pr_len = vec_len (sa->sk_pr); + clib_memcpy (&k->sk_pr, sa->sk_pr, k->sk_pr_len); + + ikev2_copy_stats (&rsa->stats, &sa->stats); + + vl_api_ikev2_sa_v3_t_endian (rsa, 1 /* to network */); + }); +} + +static void +vl_api_ikev2_sa_v3_dump_t_handler (vl_api_ikev2_sa_v3_dump_t *mp) +{ + ikev2_main_t *km = &ikev2_main; + ikev2_main_per_thread_data_t *tkm; + ikev2_sa_t *sa; + + vec_foreach (tkm, km->per_thread_data) + { + pool_foreach (sa, tkm->sas) + { + u32 api_sa_index = + ikev2_encode_sa_index (sa - tkm->sas, tkm - km->per_thread_data); + send_sa_v3 (sa, mp, api_sa_index); + } + } +} + +static void send_child_sa (ikev2_child_sa_t * child, vl_api_ikev2_child_sa_dump_t * mp, u32 child_sa_index, u32 sa_index) @@ -455,7 +549,7 @@ send_child_sa (ikev2_child_sa_t * child, k->sk_ar_len); } - vl_api_ikev2_child_sa_t_endian (&rmp->child_sa); + vl_api_ikev2_child_sa_t_endian (&rmp->child_sa, 1 /* to network */); }); } @@ -488,6 +582,85 @@ vl_api_ikev2_child_sa_dump_t_handler (vl_api_ikev2_child_sa_dump_t * mp) } static void +send_child_sa_v2 (ikev2_child_sa_t *child, vl_api_ikev2_child_sa_v2_dump_t *mp, + u32 child_sa_index, u32 sa_index) +{ + vl_api_ikev2_child_sa_v2_details_t *rmp = 0; + int rv = 0; + ikev2_sa_transform_t *tr; + vlib_main_t *vm = vlib_get_main (); + + REPLY_MACRO2_ZERO (VL_API_IKEV2_CHILD_SA_V2_DETAILS, { + vl_api_ikev2_keys_t *k = &rmp->child_sa.keys; + rmp->child_sa.child_sa_index = child_sa_index; + rmp->child_sa.uptime = vlib_time_now (vm) - child->timestamp; + rmp->child_sa.sa_index = sa_index; + rmp->child_sa.i_spi = child->i_proposals ? child->i_proposals[0].spi : 0; + rmp->child_sa.r_spi = child->r_proposals ? child->r_proposals[0].spi : 0; + + tr = + ikev2_sa_get_td_for_type (child->r_proposals, IKEV2_TRANSFORM_TYPE_ENCR); + if (tr) + cp_sa_transform (&rmp->child_sa.encryption, tr); + + tr = ikev2_sa_get_td_for_type (child->r_proposals, + IKEV2_TRANSFORM_TYPE_INTEG); + if (tr) + cp_sa_transform (&rmp->child_sa.integrity, tr); + + tr = + ikev2_sa_get_td_for_type (child->r_proposals, IKEV2_TRANSFORM_TYPE_ESN); + if (tr) + cp_sa_transform (&rmp->child_sa.esn, tr); + + k->sk_ei_len = vec_len (child->sk_ei); + clib_memcpy (&k->sk_ei, child->sk_ei, k->sk_ei_len); + + k->sk_er_len = vec_len (child->sk_er); + clib_memcpy (&k->sk_er, child->sk_er, k->sk_er_len); + + if (vec_len (child->sk_ai)) + { + k->sk_ai_len = vec_len (child->sk_ai); + clib_memcpy (&k->sk_ai, child->sk_ai, k->sk_ai_len); + + k->sk_ar_len = vec_len (child->sk_ar); + clib_memcpy (&k->sk_ar, child->sk_ar, k->sk_ar_len); + } + + vl_api_ikev2_child_sa_v2_t_endian (&rmp->child_sa, 1 /* to network */); + }); +} + +static void +vl_api_ikev2_child_sa_v2_dump_t_handler (vl_api_ikev2_child_sa_v2_dump_t *mp) +{ + ikev2_main_t *im = &ikev2_main; + ikev2_main_per_thread_data_t *tkm; + ikev2_sa_t *sa; + ikev2_child_sa_t *child; + u32 sai = ~0, ti = ~0; + + ikev2_decode_sa_index (clib_net_to_host_u32 (mp->sa_index), &sai, &ti); + + if (vec_len (im->per_thread_data) <= ti) + return; + + tkm = vec_elt_at_index (im->per_thread_data, ti); + + if (pool_len (tkm->sas) <= sai || pool_is_free_index (tkm->sas, sai)) + return; + + sa = pool_elt_at_index (tkm->sas, sai); + + vec_foreach (child, sa->childs) + { + u32 child_sa_index = child - sa->childs; + send_child_sa_v2 (child, mp, child_sa_index, sai); + } +} + +static void vl_api_ikev2_traffic_selector_dump_t_handler (vl_api_ikev2_traffic_selector_dump_t * mp) { @@ -527,7 +700,7 @@ static void rmp->ts.sa_index = api_sa_index; rmp->ts.child_sa_index = child_sa_index; cp_ts (&rmp->ts, ts, mp->is_initiator); - vl_api_ikev2_ts_t_endian (&rmp->ts); + vl_api_ikev2_ts_t_endian (&rmp->ts, 1 /* to network */); }); } } diff --git a/src/plugins/ikev2/ikev2_cli.c b/src/plugins/ikev2/ikev2_cli.c index 733ae478b83..975774c48d5 100644 --- a/src/plugins/ikev2/ikev2_cli.c +++ b/src/plugins/ikev2/ikev2_cli.c @@ -74,12 +74,16 @@ format_ikev2_child_sa (u8 * s, va_list * va) ikev2_ts_t *ts; ikev2_sa_transform_t *tr; u8 *c = 0; + vlib_main_t *vm = vlib_get_main (); u32 indent = format_get_indent (s); indent += 1; s = format (s, "child sa %u:", index); + s = format (s, "\n uptime: %f (s)\n ", + vlib_time_now (vm) - child->timestamp); + tr = ikev2_sa_get_td_for_type (child->r_proposals, IKEV2_TRANSFORM_TYPE_ENCR); c = format (c, "%U ", format_ikev2_sa_transform, tr); @@ -135,6 +139,7 @@ format_ikev2_sa (u8 * s, va_list * va) ikev2_sa_transform_t *tr; ikev2_child_sa_t *child; u32 indent = 1; + vlib_main_t *vm = vlib_get_main (); ikev2_main_t *km = &ikev2_main; ikev2_profile_t *p; @@ -168,6 +173,9 @@ format_ikev2_sa (u8 * s, va_list * va) s = format (s, "\n state: %s", stateNames[sa->state]); } + s = + format (s, "\n uptime: %f (s)\n", vlib_time_now (vm) - sa->auth_timestamp); + s = format (s, "\n%U", format_white_space, indent); s = format (s, "nonce i:%U\n%Ur:%U\n", diff --git a/src/plugins/ikev2/ikev2_priv.h b/src/plugins/ikev2/ikev2_priv.h index a11538f92c7..0639809e9b1 100644 --- a/src/plugins/ikev2/ikev2_priv.h +++ b/src/plugins/ikev2/ikev2_priv.h @@ -307,6 +307,8 @@ typedef struct f64 time_to_expiration; u8 is_expired; i8 rekey_retries; + + f64 timestamp; } ikev2_child_sa_t; typedef struct @@ -488,6 +490,8 @@ typedef struct u8 keys_generated; ikev2_stats_t stats; + + f64 auth_timestamp; } ikev2_sa_t; diff --git a/src/plugins/ikev2/ikev2_test.c b/src/plugins/ikev2/ikev2_test.c index 18d01dc6ffb..93683a5b5dc 100644 --- a/src/plugins/ikev2/ikev2_test.c +++ b/src/plugins/ikev2/ikev2_test.c @@ -391,7 +391,7 @@ vl_api_ikev2_sa_details_t_handler (vl_api_ikev2_sa_details_t * mp) ip_address_t iaddr; ip_address_t raddr; vl_api_ikev2_keys_t *k = &sa->keys; - vl_api_ikev2_sa_t_endian (sa); + vl_api_ikev2_sa_t_endian (sa, 0 /* from network */); ip_address_decode2 (&sa->iaddr, &iaddr); ip_address_decode2 (&sa->raddr, &raddr); @@ -461,7 +461,7 @@ vl_api_ikev2_sa_v2_details_t_handler (vl_api_ikev2_sa_v2_details_t *mp) ip_address_t iaddr; ip_address_t raddr; vl_api_ikev2_keys_t *k = &sa->keys; - vl_api_ikev2_sa_v2_t_endian (sa); + vl_api_ikev2_sa_v2_t_endian (sa, 0 /* from network */); ip_address_decode2 (&sa->iaddr, &iaddr); ip_address_decode2 (&sa->raddr, &raddr); @@ -497,6 +497,76 @@ vl_api_ikev2_sa_v2_details_t_handler (vl_api_ikev2_sa_v2_details_t *mp) } static int +api_ikev2_sa_v3_dump (vat_main_t *vam) +{ + ikev2_test_main_t *im = &ikev2_test_main; + vl_api_ikev2_sa_v3_dump_t *mp; + vl_api_control_ping_t *mp_ping; + int ret; + + /* Construct the API message */ + M (IKEV2_SA_V3_DUMP, mp); + + /* send it... */ + S (mp); + + /* Use a control ping for synchronization */ + if (!im->ping_id) + im->ping_id = vl_msg_api_get_msg_index ((u8 *) (VL_API_CONTROL_PING_CRC)); + mp_ping = vl_msg_api_alloc_as_if_client (sizeof (*mp_ping)); + mp_ping->_vl_msg_id = htons (im->ping_id); + mp_ping->client_index = vam->my_client_index; + vam->result_ready = 0; + + S (mp_ping); + + /* Wait for a reply... */ + W (ret); + return ret; +} + +static void +vl_api_ikev2_sa_v3_details_t_handler (vl_api_ikev2_sa_v3_details_t *mp) +{ + vat_main_t *vam = ikev2_test_main.vat_main; + vl_api_ikev2_sa_v3_t *sa = &mp->sa; + ip_address_t iaddr; + ip_address_t raddr; + vl_api_ikev2_keys_t *k = &sa->keys; + vl_api_ikev2_sa_v3_t_endian (sa, 0 /* from network */); + + ip_address_decode2 (&sa->iaddr, &iaddr); + ip_address_decode2 (&sa->raddr, &raddr); + + fformat (vam->ofp, "profile name %s sa index: %d\n", mp->sa.profile_name, + mp->sa.sa_index); + fformat (vam->ofp, " iip %U ispi %lx rip %U rspi %lx\n", format_ip_address, + &iaddr, sa->ispi, format_ip_address, &raddr, sa->rspi); + fformat (vam->ofp, " %U ", format_ikev2_sa_transform, &sa->encryption); + fformat (vam->ofp, "%U ", format_ikev2_sa_transform, &sa->prf); + fformat (vam->ofp, "%U ", format_ikev2_sa_transform, &sa->integrity); + fformat (vam->ofp, "%U \n", format_ikev2_sa_transform, &sa->dh); + + fformat (vam->ofp, " SK_d %U\n", format_hex_bytes, k->sk_d, k->sk_d_len); + + fformat (vam->ofp, " SK_a i:%U\n r:%U\n", format_hex_bytes, + k->sk_ai, k->sk_ai_len, format_hex_bytes, k->sk_ar, k->sk_ar_len); + + fformat (vam->ofp, " SK_e i:%U\n r:%U\n", format_hex_bytes, + k->sk_ei, k->sk_ei_len, format_hex_bytes, k->sk_er, k->sk_er_len); + + fformat (vam->ofp, " SK_p i:%U\n r:%U\n", format_hex_bytes, + k->sk_pi, k->sk_pi_len, format_hex_bytes, k->sk_pr, k->sk_pr_len); + + fformat (vam->ofp, " identifier (i) %U\n", format_ikev2_id_type_and_data, + &sa->i_id); + fformat (vam->ofp, " identifier (r) %U\n", format_ikev2_id_type_and_data, + &sa->r_id); + + vam->result_ready = 1; +} + +static int api_ikev2_child_sa_dump (vat_main_t * vam) { unformat_input_t *i = vam->input; @@ -549,7 +619,84 @@ vl_api_ikev2_child_sa_details_t_handler (vl_api_ikev2_child_sa_details_t * mp) vat_main_t *vam = ikev2_test_main.vat_main; vl_api_ikev2_child_sa_t *child_sa = &mp->child_sa; vl_api_ikev2_keys_t *k = &child_sa->keys; - vl_api_ikev2_child_sa_t_endian (child_sa); + vl_api_ikev2_child_sa_t_endian (child_sa, 0 /* from network */); + + fformat (vam->ofp, " child sa %u:\n", child_sa->child_sa_index); + + fformat (vam->ofp, " %U ", format_ikev2_sa_transform, + &child_sa->encryption); + fformat (vam->ofp, "%U ", format_ikev2_sa_transform, &child_sa->integrity); + fformat (vam->ofp, "%U \n", format_ikev2_sa_transform, &child_sa->esn); + + fformat (vam->ofp, " spi(i) %lx spi(r) %lx\n", child_sa->i_spi, + child_sa->r_spi); + + fformat (vam->ofp, " SK_e i:%U\n r:%U\n", format_hex_bytes, + k->sk_ei, k->sk_ei_len, format_hex_bytes, k->sk_er, k->sk_er_len); + if (k->sk_ai_len) + { + fformat (vam->ofp, " SK_a i:%U\n r:%U\n", format_hex_bytes, + k->sk_ai, k->sk_ai_len, format_hex_bytes, k->sk_ar, + k->sk_ar_len); + } + vam->result_ready = 1; +} + +static int +api_ikev2_child_sa_v2_dump (vat_main_t *vam) +{ + unformat_input_t *i = vam->input; + ikev2_test_main_t *im = &ikev2_test_main; + vl_api_ikev2_child_sa_dump_t *mp; + vl_api_control_ping_t *mp_ping; + int ret; + u32 sa_index = ~0; + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "sa_index %d", &sa_index)) + ; + else + { + errmsg ("parse error '%U'", format_unformat_error, i); + return -99; + } + } + + if (sa_index == ~0) + return -99; + + /* Construct the API message */ + M (IKEV2_CHILD_SA_DUMP, mp); + + mp->sa_index = clib_net_to_host_u32 (sa_index); + + /* send it... */ + S (mp); + + /* Use a control ping for synchronization */ + if (!im->ping_id) + im->ping_id = vl_msg_api_get_msg_index ((u8 *) (VL_API_CONTROL_PING_CRC)); + mp_ping = vl_msg_api_alloc_as_if_client (sizeof (*mp_ping)); + mp_ping->_vl_msg_id = htons (im->ping_id); + mp_ping->client_index = vam->my_client_index; + vam->result_ready = 0; + + S (mp_ping); + + /* Wait for a reply... */ + W (ret); + return ret; +} + +static void +vl_api_ikev2_child_sa_v2_details_t_handler ( + vl_api_ikev2_child_sa_details_t *mp) +{ + vat_main_t *vam = ikev2_test_main.vat_main; + vl_api_ikev2_child_sa_t *child_sa = &mp->child_sa; + vl_api_ikev2_keys_t *k = &child_sa->keys; + vl_api_ikev2_child_sa_t_endian (child_sa, 0 /* from network */); fformat (vam->ofp, " child sa %u:\n", child_sa->child_sa_index); @@ -637,7 +784,7 @@ static void vat_main_t *vam = ikev2_test_main.vat_main; vl_api_ikev2_ts_t *ts = &mp->ts; ip_address_t start_addr, end_addr; - vl_api_ikev2_ts_t_endian (ts); + vl_api_ikev2_ts_t_endian (ts, 0 /* from network */); ip_address_decode2 (&ts->start_addr, &start_addr); ip_address_decode2 (&ts->end_addr, &end_addr); diff --git a/src/plugins/ikev2/ikev2_types.api b/src/plugins/ikev2/ikev2_types.api index f0e50165501..2492611703d 100644 --- a/src/plugins/ikev2/ikev2_types.api +++ b/src/plugins/ikev2/ikev2_types.api @@ -128,6 +128,19 @@ typedef ikev2_child_sa vl_api_ikev2_sa_transform_t esn; }; +typedef ikev2_child_sa_v2 +{ + u32 sa_index; + u32 child_sa_index; + u32 i_spi; + u32 r_spi; + vl_api_ikev2_keys_t keys; + vl_api_ikev2_sa_transform_t encryption; + vl_api_ikev2_sa_transform_t integrity; + vl_api_ikev2_sa_transform_t esn; + f64 uptime; +}; + typedef ikev2_sa_stats { u16 n_keepalives; @@ -197,4 +210,30 @@ typedef ikev2_sa_v2 vl_api_ikev2_sa_transform_t dh; vl_api_ikev2_sa_stats_t stats; +}; + +typedef ikev2_sa_v3 +{ + u32 sa_index; + string profile_name[64]; + vl_api_ikev2_state_t state; + + u64 ispi; + u64 rspi; + vl_api_address_t iaddr; + vl_api_address_t raddr; + + vl_api_ikev2_keys_t keys; + + /* ID */ + vl_api_ikev2_id_t i_id; + vl_api_ikev2_id_t r_id; + + vl_api_ikev2_sa_transform_t encryption; + vl_api_ikev2_sa_transform_t integrity; + vl_api_ikev2_sa_transform_t prf; + vl_api_ikev2_sa_transform_t dh; + + vl_api_ikev2_sa_stats_t stats; + f64 uptime; };
\ No newline at end of file diff --git a/src/plugins/lb/lb.c b/src/plugins/lb/lb.c index f8d5fe8d833..7ae1884ff31 100644 --- a/src/plugins/lb/lb.c +++ b/src/plugins/lb/lb.c @@ -373,9 +373,9 @@ void lb_garbage_collection() } vec_foreach(i, to_be_removed_vips) { - vip = &lbm->vips[*i]; - pool_put(lbm->vips, vip); - pool_free(vip->as_indexes); + vip = &lbm->vips[*i]; + pool_free (vip->as_indexes); + pool_put (lbm->vips, vip); } vec_free(to_be_removed_vips); diff --git a/src/plugins/linux-cp/lcp.c b/src/plugins/linux-cp/lcp.c index 561ebb152ae..34e8550a13f 100644 --- a/src/plugins/linux-cp/lcp.c +++ b/src/plugins/linux-cp/lcp.c @@ -161,6 +161,28 @@ lcp_get_netlink_processing_active (void) return lcpm->netlink_processing_active; } +void +lcp_set_default_num_queues (u16 num_queues, u8 is_tx) +{ + lcp_main_t *lcpm = &lcp_main; + + if (is_tx) + lcpm->num_tx_queues = num_queues; + else + lcpm->num_rx_queues = num_queues; +} + +u16 +lcp_get_default_num_queues (u8 is_tx) +{ + lcp_main_t *lcpm = &lcp_main; + + if (is_tx) + return lcpm->num_tx_queues; + + return lcpm->num_rx_queues ?: vlib_num_workers (); +} + /* * fd.io coding-style-patch-verification: ON * diff --git a/src/plugins/linux-cp/lcp.h b/src/plugins/linux-cp/lcp.h index 3b6b4ec00d4..e89b149f67d 100644 --- a/src/plugins/linux-cp/lcp.h +++ b/src/plugins/linux-cp/lcp.h @@ -28,6 +28,8 @@ typedef struct lcp_main_s u8 lcp_sync; /* Automatically sync VPP changes to LCP */ u8 del_static_on_link_down; /* Delete static routes when link goes down */ u8 del_dynamic_on_link_down; /* Delete dynamic routes when link goes down */ + u16 num_rx_queues; + u16 num_tx_queues; u8 test_mode; /* Set when Unit testing */ u8 netlink_processing_active; /* Set while a batch of Netlink messages are being processed */ @@ -61,6 +63,12 @@ u8 lcp_get_del_dynamic_on_link_down (void); void lcp_set_netlink_processing_active (u8 is_processing); u8 lcp_get_netlink_processing_active (void); +/** + * Get/Set the default queue number for LCP host taps. + */ +void lcp_set_default_num_queues (u16 num_queues, u8 is_tx); +u16 lcp_get_default_num_queues (u8 is_tx); + #endif /* diff --git a/src/plugins/linux-cp/lcp_api.c b/src/plugins/linux-cp/lcp_api.c index 991516a3ecd..74421230e9d 100644 --- a/src/plugins/linux-cp/lcp_api.c +++ b/src/plugins/linux-cp/lcp_api.c @@ -214,9 +214,16 @@ vl_api_lcp_itf_pair_get_v2_t_handler (vl_api_lcp_itf_pair_get_v2_t *mp) else { VALIDATE_SW_IF_INDEX_END (mp); + + u32 pair_index = lcp_itf_pair_find_by_phy (mp->sw_if_index); + if (pair_index == INDEX_INVALID) + { + rv = VNET_API_ERROR_INVALID_SW_IF_INDEX; + goto bad_sw_if_index; + } send_lcp_itf_pair_details ( - lcp_itf_pair_find_by_phy (mp->sw_if_index), - vl_api_client_index_to_registration (mp->client_index), mp->context); + pair_index, vl_api_client_index_to_registration (mp->client_index), + mp->context); BAD_SW_IF_INDEX_LABEL; REPLY_MACRO2_END (VL_API_LCP_ITF_PAIR_GET_V2_REPLY, diff --git a/src/plugins/linux-cp/lcp_interface.c b/src/plugins/linux-cp/lcp_interface.c index 921dccf5519..e1f4a6a1d69 100644 --- a/src/plugins/linux-cp/lcp_interface.c +++ b/src/plugins/linux-cp/lcp_interface.c @@ -555,6 +555,7 @@ static clib_error_t * lcp_itf_pair_config (vlib_main_t *vm, unformat_input_t *input) { u8 *default_ns; + u32 tmp; default_ns = NULL; @@ -579,6 +580,10 @@ lcp_itf_pair_config (vlib_main_t *vm, unformat_input_t *input) lcp_set_del_static_on_link_down (1 /* is_del */); else if (unformat (input, "del-dynamic-on-link-down")) lcp_set_del_dynamic_on_link_down (1 /* is_del */); + else if (unformat (input, "num-rx-queues %d", &tmp)) + lcp_set_default_num_queues (tmp, 0 /* is_tx */); + else if (unformat (input, "num-tx-queues %d", &tmp)) + lcp_set_default_num_queues (tmp, 1 /* is_tx */); else return clib_error_return (0, "interfaces not found"); } @@ -988,8 +993,10 @@ lcp_itf_pair_create (u32 phy_sw_if_index, u8 *host_if_name, else { tap_create_if_args_t args = { - .num_rx_queues = clib_max (1, vlib_num_workers ()), - .num_tx_queues = 1, + .num_rx_queues = + clib_max (1, lcp_get_default_num_queues (0 /* is_tx */)), + .num_tx_queues = + clib_max (1, lcp_get_default_num_queues (1 /* is_tx */)), .id = hw->hw_if_index, .sw_if_index = ~0, .rx_ring_sz = 256, diff --git a/src/plugins/lisp/lisp-cp/lisp_types.h b/src/plugins/lisp/lisp-cp/lisp_types.h index 21bd72178d7..e92f8f80c70 100644 --- a/src/plugins/lisp/lisp-cp/lisp_types.h +++ b/src/plugins/lisp/lisp-cp/lisp_types.h @@ -198,7 +198,8 @@ u8 gid_address_len (gid_address_t * a); void *gid_address_cast (gid_address_t * gid, gid_address_type_t type); void gid_address_copy (gid_address_t * dst, gid_address_t * src); u32 gid_address_parse (u8 * offset, gid_address_t * a); -void gid_address_ip_set (gid_address_t * dst, void *src, u8 version); +void gid_address_ip_set (gid_address_t *dst, void *src, + ip_address_family_t version); #define gid_address_type(_a) (_a)->type #define gid_address_ippref(_a) (_a)->ippref diff --git a/src/plugins/map/ip6_map_t.c b/src/plugins/map/ip6_map_t.c index 6bfc8e0f952..51853d619e6 100644 --- a/src/plugins/map/ip6_map_t.c +++ b/src/plugins/map/ip6_map_t.c @@ -529,7 +529,10 @@ ip6_map_t (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) ip60 = vlib_buffer_get_current (p0); d0 = - ip6_map_get_domain (&ip60->dst_address, + /* Originally using the IPv6 dest for rule lookup, now source + * [dgeist] ip6_map_get_domain (&ip60->dst_address, + */ + ip6_map_get_domain (&ip60->src_address, &vnet_buffer (p0)->map_t.map_domain_index, &error0); if (!d0) diff --git a/src/plugins/map/lpm.c b/src/plugins/map/lpm.c index c0e5bad1417..a2fc3337167 100644 --- a/src/plugins/map/lpm.c +++ b/src/plugins/map/lpm.c @@ -28,7 +28,13 @@ masked_address32 (uint32_t addr, uint8_t len) static uint64_t masked_address64 (uint64_t addr, uint8_t len) { - return len == 64 ? addr : addr & ~(~0ull >> len); + /* This was originally causing non-64-bit masks to not match due to LSB vs + * MSB masking (0s at the head of the value) Probably needs some corner case + * checking in case my masking logic was off [dgeist] + * + * return len == 64 ? addr : addr & ~(~0ull >> len); + */ + return len == 64 ? addr : addr & ((1ull << (len)) - 1); } static void @@ -126,13 +132,25 @@ lpm_128_add (lpm_t *lpm, void *addr_v, u8 pfxlen, u32 value) BVT(clib_bihash_kv) kv; ip6_address_t *addr = addr_v; - kv.key[0] = masked_address64(addr->as_u64[0], pfxlen > 64 ? 64 : pfxlen); + /* This is a quick hack. It works for pfxlen < 64 but needs validation for + * other [dgeist] + * + * kv.key[0] = masked_address64(addr->as_u64[0], pfxlen > 64 ? 64 : pfxlen); + */ + kv.key[0] = masked_address64 (addr->as_u64[0], pfxlen > 64 ? 64 : 64); kv.key[1] = masked_address64(addr->as_u64[1], pfxlen > 64 ? pfxlen - 64 : 0); kv.key[2] = pfxlen; kv.value = value; BV(clib_bihash_add_del)(&lpm->bihash, &kv, 1); lpm->prefix_length_refcount[pfxlen]++; - lpm->prefix_lengths_bitmap = clib_bitmap_set (lpm->prefix_lengths_bitmap, 128 - pfxlen, 1); + /* Populating the lengths bitmap table with prefix of 48 instead of 80 + * (128 - 48) [dgeist] + * + * lpm->prefix_lengths_bitmap = clib_bitmap_set ( + * lpm->prefix_lengths_bitmap, 128 - pfxlen, 1); + */ + lpm->prefix_lengths_bitmap = clib_bitmap_set ( + lpm->prefix_lengths_bitmap, pfxlen > 64 ? 128 - pfxlen : pfxlen, 1); } static void @@ -148,8 +166,8 @@ lpm_128_delete (lpm_t *lpm, void *addr_v, u8 pfxlen) /* refcount accounting */ ASSERT (lpm->prefix_length_refcount[pfxlen] > 0); if (--lpm->prefix_length_refcount[pfxlen] == 0) { - lpm->prefix_lengths_bitmap = clib_bitmap_set (lpm->prefix_lengths_bitmap, - 128 - pfxlen, 0); + lpm->prefix_lengths_bitmap = + clib_bitmap_set (lpm->prefix_lengths_bitmap, 128 - pfxlen, 0); } } diff --git a/src/plugins/map/map.c b/src/plugins/map/map.c index 6c9668dde51..3cffadd39e8 100644 --- a/src/plugins/map/map.c +++ b/src/plugins/map/map.c @@ -176,6 +176,10 @@ map_create_domain (ip4_address_t * ip4_prefix, mm->ip6_src_prefix_tbl->add (mm->ip6_src_prefix_tbl, &d->ip6_src, d->ip6_src_len, *map_domain_index); + /* Let's build a table with the MAP rule ip6 prefixes as well [dgeist] */ + mm->ip6_prefix_tbl->add (mm->ip6_prefix_tbl, &d->ip6_prefix, + d->ip6_prefix_len, *map_domain_index); + /* Validate packet/byte counters */ map_domain_counter_lock (mm); int i; @@ -218,6 +222,9 @@ map_delete_domain (u32 map_domain_index) d->ip4_prefix_len); mm->ip6_src_prefix_tbl->delete (mm->ip6_src_prefix_tbl, &d->ip6_src, d->ip6_src_len); + /* Addition to remove the new table [dgeist] */ + mm->ip6_prefix_tbl->delete (mm->ip6_prefix_tbl, &d->ip6_prefix, + d->ip6_prefix_len); /* Release user-assigned MAP domain name. */ map_free_extras (map_domain_index); diff --git a/src/plugins/map/map.h b/src/plugins/map/map.h index d874aa47b3a..86b8ec22df4 100644 --- a/src/plugins/map/map.h +++ b/src/plugins/map/map.h @@ -335,7 +335,11 @@ ip6_map_get_domain (ip6_address_t * addr, u32 * map_domain_index, u8 * error) { map_main_t *mm = &map_main; u32 mdi = - mm->ip6_src_prefix_tbl->lookup (mm->ip6_src_prefix_tbl, addr, 128); + /* This is the old src (ip6 destination) hash lookup [dgeist] + * + * mm->ip6_src_prefix_tbl->lookup (mm->ip6_src_prefix_tbl, addr, 128); + */ + mm->ip6_prefix_tbl->lookup (mm->ip6_prefix_tbl, addr, 128); if (mdi == ~0) { *error = MAP_ERROR_NO_DOMAIN; diff --git a/src/plugins/marvell/pp2/cli.c b/src/plugins/marvell/pp2/cli.c index f4ecb1873c9..5072a3c035b 100644 --- a/src/plugins/marvell/pp2/cli.c +++ b/src/plugins/marvell/pp2/cli.c @@ -31,7 +31,7 @@ mrvl_pp2_create_command_fn (vlib_main_t * vm, unformat_input_t * input, { unformat_input_t _line_input, *line_input = &_line_input; mrvl_pp2_create_if_args_t args = { 0 }; - uint val; + unsigned int val; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) diff --git a/src/plugins/nat/nat44-ed/nat44_ed_api.c b/src/plugins/nat/nat44-ed/nat44_ed_api.c index 1f01410afce..b6c9d51d777 100644 --- a/src/plugins/nat/nat44-ed/nat44_ed_api.c +++ b/src/plugins/nat/nat44-ed/nat44_ed_api.c @@ -442,7 +442,8 @@ send_nat44_ed_output_interface_details (u32 index, vl_api_registration_t *rp, /* Endian hack until apigen registers _details * endian functions */ - vl_api_nat44_ed_output_interface_details_t_endian (rmp); + vl_api_nat44_ed_output_interface_details_t_endian (rmp, + 1 /* to network */); rmp->_vl_msg_id = htons (rmp->_vl_msg_id); rmp->context = htonl (rmp->context); })); diff --git a/src/plugins/nat/nat44-ed/nat44_ed_in2out.c b/src/plugins/nat/nat44-ed/nat44_ed_in2out.c index 4ba51bcaea6..9b4dac3b356 100644 --- a/src/plugins/nat/nat44-ed/nat44_ed_in2out.c +++ b/src/plugins/nat/nat44-ed/nat44_ed_in2out.c @@ -523,6 +523,7 @@ slow_path_ed (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b, nat_6t_flow_dport_rewrite_set (&s->o2i, l_port); } nat_6t_flow_txfib_rewrite_set (&s->o2i, rx_fib_index); + nat_6t_flow_saddr_rewrite_set (&s->o2i, r_addr.as_u32); if (nat_ed_alloc_addr_and_port ( sm, rx_fib_index, tx_sw_if_index, proto, thread_index, l_addr, @@ -567,6 +568,7 @@ slow_path_ed (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b, } nat_6t_flow_daddr_rewrite_set (&s->o2i, l_addr.as_u32); nat_6t_flow_txfib_rewrite_set (&s->o2i, rx_fib_index); + nat_6t_flow_saddr_rewrite_set (&s->o2i, r_addr.as_u32); if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 2)) { nat_elog_notice (sm, "out2in key add failed"); diff --git a/src/plugins/nat/nat44-ei/nat44_ei_api.c b/src/plugins/nat/nat44-ei/nat44_ei_api.c index 8671a556929..454a5032c6a 100644 --- a/src/plugins/nat/nat44-ei/nat44_ei_api.c +++ b/src/plugins/nat/nat44-ei/nat44_ei_api.c @@ -751,7 +751,8 @@ send_nat44_ei_output_interface_details (u32 index, vl_api_registration_t *rp, /* Endian hack until apigen registers _details * endian functions */ - vl_api_nat44_ei_output_interface_details_t_endian (rmp); + vl_api_nat44_ei_output_interface_details_t_endian (rmp, + 1 /* to network */); rmp->_vl_msg_id = htons (rmp->_vl_msg_id); rmp->context = htonl (rmp->context); })); diff --git a/src/plugins/nat/nat44-ei/nat44_ei_in2out.c b/src/plugins/nat/nat44-ei/nat44_ei_in2out.c index 01b333a5234..3b981d69986 100644 --- a/src/plugins/nat/nat44-ei/nat44_ei_in2out.c +++ b/src/plugins/nat/nat44-ei/nat44_ei_in2out.c @@ -859,7 +859,7 @@ nat44_ei_icmp_in2out (vlib_buffer_t *b0, ip4_header_t *ip0, nat44_ei_main_t *nm = &nat44_ei_main; vlib_main_t *vm = vlib_get_main (); ip4_address_t addr; - u16 port; + u16 port = 0; u32 fib_index; nat_protocol_t proto; icmp_echo_header_t *echo0, *inner_echo0 = 0; diff --git a/src/plugins/nat/pnat/pnat_api.c b/src/plugins/nat/pnat/pnat_api.c index 02e61219d1e..a4e7ff192bf 100644 --- a/src/plugins/nat/pnat/pnat_api.c +++ b/src/plugins/nat/pnat/pnat_api.c @@ -116,7 +116,8 @@ static void send_bindings_details(u32 index, vl_api_registration_t *rp, /* Endian hack until apigen registers _details * endian functions */ - vl_api_pnat_bindings_details_t_endian(rmp); + vl_api_pnat_bindings_details_t_endian( + rmp, 1 /* to network */); rmp->_vl_msg_id = htons(rmp->_vl_msg_id); rmp->context = htonl(rmp->context); })); @@ -158,7 +159,7 @@ static void send_interfaces_details(u32 index, vl_api_registration_t *rp, /* Endian hack until apigen registers _details * endian functions */ - vl_api_pnat_interfaces_details_t_endian(rmp); + vl_api_pnat_interfaces_details_t_endian(rmp, 1 /* to network */); rmp->_vl_msg_id = htons(rmp->_vl_msg_id); rmp->context = htonl(rmp->context); })); diff --git a/src/plugins/netmap/CMakeLists.txt b/src/plugins/netmap/CMakeLists.txt new file mode 100644 index 00000000000..d53a9e0911a --- /dev/null +++ b/src/plugins/netmap/CMakeLists.txt @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Tom Jones <thj@freebsd.org> +# +# This software was developed by Tom Jones <thj@freebsd.org> under sponsorship +# from the FreeBSD Foundation. +# + +if (NOT "${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD") + message(WARNING "Netmap is only currently support on FreeBSD - netmap plugin disabled") + return() +endif() + +add_vpp_plugin(netmap + SOURCES + plugin.c + netmap.c + node.c + device.c + cli.c + netmap_api.c + + MULTIARCH_SOURCES + node.c + device.c + + INSTALL_HEADERS + netmap.h + net_netmap.h + + API_FILES + netmap.api +) diff --git a/src/plugins/netmap/FEATURE.yaml b/src/plugins/netmap/FEATURE.yaml new file mode 100644 index 00000000000..a9dfb2163e4 --- /dev/null +++ b/src/plugins/netmap/FEATURE.yaml @@ -0,0 +1,12 @@ +--- +name: Netmap Device +maintainer: Tom Jones <thj@freebsd.org> +features: + - L4 checksum offload +description: "Create a netmap interface, which is a high speed user-space + interface that allows VPP to patch to a physical or virtual NIC + without the use of DPDK" +missing: + - API dump +state: production +properties: [API, CLI, STATS, MULTITHREAD] diff --git a/src/plugins/netmap/cli.c b/src/plugins/netmap/cli.c new file mode 100644 index 00000000000..b54d397ecbe --- /dev/null +++ b/src/plugins/netmap/cli.c @@ -0,0 +1,236 @@ +/* + *------------------------------------------------------------------ + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------ + */ +#include <stdint.h> +#include <net/if.h> +#include <sys/ioctl.h> + +#include <vlib/vlib.h> +#include <vlib/unix/unix.h> +#include <vnet/ethernet/ethernet.h> + +#include <netmap/net_netmap.h> +#include <netmap/netmap.h> + +static clib_error_t * +netmap_create_command_fn (vlib_main_t * vm, unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + unformat_input_t _line_input, *line_input = &_line_input; + u8 *host_if_name = NULL; + u8 hwaddr[6]; + u8 *hw_addr_ptr = 0; + int r; + u8 is_pipe = 0; + u8 is_master = 0; + u32 sw_if_index = ~0; + clib_error_t *error = NULL; + + /* Get a line of input. */ + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "name %s", &host_if_name)) + ; + else + if (unformat + (line_input, "hw-addr %U", unformat_ethernet_address, hwaddr)) + hw_addr_ptr = hwaddr; + else if (unformat (line_input, "pipe")) + is_pipe = 1; + else if (unformat (line_input, "master")) + is_master = 1; + else if (unformat (line_input, "slave")) + is_master = 0; + else + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } + } + + if (host_if_name == NULL) + { + error = clib_error_return (0, "missing host interface name"); + goto done; + } + + r = + netmap_create_if (vm, host_if_name, hw_addr_ptr, is_pipe, is_master, + &sw_if_index); + + if (r == VNET_API_ERROR_SYSCALL_ERROR_1) + { + error = clib_error_return (0, "%s (errno %d)", strerror (errno), errno); + goto done; + } + + if (r == VNET_API_ERROR_INVALID_INTERFACE) + { + error = clib_error_return (0, "Invalid interface name"); + goto done; + } + + if (r == VNET_API_ERROR_SUBIF_ALREADY_EXISTS) + { + error = clib_error_return (0, "Interface already exists"); + goto done; + } + + vlib_cli_output (vm, "%U\n", format_vnet_sw_if_index_name, vnet_get_main (), + sw_if_index); + +done: + unformat_free (line_input); + + return error; +} + +/*? + * '<em>netmap</em>' is a framework for very fast packet I/O from userspace. + * '<em>VALE</em>' is an equally fast in-kernel software switch using the + * netmap API. '<em>netmap</em>' includes '<em>netmap pipes</em>', a shared + * memory packet transport channel. Together, they provide a high speed + * user-space interface that allows VPP to patch into a linux namespace, a + * linux container, or a physical NIC without the use of DPDK. Netmap/VALE + * generates the '<em>netmap.ko</em>' kernel module that needs to be loaded + * before netmap interfaces can be created. + * - https://github.com/luigirizzo/netmap - Netmap/VALE repo. + * - https://github.com/vpp-dev/netmap - VPP development package for Netmap/VALE, + * which is a snapshot of the Netmap/VALE repo with minor changes to work + * with containers and modified kernel drivers to work with NICs. + * + * Create a netmap interface that will attach to a linux interface. + * The interface must already exist. Once created, a new netmap interface + * will exist in VPP with the name '<em>netmap-<ifname></em>', where + * '<em><ifname></em>' takes one of two forms: + * - <b>ifname</b> - Linux interface to bind too. + * - <b>valeXXX:YYY</b> - + * - Where '<em>valeXXX</em>' is an arbitrary name for a VALE + * interface that must start with '<em>vale</em>' and is less + * than 16 characters. + * - Where '<em>YYY</em>' is an existing linux namespace. + * + * This command has the following optional parameters: + * + * - <b>hw-addr <mac-addr></b> - Optional ethernet address, can be in either + * X:X:X:X:X:X unix or X.X.X cisco format. + * + * - <b>pipe</b> - Optional flag to indicate that a '<em>netmap pipe</em>' + * instance should be created. + * + * - <b>master | slave</b> - Optional flag to indicate whether VPP should + * be the master or slave of the '<em>netmap pipe</em>'. Only considered + * if '<em>pipe</em>' is entered. Defaults to '<em>slave</em>' if not entered. + * + * @cliexpar + * Example of how to create a netmap interface tied to the linux + * namespace '<em>vpp1</em>': + * @cliexstart{create netmap name vale00:vpp1 hw-addr 02:FE:3F:34:15:9B pipe master} + * netmap-vale00:vpp1 + * @cliexend + * Once the netmap interface is created, enable the interface using: + * @cliexcmd{set interface state netmap-vale00:vpp1 up} +?*/ +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (netmap_create_command, static) = { + .path = "create netmap", + .short_help = "create netmap name <ifname>|valeXXX:YYY " + "[hw-addr <mac-addr>] [pipe] [master|slave]", + .function = netmap_create_command_fn, +}; +/* *INDENT-ON* */ + +static clib_error_t * +netmap_delete_command_fn (vlib_main_t * vm, unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + unformat_input_t _line_input, *line_input = &_line_input; + u8 *host_if_name = NULL; + clib_error_t *error = NULL; + + /* Get a line of input. */ + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "name %s", &host_if_name)) + ; + else + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } + } + + if (host_if_name == NULL) + { + error = clib_error_return (0, "missing host interface name"); + goto done; + } + + netmap_delete_if (vm, host_if_name); + +done: + unformat_free (line_input); + + return error; +} + +/*? + * Delete a netmap interface. Use the '<em><ifname></em>' to identify + * the netmap interface to be deleted. In VPP, netmap interfaces are + * named as '<em>netmap-<ifname></em>', where '<em><ifname></em>' + * takes one of two forms: + * - <b>ifname</b> - Linux interface to bind too. + * - <b>valeXXX:YYY</b> - + * - Where '<em>valeXXX</em>' is an arbitrary name for a VALE + * interface that must start with '<em>vale</em>' and is less + * than 16 characters. + * - Where '<em>YYY</em>' is an existing linux namespace. + * + * @cliexpar + * Example of how to delete a netmap interface named '<em>netmap-vale00:vpp1</em>': + * @cliexcmd{delete netmap name vale00:vpp1} +?*/ +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (netmap_delete_command, static) = { + .path = "delete netmap", + .short_help = "delete netmap name <ifname>|valeXXX:YYY", + .function = netmap_delete_command_fn, +}; +/* *INDENT-ON* */ + +clib_error_t * +netmap_cli_init (vlib_main_t * vm) +{ + return 0; +} + +VLIB_INIT_FUNCTION (netmap_cli_init); + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/netmap/device.c b/src/plugins/netmap/device.c new file mode 100644 index 00000000000..505deb988c4 --- /dev/null +++ b/src/plugins/netmap/device.c @@ -0,0 +1,252 @@ +/* + *------------------------------------------------------------------ + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------ + */ + +#include <stdint.h> +#include <net/if.h> +#include <sys/ioctl.h> + +#include <vlib/vlib.h> +#include <vlib/unix/unix.h> +#include <vnet/ethernet/ethernet.h> + +#include <netmap/net_netmap.h> +#include <netmap/netmap.h> + +#define foreach_netmap_tx_func_error \ +_(NO_FREE_SLOTS, "no free tx slots") \ +_(PENDING_MSGS, "pending msgs in tx ring") + +typedef enum +{ +#define _(f,s) NETMAP_TX_ERROR_##f, + foreach_netmap_tx_func_error +#undef _ + NETMAP_TX_N_ERROR, +} netmap_tx_func_error_t; + +static char *netmap_tx_func_error_strings[] = { +#define _(n,s) s, + foreach_netmap_tx_func_error +#undef _ +}; + + +static u8 * +format_netmap_device_name (u8 * s, va_list * args) +{ + u32 i = va_arg (*args, u32); + netmap_main_t *apm = &netmap_main; + netmap_if_t *nif = pool_elt_at_index (apm->interfaces, i); + + s = format (s, "netmap-%s", nif->host_if_name); + return s; +} + +static u8 * +format_netmap_device (u8 * s, va_list * args) +{ + u32 dev_instance = va_arg (*args, u32); + int verbose = va_arg (*args, int); + netmap_main_t *nm = &netmap_main; + netmap_if_t *nif = vec_elt_at_index (nm->interfaces, dev_instance); + u32 indent = format_get_indent (s); + + s = format (s, "NETMAP interface"); + if (verbose) + { + s = format (s, "\n%U version %d flags 0x%x" + "\n%U region %u memsize 0x%x offset 0x%x" + "\n%U tx_slots %u rx_slots %u tx_rings %u rx_rings %u", + format_white_space, indent + 2, + nif->req->nr_version, + nif->req->nr_flags, + format_white_space, indent + 2, + nif->mem_region, + nif->req->nr_memsize, + nif->req->nr_offset, + format_white_space, indent + 2, + nif->req->nr_tx_slots, + nif->req->nr_rx_slots, + nif->req->nr_tx_rings, nif->req->nr_rx_rings); + } + return s; +} + +static u8 * +format_netmap_tx_trace (u8 * s, va_list * args) +{ + s = format (s, "Unimplemented..."); + return s; +} + +VNET_DEVICE_CLASS_TX_FN (netmap_device_class) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + netmap_main_t *nm = &netmap_main; + u32 *buffers = vlib_frame_vector_args (frame); + u32 n_left = frame->n_vectors; + f64 const time_constant = 1e3; + vnet_interface_output_runtime_t *rd = (void *) node->runtime_data; + netmap_if_t *nif = pool_elt_at_index (nm->interfaces, rd->dev_instance); + int cur_ring; + + clib_spinlock_lock_if_init (&nif->lockp); + + cur_ring = nif->first_tx_ring; + + while (n_left && cur_ring <= nif->last_tx_ring) + { + struct netmap_ring *ring = NETMAP_TXRING (nif->nifp, cur_ring); + int n_free_slots = nm_ring_space (ring); + uint cur = ring->cur; + + if (nm_tx_pending (ring)) + { + if (ioctl (nif->fd, NIOCTXSYNC, NULL) < 0) + clib_unix_warning ("NIOCTXSYNC"); + clib_cpu_time_wait (time_constant); + + if (nm_tx_pending (ring) && !n_free_slots) + { + cur_ring++; + continue; + } + } + + while (n_left && n_free_slots) + { + vlib_buffer_t *b0 = 0; + u32 bi = buffers[0]; + u32 len; + u32 offset = 0; + buffers++; + + struct netmap_slot *slot = &ring->slot[cur]; + + do + { + b0 = vlib_get_buffer (vm, bi); + len = b0->current_length; + /* memcpy */ + clib_memcpy_fast ((u8 *) NETMAP_BUF (ring, slot->buf_idx) + + offset, vlib_buffer_get_current (b0), len); + offset += len; + } + while ((bi = b0->next_buffer)); + + slot->len = offset; + cur = (cur + 1) % ring->num_slots; + n_free_slots--; + n_left--; + } + CLIB_MEMORY_BARRIER (); + ring->head = ring->cur = cur; + } + + if (n_left < frame->n_vectors) + ioctl (nif->fd, NIOCTXSYNC, NULL); + + clib_spinlock_unlock_if_init (&nif->lockp); + + if (n_left) + vlib_error_count (vm, node->node_index, + (n_left == + frame->n_vectors ? NETMAP_TX_ERROR_PENDING_MSGS : + NETMAP_TX_ERROR_NO_FREE_SLOTS), n_left); + + vlib_buffer_free (vm, vlib_frame_vector_args (frame), frame->n_vectors); + return frame->n_vectors; +} + +static void +netmap_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index, + u32 node_index) +{ + netmap_main_t *apm = &netmap_main; + vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); + netmap_if_t *nif = pool_elt_at_index (apm->interfaces, hw->dev_instance); + + /* Shut off redirection */ + if (node_index == ~0) + { + nif->per_interface_next_index = node_index; + return; + } + + nif->per_interface_next_index = + vlib_node_add_next (vlib_get_main (), netmap_input_node.index, + node_index); +} + +static void +netmap_clear_hw_interface_counters (u32 instance) +{ + /* Nothing for now */ +} + +static clib_error_t * +netmap_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags) +{ + netmap_main_t *apm = &netmap_main; + vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); + netmap_if_t *nif = pool_elt_at_index (apm->interfaces, hw->dev_instance); + u32 hw_flags; + + nif->is_admin_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0; + + if (nif->is_admin_up) + hw_flags = VNET_HW_INTERFACE_FLAG_LINK_UP; + else + hw_flags = 0; + + vnet_hw_interface_set_flags (vnm, hw_if_index, hw_flags); + + return 0; +} + +static clib_error_t * +netmap_subif_add_del_function (vnet_main_t * vnm, + u32 hw_if_index, + struct vnet_sw_interface_t *st, int is_add) +{ + /* Nothing for now */ + return 0; +} + +/* *INDENT-OFF* */ +VNET_DEVICE_CLASS (netmap_device_class) = { + .name = "netmap", + .format_device_name = format_netmap_device_name, + .format_device = format_netmap_device, + .format_tx_trace = format_netmap_tx_trace, + .tx_function_n_errors = NETMAP_TX_N_ERROR, + .tx_function_error_strings = netmap_tx_func_error_strings, + .rx_redirect_to_node = netmap_set_interface_next_node, + .clear_counters = netmap_clear_hw_interface_counters, + .admin_up_down_function = netmap_interface_admin_up_down, + .subif_add_del_function = netmap_subif_add_del_function, +}; +/* *INDENT-ON* */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/netmap/net_netmap.h b/src/plugins/netmap/net_netmap.h new file mode 100644 index 00000000000..ecccedd4484 --- /dev/null +++ b/src/plugins/netmap/net_netmap.h @@ -0,0 +1,650 @@ +/* + * Copyright (C) 2011-2014 Matteo Landi, Luigi Rizzo. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``S IS''AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * $FreeBSD: head/sys/net/netmap.h 251139 2013-05-30 14:07:14Z luigi $ + * + * Definitions of constants and the structures used by the netmap + * framework, for the part visible to both kernel and userspace. + * Detailed info on netmap is available with "man netmap" or at + * + * http://info.iet.unipi.it/~luigi/netmap/ + * + * This API is also used to communicate with the VALE software switch + */ + +#ifndef _NET_NETMAP_H_ +#define _NET_NETMAP_H_ + +#define NETMAP_API 14 /* current API version */ + +#define NETMAP_MIN_API 14 /* min and max versions accepted */ +#define NETMAP_MAX_API 15 +/* + * Some fields should be cache-aligned to reduce contention. + * The alignment is architecture and OS dependent, but rather than + * digging into OS headers to find the exact value we use an estimate + * that should cover most architectures. + */ +#define NM_CACHE_ALIGN 128 + +/* + * --- Netmap data structures --- + * + * The userspace data structures used by netmap are shown below. + * They are allocated by the kernel and mmap()ed by userspace threads. + * Pointers are implemented as memory offsets or indexes, + * so that they can be easily dereferenced in kernel and userspace. + + KERNEL (opaque, obviously) + + ==================================================================== + | + USERSPACE | struct netmap_ring + +---->+---------------+ + / | head,cur,tail | + struct netmap_if (nifp, 1 per fd) / | buf_ofs | + +---------------+ / | other fields | + | ni_tx_rings | / +===============+ + | ni_rx_rings | / | buf_idx, len | slot[0] + | | / | flags, ptr | + | | / +---------------+ + +===============+ / | buf_idx, len | slot[1] + | txring_ofs[0] | (rel.to nifp)--' | flags, ptr | + | txring_ofs[1] | +---------------+ + (tx+1 entries) (num_slots entries) + | txring_ofs[t] | | buf_idx, len | slot[n-1] + +---------------+ | flags, ptr | + | rxring_ofs[0] | +---------------+ + | rxring_ofs[1] | + (rx+1 entries) + | rxring_ofs[r] | + +---------------+ + + * For each "interface" (NIC, host stack, PIPE, VALE switch port) bound to + * a file descriptor, the mmap()ed region contains a (logically readonly) + * struct netmap_if pointing to struct netmap_ring's. + * + * There is one netmap_ring per physical NIC ring, plus one tx/rx ring + * pair attached to the host stack (this pair is unused for non-NIC ports). + * + * All physical/host stack ports share the same memory region, + * so that zero-copy can be implemented between them. + * VALE switch ports instead have separate memory regions. + * + * The netmap_ring is the userspace-visible replica of the NIC ring. + * Each slot has the index of a buffer (MTU-sized and residing in the + * mmapped region), its length and some flags. An extra 64-bit pointer + * is provided for user-supplied buffers in the tx path. + * + * In user space, the buffer address is computed as + * (char *)ring + buf_ofs + index * NETMAP_BUF_SIZE + * + * Added in NETMAP_API 11: + * + * + NIOCREGIF can request the allocation of extra spare buffers from + * the same memory pool. The desired number of buffers must be in + * nr_arg3. The ioctl may return fewer buffers, depending on memory + * availability. nr_arg3 will return the actual value, and, once + * mapped, nifp->ni_bufs_head will be the index of the first buffer. + * + * The buffers are linked to each other using the first uint32_t + * as the index. On close, ni_bufs_head must point to the list of + * buffers to be released. + * + * + NIOCREGIF can request space for extra rings (and buffers) + * allocated in the same memory space. The number of extra rings + * is in nr_arg1, and is advisory. This is a no-op on NICs where + * the size of the memory space is fixed. + * + * + NIOCREGIF can attach to PIPE rings sharing the same memory + * space with a parent device. The ifname indicates the parent device, + * which must already exist. Flags in nr_flags indicate if we want to + * bind the master or slave side, the index (from nr_ringid) + * is just a cookie and does not need to be sequential. + * + * + NIOCREGIF can also attach to 'monitor' rings that replicate + * the content of specific rings, also from the same memory space. + * + * Extra flags in nr_flags support the above functions. + * Application libraries may use the following naming scheme: + * netmap:foo all NIC ring pairs + * netmap:foo^ only host ring pair + * netmap:foo+ all NIC ring + host ring pairs + * netmap:foo-k the k-th NIC ring pair + * netmap:foo{k PIPE ring pair k, master side + * netmap:foo}k PIPE ring pair k, slave side + */ + +/* + * struct netmap_slot is a buffer descriptor + */ +struct netmap_slot { + uint32_t buf_idx; /* buffer index */ + uint16_t len; /* length for this slot */ + uint16_t flags; /* buf changed, etc. */ + uint64_t ptr; /* pointer for indirect buffers */ +}; + +/* + * The following flags control how the slot is used + */ + +#define NS_BUF_CHANGED 0x0001 /* buf_idx changed */ + /* + * must be set whenever buf_idx is changed (as it might be + * necessary to recompute the physical address and mapping) + * + * It is also set by the kernel whenever the buf_idx is + * changed internally (e.g., by pipes). Applications may + * use this information to know when they can reuse the + * contents of previously prepared buffers. + */ + +#define NS_REPORT 0x0002 /* ask the hardware to report results */ + /* + * Request notification when slot is used by the hardware. + * Normally transmit completions are handled lazily and + * may be unreported. This flag lets us know when a slot + * has been sent (e.g. to terminate the sender). + */ + +#define NS_FORWARD 0x0004 /* pass packet 'forward' */ + /* + * (Only for physical ports, rx rings with NR_FORWARD set). + * Slot released to the kernel (i.e. before ring->head) with + * this flag set are passed to the peer ring (host/NIC), + * thus restoring the host-NIC connection for these slots. + * This supports efficient traffic monitoring or firewalling. + */ + +#define NS_NO_LEARN 0x0008 /* disable bridge learning */ + /* + * On a VALE switch, do not 'learn' the source port for + * this buffer. + */ + +#define NS_INDIRECT 0x0010 /* userspace buffer */ + /* + * (VALE tx rings only) data is in a userspace buffer, + * whose address is in the 'ptr' field in the slot. + */ + +#define NS_MOREFRAG 0x0020 /* packet has more fragments */ + /* + * (VALE ports only) + * Set on all but the last slot of a multi-segment packet. + * The 'len' field refers to the individual fragment. + */ + +#define NS_PORT_SHIFT 8 +#define NS_PORT_MASK (0xff << NS_PORT_SHIFT) + /* + * The high 8 bits of the flag, if not zero, indicate the + * destination port for the VALE switch, overriding + * the lookup table. + */ + +#define NS_RFRAGS(_slot) ( ((_slot)->flags >> 8) & 0xff) + /* + * (VALE rx rings only) the high 8 bits + * are the number of fragments. + */ + + +/* + * struct netmap_ring + * + * Netmap representation of a TX or RX ring (also known as "queue"). + * This is a queue implemented as a fixed-size circular array. + * At the software level the important fields are: head, cur, tail. + * + * In TX rings: + * + * head first slot available for transmission. + * cur wakeup point. select() and poll() will unblock + * when 'tail' moves past 'cur' + * tail (readonly) first slot reserved to the kernel + * + * [head .. tail-1] can be used for new packets to send; + * 'head' and 'cur' must be incremented as slots are filled + * with new packets to be sent; + * 'cur' can be moved further ahead if we need more space + * for new transmissions. XXX todo (2014-03-12) + * + * In RX rings: + * + * head first valid received packet + * cur wakeup point. select() and poll() will unblock + * when 'tail' moves past 'cur' + * tail (readonly) first slot reserved to the kernel + * + * [head .. tail-1] contain received packets; + * 'head' and 'cur' must be incremented as slots are consumed + * and can be returned to the kernel; + * 'cur' can be moved further ahead if we want to wait for + * new packets without returning the previous ones. + * + * DATA OWNERSHIP/LOCKING: + * The netmap_ring, and all slots and buffers in the range + * [head .. tail-1] are owned by the user program; + * the kernel only accesses them during a netmap system call + * and in the user thread context. + * + * Other slots and buffers are reserved for use by the kernel + */ +struct netmap_ring { + /* + * buf_ofs is meant to be used through macros. + * It contains the offset of the buffer region from this + * descriptor. + */ + const int64_t buf_ofs; + const uint32_t num_slots; /* number of slots in the ring. */ + const uint32_t nr_buf_size; + const uint16_t ringid; + const uint16_t dir; /* 0: tx, 1: rx */ + + uint32_t head; /* (u) first user slot */ + uint32_t cur; /* (u) wakeup point */ + uint32_t tail; /* (k) first kernel slot */ + + uint32_t flags; + + struct timeval ts; /* (k) time of last *sync() */ + + /* opaque room for a mutex or similar object */ +#if !defined(_WIN32) || defined(__CYGWIN__) + uint8_t __attribute__((__aligned__(NM_CACHE_ALIGN))) sem[128]; +#else + uint8_t __declspec(align(NM_CACHE_ALIGN)) sem[128]; +#endif + + /* the slots follow. This struct has variable size */ + struct netmap_slot slot[0]; /* array of slots. */ +}; + + +/* + * RING FLAGS + */ +#define NR_TIMESTAMP 0x0002 /* set timestamp on *sync() */ + /* + * updates the 'ts' field on each netmap syscall. This saves + * saves a separate gettimeofday(), and is not much worse than + * software timestamps generated in the interrupt handler. + */ + +#define NR_FORWARD 0x0004 /* enable NS_FORWARD for ring */ + /* + * Enables the NS_FORWARD slot flag for the ring. + */ + + +/* + * Netmap representation of an interface and its queue(s). + * This is initialized by the kernel when binding a file + * descriptor to a port, and should be considered as readonly + * by user programs. The kernel never uses it. + * + * There is one netmap_if for each file descriptor on which we want + * to select/poll. + * select/poll operates on one or all pairs depending on the value of + * nmr_queueid passed on the ioctl. + */ +struct netmap_if { + char ni_name[IFNAMSIZ]; /* name of the interface. */ + const uint32_t ni_version; /* API version, currently unused */ + const uint32_t ni_flags; /* properties */ +#define NI_PRIV_MEM 0x1 /* private memory region */ + + /* + * The number of packet rings available in netmap mode. + * Physical NICs can have different numbers of tx and rx rings. + * Physical NICs also have a 'host' ring pair. + * Additionally, clients can request additional ring pairs to + * be used for internal communication. + */ + const uint32_t ni_tx_rings; /* number of HW tx rings */ + const uint32_t ni_rx_rings; /* number of HW rx rings */ + + uint32_t ni_bufs_head; /* head index for extra bufs */ + uint32_t ni_spare1[5]; + /* + * The following array contains the offset of each netmap ring + * from this structure, in the following order: + * NIC tx rings (ni_tx_rings); host tx ring (1); extra tx rings; + * NIC rx rings (ni_rx_rings); host tx ring (1); extra rx rings. + * + * The area is filled up by the kernel on NIOCREGIF, + * and then only read by userspace code. + */ + const ssize_t ring_ofs[0]; +}; + + +#ifndef NIOCREGIF +/* + * ioctl names and related fields + * + * NIOCTXSYNC, NIOCRXSYNC synchronize tx or rx queues, + * whose identity is set in NIOCREGIF through nr_ringid. + * These are non blocking and take no argument. + * + * NIOCGINFO takes a struct ifreq, the interface name is the input, + * the outputs are number of queues and number of descriptor + * for each queue (useful to set number of threads etc.). + * The info returned is only advisory and may change before + * the interface is bound to a file descriptor. + * + * NIOCREGIF takes an interface name within a struct nmre, + * and activates netmap mode on the interface (if possible). + * + * The argument to NIOCGINFO/NIOCREGIF overlays struct ifreq so we + * can pass it down to other NIC-related ioctls. + * + * The actual argument (struct nmreq) has a number of options to request + * different functions. + * The following are used in NIOCREGIF when nr_cmd == 0: + * + * nr_name (in) + * The name of the port (em0, valeXXX:YYY, etc.) + * limited to IFNAMSIZ for backward compatibility. + * + * nr_version (in/out) + * Must match NETMAP_API as used in the kernel, error otherwise. + * Always returns the desired value on output. + * + * nr_tx_slots, nr_tx_slots, nr_tx_rings, nr_rx_rings (in/out) + * On input, non-zero values may be used to reconfigure the port + * according to the requested values, but this is not guaranteed. + * On output the actual values in use are reported. + * + * nr_ringid (in) + * Indicates how rings should be bound to the file descriptors. + * If nr_flags != 0, then the low bits (in NETMAP_RING_MASK) + * are used to indicate the ring number, and nr_flags specifies + * the actual rings to bind. NETMAP_NO_TX_POLL is unaffected. + * + * NOTE: THE FOLLOWING (nr_flags == 0) IS DEPRECATED: + * If nr_flags == 0, NETMAP_HW_RING and NETMAP_SW_RING control + * the binding as follows: + * 0 (default) binds all physical rings + * NETMAP_HW_RING | ring number binds a single ring pair + * NETMAP_SW_RING binds only the host tx/rx rings + * + * NETMAP_NO_TX_POLL can be OR-ed to make select()/poll() push + * packets on tx rings only if POLLOUT is set. + * The default is to push any pending packet. + * + * NETMAP_DO_RX_POLL can be OR-ed to make select()/poll() release + * packets on rx rings also when POLLIN is NOT set. + * The default is to touch the rx ring only with POLLIN. + * Note that this is the opposite of TX because it + * reflects the common usage. + * + * NOTE: NETMAP_PRIV_MEM IS DEPRECATED, use nr_arg2 instead. + * NETMAP_PRIV_MEM is set on return for ports that do not use + * the global memory allocator. + * This information is not significant and applications + * should look at the region id in nr_arg2 + * + * nr_flags is the recommended mode to indicate which rings should + * be bound to a file descriptor. Values are NR_REG_* + * + * nr_arg1 (in) The number of extra rings to be reserved. + * Especially when allocating a VALE port the system only + * allocates the amount of memory needed for the port. + * If more shared memory rings are desired (e.g. for pipes), + * the first invocation for the same basename/allocator + * should specify a suitable number. Memory cannot be + * extended after the first allocation without closing + * all ports on the same region. + * + * nr_arg2 (in/out) The identity of the memory region used. + * On input, 0 means the system decides autonomously, + * other values may try to select a specific region. + * On return the actual value is reported. + * Region '1' is the global allocator, normally shared + * by all interfaces. Other values are private regions. + * If two ports the same region zero-copy is possible. + * + * nr_arg3 (in/out) number of extra buffers to be allocated. + * + * + * + * nr_cmd (in) if non-zero indicates a special command: + * NETMAP_BDG_ATTACH and nr_name = vale*:ifname + * attaches the NIC to the switch; nr_ringid specifies + * which rings to use. Used by vale-ctl -a ... + * nr_arg1 = NETMAP_BDG_HOST also attaches the host port + * as in vale-ctl -h ... + * + * NETMAP_BDG_DETACH and nr_name = vale*:ifname + * disconnects a previously attached NIC. + * Used by vale-ctl -d ... + * + * NETMAP_BDG_LIST + * list the configuration of VALE switches. + * + * NETMAP_BDG_VNET_HDR + * Set the virtio-net header length used by the client + * of a VALE switch port. + * + * NETMAP_BDG_NEWIF + * create a persistent VALE port with name nr_name. + * Used by vale-ctl -n ... + * + * NETMAP_BDG_DELIF + * delete a persistent VALE port. Used by vale-ctl -d ... + * + * nr_arg1, nr_arg2, nr_arg3 (in/out) command specific + * + * + * + */ + + +/* + * struct nmreq overlays a struct ifreq (just the name) + */ +struct nmreq { + char nr_name[IFNAMSIZ]; + uint32_t nr_version; /* API version */ + uint32_t nr_offset; /* nifp offset in the shared region */ + uint32_t nr_memsize; /* size of the shared region */ + uint32_t nr_tx_slots; /* slots in tx rings */ + uint32_t nr_rx_slots; /* slots in rx rings */ + uint16_t nr_tx_rings; /* number of tx rings */ + uint16_t nr_rx_rings; /* number of rx rings */ + + uint16_t nr_ringid; /* ring(s) we care about */ +#define NETMAP_HW_RING 0x4000 /* single NIC ring pair */ +#define NETMAP_SW_RING 0x2000 /* only host ring pair */ + +#define NETMAP_RING_MASK 0x0fff /* the ring number */ + +#define NETMAP_NO_TX_POLL 0x1000 /* no automatic txsync on poll */ + +#define NETMAP_DO_RX_POLL 0x8000 /* DO automatic rxsync on poll */ + + uint16_t nr_cmd; +#define NETMAP_BDG_ATTACH 1 /* attach the NIC */ +#define NETMAP_BDG_DETACH 2 /* detach the NIC */ +#define NETMAP_BDG_REGOPS 3 /* register bridge callbacks */ +#define NETMAP_BDG_LIST 4 /* get bridge's info */ +#define NETMAP_BDG_VNET_HDR 5 /* set the port virtio-net-hdr length */ +#define NETMAP_BDG_OFFSET NETMAP_BDG_VNET_HDR /* deprecated alias */ +#define NETMAP_BDG_NEWIF 6 /* create a virtual port */ +#define NETMAP_BDG_DELIF 7 /* destroy a virtual port */ +#define NETMAP_PT_HOST_CREATE 8 /* create ptnetmap kthreads */ +#define NETMAP_PT_HOST_DELETE 9 /* delete ptnetmap kthreads */ +#define NETMAP_BDG_POLLING_ON 10 /* delete polling kthread */ +#define NETMAP_BDG_POLLING_OFF 11 /* delete polling kthread */ +#define NETMAP_VNET_HDR_GET 12 /* get the port virtio-net-hdr length */ + uint16_t nr_arg1; /* reserve extra rings in NIOCREGIF */ +#define NETMAP_BDG_HOST 1 /* attach the host stack on ATTACH */ + + uint16_t nr_arg2; + uint32_t nr_arg3; /* req. extra buffers in NIOCREGIF */ + uint32_t nr_flags; + /* various modes, extends nr_ringid */ + uint32_t spare2[1]; +}; + +#define NR_REG_MASK 0xf /* values for nr_flags */ +enum { NR_REG_DEFAULT = 0, /* backward compat, should not be used. */ + NR_REG_ALL_NIC = 1, + NR_REG_SW = 2, + NR_REG_NIC_SW = 3, + NR_REG_ONE_NIC = 4, + NR_REG_PIPE_MASTER = 5, + NR_REG_PIPE_SLAVE = 6, +}; +/* monitor uses the NR_REG to select the rings to monitor */ +#define NR_MONITOR_TX 0x100 +#define NR_MONITOR_RX 0x200 +#define NR_ZCOPY_MON 0x400 +/* request exclusive access to the selected rings */ +#define NR_EXCLUSIVE 0x800 +/* request ptnetmap host support */ +#define NR_PASSTHROUGH_HOST NR_PTNETMAP_HOST /* deprecated */ +#define NR_PTNETMAP_HOST 0x1000 +#define NR_RX_RINGS_ONLY 0x2000 +#define NR_TX_RINGS_ONLY 0x4000 +/* Applications set this flag if they are able to deal with virtio-net headers, + * that is send/receive frames that start with a virtio-net header. + * If not set, NIOCREGIF will fail with netmap ports that require applications + * to use those headers. If the flag is set, the application can use the + * NETMAP_VNET_HDR_GET command to figure out the header length. */ +#define NR_ACCEPT_VNET_HDR 0x8000 + + +/* + * Windows does not have _IOWR(). _IO(), _IOW() and _IOR() are defined + * in ws2def.h but not sure if they are in the form we need. + * XXX so we redefine them + * in a convenient way to use for DeviceIoControl signatures + */ +#ifdef _WIN32 +#undef _IO // ws2def.h +#define _WIN_NM_IOCTL_TYPE 40000 +#define _IO(_c, _n) CTL_CODE(_WIN_NM_IOCTL_TYPE, ((_n) + 0x800) , \ + METHOD_BUFFERED, FILE_ANY_ACCESS ) +#define _IO_direct(_c, _n) CTL_CODE(_WIN_NM_IOCTL_TYPE, ((_n) + 0x800) , \ + METHOD_OUT_DIRECT, FILE_ANY_ACCESS ) + +#define _IOWR(_c, _n, _s) _IO(_c, _n) + +/* We havesome internal sysctl in addition to the externally visible ones */ +#define NETMAP_MMAP _IO_direct('i', 160) // note METHOD_OUT_DIRECT +#define NETMAP_POLL _IO('i', 162) + +/* and also two setsockopt for sysctl emulation */ +#define NETMAP_SETSOCKOPT _IO('i', 140) +#define NETMAP_GETSOCKOPT _IO('i', 141) + + +//These linknames are for the Netmap Core Driver +#define NETMAP_NT_DEVICE_NAME L"\\Device\\NETMAP" +#define NETMAP_DOS_DEVICE_NAME L"\\DosDevices\\netmap" + +//Definition of a structure used to pass a virtual address within an IOCTL +typedef struct _MEMORY_ENTRY { + PVOID pUsermodeVirtualAddress; +} MEMORY_ENTRY, *PMEMORY_ENTRY; + +typedef struct _POLL_REQUEST_DATA { + int events; + int timeout; + int revents; +} POLL_REQUEST_DATA; + +#endif /* _WIN32 */ + +/* + * FreeBSD uses the size value embedded in the _IOWR to determine + * how much to copy in/out. So we need it to match the actual + * data structure we pass. We put some spares in the structure + * to ease compatibility with other versions + */ +#define NIOCGINFO _IOWR('i', 145, struct nmreq) /* return IF info */ +#define NIOCREGIF _IOWR('i', 146, struct nmreq) /* interface register */ +#define NIOCTXSYNC _IO('i', 148) /* sync tx queues */ +#define NIOCRXSYNC _IO('i', 149) /* sync rx queues */ +#define NIOCCONFIG _IOWR('i',150, struct nm_ifreq) /* for ext. modules */ +#endif /* !NIOCREGIF */ + + +/* + * Helper functions for kernel and userspace + */ + +/* + * check if space is available in the ring. + */ +static inline int +nm_ring_empty(struct netmap_ring *ring) +{ + return (ring->cur == ring->tail); +} + +/* + * Opaque structure that is passed to an external kernel + * module via ioctl(fd, NIOCCONFIG, req) for a user-owned + * bridge port (at this point ephemeral VALE interface). + */ +#define NM_IFRDATA_LEN 256 +struct nm_ifreq { + char nifr_name[IFNAMSIZ]; + char data[NM_IFRDATA_LEN]; +}; + +/* + * netmap kernel thread configuration + */ +/* bhyve/vmm.ko MSIX parameters for IOCTL */ +struct ptn_vmm_ioctl_msix { + uint64_t msg; + uint64_t addr; +}; + +/* IOCTL parameters */ +struct nm_kth_ioctl { + u_long com; + /* TODO: use union */ + union { + struct ptn_vmm_ioctl_msix msix; + } data; +}; + +/* Configuration of a ptnetmap ring */ +struct ptnet_ring_cfg { + uint64_t ioeventfd; /* eventfd in linux, tsleep() parameter in FreeBSD */ + uint64_t irqfd; /* eventfd in linux, ioctl fd in FreeBSD */ + struct nm_kth_ioctl ioctl; /* ioctl parameter to send irq (only used in bhyve/FreeBSD) */ +}; +#endif /* _NET_NETMAP_H_ */ diff --git a/src/plugins/netmap/netmap.api b/src/plugins/netmap/netmap.api new file mode 100644 index 00000000000..a14753cad9c --- /dev/null +++ b/src/plugins/netmap/netmap.api @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2015-2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +option version = "1.0.0"; + +/** \brief Create netmap + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param netmap_if_name - interface name + @param hw_addr - interface MAC + @param use_random_hw_addr - use random generated MAC + @param is_pipe - is pipe + @param is_master - 0=slave, 1=master +*/ +autoreply define netmap_create +{ + u32 client_index; + u32 context; + + u8 netmap_if_name[64]; + u8 hw_addr[6]; + u8 use_random_hw_addr; + u8 is_pipe; + u8 is_master; +}; + +/** \brief Delete netmap + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param netmap_if_name - interface name +*/ +autoreply define netmap_delete +{ + u32 client_index; + u32 context; + + u8 netmap_if_name[64]; +}; + +/* + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/netmap/netmap.c b/src/plugins/netmap/netmap.c new file mode 100644 index 00000000000..ebef215eb3b --- /dev/null +++ b/src/plugins/netmap/netmap.c @@ -0,0 +1,334 @@ +/* + *------------------------------------------------------------------ + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------ + */ + +#include <stdint.h> +#include <net/if.h> +#include <sys/ioctl.h> +#include <sys/types.h> +#include <fcntl.h> + +#include <vlib/vlib.h> +#include <vlib/unix/unix.h> +#include <vnet/ethernet/ethernet.h> + +#include <netmap/net_netmap.h> +#include <netmap/netmap.h> +#include <netmap/netmap.api_enum.h> +#include <netmap/netmap.api_types.h> + +netmap_main_t netmap_main; + +static clib_error_t * +netmap_fd_read_ready (clib_file_t * uf) +{ + vlib_main_t *vm = vlib_get_main (); + netmap_main_t *nm = &netmap_main; + u32 idx = uf->private_data; + + nm->pending_input_bitmap = + clib_bitmap_set (nm->pending_input_bitmap, idx, 1); + + /* Schedule the rx node */ + vlib_node_set_interrupt_pending (vm, netmap_input_node.index); + + return 0; +} + +static void +close_netmap_if (netmap_main_t * nm, netmap_if_t * nif) +{ + if (nif->clib_file_index != ~0) + { + clib_file_del (&file_main, file_main.file_pool + nif->clib_file_index); + nif->clib_file_index = ~0; + } + else if (nif->fd > -1) + close (nif->fd); + + if (nif->mem_region) + { + netmap_mem_region_t *reg = &nm->mem_regions[nif->mem_region]; + if (--reg->refcnt == 0) + { + munmap (reg->mem, reg->region_size); + reg->region_size = 0; + } + } + + + mhash_unset (&nm->if_index_by_host_if_name, nif->host_if_name, + &nif->if_index); + vec_free (nif->host_if_name); + vec_free (nif->req); + + clib_memset (nif, 0, sizeof (*nif)); + pool_put (nm->interfaces, nif); +} + +int +netmap_worker_thread_enable () +{ + /* if worker threads are enabled, switch to polling mode */ + foreach_vlib_main () + { + vlib_node_set_state (this_vlib_main, netmap_input_node.index, + VLIB_NODE_STATE_POLLING); + } + + return 0; +} + +int +netmap_worker_thread_disable () +{ + foreach_vlib_main () + { + vlib_node_set_state (this_vlib_main, netmap_input_node.index, + VLIB_NODE_STATE_INTERRUPT); + } + + return 0; +} + +int +netmap_create_if (vlib_main_t * vm, u8 * if_name, u8 * hw_addr_set, + u8 is_pipe, u8 is_master, u32 * sw_if_index) +{ + netmap_main_t *nm = &netmap_main; + int ret = 0; + uint32_t nr_reg; + netmap_if_t *nif = 0; + u8 hw_addr[6]; + vnet_sw_interface_t *sw; + vnet_main_t *vnm = vnet_get_main (); + uword *p; + struct nmreq *req = 0; + netmap_mem_region_t *reg; + vlib_thread_main_t *tm = vlib_get_thread_main (); + int fd; + + p = mhash_get (&nm->if_index_by_host_if_name, if_name); + if (p) + return VNET_API_ERROR_SUBIF_ALREADY_EXISTS; + + fd = open ("/dev/netmap", O_RDWR); + if (fd < 0) + return VNET_API_ERROR_SUBIF_ALREADY_EXISTS; + + pool_get (nm->interfaces, nif); + nif->if_index = nif - nm->interfaces; + nif->fd = fd; + nif->clib_file_index = ~0; + + vec_validate (req, 0); + nif->req = req; + req->nr_version = NETMAP_API; + req->nr_flags = NR_REG_ALL_NIC; + + if (is_pipe) + req->nr_flags = is_master ? NR_REG_PIPE_MASTER : NR_REG_PIPE_SLAVE; + else + req->nr_flags = NR_REG_ALL_NIC; + + req->nr_flags |= NR_ACCEPT_VNET_HDR; + snprintf (req->nr_name, IFNAMSIZ, "%s", if_name); + req->nr_name[IFNAMSIZ - 1] = 0; + + if (ioctl (nif->fd, NIOCREGIF, req)) + { + ret = VNET_API_ERROR_NOT_CONNECTED; + goto error; + } + + nif->mem_region = req->nr_arg2; + vec_validate (nm->mem_regions, nif->mem_region); + reg = &nm->mem_regions[nif->mem_region]; + if (reg->region_size == 0) + { + reg->mem = mmap (NULL, req->nr_memsize, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + clib_warning ("mem %p", reg->mem); + if (reg->mem == MAP_FAILED) + { + ret = VNET_API_ERROR_NOT_CONNECTED; + goto error; + } + reg->region_size = req->nr_memsize; + } + reg->refcnt++; + + nif->nifp = NETMAP_IF (reg->mem, req->nr_offset); + nr_reg = nif->req->nr_flags & NR_REG_MASK; + + if (nr_reg == NR_REG_SW) + { /* host stack */ + nif->first_tx_ring = nif->last_tx_ring = nif->req->nr_tx_rings; + nif->first_rx_ring = nif->last_rx_ring = nif->req->nr_rx_rings; + } + else if (nr_reg == NR_REG_ALL_NIC) + { /* only nic */ + nif->first_tx_ring = 0; + nif->first_rx_ring = 0; + nif->last_tx_ring = nif->req->nr_tx_rings - 1; + nif->last_rx_ring = nif->req->nr_rx_rings - 1; + } + else if (nr_reg == NR_REG_NIC_SW) + { + nif->first_tx_ring = 0; + nif->first_rx_ring = 0; + nif->last_tx_ring = nif->req->nr_tx_rings; + nif->last_rx_ring = nif->req->nr_rx_rings; + } + else if (nr_reg == NR_REG_ONE_NIC) + { + /* XXX check validity */ + nif->first_tx_ring = nif->last_tx_ring = nif->first_rx_ring = + nif->last_rx_ring = nif->req->nr_ringid & NETMAP_RING_MASK; + } + else + { /* pipes */ + nif->first_tx_ring = nif->last_tx_ring = 0; + nif->first_rx_ring = nif->last_rx_ring = 0; + } + + nif->host_if_name = if_name; + nif->per_interface_next_index = ~0; + + if (tm->n_vlib_mains > 1) + clib_spinlock_init (&nif->lockp); + + { + clib_file_t template = { 0 }; + template.read_function = netmap_fd_read_ready; + template.file_descriptor = nif->fd; + template.private_data = nif->if_index; + template.description = format (0, "netmap socket"); + nif->clib_file_index = clib_file_add (&file_main, &template); + } + + /*use configured or generate random MAC address */ + if (hw_addr_set) + memcpy (hw_addr, hw_addr_set, 6); + else + { + f64 now = vlib_time_now (vm); + u32 rnd; + rnd = (u32) (now * 1e6); + rnd = random_u32 (&rnd); + + memcpy (hw_addr + 2, &rnd, sizeof (rnd)); + hw_addr[0] = 2; + hw_addr[1] = 0xfe; + } + + vnet_eth_interface_registration_t eir = {}; + + eir.dev_class_index = netmap_device_class.index; + eir.dev_instance = nif->if_index; + eir.address = hw_addr; + eir.cb.set_max_frame_size = NULL; + + nif->hw_if_index = vnet_eth_register_interface (vnm, &eir); + + sw = vnet_get_hw_sw_interface (vnm, nif->hw_if_index); + nif->sw_if_index = sw->sw_if_index; + + mhash_set_mem (&nm->if_index_by_host_if_name, if_name, &nif->if_index, 0); + + if (sw_if_index) + *sw_if_index = nif->sw_if_index; + + if (tm->n_vlib_mains > 1 && pool_elts (nm->interfaces) == 1) + netmap_worker_thread_enable (); + + return 0; + +error: + close_netmap_if (nm, nif); + return ret; +} + +int +netmap_delete_if (vlib_main_t * vm, u8 * host_if_name) +{ + vnet_main_t *vnm = vnet_get_main (); + netmap_main_t *nm = &netmap_main; + netmap_if_t *nif; + uword *p; + vlib_thread_main_t *tm = vlib_get_thread_main (); + + p = mhash_get (&nm->if_index_by_host_if_name, host_if_name); + if (p == NULL) + { + clib_warning ("Host interface %s does not exist", host_if_name); + return VNET_API_ERROR_SYSCALL_ERROR_1; + } + nif = pool_elt_at_index (nm->interfaces, p[0]); + + /* bring down the interface */ + vnet_hw_interface_set_flags (vnm, nif->hw_if_index, 0); + + ethernet_delete_interface (vnm, nif->hw_if_index); + + close_netmap_if (nm, nif); + + if (tm->n_vlib_mains > 1 && pool_elts (nm->interfaces) == 0) + netmap_worker_thread_disable (); + + return 0; +} + +static clib_error_t * +netmap_init (vlib_main_t * vm) +{ + netmap_main_t *nm = &netmap_main; + vlib_thread_main_t *tm = vlib_get_thread_main (); + vlib_thread_registration_t *tr; + uword *p; + + clib_memset (nm, 0, sizeof (netmap_main_t)); + + nm->input_cpu_first_index = 0; + nm->input_cpu_count = 1; + + /* find out which cpus will be used for input */ + p = hash_get_mem (tm->thread_registrations_by_name, "workers"); + tr = p ? (vlib_thread_registration_t *) p[0] : 0; + + if (tr && tr->count > 0) + { + nm->input_cpu_first_index = tr->first_index; + nm->input_cpu_count = tr->count; + } + + mhash_init_vec_string (&nm->if_index_by_host_if_name, sizeof (uword)); + + vec_validate_aligned (nm->rx_buffers, tm->n_vlib_mains - 1, + CLIB_CACHE_LINE_BYTES); + + return 0; +} + +VLIB_INIT_FUNCTION (netmap_init); + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/netmap/netmap.h b/src/plugins/netmap/netmap.h new file mode 100644 index 00000000000..29f855fda8e --- /dev/null +++ b/src/plugins/netmap/netmap.h @@ -0,0 +1,166 @@ +/* + *------------------------------------------------------------------ + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------ + */ +/* + * Copyright (C) 2011-2014 Universita` di Pisa. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <vppinfra/lock.h> + +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + clib_spinlock_t lockp; + u8 *host_if_name; + uword if_index; + u32 hw_if_index; + u32 sw_if_index; + u32 clib_file_index; + + u32 per_interface_next_index; + u8 is_admin_up; + + /* netmap */ + struct nmreq *req; + u16 mem_region; + int fd; + struct netmap_if *nifp; + u16 first_tx_ring; + u16 last_tx_ring; + u16 first_rx_ring; + u16 last_rx_ring; + +} netmap_if_t; + +typedef struct +{ + char *mem; + u32 region_size; + int refcnt; +} netmap_mem_region_t; + +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + netmap_if_t *interfaces; + + /* bitmap of pending rx interfaces */ + uword *pending_input_bitmap; + + /* rx buffer cache */ + u32 **rx_buffers; + + /* hash of host interface names */ + mhash_t if_index_by_host_if_name; + + /* vector of memory regions */ + netmap_mem_region_t *mem_regions; + + /* first cpu index */ + u32 input_cpu_first_index; + + /* total cpu count */ + u32 input_cpu_count; +} netmap_main_t; + +extern netmap_main_t netmap_main; +extern vnet_device_class_t netmap_device_class; +extern vlib_node_registration_t netmap_input_node; + +int netmap_create_if (vlib_main_t * vm, u8 * host_if_name, u8 * hw_addr_set, + u8 is_pipe, u8 is_master, u32 * sw_if_index); +int netmap_delete_if (vlib_main_t * vm, u8 * host_if_name); + + +/* Macros and helper functions from sys/net/netmap_user.h */ + +#ifdef _NET_NETMAP_H_ + +#define _NETMAP_OFFSET(type, ptr, offset) \ + ((type)(void *)((char *)(ptr) + (offset))) + +#define NETMAP_IF(_base, _ofs) _NETMAP_OFFSET(struct netmap_if *, _base, _ofs) + +#define NETMAP_TXRING(nifp, index) _NETMAP_OFFSET(struct netmap_ring *, \ + nifp, (nifp)->ring_ofs[index] ) + +#define NETMAP_RXRING(nifp, index) _NETMAP_OFFSET(struct netmap_ring *, \ + nifp, (nifp)->ring_ofs[index + (nifp)->ni_tx_rings + 1] ) + +#define NETMAP_BUF(ring, index) \ + ((char *)(ring) + (ring)->buf_ofs + ((index)*(ring)->nr_buf_size)) + +#define NETMAP_BUF_IDX(ring, buf) \ + ( ((char *)(buf) - ((char *)(ring) + (ring)->buf_ofs) ) / \ + (ring)->nr_buf_size ) + +static inline uint32_t +nm_ring_next (struct netmap_ring *ring, uint32_t i) +{ + return (PREDICT_FALSE (i + 1 == ring->num_slots) ? 0 : i + 1); +} + + +/* + * Return 1 if we have pending transmissions in the tx ring. + * When everything is complete ring->head = ring->tail + 1 (modulo ring size) + */ +static inline int +nm_tx_pending (struct netmap_ring *ring) +{ + return nm_ring_next (ring, ring->tail) != ring->head; +} + +static inline uint32_t +nm_ring_space (struct netmap_ring *ring) +{ + int ret = ring->tail - ring->cur; + if (ret < 0) + ret += ring->num_slots; + return ret; +} +#endif + + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/netmap/netmap_api.c b/src/plugins/netmap/netmap_api.c new file mode 100644 index 00000000000..51f572a23e6 --- /dev/null +++ b/src/plugins/netmap/netmap_api.c @@ -0,0 +1,95 @@ +/* + *------------------------------------------------------------------ + * netmap_api.c - netmap api + * + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------ + */ + +#include <vnet/vnet.h> +#include <vlibmemory/api.h> + +#include <vnet/interface.h> +#include <vnet/api_errno.h> +#include <netmap/netmap.h> + +#include <vnet/format_fns.h> +#include <netmap/netmap.api_enum.h> +#include <netmap/netmap.api_types.h> + +#include <vlibapi/api_helper_macros.h> + +#define foreach_vpe_api_msg \ +_(NETMAP_CREATE, netmap_create) \ +_(NETMAP_DELETE, netmap_delete) \ + +static void +vl_api_netmap_create_t_handler (vl_api_netmap_create_t * mp) +{ + vlib_main_t *vm = vlib_get_main (); + vl_api_netmap_create_reply_t *rmp; + int rv = 0; + u8 *if_name = NULL; + + if_name = format (0, "%s", mp->netmap_if_name); + vec_add1 (if_name, 0); + + rv = + netmap_create_if (vm, if_name, mp->use_random_hw_addr ? 0 : mp->hw_addr, + mp->is_pipe, mp->is_master, 0); + + vec_free (if_name); + + REPLY_MACRO (VL_API_NETMAP_CREATE_REPLY); +} + +static void +vl_api_netmap_delete_t_handler (vl_api_netmap_delete_t * mp) +{ + vlib_main_t *vm = vlib_get_main (); + vl_api_netmap_delete_reply_t *rmp; + int rv = 0; + u8 *if_name = NULL; + + if_name = format (0, "%s", mp->netmap_if_name); + vec_add1 (if_name, 0); + + rv = netmap_delete_if (vm, if_name); + + vec_free (if_name); + + REPLY_MACRO (VL_API_NETMAP_DELETE_REPLY); +} + +#include <netmap/netmap.api.c> +static clib_error_t * +netmap_api_hookup (vlib_main_t * vm) +{ + /* + * Set up the (msg_name, crc, message-id) table + */ + setup_message_id_table (); + + return 0; +} + +VLIB_API_INIT_FUNCTION (netmap_api_hookup); + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/netmap/node.c b/src/plugins/netmap/node.c new file mode 100644 index 00000000000..6169847fa79 --- /dev/null +++ b/src/plugins/netmap/node.c @@ -0,0 +1,295 @@ +/* + *------------------------------------------------------------------ + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------ + */ + +#include <stdint.h> +#include <net/if.h> +#include <sys/ioctl.h> + +#include <vlib/vlib.h> +#include <vlib/unix/unix.h> +#include <vnet/ethernet/ethernet.h> +#include <vnet/devices/devices.h> +#include <vnet/feature/feature.h> + +#include <netmap/net_netmap.h> +#include <netmap/netmap.h> + +#define foreach_netmap_input_error + +typedef enum +{ +#define _(f,s) NETMAP_INPUT_ERROR_##f, + foreach_netmap_input_error +#undef _ + NETMAP_INPUT_N_ERROR, +} netmap_input_error_t; + +static char *netmap_input_error_strings[] = { +#define _(n,s) s, + foreach_netmap_input_error +#undef _ +}; + +typedef struct +{ + u32 next_index; + u32 hw_if_index; + struct netmap_slot slot; +} netmap_input_trace_t; + +static u8 * +format_netmap_input_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + netmap_input_trace_t *t = va_arg (*args, netmap_input_trace_t *); + u32 indent = format_get_indent (s); + + s = format (s, "netmap: hw_if_index %d next-index %d", + t->hw_if_index, t->next_index); + s = format (s, "\n%Uslot: flags 0x%x len %u buf_idx %u", + format_white_space, indent + 2, + t->slot.flags, t->slot.len, t->slot.buf_idx); + return s; +} + +always_inline void +buffer_add_to_chain (vlib_main_t * vm, u32 bi, u32 first_bi, u32 prev_bi) +{ + vlib_buffer_t *b = vlib_get_buffer (vm, bi); + vlib_buffer_t *first_b = vlib_get_buffer (vm, first_bi); + vlib_buffer_t *prev_b = vlib_get_buffer (vm, prev_bi); + + /* update first buffer */ + first_b->total_length_not_including_first_buffer += b->current_length; + + /* update previous buffer */ + prev_b->next_buffer = bi; + prev_b->flags |= VLIB_BUFFER_NEXT_PRESENT; + + /* update current buffer */ + b->next_buffer = 0; +} + +always_inline uword +netmap_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * frame, netmap_if_t * nif) +{ + u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; + uword n_trace = vlib_get_trace_count (vm, node); + netmap_main_t *nm = &netmap_main; + u32 n_rx_packets = 0; + u32 n_rx_bytes = 0; + u32 *to_next = 0; + u32 n_free_bufs; + struct netmap_ring *ring; + int cur_ring; + u32 thread_index = vm->thread_index; + u32 n_buffer_bytes = vlib_buffer_get_default_data_size (vm); + + if (nif->per_interface_next_index != ~0) + next_index = nif->per_interface_next_index; + + n_free_bufs = vec_len (nm->rx_buffers[thread_index]); + if (PREDICT_FALSE (n_free_bufs < VLIB_FRAME_SIZE)) + { + vec_validate (nm->rx_buffers[thread_index], + VLIB_FRAME_SIZE + n_free_bufs - 1); + n_free_bufs += + vlib_buffer_alloc (vm, &nm->rx_buffers[thread_index][n_free_bufs], + VLIB_FRAME_SIZE); + vec_set_len (nm->rx_buffers[thread_index], n_free_bufs); + } + + cur_ring = nif->first_rx_ring; + while (cur_ring <= nif->last_rx_ring && n_free_bufs) + { + int r = 0; + u32 cur_slot_index; + ring = NETMAP_RXRING (nif->nifp, cur_ring); + r = nm_ring_space (ring); + + if (!r) + { + cur_ring++; + continue; + } + + if (r > n_free_bufs) + r = n_free_bufs; + + cur_slot_index = ring->cur; + while (r) + { + u32 n_left_to_next; + u32 next0 = next_index; + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + + while (r && n_left_to_next) + { + vlib_buffer_t *first_b0 = 0; + u32 offset = 0; + u32 bi0 = 0, first_bi0 = 0, prev_bi0; + u32 next_slot_index = (cur_slot_index + 1) % ring->num_slots; + u32 next2_slot_index = (cur_slot_index + 2) % ring->num_slots; + struct netmap_slot *slot = &ring->slot[cur_slot_index]; + u32 data_len = slot->len; + + /* prefetch 2 slots in advance */ + CLIB_PREFETCH (&ring->slot[next2_slot_index], + CLIB_CACHE_LINE_BYTES, LOAD); + /* prefetch start of next packet */ + CLIB_PREFETCH (NETMAP_BUF + (ring, ring->slot[next_slot_index].buf_idx), + CLIB_CACHE_LINE_BYTES, LOAD); + + while (data_len && n_free_bufs) + { + vlib_buffer_t *b0; + /* grab free buffer */ + u32 last_empty_buffer = + vec_len (nm->rx_buffers[thread_index]) - 1; + prev_bi0 = bi0; + bi0 = nm->rx_buffers[thread_index][last_empty_buffer]; + b0 = vlib_get_buffer (vm, bi0); + vec_set_len (nm->rx_buffers[thread_index], + last_empty_buffer); + n_free_bufs--; + + /* copy data */ + u32 bytes_to_copy = + data_len > n_buffer_bytes ? n_buffer_bytes : data_len; + b0->current_data = 0; + clib_memcpy_fast (vlib_buffer_get_current (b0), + (u8 *) NETMAP_BUF (ring, slot->buf_idx) + + offset, bytes_to_copy); + + /* fill buffer header */ + b0->current_length = bytes_to_copy; + + if (offset == 0) + { + b0->total_length_not_including_first_buffer = 0; + b0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID; + vnet_buffer (b0)->sw_if_index[VLIB_RX] = + nif->sw_if_index; + vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0; + first_bi0 = bi0; + first_b0 = vlib_get_buffer (vm, first_bi0); + } + else + buffer_add_to_chain (vm, bi0, first_bi0, prev_bi0); + + offset += bytes_to_copy; + data_len -= bytes_to_copy; + } + + /* trace */ + if (PREDICT_FALSE (n_trace > 0)) + { + if (PREDICT_TRUE (first_b0 != 0) && + vlib_trace_buffer (vm, node, next0, first_b0, + /* follow_chain */ 0)) + { + netmap_input_trace_t *tr; + + vlib_set_trace_count (vm, node, --n_trace); + tr = vlib_add_trace (vm, node, first_b0, sizeof (*tr)); + tr->next_index = next0; + tr->hw_if_index = nif->hw_if_index; + memcpy (&tr->slot, slot, sizeof (struct netmap_slot)); + } + } + + /* enque and take next packet */ + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, + n_left_to_next, first_bi0, + next0); + + /* next packet */ + n_rx_packets++; + n_rx_bytes += slot->len; + to_next[0] = first_bi0; + to_next += 1; + n_left_to_next--; + cur_slot_index = next_slot_index; + + r--; + } + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + ring->head = ring->cur = cur_slot_index; + cur_ring++; + } + + if (n_rx_packets) + ioctl (nif->fd, NIOCRXSYNC, NULL); + + vlib_increment_combined_counter + (vnet_get_main ()->interface_main.combined_sw_if_counters + + VNET_INTERFACE_COUNTER_RX, + vlib_get_thread_index (), nif->hw_if_index, n_rx_packets, n_rx_bytes); + + vnet_device_increment_rx_packets (thread_index, n_rx_packets); + + return n_rx_packets; +} + +VLIB_NODE_FN (netmap_input_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + int i; + u32 n_rx_packets = 0; + u32 thread_index = vm->thread_index; + netmap_main_t *nm = &netmap_main; + netmap_if_t *nmi; + + for (i = 0; i < vec_len (nm->interfaces); i++) + { + nmi = vec_elt_at_index (nm->interfaces, i); + if (nmi->is_admin_up && + (i % nm->input_cpu_count) == + (thread_index - nm->input_cpu_first_index)) + n_rx_packets += netmap_device_input_fn (vm, node, frame, nmi); + } + + return n_rx_packets; +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (netmap_input_node) = { + .name = "netmap-input", + .sibling_of = "device-input", + .flags = VLIB_NODE_FLAG_TRACE_SUPPORTED, + .format_trace = format_netmap_input_trace, + .type = VLIB_NODE_TYPE_INPUT, + /* default state is INTERRUPT mode, switch to POLLING if worker threads are enabled */ + .state = VLIB_NODE_STATE_INTERRUPT, + .n_errors = NETMAP_INPUT_N_ERROR, + .error_strings = netmap_input_error_strings, +}; +/* *INDENT-ON* */ + + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/netmap/plugin.c b/src/plugins/netmap/plugin.c new file mode 100644 index 00000000000..1673225b683 --- /dev/null +++ b/src/plugins/netmap/plugin.c @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Tom Jones <thj@freebsd.org> + * + * This software was developed by Tom Jones <thj@freebsd.org> under sponsorship + * from the FreeBSD Foundation. + * + */ + +#include <vlib/vlib.h> +#include <vnet/plugin/plugin.h> +#include <vpp/app/version.h> + +VLIB_PLUGIN_REGISTER () = { + .version = VPP_BUILD_VER, + .description = "netmap", +}; diff --git a/src/plugins/srmpls/CMakeLists.txt b/src/plugins/srmpls/CMakeLists.txt new file mode 100644 index 00000000000..25905d31e1b --- /dev/null +++ b/src/plugins/srmpls/CMakeLists.txt @@ -0,0 +1,30 @@ +# Copyright (c) 2024 Cisco and/or its affiliates +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_vpp_plugin(srmpls + SOURCES + sr_mpls_policy.c + sr_mpls_steering.c + sr_mpls_api.c + plugin.c + + INSTALL_HEADERS + sr_mpls.h + + API_FILES + sr_mpls.api + + # This might need to be VAT_AUTO_TEST? Not documented + API_TEST_SOURCES + sr_mpls_test.c +) diff --git a/src/vnet/srmpls/FEATURE.yaml b/src/plugins/srmpls/FEATURE.yaml index c5b958224c7..c5b958224c7 100644 --- a/src/vnet/srmpls/FEATURE.yaml +++ b/src/plugins/srmpls/FEATURE.yaml diff --git a/src/vnet/srmpls/dir.dox b/src/plugins/srmpls/dir.dox index 76ec1d6a41b..76ec1d6a41b 100644 --- a/src/vnet/srmpls/dir.dox +++ b/src/plugins/srmpls/dir.dox diff --git a/src/plugins/srmpls/plugin.c b/src/plugins/srmpls/plugin.c new file mode 100644 index 00000000000..af87607764f --- /dev/null +++ b/src/plugins/srmpls/plugin.c @@ -0,0 +1,26 @@ +/* + * plugin.c: srmpls + * + * Copyright (c) 2024 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <vlib/vlib.h> +#include <vnet/plugin/plugin.h> +#include <vpp/app/version.h> + +// register a plugin +VLIB_PLUGIN_REGISTER () = { + .version = VPP_BUILD_VER, + .description = "Segment Routing for MPLS plugin", +}; diff --git a/src/vnet/srmpls/sr_doc.rst b/src/plugins/srmpls/sr_doc.rst index ed847fa0d42..ed847fa0d42 100644 --- a/src/vnet/srmpls/sr_doc.rst +++ b/src/plugins/srmpls/sr_doc.rst diff --git a/src/vnet/srmpls/sr_mpls.api b/src/plugins/srmpls/sr_mpls.api index 742f135d493..742f135d493 100644 --- a/src/vnet/srmpls/sr_mpls.api +++ b/src/plugins/srmpls/sr_mpls.api diff --git a/src/vnet/srmpls/sr_mpls.h b/src/plugins/srmpls/sr_mpls.h index a8f9494428f..a8f9494428f 100644 --- a/src/vnet/srmpls/sr_mpls.h +++ b/src/plugins/srmpls/sr_mpls.h diff --git a/src/vnet/srmpls/sr_mpls_api.c b/src/plugins/srmpls/sr_mpls_api.c index 920856acff6..3e89017dbc1 100644 --- a/src/vnet/srmpls/sr_mpls_api.c +++ b/src/plugins/srmpls/sr_mpls_api.c @@ -17,7 +17,7 @@ */ #include <vnet/vnet.h> -#include <vnet/srmpls/sr_mpls.h> +#include "sr_mpls.h" #include <vlibmemory/api.h> #include <vnet/interface.h> @@ -26,28 +26,27 @@ #include <vnet/ip/ip_types_api.h> #include <vnet/format_fns.h> -#include <vnet/srmpls/sr_mpls.api_enum.h> -#include <vnet/srmpls/sr_mpls.api_types.h> - +#include <plugins/srmpls/sr_mpls.api_enum.h> +#include <plugins/srmpls/sr_mpls.api_types.h> #define vl_api_version(n, v) static u32 api_version = v; -#include <vnet/srmpls/sr_mpls.api.h> +#include <plugins/srmpls/sr_mpls.api.h> #undef vl_api_version #define vl_endianfun -#include <vnet/srmpls/sr_mpls.api.h> +#include <plugins/srmpls/sr_mpls.api.h> #undef vl_endianfun #define vl_calcsizefun -#include <vnet/srmpls/sr_mpls.api.h> +#include <plugins/srmpls/sr_mpls.api.h> #undef vl_calcsizefun #define vl_printfun -#include <vnet/srmpls/sr_mpls.api.h> +#include <plugins/srmpls/sr_mpls.api.h> #undef vl_printfun #define vl_msg_name_crc_list -#include <vnet/srmpls/sr_mpls.api.h> +#include <plugins/srmpls/sr_mpls.api.h> #undef vl_msg_name_crc_list #define REPLY_MSG_ID_BASE msg_id_base diff --git a/src/vnet/srmpls/sr_mpls_policy.c b/src/plugins/srmpls/sr_mpls_policy.c index 41cb71601e9..af24acd8cf6 100644 --- a/src/vnet/srmpls/sr_mpls_policy.c +++ b/src/plugins/srmpls/sr_mpls_policy.c @@ -31,7 +31,7 @@ #include <vlib/vlib.h> #include <vnet/vnet.h> -#include <vnet/srmpls/sr_mpls.h> +#include "sr_mpls.h" #include <vnet/fib/mpls_fib.h> #include <vnet/dpo/dpo.h> #include <vnet/ip/ip.h> diff --git a/src/vnet/srmpls/sr_mpls_steering.c b/src/plugins/srmpls/sr_mpls_steering.c index e8920df542b..24c8b0e2d9f 100644 --- a/src/vnet/srmpls/sr_mpls_steering.c +++ b/src/plugins/srmpls/sr_mpls_steering.c @@ -31,7 +31,7 @@ #include <vlib/vlib.h> #include <vnet/vnet.h> -#include <vnet/srmpls/sr_mpls.h> +#include "sr_mpls.h" #include <vnet/ip/ip4_packet.h> #include <vnet/ip/ip6_packet.h> #include <vnet/fib/mpls_fib.h> diff --git a/src/vnet/srmpls/sr_mpls_test.c b/src/plugins/srmpls/sr_mpls_test.c index e5d68462443..7aff4c32b06 100644 --- a/src/vnet/srmpls/sr_mpls_test.c +++ b/src/plugins/srmpls/sr_mpls_test.c @@ -25,11 +25,11 @@ /* Declare message IDs */ #include <vnet/format_fns.h> -#include <vnet/srmpls/sr_mpls.api_enum.h> -#include <vnet/srmpls/sr_mpls.api_types.h> +#include <plugins/srmpls/sr_mpls.api_enum.h> +#include <plugins/srmpls/sr_mpls.api_types.h> #define vl_endianfun /* define message structures */ -#include <vnet/srmpls/sr_mpls.api.h> +#include <plugins/srmpls/sr_mpls.api.h> #undef vl_endianfun typedef struct @@ -163,7 +163,7 @@ api_sr_mpls_policy_del (vat_main_t *vam) return ret; } -#include <vnet/srmpls/sr_mpls.api_test.c> +#include <plugins/srmpls/sr_mpls.api_test.c> /* * fd.io coding-style-patch-verification: ON diff --git a/src/plugins/stn/stn.c b/src/plugins/stn/stn.c index 6e789feca5d..c0ac0d0b3a6 100644 --- a/src/plugins/stn/stn.c +++ b/src/plugins/stn/stn.c @@ -49,7 +49,7 @@ format_stn_rule (u8 * s, va_list * args) s = format (s, "%Uiface: %U (%d)\n", format_white_space, indent, format_vnet_sw_if_index_name, vnet_get_main(), r->sw_if_index, r->sw_if_index); - s = format (s, "%Unext_node: %s (%d)", format_white_space, indent, + s = format (s, "%Unext_node: %v (%d)", format_white_space, indent, next_node->name, next_node->index); return s; } diff --git a/src/plugins/tlsopenssl/tls_openssl.c b/src/plugins/tlsopenssl/tls_openssl.c index a21e3bb54c1..5d172a0adcf 100644 --- a/src/plugins/tlsopenssl/tls_openssl.c +++ b/src/plugins/tlsopenssl/tls_openssl.c @@ -455,8 +455,11 @@ check_tls_fifo: sp->flags |= TRANSPORT_SND_F_DESCHED; } else - /* Request tx reschedule of the app session */ - app_session->flags |= SESSION_F_CUSTOM_TX; + { + /* Request tx reschedule of the app session */ + if (wrote) + app_session->flags |= SESSION_F_CUSTOM_TX; + } return wrote; } @@ -1176,18 +1179,13 @@ int tls_openssl_set_ciphers (char *ciphers) { openssl_main_t *om = &openssl_main; - int i; if (!ciphers) { return -1; } - vec_validate (om->ciphers, strlen (ciphers)); - for (i = 0; i < vec_len (om->ciphers) - 1; i++) - { - om->ciphers[i] = toupper (ciphers[i]); - } + vec_validate_init_c_string (om->ciphers, ciphers, strlen (ciphers)); return 0; diff --git a/src/plugins/unittest/gso_test.c b/src/plugins/unittest/gso_test.c index 54eb7422c87..43c614341d2 100644 --- a/src/plugins/unittest/gso_test.c +++ b/src/plugins/unittest/gso_test.c @@ -96,12 +96,94 @@ GSO_TEST_REGISTER_DATA (gso_ipv6_tcp, static) = { .is_ip6 = 1, }; +/* + * this does not support tunnel packets + */ +static void +set_hdr_offsets (vlib_buffer_t *b0, u8 is_l2) +{ + u16 ethertype = 0, l2hdr_sz = 0; + vnet_buffer_oflags_t oflags = 0; + u8 l4_proto = 0; + + if (!is_l2) + { + switch (b0->data[0] & 0xf0) + { + case 0x40: + ethertype = ETHERNET_TYPE_IP4; + break; + case 0x60: + ethertype = ETHERNET_TYPE_IP6; + break; + } + } + else + { + ethernet_header_t *eh = (ethernet_header_t *) b0->data; + ethertype = clib_net_to_host_u16 (eh->type); + l2hdr_sz = sizeof (ethernet_header_t); + + if (ethernet_frame_is_tagged (ethertype)) + { + ethernet_vlan_header_t *vlan = (ethernet_vlan_header_t *) (eh + 1); + + ethertype = clib_net_to_host_u16 (vlan->type); + l2hdr_sz += sizeof (*vlan); + if (ethertype == ETHERNET_TYPE_VLAN) + { + vlan++; + ethertype = clib_net_to_host_u16 (vlan->type); + l2hdr_sz += sizeof (*vlan); + } + } + } + + vnet_buffer (b0)->l2_hdr_offset = 0; + vnet_buffer (b0)->l3_hdr_offset = l2hdr_sz; + + if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP4)) + { + ip4_header_t *ip4 = (ip4_header_t *) (b0->data + l2hdr_sz); + vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + ip4_header_bytes (ip4); + l4_proto = ip4->protocol; + oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM; + b0->flags |= (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID | + VNET_BUFFER_F_L3_HDR_OFFSET_VALID | + VNET_BUFFER_F_L4_HDR_OFFSET_VALID); + } + else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6)) + { + ip6_header_t *ip6 = (ip6_header_t *) (b0->data + l2hdr_sz); + vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + sizeof (ip6_header_t); + /* FIXME IPv6 EH traversal */ + l4_proto = ip6->protocol; + b0->flags |= (VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID | + VNET_BUFFER_F_L3_HDR_OFFSET_VALID | + VNET_BUFFER_F_L4_HDR_OFFSET_VALID); + } + if (l4_proto == IP_PROTOCOL_TCP) + { + oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; + } + else if (l4_proto == IP_PROTOCOL_UDP) + { + oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; + } + if (oflags) + vnet_buffer_offload_flags_set (b0, oflags); +} + static u32 -fill_buffers (vlib_main_t *vm, u32 *buffer_indices, u8 *data, u32 data_size, - u32 n_buffers, u32 buffer_size, u32 packet_size, u32 gso_size, - u32 l4_hdr_len) +fill_buffers (vlib_main_t *vm, u32 *buffer_indices, + gso_test_data_t *gso_test_data, u32 n_buffers, u32 buffer_size, + u32 packet_size, u32 gso_size) { u32 i; + u8 *data = gso_test_data->data; + u32 data_size = gso_test_data->data_size; + u32 l4_hdr_len = gso_test_data->l4_hdr_len; + u8 is_l2 = gso_test_data->is_l2; for (i = 0; i < n_buffers; i++) { @@ -153,6 +235,8 @@ fill_buffers (vlib_main_t *vm, u32 *buffer_indices, u8 *data, u32 data_size, len += fill_data_size; } while (k < n_bufs); + + set_hdr_offsets (b, is_l2); b->flags |= VNET_BUFFER_F_GSO; vnet_buffer2 (b)->gso_size = gso_size; vnet_buffer2 (b)->gso_l4_hdr_sz = l4_hdr_len; @@ -165,17 +249,14 @@ fill_buffers (vlib_main_t *vm, u32 *buffer_indices, u8 *data, u32 data_size, static_always_inline u32 gso_segment_buffer_test (vlib_main_t *vm, u32 bi, - vnet_interface_per_thread_data_t *ptd, u8 is_l2, - u8 is_ip6) + vnet_interface_per_thread_data_t *ptd, u8 is_l2) { vlib_buffer_t *b = vlib_get_buffer (vm, bi); - generic_header_offset_t gho = { 0 }; u32 n_tx_bytes = 0; if (PREDICT_TRUE (b->flags & VNET_BUFFER_F_GSO)) { - vnet_generic_header_offset_parser (b, &gho, is_l2, !is_ip6, is_ip6); - n_tx_bytes = gso_segment_buffer_inline (vm, ptd, b, &gho, is_l2, is_ip6); + n_tx_bytes = gso_segment_buffer_inline (vm, ptd, b, is_l2); } return n_tx_bytes; @@ -237,19 +318,16 @@ test_gso_perf (vlib_main_t *vm, gso_test_main_t *gtm) vlib_buffer_free (vm, buffer_indices, n_alloc); goto done; } - n_filled = - fill_buffers (vm, buffer_indices, gso_test_data->data, - gso_test_data->data_size, n_buffers, buffer_size, - packet_size, gso_size, gso_test_data->l4_hdr_len); + n_filled = fill_buffers (vm, buffer_indices, gso_test_data, n_buffers, + buffer_size, packet_size, gso_size); u8 is_l2 = gso_test_data->is_l2; - u8 is_ip6 = gso_test_data->is_ip6; for (k = 0; k < warmup_rounds; k++) { for (j = 0; j < n_filled; j++) - gso_segment_buffer_test (vm, buffer_indices[j], &ptd[j], is_l2, - is_ip6); + gso_segment_buffer_test (vm, buffer_indices[j], &ptd[j], is_l2); + for (j = 0; j < n_filled; j++) { vlib_buffer_free (vm, ptd[j].split_buffers, @@ -264,8 +342,9 @@ test_gso_perf (vlib_main_t *vm, gso_test_main_t *gtm) { t0 = clib_cpu_time_now (); for (j = 0; j < n_filled; j++) - gso_segment_buffer_test (vm, buffer_indices[j], &ptd[j], is_l2, - is_ip6); + gso_segment_buffer_test (vm, buffer_indices[j], &ptd[j], + is_l2); + t1 = clib_cpu_time_now (); t2[i] += (t1 - t0); for (j = 0; j < n_filled; j++) diff --git a/src/plugins/unittest/policer_test.c b/src/plugins/unittest/policer_test.c index 2b14bf687bf..41f769960a3 100644 --- a/src/plugins/unittest/policer_test.c +++ b/src/plugins/unittest/policer_test.c @@ -21,7 +21,7 @@ policer_test (vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd_arg) { int policer_index, i; - uint rate_kbps, burst, num_pkts; + unsigned int rate_kbps, burst, num_pkts; double total_bytes, cpu_ticks_per_pkt, time = 0; double cpu_speed, cpu_ticks_per_byte; policer_result_e result, input_colour = POLICE_CONFORM; diff --git a/src/plugins/vhost/CMakeLists.txt b/src/plugins/vhost/CMakeLists.txt index f72d9f20346..6b86c8c98d1 100644 --- a/src/plugins/vhost/CMakeLists.txt +++ b/src/plugins/vhost/CMakeLists.txt @@ -29,4 +29,6 @@ add_vpp_plugin(vhost API_FILES vhost_user.api + + SUPPORTED_OS_LIST Linux ) diff --git a/src/plugins/wireguard/wireguard_chachapoly.c b/src/plugins/wireguard/wireguard_chachapoly.c index 0dd7908d2e2..ad644ff6cb8 100644 --- a/src/plugins/wireguard/wireguard_chachapoly.c +++ b/src/plugins/wireguard/wireguard_chachapoly.c @@ -72,11 +72,11 @@ wg_xchacha20poly1305_encrypt (vlib_main_t *vm, u8 *src, u32 src_len, u8 *dst, u64 h_nonce; clib_memcpy (&h_nonce, nonce + 16, sizeof (h_nonce)); - h_nonce = le64toh (h_nonce); + h_nonce = clib_little_to_host_u64 (h_nonce); hchacha20 (derived_key, nonce, key); for (i = 0; i < (sizeof (derived_key) / sizeof (derived_key[0])); i++) - (derived_key[i]) = htole32 ((derived_key[i])); + (derived_key[i]) = clib_host_to_little_u32 ((derived_key[i])); uint32_t key_idx; @@ -102,11 +102,11 @@ wg_xchacha20poly1305_decrypt (vlib_main_t *vm, u8 *src, u32 src_len, u8 *dst, u64 h_nonce; clib_memcpy (&h_nonce, nonce + 16, sizeof (h_nonce)); - h_nonce = le64toh (h_nonce); + h_nonce = clib_little_to_host_u64 (h_nonce); hchacha20 (derived_key, nonce, key); for (i = 0; i < (sizeof (derived_key) / sizeof (derived_key[0])); i++) - (derived_key[i]) = htole32 ((derived_key[i])); + (derived_key[i]) = clib_host_to_little_u32 ((derived_key[i])); uint32_t key_idx; diff --git a/src/plugins/wireguard/wireguard_noise.c b/src/plugins/wireguard/wireguard_noise.c index 5fe2e44b03b..c3f28f442f5 100644 --- a/src/plugins/wireguard/wireguard_noise.c +++ b/src/plugins/wireguard/wireguard_noise.c @@ -751,8 +751,8 @@ noise_tai64n_now (uint8_t output[NOISE_TIMESTAMP_LEN]) unix_nanosec &= REJECT_INTERVAL_MASK; /* https://cr.yp.to/libtai/tai64.html */ - sec = htobe64 (0x400000000000000aULL + unix_sec); - nsec = htobe32 (unix_nanosec); + sec = clib_host_to_big_u64 (0x400000000000000aULL + unix_sec); + nsec = clib_host_to_big_u32 (unix_nanosec); /* memcpy to output buffer, assuming output could be unaligned. */ clib_memcpy (output, &sec, sizeof (sec)); diff --git a/src/tools/appimage/CMakeLists.txt b/src/tools/appimage/CMakeLists.txt index 1b83656dbf8..26ef77d1c91 100644 --- a/src/tools/appimage/CMakeLists.txt +++ b/src/tools/appimage/CMakeLists.txt @@ -18,7 +18,7 @@ if(VPP_BUILD_APPIMAGE) WORLD_READ WORLD_EXECUTE) install(FILES vpp.desktop DESTINATION .) install(FILES vpp.png DESTINATION .) - install(FILES vpp.svg DESTINATION share/icons/hicolor/scalable/vpp.svg) + install(FILES vpp.svg DESTINATION ${CMAKE_INSTALL_DATADIR}/icons/hicolor/scalable/vpp.svg) install(CODE "EXECUTE_PROCESS(COMMAND ln -s . ./usr WORKING_DIRECTORY ${CMAKE_INSTALL_PREFIX})") install(CODE "EXECUTE_PROCESS( diff --git a/src/tools/g2/pointsel.c b/src/tools/g2/pointsel.c index 59822377219..fae93365e3d 100644 --- a/src/tools/g2/pointsel.c +++ b/src/tools/g2/pointsel.c @@ -169,7 +169,7 @@ static void down_button(void) static void button_click_callback(GtkButton *item, gpointer data) { int i; - enum button_click click = (enum button_click)data; + enum button_click click = (enum button_click) (long int) data; switch (click) { case ALL_BUTTON: diff --git a/src/tools/g2/view1.c b/src/tools/g2/view1.c index 3902c0a2dc1..7a6ae714e3f 100644 --- a/src/tools/g2/view1.c +++ b/src/tools/g2/view1.c @@ -2329,21 +2329,22 @@ out: static void view1_button_click_callback(GtkButton *item, gpointer data) { - enum view1_button_click click = (enum view1_button_click) data; - event_t *ep; - ulonglong event_incdec; - ulonglong current_width; - ulonglong zoom_delta; + enum view1_button_click click = (enum view1_button_click) (long int) data; + event_t *ep; + ulonglong event_incdec; + ulonglong current_width; + ulonglong zoom_delta; - current_width = s_v1->maxvistime - s_v1->minvistime; - event_incdec = (current_width) / 3; + current_width = s_v1->maxvistime - s_v1->minvistime; + event_incdec = (current_width) / 3; - if (event_incdec == 0LL) - event_incdec = 1; + if (event_incdec == 0LL) + event_incdec = 1; - zoom_delta = (s_v1->maxvistime - s_v1->minvistime) / 6; + zoom_delta = (s_v1->maxvistime - s_v1->minvistime) / 6; - switch(click) { + switch (click) + { case TOP_BUTTON: /* First PID to top of window */ s_v1->first_pid_index = 0; diff --git a/src/tools/vppapigen/CMakeLists.txt b/src/tools/vppapigen/CMakeLists.txt index 04ebed54862..97a6d35f9b5 100644 --- a/src/tools/vppapigen/CMakeLists.txt +++ b/src/tools/vppapigen/CMakeLists.txt @@ -43,7 +43,7 @@ install( vppapigen_json.py generate_json.py DESTINATION - share/vpp + ${CMAKE_INSTALL_DATADIR}/vpp COMPONENT vpp-dev ) diff --git a/src/tools/vppapigen/vppapigen_c.py b/src/tools/vppapigen/vppapigen_c.py index fb7de0a023f..c2e1e7da7b7 100755 --- a/src/tools/vppapigen/vppapigen_c.py +++ b/src/tools/vppapigen/vppapigen_c.py @@ -365,7 +365,7 @@ class FromJSON: write(" char *p = cJSON_GetStringValue(item);\n") write(" size_t plen = strlen(p);\n") write( - " {msgvar} = cJSON_realloc({msgvar}, {msgsize} + plen, {msgsize});\n".format( + " {msgvar} = cJSON_realloc({msgvar}, {msgsize} + plen);\n".format( msgvar=msgvar, msgsize=msgsize ) ) @@ -434,7 +434,7 @@ class FromJSON: cJSON *array = cJSON_GetObjectItem(o, "{n}"); int size = cJSON_GetArraySize(array); {lfield} = size; - {realloc} = cJSON_realloc({realloc}, {msgsize} + sizeof({t}) * size, {msgsize}); + {realloc} = cJSON_realloc({realloc}, {msgsize} + sizeof({t}) * size); {t} *d = (void *){realloc} + {msgsize}; {msgsize} += sizeof({t}) * size; for (i = 0; i < size; i++) {{ @@ -461,12 +461,12 @@ class FromJSON: write( " {realloc} = cJSON_realloc({realloc}, {msgsize} + " - "vec_len(s), {msgsize});\n".format( + "vec_len(s));\n".format( msgvar=msgvar, msgsize=msgsize, realloc=realloc ) ) write( - " memcpy((void *){realloc} + {msgsize}, s, " + " clib_memcpy((void *){realloc} + {msgsize}, s, " "vec_len(s));\n".format(realloc=realloc, msgsize=msgsize) ) write(" {msgsize} += vec_len(s);\n".format(msgsize=msgsize)) @@ -1143,20 +1143,14 @@ ENDIAN_STRINGS = { } -def get_endian_string(o, type): +def get_endian_string(o, fieldtype): """Return proper endian string conversion function""" - try: - if o.to_network: - return ENDIAN_STRINGS[type].replace("net_to_host", "host_to_net") - except: - pass - return ENDIAN_STRINGS[type] + return ENDIAN_STRINGS[fieldtype] def endianfun_array(o): """Generate endian functions for arrays""" forloop = """\ - {comment} ASSERT((u32){length} <= (u32)VL_API_MAX_ARRAY_SIZE); for (i = 0; i < {length}; i++) {{ a->{name}[i] = {format}(a->{name}[i]); @@ -1165,31 +1159,26 @@ def endianfun_array(o): forloop_format = """\ for (i = 0; i < {length}; i++) {{ - {type}_endian(&a->{name}[i]); + {type}_endian(&a->{name}[i], to_net); }} """ - to_network_comment = "" - try: - if o.to_network: - to_network_comment = """/* - * Array fields processed first to handle variable length arrays and size - * field endian conversion in the proper order for to-network messages. - * Message fields have been sorted by type in the code generator, thus fields - * in this generated code may be converted in a different order than specified - * in the *.api file. - */""" - except: - pass - output = "" if o.fieldtype == "u8" or o.fieldtype == "string" or o.fieldtype == "bool": output += " /* a->{n} = a->{n} (no-op) */\n".format(n=o.fieldname) else: lfield = "a->" + o.lengthfield if o.lengthfield else o.length + if o.lengthfield: + output += ( + f" u32 count = to_net ? clib_host_to_net_u32(a->{o.lengthfield}) : " + f"a->{o.lengthfield};\n" + ) + lfield = "count" + else: + lfield = o.length + if o.fieldtype in ENDIAN_STRINGS: output += forloop.format( - comment=to_network_comment, length=lfield, format=get_endian_string(o, o.fieldtype), name=o.fieldname, @@ -1222,7 +1211,7 @@ def endianfun_obj(o): name=o.fieldname, format=get_endian_string(o, o.fieldtype) ) elif o.fieldtype.startswith("vl_api_"): - output += " {type}_endian(&a->{name});\n".format( + output += " {type}_endian(&a->{name}, to_net);\n".format( type=o.fieldtype, name=o.fieldname ) else: @@ -1254,19 +1243,12 @@ def endianfun(objs, modulename): output = output.format(module=modulename) signature = """\ -static inline void vl_api_{name}_t_endian (vl_api_{name}_t *a) +static inline void vl_api_{name}_t_endian (vl_api_{name}_t *a, bool to_net) {{ int i __attribute__((unused)); """ for t in objs: - # Outbound (to network) messages are identified by message nomenclature - # i.e. message names ending with these suffixes are 'to network' - if t.name.endswith("_reply") or t.name.endswith("_details"): - t.to_network = True - else: - t.to_network = False - if t.__class__.__name__ == "Enum" or t.__class__.__name__ == "EnumFlag": output += signature.format(name=t.name) if t.enumtype in ENDIAN_STRINGS: @@ -1300,15 +1282,7 @@ static inline void vl_api_{name}_t_endian (vl_api_{name}_t *a) output += signature.format(name=t.name) - # For outbound (to network) messages: - # some arrays have dynamic length -- iterate over - # them before changing endianness for the length field - # by making the Array types show up first - if t.to_network: - t.block.sort(key=lambda x: x.type) - for o in t.block: - o.to_network = t.to_network output += endianfun_obj(o) output += "}\n\n" @@ -1852,7 +1826,7 @@ api_{n} (cJSON *o) }} mp->_vl_msg_id = vac_get_msg_index(VL_API_{N}_CRC); - vl_api_{n}_t_endian(mp); + vl_api_{n}_t_endian(mp, 1); vac_write((char *)mp, len); cJSON_free(mp); @@ -1867,7 +1841,7 @@ api_{n} (cJSON *o) return 0; }} vl_api_{r}_t *rmp = (vl_api_{r}_t *)p; - vl_api_{r}_t_endian(rmp); + vl_api_{r}_t_endian(rmp, 0); return vl_api_{r}_t_tojson(rmp); }} @@ -1885,7 +1859,7 @@ api_{n} (cJSON *o) return 0; }} mp->_vl_msg_id = msg_id; - vl_api_{n}_t_endian(mp); + vl_api_{n}_t_endian(mp, 1); vac_write((char *)mp, len); cJSON_free(mp); @@ -1919,7 +1893,7 @@ api_{n} (cJSON *o) return 0; }} vl_api_{r}_t *rmp = (vl_api_{r}_t *)p; - vl_api_{r}_t_endian(rmp); + vl_api_{r}_t_endian(rmp, 0); cJSON_AddItemToArray(reply, vl_api_{r}_t_tojson(rmp)); }} }} @@ -1941,7 +1915,7 @@ api_{n} (cJSON *o) }} mp->_vl_msg_id = msg_id; - vl_api_{n}_t_endian(mp); + vl_api_{n}_t_endian(mp, 1); vac_write((char *)mp, len); cJSON_free(mp); @@ -1962,14 +1936,14 @@ api_{n} (cJSON *o) u16 msg_id = ntohs(*((u16 *)p)); if (msg_id == reply_msg_id) {{ vl_api_{r}_t *rmp = (vl_api_{r}_t *)p; - vl_api_{r}_t_endian(rmp); + vl_api_{r}_t_endian(rmp, 0); cJSON_AddItemToArray(reply, vl_api_{r}_t_tojson(rmp)); break; }} if (msg_id == details_msg_id) {{ vl_api_{d}_t *rmp = (vl_api_{d}_t *)p; - vl_api_{d}_t_endian(rmp); + vl_api_{d}_t_endian(rmp, 0); cJSON_AddItemToArray(reply, vl_api_{d}_t_tojson(rmp)); }} }} diff --git a/src/vat/ip_types.c b/src/vat/ip_types.c index abcd2eaf648..248205287a4 100644 --- a/src/vat/ip_types.c +++ b/src/vat/ip_types.c @@ -205,9 +205,9 @@ ip_address_family_to_link_type (ip_address_family_t af) return (VNET_LINK_IP4); } - void -ip_address_set (ip_address_t * dst, const void *src, u8 version) +ip_address_set (ip_address_t *dst, const void *src, + ip_address_family_t version) { ip_addr_version (dst) = version; diff --git a/src/vat2/vat2_helpers.h b/src/vat2/vat2_helpers.h index 7b197608a7b..d9ce2af6b35 100644 --- a/src/vat2/vat2_helpers.h +++ b/src/vat2/vat2_helpers.h @@ -29,7 +29,7 @@ vat2_control_ping (u32 context) vl_api_control_ping_t mp = {0}; mp._vl_msg_id = vac_get_msg_index(VL_API_CONTROL_PING_CRC); mp.context = context; - vl_api_control_ping_t_endian(&mp); + vl_api_control_ping_t_endian (&mp, 1 /* to network */); vac_write((char *)&mp, sizeof(mp)); } diff --git a/src/vcl/CMakeLists.txt b/src/vcl/CMakeLists.txt index 2f738f39d1a..c8835e771c1 100644 --- a/src/vcl/CMakeLists.txt +++ b/src/vcl/CMakeLists.txt @@ -11,11 +11,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -if(NOT "${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") - message(WARNING "-- vppcom is currently only support on Linux - disabled") - return() -endif() - ############################################################################## # vppcom shared library ############################################################################## @@ -40,6 +35,9 @@ if (LDP_HAS_GNU_SOURCE) add_compile_definitions(HAVE_GNU_SOURCE) endif(LDP_HAS_GNU_SOURCE) +if("${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD") + message("WARNING: vcl_ldpreload isn't supported on FreeBSD - disabled") +else() add_vpp_library(vcl_ldpreload SOURCES ldp_socket_wrapper.c @@ -48,6 +46,7 @@ add_vpp_library(vcl_ldpreload LINK_LIBRARIES vppinfra svm vlibmemoryclient rt pthread vppcom dl ) +endif() add_vpp_headers(vcl ldp.h diff --git a/src/vcl/vcl_private.h b/src/vcl/vcl_private.h index 41a11b7123f..b89052f96af 100644 --- a/src/vcl/vcl_private.h +++ b/src/vcl/vcl_private.h @@ -159,8 +159,8 @@ typedef struct vcl_session_ svm_fifo_t *ct_tx_fifo; vcl_session_msg_t *accept_evts_fifo; - u64 vpp_handle; - u64 parent_handle; + session_handle_t vpp_handle; + session_handle_t parent_handle; u32 listener_index; /**< index of parent listener (if any) */ int n_accepted_sessions; /**< sessions accepted by this listener */ vppcom_epoll_t vep; diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c index a0bbae170f8..a557093e897 100644 --- a/src/vcl/vppcom.c +++ b/src/vcl/vppcom.c @@ -1266,7 +1266,7 @@ vppcom_session_unbind (u32 session_handle) session->vpp_handle); vcl_evt (VCL_EVT_UNBIND, session); - session->vpp_handle = ~0; + session->vpp_handle = SESSION_INVALID_HANDLE; session->session_state = VCL_STATE_DISCONNECT; return VPPCOM_OK; @@ -1471,7 +1471,7 @@ vppcom_session_create (u8 proto, u8 is_nonblocking) session->session_type = proto; session->session_state = VCL_STATE_CLOSED; - session->vpp_handle = ~0; + session->vpp_handle = SESSION_INVALID_HANDLE; session->is_dgram = vcl_proto_is_dgram (proto); session->vpp_error = SESSION_E_NONE; @@ -2087,7 +2087,16 @@ read_again: ASSERT (rv >= 0); if (peek) - return rv; + { + /* Request new notifications if more data enqueued */ + if (rv < n || rv == svm_fifo_max_dequeue_cons (rx_fifo)) + { + if (is_ct) + svm_fifo_unset_event (s->rx_fifo); + svm_fifo_unset_event (rx_fifo); + } + return rv; + } n_read += rv; @@ -2818,7 +2827,7 @@ vppcom_epoll_create (void) vep_session->vep.vep_sh = ~0; vep_session->vep.next_sh = ~0; vep_session->vep.prev_sh = ~0; - vep_session->vpp_handle = ~0; + vep_session->vpp_handle = SESSION_INVALID_HANDLE; vcl_evt (VCL_EVT_EPOLL_CREATE, vep_session, vep_session->session_index); VDBG (0, "Created vep_idx %u", vep_session->session_index); diff --git a/src/vlib/CMakeLists.txt b/src/vlib/CMakeLists.txt index 7ec9b2050e9..3c354b764dd 100644 --- a/src/vlib/CMakeLists.txt +++ b/src/vlib/CMakeLists.txt @@ -55,6 +55,7 @@ install( # vlib shared library ############################################################################## +if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") set(PLATFORM_SOURCES linux/pci.c linux/vfio.c @@ -64,6 +65,11 @@ set(PLATFORM_SOURCES set(PLATFORM_HEADERS linux/vfio.h ) +elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD") +set(PLATFORM_SOURCES + freebsd/pci.c +) +endif() add_vpp_library(vlib SOURCES diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h index b3861a2f695..010289ce2be 100644 --- a/src/vlib/buffer_funcs.h +++ b/src/vlib/buffer_funcs.h @@ -777,7 +777,7 @@ vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers, const int queue_size = 128; vlib_buffer_pool_t *bp = 0; u8 buffer_pool_index = ~0; - u32 n_queue = 0, queue[queue_size + 4]; + u32 n_queue = 0, queue[queue_size + 8]; vlib_buffer_template_t bt = {}; #if defined(CLIB_HAVE_VEC128) vlib_buffer_t bpi_mask = {.buffer_pool_index = ~0 }; diff --git a/src/vlib/drop.c b/src/vlib/drop.c index 66eb98fb899..3fda1d9b3b6 100644 --- a/src/vlib/drop.c +++ b/src/vlib/drop.c @@ -93,10 +93,12 @@ format_error_trace (u8 * s, va_list * va) u32 i; error_node = vlib_get_node (vm, vlib_error_get_node (&vm->node_main, e[0])); - i = counter_index (vm, vlib_error_get_code (&vm->node_main, e[0])) + - error_node->error_heap_index; + i = counter_index (vm, vlib_error_get_code (&vm->node_main, e[0])); if (i != CLIB_U32_MAX) - s = format (s, "%v: %s", error_node->name, em->counters_heap[i].desc); + { + i += error_node->error_heap_index; + s = format (s, "%v: %s", error_node->name, em->counters_heap[i].desc); + } return s; } diff --git a/src/vlib/freebsd/pci.c b/src/vlib/freebsd/pci.c new file mode 100644 index 00000000000..a4e9eb2dda6 --- /dev/null +++ b/src/vlib/freebsd/pci.c @@ -0,0 +1,380 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Tom Jones <thj@freebsd.org> + * + * This software was developed by Tom Jones <thj@freebsd.org> under sponsorship + * from the FreeBSD Foundation. + * + */ + +#include <vlib/vlib.h> +#include <vlib/pci/pci.h> +#include <vlib/unix/unix.h> + +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/ioctl.h> +#include <sys/eventfd.h> + +#include <sys/pciio.h> + +#include <fcntl.h> +#include <dirent.h> +#include <net/if.h> + +extern vlib_pci_main_t freebsd_pci_main; + +uword +vlib_pci_get_private_data (vlib_main_t *vm, vlib_pci_dev_handle_t h) +{ + return 0; +} + +void +vlib_pci_set_private_data (vlib_main_t *vm, vlib_pci_dev_handle_t h, + uword private_data) +{ +} + +vlib_pci_addr_t * +vlib_pci_get_addr (vlib_main_t *vm, vlib_pci_dev_handle_t h) +{ + return NULL; +} + +u32 +vlib_pci_get_numa_node (vlib_main_t *vm, vlib_pci_dev_handle_t h) +{ + return 0; +} + +u32 +vlib_pci_get_num_msix_interrupts (vlib_main_t *vm, vlib_pci_dev_handle_t h) +{ + return 0; +} + +/* Call to allocate/initialize the pci subsystem. + This is not an init function so that users can explicitly enable + pci only when it's needed. */ +clib_error_t *pci_bus_init (vlib_main_t *vm); + +vlib_pci_device_info_t * +vlib_pci_get_device_info (vlib_main_t *vm, vlib_pci_addr_t *addr, + clib_error_t **error) +{ + /* Populate a vlib_pci_device_info_t from the given address */ + clib_error_t *err = NULL; + vlib_pci_device_info_t *di = NULL; + + int fd = -1; + struct pci_conf_io pci; + struct pci_conf match; + struct pci_match_conf pattern; + bzero (&match, sizeof (match)); + bzero (&pattern, sizeof (pattern)); + + pattern.pc_sel.pc_domain = addr->domain; + pattern.pc_sel.pc_bus = addr->bus; + pattern.pc_sel.pc_dev = addr->slot; + pattern.pc_sel.pc_func = addr->function; + pattern.flags = PCI_GETCONF_MATCH_DOMAIN | PCI_GETCONF_MATCH_BUS | + PCI_GETCONF_MATCH_DEV | PCI_GETCONF_MATCH_FUNC; + + pci.pat_buf_len = sizeof (pattern); + pci.num_patterns = 1; + pci.patterns = &pattern; + pci.match_buf_len = sizeof (match); + pci.num_matches = 1; + pci.matches = &match; + pci.offset = 0; + pci.generation = 0; + pci.status = 0; + + fd = open ("/dev/pci", 0); + if (fd == -1) + { + err = clib_error_return_unix (0, "open '/dev/pci'"); + goto error; + } + + if (ioctl (fd, PCIOCGETCONF, &pci) == -1) + { + err = clib_error_return_unix (0, "reading PCIOCGETCONF"); + goto error; + } + + di = clib_mem_alloc (sizeof (vlib_pci_device_info_t)); + clib_memset (di, 0, sizeof (vlib_pci_device_info_t)); + + di->addr.as_u32 = addr->as_u32; + di->numa_node = 0; /* TODO: Place holder until we have NUMA on FreeBSD */ + + di->device_class = match.pc_class; + di->vendor_id = match.pc_vendor; + di->device_id = match.pc_device; + di->revision = match.pc_revid; + + di->product_name = NULL; + di->vpd_r = 0; + di->vpd_w = 0; + di->driver_name = format (0, "%s", &match.pd_name); + di->iommu_group = -1; + + goto done; + +error: + vlib_pci_free_device_info (di); + di = NULL; +done: + if (error) + *error = err; + close (fd); + return di; +} + +clib_error_t *__attribute__ ((weak)) +vlib_pci_get_device_root_bus (vlib_pci_addr_t *addr, vlib_pci_addr_t *root_bus) +{ + return NULL; +} + +clib_error_t * +vlib_pci_bind_to_uio (vlib_main_t *vm, vlib_pci_addr_t *addr, + char *uio_drv_name, int force) +{ + clib_error_t *error = 0; + + if (error) + { + return error; + } + + if (strncmp ("auto", uio_drv_name, 5) == 0) + { + /* TODO: We should confirm that nic_uio is loaded and return an error. */ + uio_drv_name = "nic_uio"; + } + return error; +} + +clib_error_t * +vlib_pci_register_intx_handler (vlib_main_t *vm, vlib_pci_dev_handle_t h, + pci_intx_handler_function_t *intx_handler) +{ + return NULL; +} + +clib_error_t * +vlib_pci_unregister_intx_handler (vlib_main_t *vm, vlib_pci_dev_handle_t h) +{ + return NULL; +} + +clib_error_t * +vlib_pci_register_msix_handler (vlib_main_t *vm, vlib_pci_dev_handle_t h, + u32 start, u32 count, + pci_msix_handler_function_t *msix_handler) +{ + return NULL; +} + +clib_error_t * +vlib_pci_unregister_msix_handler (vlib_main_t *vm, vlib_pci_dev_handle_t h, + u32 start, u32 count) +{ + return NULL; +} + +clib_error_t * +vlib_pci_enable_msix_irq (vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 start, + u16 count) +{ + return NULL; +} + +uword +vlib_pci_get_msix_file_index (vlib_main_t *vm, vlib_pci_dev_handle_t h, + u16 index) +{ + return 0; +} + +clib_error_t * +vlib_pci_disable_msix_irq (vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 start, + u16 count) +{ + return NULL; +} + +/* Configuration space read/write. */ +clib_error_t * +vlib_pci_read_write_config (vlib_main_t *vm, vlib_pci_dev_handle_t h, + vlib_read_or_write_t read_or_write, uword address, + void *data, u32 n_bytes) +{ + return NULL; +} + +clib_error_t * +vlib_pci_map_region (vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource, + void **result) +{ + return NULL; +} + +clib_error_t * +vlib_pci_map_region_fixed (vlib_main_t *vm, vlib_pci_dev_handle_t h, + u32 resource, u8 *addr, void **result) +{ + return NULL; +} + +clib_error_t * +vlib_pci_io_region (vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource) +{ + return NULL; +} + +clib_error_t * +vlib_pci_read_write_io (vlib_main_t *vm, vlib_pci_dev_handle_t h, + vlib_read_or_write_t read_or_write, uword offset, + void *data, u32 length) +{ + return NULL; +} + +clib_error_t * +vlib_pci_map_dma (vlib_main_t *vm, vlib_pci_dev_handle_t h, void *ptr) +{ + return NULL; +} + +int +vlib_pci_supports_virtual_addr_dma (vlib_main_t *vm, vlib_pci_dev_handle_t h) +{ + return 0; +} + +clib_error_t * +vlib_pci_device_open (vlib_main_t *vm, vlib_pci_addr_t *addr, + pci_device_id_t ids[], vlib_pci_dev_handle_t *handle) +{ + return NULL; +} + +void +vlib_pci_device_close (vlib_main_t *vm, vlib_pci_dev_handle_t h) +{ +} + +void +init_device_from_registered (vlib_main_t *vm, vlib_pci_device_info_t *di) +{ +} + +static int +pci_addr_cmp (void *v1, void *v2) +{ + vlib_pci_addr_t *a1 = v1; + vlib_pci_addr_t *a2 = v2; + + if (a1->domain > a2->domain) + return 1; + if (a1->domain < a2->domain) + return -1; + if (a1->bus > a2->bus) + return 1; + if (a1->bus < a2->bus) + return -1; + if (a1->slot > a2->slot) + return 1; + if (a1->slot < a2->slot) + return -1; + if (a1->function > a2->function) + return 1; + if (a1->function < a2->function) + return -1; + return 0; +} + +vlib_pci_addr_t * +vlib_pci_get_all_dev_addrs () +{ + vlib_pci_addr_t *addrs = 0; + + int fd = -1; + struct pci_conf_io pci; + struct pci_conf matches[32]; + bzero (matches, sizeof (matches)); + + pci.pat_buf_len = 0; + pci.num_patterns = 0; + pci.patterns = NULL; + pci.match_buf_len = sizeof (matches); + pci.num_matches = 32; + pci.matches = (struct pci_conf *) &matches; + pci.offset = 0; + pci.generation = 0; + pci.status = 0; + + fd = open ("/dev/pci", 0); + if (fd == -1) + { + clib_error_return_unix (0, "opening /dev/pci"); + return (NULL); + } + + if (ioctl (fd, PCIOCGETCONF, &pci) == -1) + { + clib_error_return_unix (0, "reading pci config"); + close (fd); + return (NULL); + } + + for (int i = 0; i < pci.num_matches; i++) + { + struct pci_conf *m = &pci.matches[i]; + vlib_pci_addr_t addr; + + addr.domain = m->pc_sel.pc_domain; + addr.bus = m->pc_sel.pc_bus; + addr.slot = m->pc_sel.pc_dev; + addr.function = m->pc_sel.pc_func; + + vec_add1 (addrs, addr); + } + + vec_sort_with_function (addrs, pci_addr_cmp); + close (fd); + + return addrs; +} + +clib_error_t * +freebsd_pci_init (vlib_main_t *vm) +{ + vlib_pci_main_t *pm = &pci_main; + vlib_pci_addr_t *addr = 0, *addrs; + + pm->vlib_main = vm; + + ASSERT (sizeof (vlib_pci_addr_t) == sizeof (u32)); + + addrs = vlib_pci_get_all_dev_addrs (); + vec_foreach (addr, addrs) + { + vlib_pci_device_info_t *d; + if ((d = vlib_pci_get_device_info (vm, addr, 0))) + { + init_device_from_registered (vm, d); + vlib_pci_free_device_info (d); + } + } + + return 0; +} + +VLIB_INIT_FUNCTION (freebsd_pci_init) = { + .runs_after = VLIB_INITS ("unix_input_init"), +}; diff --git a/src/vlib/linux/pci.c b/src/vlib/linux/pci.c index 08923bebcdf..29ca3d97523 100644 --- a/src/vlib/linux/pci.c +++ b/src/vlib/linux/pci.c @@ -55,6 +55,7 @@ #include <linux/ethtool.h> #include <linux/sockios.h> #include <linux/vfio.h> +#include <limits.h> #include <sys/eventfd.h> #define SYSFS_DEVICES_PCI "/sys/devices/pci" @@ -1560,14 +1561,17 @@ linux_pci_init (vlib_main_t * vm) ASSERT (sizeof (vlib_pci_addr_t) == sizeof (u32)); - addrs = vlib_pci_get_all_dev_addrs (); - vec_foreach (addr, addrs) + if (pm->pci_device_registrations) { - vlib_pci_device_info_t *d; - if ((d = vlib_pci_get_device_info (vm, addr, 0))) + addrs = vlib_pci_get_all_dev_addrs (); + vec_foreach (addr, addrs) { - init_device_from_registered (vm, d); - vlib_pci_free_device_info (d); + vlib_pci_device_info_t *d; + if ((d = vlib_pci_get_device_info (vm, addr, 0))) + { + init_device_from_registered (vm, d); + vlib_pci_free_device_info (d); + } } } diff --git a/src/vlib/pci/pci.c b/src/vlib/pci/pci.c index cd580f04176..7284e6675fd 100644 --- a/src/vlib/pci/pci.c +++ b/src/vlib/pci/pci.c @@ -47,8 +47,10 @@ #include <dirent.h> #include <sys/ioctl.h> #include <net/if.h> +#ifdef __linux__ #include <linux/ethtool.h> #include <linux/sockios.h> +#endif /* __linux__ */ vlib_pci_main_t pci_main; @@ -117,7 +119,7 @@ vlib_pci_intr_enable (vlib_main_t *vm, vlib_pci_dev_handle_t h) { const vlib_pci_config_reg_command_t cmd = { .intx_disable = 1 }; clib_error_t *err; - int already_set; + int already_set = 0; err = _vlib_pci_config_set_control_bit (vm, h, cmd.as_u16, 0, &already_set); log_debug (h, "interrupt%senabled", already_set ? " " : " already "); @@ -129,7 +131,7 @@ vlib_pci_intr_disable (vlib_main_t *vm, vlib_pci_dev_handle_t h) { const vlib_pci_config_reg_command_t cmd = { .intx_disable = 1 }; clib_error_t *err; - int already_set; + int already_set = 0; err = _vlib_pci_config_set_control_bit (vm, h, cmd.as_u16, 1, &already_set); log_debug (h, "interrupt%sdisabled", already_set ? " " : " already "); @@ -141,7 +143,7 @@ vlib_pci_bus_master_enable (vlib_main_t *vm, vlib_pci_dev_handle_t h) { const vlib_pci_config_reg_command_t cmd = { .bus_master = 1 }; clib_error_t *err; - int already_set; + int already_set = 0; err = _vlib_pci_config_set_control_bit (vm, h, cmd.as_u16, 1, &already_set); log_debug (h, "bus-master%senabled", already_set ? " " : " already "); @@ -153,7 +155,7 @@ vlib_pci_bus_master_disable (vlib_main_t *vm, vlib_pci_dev_handle_t h) { const vlib_pci_config_reg_command_t cmd = { .bus_master = 1 }; clib_error_t *err; - int already_set; + int already_set = 0; err = _vlib_pci_config_set_control_bit (vm, h, cmd.as_u16, 0, &already_set); log_debug (h, "bus-master%sdisabled", already_set ? " " : " already "); diff --git a/src/vlib/threads.c b/src/vlib/threads.c index 713e1927d1f..87b71adc2bc 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -16,6 +16,9 @@ #include <signal.h> #include <math.h> +#ifdef __FreeBSD__ +#include <pthread_np.h> +#endif /* __FreeBSD__ */ #include <vppinfra/format.h> #include <vppinfra/time_range.h> #include <vppinfra/interrupt.h> @@ -236,7 +239,11 @@ vlib_thread_init (vlib_main_t * vm) w->thread_mheap = clib_mem_get_heap (); w->thread_stack = vlib_thread_stacks[0]; w->cpu_id = tm->main_lcore; +#ifdef __FreeBSD__ + w->lwp = pthread_getthreadid_np (); +#else w->lwp = syscall (SYS_gettid); +#endif /* __FreeBSD__ */ w->thread_id = pthread_self (); tm->n_vlib_mains = 1; @@ -401,7 +408,11 @@ vlib_worker_thread_bootstrap_fn (void *arg) { vlib_worker_thread_t *w = arg; +#ifdef __FreeBSD__ + w->lwp = pthread_getthreadid_np (); +#else w->lwp = syscall (SYS_gettid); +#endif /* __FreeBSD__ */ w->thread_id = pthread_self (); __os_thread_index = w - vlib_worker_threads; diff --git a/src/vlib/unix/main.c b/src/vlib/unix/main.c index 38eef4125a9..ee28ca8f1aa 100644 --- a/src/vlib/unix/main.c +++ b/src/vlib/unix/main.c @@ -39,6 +39,7 @@ #include <vlib/vlib.h> #include <vlib/unix/unix.h> #include <vlib/unix/plugin.h> +#include <vppinfra/unix.h> #include <limits.h> #include <signal.h> @@ -612,22 +613,23 @@ vlib_unix_main (int argc, char *argv[]) vlib_main_t *vm = vlib_get_first_main (); /* one and only time for this! */ unformat_input_t input; clib_error_t *e; - char buffer[PATH_MAX]; int i; vec_validate_aligned (vgm->vlib_mains, 0, CLIB_CACHE_LINE_BYTES); - if ((i = readlink ("/proc/self/exe", buffer, sizeof (buffer) - 1)) > 0) + vgm->exec_path = (char *) os_get_exec_path (); + + if (vgm->exec_path) { - int j; - buffer[i] = 0; - vgm->exec_path = vec_new (char, i + 1); - clib_memcpy_fast (vgm->exec_path, buffer, i + 1); - for (j = i - 1; j > 0; j--) - if (buffer[j - 1] == '/') + for (i = vec_len (vgm->exec_path) - 1; i > 0; i--) + if (vgm->exec_path[i - 1] == '/') break; - vgm->name = vec_new (char, i - j + 1); - clib_memcpy_fast (vgm->name, buffer + j, i - j + 1); + + vgm->name = 0; + + vec_add (vgm->name, vgm->exec_path + i, vec_len (vgm->exec_path) - i); + vec_add1 (vgm->exec_path, 0); + vec_add1 (vgm->name, 0); } else vgm->exec_path = vgm->name = argv[0]; diff --git a/src/vlib/unix/plugin.c b/src/vlib/unix/plugin.c index fd3a050b944..5cac9abc8fe 100644 --- a/src/vlib/unix/plugin.c +++ b/src/vlib/unix/plugin.c @@ -306,8 +306,12 @@ process_reg: } vec_free (version_required); +#if defined(RTLD_DEEPBIND) handle = dlopen ((char *) pi->filename, RTLD_LAZY | (reg->deep_bind ? RTLD_DEEPBIND : 0)); +#else + handle = dlopen ((char *) pi->filename, RTLD_LAZY); +#endif if (handle == 0) { diff --git a/src/vlibapi/api_common.h b/src/vlibapi/api_common.h index 62a8d4c62d8..c09334136c0 100644 --- a/src/vlibapi/api_common.h +++ b/src/vlibapi/api_common.h @@ -235,8 +235,8 @@ typedef struct /** Message convert function vector */ void *(*fromjson_handler) (cJSON *, int *); - /** Message endian handler vector */ - void (*endian_handler) (void *); + /** Message endian handler vector. */ + void (*endian_handler) (void *, bool to_net); /** Message calc size function vector */ uword (*calc_size_func) (void *); diff --git a/src/vlibapi/api_helper_macros.h b/src/vlibapi/api_helper_macros.h index 9c93d33934b..0380692b80e 100644 --- a/src/vlibapi/api_helper_macros.h +++ b/src/vlibapi/api_helper_macros.h @@ -29,9 +29,9 @@ #define _NATIVE_TO_NETWORK(t, rmp) \ api_main_t *am = vlibapi_get_main (); \ - void (*endian_fp) (void *); \ + void (*endian_fp) (void *, bool); \ endian_fp = am->msg_data[t + (REPLY_MSG_ID_BASE)].endian_handler; \ - (*endian_fp) (rmp); + (*endian_fp) (rmp, 1 /* to network */); #define REPLY_MACRO(msg) \ do \ diff --git a/src/vlibapi/api_shared.c b/src/vlibapi/api_shared.c index 7de1906f17a..79064b292c9 100644 --- a/src/vlibapi/api_shared.c +++ b/src/vlibapi/api_shared.c @@ -230,7 +230,7 @@ vl_msg_api_trace_write_one (api_main_t *am, u8 *msg, FILE *fp) if (m && m->endian_handler) { - m->endian_handler (tmpmem); + m->endian_handler (tmpmem, 1); } if (m && m->tojson_handler) @@ -561,7 +561,7 @@ msg_handler_internal (api_main_t *am, void *the_msg, uword msg_len, } if (m->is_autoendian) - m->endian_handler (the_msg); + m->endian_handler (the_msg, 0); if (PREDICT_FALSE (vec_len (am->perf_counter_cbs) != 0)) clib_call_callbacks (am->perf_counter_cbs, am, id, diff --git a/src/vlibmemory/memclnt_api.c b/src/vlibmemory/memclnt_api.c index d4106b10559..b0b0c72eae0 100644 --- a/src/vlibmemory/memclnt_api.c +++ b/src/vlibmemory/memclnt_api.c @@ -197,6 +197,7 @@ vlib_api_init (void) cJSON_Hooks cjson_hooks = { .malloc_fn = clib_mem_alloc, .free_fn = clib_mem_free, + .realloc_fn = clib_mem_realloc, }; cJSON_InitHooks (&cjson_hooks); diff --git a/src/vlibmemory/memory_api.c b/src/vlibmemory/memory_api.c index 39c6b0fd15b..57373b91e31 100644 --- a/src/vlibmemory/memory_api.c +++ b/src/vlibmemory/memory_api.c @@ -823,9 +823,9 @@ vl_mem_api_handler_with_vm_node (api_main_t *am, svm_region_t *vlib_rp, if (m->is_autoendian) { - void (*endian_fp) (void *); + void (*endian_fp) (void *, bool); endian_fp = am->msg_data[id].endian_handler; - (*endian_fp) (the_msg); + (*endian_fp) (the_msg, 0); } if (PREDICT_FALSE (vec_len (am->perf_counter_cbs) != 0)) clib_call_callbacks (am->perf_counter_cbs, am, id, 0 /* before */); diff --git a/src/vlibmemory/socket_client.c b/src/vlibmemory/socket_client.c index 5fa19c6a9c0..ad28136dc07 100644 --- a/src/vlibmemory/socket_client.c +++ b/src/vlibmemory/socket_client.c @@ -22,6 +22,14 @@ #define _GNU_SOURCE #include <sys/socket.h> +#ifdef __FreeBSD__ +#define _WANT_UCRED +#include <sys/types.h> +#include <sys/param.h> +#include <sys/ucred.h> +#include <sys/un.h> +#endif /* __FreeBSD__ */ + #include <svm/ssvm.h> #include <vlibmemory/socket_client.h> #include <vlibmemory/memory_client.h> @@ -278,7 +286,11 @@ vl_sock_api_recv_fd_msg_internal (socket_client_main_t * scm, int fds[], struct msghdr mh = { 0 }; struct iovec iov[1]; ssize_t size = 0; +#ifdef __linux__ struct ucred *cr = 0; +#elif __FreeBSD__ + struct cmsgcred *cr = 0; +#endif /* __linux__ */ struct cmsghdr *cmsg; pid_t pid __attribute__ ((unused)); uid_t uid __attribute__ ((unused)); @@ -318,6 +330,7 @@ vl_sock_api_recv_fd_msg_internal (socket_client_main_t * scm, int fds[], { if (cmsg->cmsg_level == SOL_SOCKET) { +#ifdef __linux__ if (cmsg->cmsg_type == SCM_CREDENTIALS) { cr = (struct ucred *) CMSG_DATA (cmsg); @@ -325,6 +338,15 @@ vl_sock_api_recv_fd_msg_internal (socket_client_main_t * scm, int fds[], gid = cr->gid; pid = cr->pid; } +#elif __FreeBSD__ + if (cmsg->cmsg_type == SCM_CREDS) + { + cr = (struct cmsgcred *) CMSG_DATA (cmsg); + uid = cr->cmcred_uid; + gid = cr->cmcred_gid; + pid = cr->cmcred_pid; + } +#endif /* __linux__ */ else if (cmsg->cmsg_type == SCM_RIGHTS) { clib_memcpy_fast (fds, CMSG_DATA (cmsg), sizeof (int) * n_fds); diff --git a/src/vlibmemory/vlib_api_cli.c b/src/vlibmemory/vlib_api_cli.c index 4492f5af980..6ae81cd13df 100644 --- a/src/vlibmemory/vlib_api_cli.c +++ b/src/vlibmemory/vlib_api_cli.c @@ -554,7 +554,7 @@ vl_msg_api_process_file (vlib_main_t * vm, u8 * filename, } if (m) { - m->endian_handler (tmpbuf + sizeof (uword)); + m->endian_handler (tmpbuf + sizeof (uword), 1 /* to network */); } } @@ -674,7 +674,7 @@ vl_msg_print_trace (u8 *msg, void *ctx) clib_memcpy_fast (tmpbuf, msg, msg_length); msg = tmpbuf; - m->endian_handler (tmpbuf); + m->endian_handler (tmpbuf, 0 /* from network */); } vlib_cli_output (a->vm, "%U\n", @@ -824,7 +824,7 @@ vl_msg_exec_json_command (vlib_main_t *vm, cJSON *o) } if (clib_arch_is_little_endian) - m->endian_handler (msg); + m->endian_handler (msg, 1 /* to network */); if (!m->handler) { diff --git a/src/vnet/CMakeLists.txt b/src/vnet/CMakeLists.txt index 973a6c9434b..ad5f44846cc 100644 --- a/src/vnet/CMakeLists.txt +++ b/src/vnet/CMakeLists.txt @@ -813,24 +813,6 @@ list(APPEND VNET_API_FILES ) ############################################################################## -# mpls segment routing -############################################################################## - -list(APPEND VNET_SOURCES - srmpls/sr_mpls_policy.c - srmpls/sr_mpls_steering.c - srmpls/sr_mpls_api.c -) - -list(APPEND VNET_HEADERS - srmpls/sr_mpls.h -) - -list(APPEND VNET_API_FILES - srmpls/sr_mpls.api -) - -############################################################################## # IPFIX / netflow v10 ############################################################################## list(APPEND VNET_SOURCES @@ -895,23 +877,6 @@ list(APPEND VNET_HEADERS ) ############################################################################## -# lawful intercept -############################################################################## - -list(APPEND VNET_SOURCES - lawful-intercept/lawful_intercept.c - lawful-intercept/node.c -) - -list(APPEND VNET_MULTIARCH_SOURCES - lawful-intercept/node.c -) - -list(APPEND VNET_HEADERS - lawful-intercept/lawful_intercept.h -) - -############################################################################## # SPAN (port mirroring) ############################################################################## @@ -1477,7 +1442,6 @@ add_vat_test_library(vnet ip/ip_test.c arp/arp_test.c ip6-nd/ip6_nd_test.c - srmpls/sr_mpls_test.c session/session_test.c l2/l2_test.c ipsec/ipsec_test.c diff --git a/src/vnet/adj/adj_glean.c b/src/vnet/adj/adj_glean.c index 0313407b6f1..ceece0d74ed 100644 --- a/src/vnet/adj/adj_glean.c +++ b/src/vnet/adj/adj_glean.c @@ -45,7 +45,7 @@ adj_glean_db_lookup (fib_protocol_t proto, { uword *p; - if (vec_len(adj_gleans[proto]) <= sw_if_index) + if ((proto >= FIB_PROTOCOL_IP_MAX) || vec_len(adj_gleans[proto]) <= sw_if_index) return (ADJ_INDEX_INVALID); p = hash_get_mem (adj_gleans[proto][sw_if_index], nh_addr); @@ -66,6 +66,7 @@ adj_glean_db_insert (fib_protocol_t proto, vlib_worker_thread_barrier_sync(vm); + ASSERT(proto < FIB_PROTOCOL_IP_MAX); vec_validate(adj_gleans[proto], sw_if_index); if (NULL == adj_gleans[proto][sw_if_index]) @@ -195,6 +196,7 @@ adj_glean_walk_proto (fib_protocol_t proto, adj_index_t ai, *aip, *ais = NULL; ip46_address_t *conn; + ASSERT(proto < FIB_PROTOCOL_IP_MAX); if (vec_len(adj_gleans[proto]) <= sw_if_index || NULL == adj_gleans[proto][sw_if_index]) return; @@ -212,7 +214,7 @@ adj_glean_walk_proto (fib_protocol_t proto, vec_foreach(aip, ais) { if (ADJ_WALK_RC_STOP == cb(*aip, data)) - return; + break; } vec_free(ais); } @@ -244,6 +246,7 @@ adj_glean_get (fib_protocol_t proto, ip46_address_t *conn; adj_index_t ai; + ASSERT(proto < FIB_PROTOCOL_IP_MAX); if (vec_len(adj_gleans[proto]) <= sw_if_index || NULL == adj_gleans[proto][sw_if_index]) return (ADJ_INDEX_INVALID); @@ -265,6 +268,7 @@ adj_glean_get_src (fib_protocol_t proto, const ip_adjacency_t *adj; adj_index_t ai; + ASSERT(proto < FIB_PROTOCOL_IP_MAX); if (vec_len(adj_gleans[proto]) <= sw_if_index || NULL == adj_gleans[proto][sw_if_index]) return (NULL); diff --git a/src/vnet/adj/rewrite.h b/src/vnet/adj/rewrite.h index 5cb90e47318..06b1b00882e 100644 --- a/src/vnet/adj/rewrite.h +++ b/src/vnet/adj/rewrite.h @@ -147,8 +147,8 @@ vnet_rewrite_set_data_internal (vnet_rewrite_header_t * rw, int max_size, void *data, int data_bytes) { /* Sanity check values carefully for this clib_memset operation */ - ASSERT ((max_size > 0) && (max_size < VNET_REWRITE_TOTAL_BYTES)); - ASSERT ((data_bytes >= 0) && (data_bytes < max_size)); + ASSERT ((max_size > 0) && (max_size <= VNET_REWRITE_TOTAL_BYTES)); + ASSERT ((data_bytes >= 0) && (data_bytes <= max_size)); rw->data_bytes = data_bytes; clib_memcpy_fast (rw->data, data, data_bytes); diff --git a/src/vnet/arp/arp.c b/src/vnet/arp/arp.c index cacdd71e381..43b2a93a7b3 100644 --- a/src/vnet/arp/arp.c +++ b/src/vnet/arp/arp.c @@ -423,6 +423,10 @@ arp_reply (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) } + dst_fei = ip4_fib_table_lookup (ip4_fib_get (fib_index0), + &arp0->ip4_over_ethernet[1].ip4, 32); + conn_sw_if_index0 = fib_entry_get_any_resolving_interface (dst_fei); + { /* * we're looking for FIB entries that indicate the source @@ -509,24 +513,20 @@ arp_reply (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) while (!attached && !fib_entry_is_sourced (src_fei, FIB_SOURCE_DEFAULT_ROUTE)); - if (!attached) + if (!attached && + !arp_unnumbered (p0, sw_if_index0, conn_sw_if_index0)) { /* - * the matching route is a not attached, i.e. it was - * added as a result of routing, rather than interface/ARP - * configuration. If the matching route is not a host route - * (i.e. a /32) + * the matching route is a not attached and not unnumbered, + * i.e. it was added as a result of routing, rather than + * interface/ARP configuration. If the matching route is not + * a host route (i.e. a /32) */ error0 = ARP_ERROR_L3_SRC_ADDRESS_NOT_LOCAL; goto drop; } } - dst_fei = ip4_fib_table_lookup (ip4_fib_get (fib_index0), - &arp0->ip4_over_ethernet[1].ip4, - 32); - conn_sw_if_index0 = fib_entry_get_any_resolving_interface (dst_fei); - switch (arp_dst_fib_check (dst_fei, &dst_flags)) { case ARP_DST_FIB_ADJ: @@ -619,9 +619,9 @@ arp_reply (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) sw_if_index0 != fib_entry_get_resolving_interface (src_fei)) { /* - * The interface the ARP is sent to or was received on is not the - * interface on which the covering prefix is configured. - * Maybe this is a case for unnumbered. + * The interface the ARP is sent to or was received on is + * not the interface on which the covering prefix is + * configured. Maybe this is a case for unnumbered. */ if (!arp_unnumbered (p0, sw_if_index0, conn_sw_if_index0)) { @@ -636,8 +636,7 @@ arp_reply (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) goto drop; } - next0 = arp_mk_reply (vnm, p0, sw_if_index0, - if_addr0, arp0, eth_rx); + next0 = arp_mk_reply (vnm, p0, sw_if_index0, if_addr0, arp0, eth_rx); /* We are going to reply to this request, so, in the absence of errors, learn the sender */ diff --git a/src/vnet/classify/vnet_classify.c b/src/vnet/classify/vnet_classify.c index fc694492f4e..77c1c81f9c4 100644 --- a/src/vnet/classify/vnet_classify.c +++ b/src/vnet/classify/vnet_classify.c @@ -1331,12 +1331,11 @@ unformat_classify_mask (unformat_input_t * input, va_list * args) return 0; } -#define foreach_l2_input_next \ -_(drop, DROP) \ -_(ethernet, ETHERNET_INPUT) \ -_(ip4, IP4_INPUT) \ -_(ip6, IP6_INPUT) \ -_(li, LI) +#define foreach_l2_input_next \ + _ (drop, DROP) \ + _ (ethernet, ETHERNET_INPUT) \ + _ (ip4, IP4_INPUT) \ + _ (ip6, IP6_INPUT) uword unformat_l2_input_next_index (unformat_input_t * input, va_list * args) diff --git a/src/vnet/dev/format.c b/src/vnet/dev/format.c index ed83a0eba95..f599c0f8b85 100644 --- a/src/vnet/dev/format.c +++ b/src/vnet/dev/format.c @@ -101,7 +101,7 @@ format_vnet_dev_port_info (u8 *s, va_list *args) u32 indent = format_get_indent (s); s = format (s, "Hardware Address is %U", format_vnet_dev_hw_addr, - &port->attr.hw_addr); + &port->primary_hw_addr); s = format (s, ", %u RX queues (max %u), %u TX queues (max %u)", pool_elts (port->rx_queues), port->attr.max_rx_queues, pool_elts (port->tx_queues), port->attr.max_tx_queues); diff --git a/src/vnet/ethernet/node.c b/src/vnet/ethernet/node.c index e2558eeca41..03cbdde1c2b 100644 --- a/src/vnet/ethernet/node.c +++ b/src/vnet/ethernet/node.c @@ -982,8 +982,31 @@ eth_input_process_frame (vlib_main_t * vm, vlib_node_runtime_t * node, else { for (int j = 0; j < 16; j++) - if (next[j] == 0) - slowpath_indices[n_slowpath++] = i + j; + { + if (next[j] == 0) + slowpath_indices[n_slowpath++] = i + j; + else if (dmac_check && main_is_l3 && dmacs_bad[i + j]) + { + next[j] = 0; + slowpath_indices[n_slowpath++] = i + j; + } + } + } + } + else + { + if (dmac_check && main_is_l3) + { + u8x16 dmac_bad = u8x16_load_unaligned (&dmacs_bad[i]); + if (!u8x16_is_all_zero (dmac_bad)) + { + for (int j = 0; j < 16; j++) + if (dmacs_bad[i + j]) + { + next[j] = 0; + slowpath_indices[n_slowpath++] = i + j; + } + } } } @@ -994,7 +1017,12 @@ eth_input_process_frame (vlib_main_t * vm, vlib_node_runtime_t * node, continue; } #endif - if (main_is_l3 && etype[0] == et_ip4) + if (dmac_check && main_is_l3 && dmacs_bad[i]) + { + next[0] = 0; + slowpath_indices[n_slowpath++] = i; + } + else if (main_is_l3 && etype[0] == et_ip4) next[0] = next_ip4; else if (main_is_l3 && etype[0] == et_ip6) next[0] = next_ip6; @@ -1052,7 +1080,7 @@ eth_input_process_frame (vlib_main_t * vm, vlib_node_runtime_t * node, } else { - /* untagged packet with not well known etyertype */ + /* untagged packet with not well known ethertype */ if (last_unknown_etype != etype) { last_unknown_etype = etype; diff --git a/src/vnet/fib/fib_table.c b/src/vnet/fib/fib_table.c index 85b17870eec..b2a32d0da56 100644 --- a/src/vnet/fib/fib_table.c +++ b/src/vnet/fib/fib_table.c @@ -25,6 +25,13 @@ const static char * fib_table_flags_strings[] = FIB_TABLE_ATTRIBUTES; +/* + * Default names for IP4, IP6, and MPLS FIB table index 0. + * Nominally like "ipv6-VRF:0", but this will override that name if set + * in a config section of the startup.conf file. + */ +char *fib_table_default_names[FIB_PROTOCOL_MAX]; + fib_table_t * fib_table_get (fib_node_index_t index, fib_protocol_t proto) @@ -1153,21 +1160,29 @@ fib_table_find_or_create_and_lock_i (fib_protocol_t proto, fib_table = fib_table_get(fi, proto); - if (NULL == fib_table->ft_desc) + if (fib_table->ft_desc) + return fi; + + if (name && name[0]) { - if (name && name[0]) - { - fib_table->ft_desc = format(NULL, "%s", name); - } - else - { - fib_table->ft_desc = format(NULL, "%U-VRF:%d", - format_fib_protocol, proto, - table_id); - } + fib_table->ft_desc = format(NULL, "%s", name); + return fi; } - return (fi); + if (table_id == 0) + { + char *default_name = fib_table_default_names[proto]; + if (default_name && default_name[0]) + { + fib_table->ft_desc = format(NULL, "%s", default_name); + return fi; + } + } + + fib_table->ft_desc = format(NULL, "%U-VRF:%d", + format_fib_protocol, proto, + table_id); + return fi; } u32 diff --git a/src/vnet/fib/fib_table.h b/src/vnet/fib/fib_table.h index 11137e173cf..0eaaa67eea2 100644 --- a/src/vnet/fib/fib_table.h +++ b/src/vnet/fib/fib_table.h @@ -122,6 +122,15 @@ typedef struct fib_table_t_ u8* ft_desc; } fib_table_t; + +/** + * @brief + * Default names for IP4, IP6, and MPLS FIB table index 0. + * Nominally like "ipv4-VRF:0", but this will override that name if set + * in a config section of the startup.conf file. + */ +extern char *fib_table_default_names[FIB_PROTOCOL_MAX]; + /** * @brief * Format the description/name of the table diff --git a/src/vnet/fib/ip4_fib.c b/src/vnet/fib/ip4_fib.c index 4211f8785ca..0eff8d0d485 100644 --- a/src/vnet/fib/ip4_fib.c +++ b/src/vnet/fib/ip4_fib.c @@ -626,3 +626,24 @@ VLIB_CLI_COMMAND (ip4_show_fib_command, static) = { .short_help = "show ip fib [summary] [table <table-id>] [index <fib-id>] [<ip4-addr>[/<mask>]] [mtrie] [detail]", .function = ip4_show_fib, }; + +static clib_error_t * +ip_config (vlib_main_t * vm, unformat_input_t * input) +{ + char *default_name = 0; + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "default-table-name %s", &default_name)) + ; + else + return clib_error_return (0, "unknown input '%U'", + format_unformat_error, input); + } + + fib_table_default_names[FIB_PROTOCOL_IP4] = default_name; + + return 0; +} + +VLIB_EARLY_CONFIG_FUNCTION (ip_config, "ip"); diff --git a/src/vnet/fib/ip6_fib.c b/src/vnet/fib/ip6_fib.c index c40491cc997..d37b77e08a4 100644 --- a/src/vnet/fib/ip6_fib.c +++ b/src/vnet/fib/ip6_fib.c @@ -873,6 +873,7 @@ ip6_config (vlib_main_t * vm, unformat_input_t * input) { uword heapsize = 0; u32 nbuckets = 0; + char *default_name = 0; while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { @@ -881,6 +882,8 @@ ip6_config (vlib_main_t * vm, unformat_input_t * input) else if (unformat (input, "heap-size %U", unformat_memory_size, &heapsize)) ; + else if (unformat (input, "default-table-name %s", &default_name)) + ; else return clib_error_return (0, "unknown input '%U'", format_unformat_error, input); @@ -888,6 +891,7 @@ ip6_config (vlib_main_t * vm, unformat_input_t * input) ip6_fib_table_nbuckets = nbuckets; ip6_fib_table_size = heapsize; + fib_table_default_names[FIB_PROTOCOL_IP6] = default_name; return 0; } diff --git a/src/vnet/fib/mpls_fib.c b/src/vnet/fib/mpls_fib.c index 5dcd70b4c53..767fc84c8a8 100644 --- a/src/vnet/fib/mpls_fib.c +++ b/src/vnet/fib/mpls_fib.c @@ -481,3 +481,24 @@ VLIB_CLI_COMMAND (mpls_fib_show_command, static) = { .short_help = "show mpls fib [summary] [table <n>]", .function = mpls_fib_show, }; + +static clib_error_t * +mpls_config (vlib_main_t * vm, unformat_input_t * input) +{ + char *default_name = 0; + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "default-table-name %s", &default_name)) + ; + else + return clib_error_return (0, "unknown input '%U'", + format_unformat_error, input); + } + + fib_table_default_names[FIB_PROTOCOL_MPLS] = default_name; + + return 0; +} + +VLIB_EARLY_CONFIG_FUNCTION (mpls_config, "mpls"); diff --git a/src/vnet/gso/gso.h b/src/vnet/gso/gso.h index 883a4941ee2..dee5da5c70b 100644 --- a/src/vnet/gso/gso.h +++ b/src/vnet/gso/gso.h @@ -39,13 +39,13 @@ gso_init_bufs_from_template_base (vlib_buffer_t **bufs, vlib_buffer_t *b0, u32 flags, u16 n_bufs, u16 hdr_sz) { u32 i = n_bufs; - while (i >= 4) + while (i >= 6) { /* prefetches */ CLIB_PREFETCH (bufs[2], 2 * CLIB_CACHE_LINE_BYTES, LOAD); CLIB_PREFETCH (bufs[3], 2 * CLIB_CACHE_LINE_BYTES, LOAD); - vlib_prefetch_buffer_data (bufs[2], LOAD); - vlib_prefetch_buffer_data (bufs[3], LOAD); + vlib_prefetch_buffer_data (bufs[4], LOAD); + vlib_prefetch_buffer_data (bufs[5], LOAD); /* copying objects from cacheline 0 */ bufs[0]->current_data = 0; @@ -70,10 +70,26 @@ gso_init_bufs_from_template_base (vlib_buffer_t **bufs, vlib_buffer_t *b0, bufs[0]->total_length_not_including_first_buffer = 0; bufs[1]->total_length_not_including_first_buffer = 0; + clib_memcpy_fast (&bufs[0]->opaque2, &b0->opaque2, sizeof (b0->opaque2)); + clib_memcpy_fast (&bufs[1]->opaque2, &b0->opaque2, sizeof (b0->opaque2)); + /* copying data */ clib_memcpy_fast (bufs[0]->data, vlib_buffer_get_current (b0), hdr_sz); clib_memcpy_fast (bufs[1]->data, vlib_buffer_get_current (b0), hdr_sz); + /* header offset fixup */ + vnet_buffer (bufs[0])->l2_hdr_offset -= b0->current_data; + vnet_buffer (bufs[0])->l3_hdr_offset -= b0->current_data; + vnet_buffer (bufs[0])->l4_hdr_offset -= b0->current_data; + vnet_buffer2 (bufs[0])->outer_l3_hdr_offset -= b0->current_data; + vnet_buffer2 (bufs[0])->outer_l4_hdr_offset -= b0->current_data; + + vnet_buffer (bufs[1])->l2_hdr_offset -= b0->current_data; + vnet_buffer (bufs[1])->l3_hdr_offset -= b0->current_data; + vnet_buffer (bufs[1])->l4_hdr_offset -= b0->current_data; + vnet_buffer2 (bufs[1])->outer_l3_hdr_offset -= b0->current_data; + vnet_buffer2 (bufs[1])->outer_l4_hdr_offset -= b0->current_data; + bufs += 2; i -= 2; } @@ -92,10 +108,18 @@ gso_init_bufs_from_template_base (vlib_buffer_t **bufs, vlib_buffer_t *b0, /* copying objects from cacheline 1 */ bufs[0]->trace_handle = b0->trace_handle; bufs[0]->total_length_not_including_first_buffer = 0; + clib_memcpy_fast (&bufs[0]->opaque2, &b0->opaque2, sizeof (b0->opaque2)); /* copying data */ clib_memcpy_fast (bufs[0]->data, vlib_buffer_get_current (b0), hdr_sz); + /* header offset fixup */ + vnet_buffer (bufs[0])->l2_hdr_offset -= b0->current_data; + vnet_buffer (bufs[0])->l3_hdr_offset -= b0->current_data; + vnet_buffer (bufs[0])->l4_hdr_offset -= b0->current_data; + vnet_buffer2 (bufs[0])->outer_l3_hdr_offset -= b0->current_data; + vnet_buffer2 (bufs[0])->outer_l4_hdr_offset -= b0->current_data; + bufs++; i--; } @@ -103,28 +127,41 @@ gso_init_bufs_from_template_base (vlib_buffer_t **bufs, vlib_buffer_t *b0, static_always_inline void gso_fixup_segmented_buf (vlib_main_t *vm, vlib_buffer_t *b0, u32 next_tcp_seq, - int is_l2, int is_ip6, generic_header_offset_t *gho, - clib_ip_csum_t *c, u8 tcp_flags) + int is_l2, u8 oflags, u16 hdr_sz, u16 l4_hdr_sz, + clib_ip_csum_t *c, u8 tcp_flags, u8 is_prefetch, + vlib_buffer_t *b1) { - ip4_header_t *ip4 = - (ip4_header_t *) (vlib_buffer_get_current (b0) + gho->l3_hdr_offset + - gho->outer_hdr_sz); - ip6_header_t *ip6 = - (ip6_header_t *) (vlib_buffer_get_current (b0) + gho->l3_hdr_offset + - gho->outer_hdr_sz); - tcp_header_t *tcp = - (tcp_header_t *) (vlib_buffer_get_current (b0) + gho->l4_hdr_offset + - gho->outer_hdr_sz); + i16 l3_hdr_offset = vnet_buffer (b0)->l3_hdr_offset; + i16 l4_hdr_offset = vnet_buffer (b0)->l4_hdr_offset; + + ip4_header_t *ip4 = (ip4_header_t *) (b0->data + l3_hdr_offset); + ip6_header_t *ip6 = (ip6_header_t *) (b0->data + l3_hdr_offset); + tcp_header_t *tcp = (tcp_header_t *) (b0->data + l4_hdr_offset); tcp->flags = tcp_flags; tcp->seq_number = clib_host_to_net_u32 (next_tcp_seq); c->odd = 0; - if (is_ip6) + if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM) + { + ip4->length = + clib_host_to_net_u16 (b0->current_length - hdr_sz + + (l4_hdr_offset - l3_hdr_offset) + l4_hdr_sz); + ip4->checksum = 0; + ip4->checksum = ip4_header_checksum (ip4); + vnet_buffer_offload_flags_clear (b0, (VNET_BUFFER_OFFLOAD_F_IP_CKSUM | + VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)); + c->sum += clib_mem_unaligned (&ip4->src_address, u32); + c->sum += clib_mem_unaligned (&ip4->dst_address, u32); + c->sum += clib_host_to_net_u32 ( + (clib_net_to_host_u16 (ip4->length) - ip4_header_bytes (ip4)) + + (ip4->protocol << 16)); + } + else { - ip6->payload_length = clib_host_to_net_u16 ( - b0->current_length - gho->l4_hdr_offset - gho->outer_hdr_sz); + ip6->payload_length = + clib_host_to_net_u16 (b0->current_length - hdr_sz + l4_hdr_sz); vnet_buffer_offload_flags_clear (b0, VNET_BUFFER_OFFLOAD_F_TCP_CKSUM); ip6_psh_t psh = { 0 }; u32 *p = (u32 *) &psh; @@ -135,24 +172,15 @@ gso_fixup_segmented_buf (vlib_main_t *vm, vlib_buffer_t *b0, u32 next_tcp_seq, for (int i = 0; i < 10; i++) c->sum += p[i]; } - else - { - ip4->length = clib_host_to_net_u16 ( - b0->current_length - gho->l3_hdr_offset - gho->outer_hdr_sz); - if (gho->gho_flags & GHO_F_IP4) - ip4->checksum = ip4_header_checksum (ip4); - vnet_buffer_offload_flags_clear (b0, (VNET_BUFFER_OFFLOAD_F_IP_CKSUM | - VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)); - c->sum += clib_mem_unaligned (&ip4->src_address, u32); - c->sum += clib_mem_unaligned (&ip4->dst_address, u32); - c->sum += clib_host_to_net_u32 ( - (clib_net_to_host_u16 (ip4->length) - ip4_header_bytes (ip4)) + - (ip4->protocol << 16)); - } - clib_ip_csum_chunk (c, (u8 *) tcp, gho->l4_hdr_sz); + + if (is_prefetch) + CLIB_PREFETCH (vlib_buffer_get_current (b1) + hdr_sz, + CLIB_CACHE_LINE_BYTES, LOAD); + + clib_ip_csum_chunk (c, (u8 *) tcp, l4_hdr_sz); tcp->checksum = clib_ip_csum_fold (c); - if (!is_l2 && ((gho->gho_flags & GHO_F_TUNNEL) == 0)) + if (!is_l2 && ((oflags & VNET_BUFFER_OFFLOAD_F_TNL_MASK) == 0)) { u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX]; @@ -169,16 +197,20 @@ gso_fixup_segmented_buf (vlib_main_t *vm, vlib_buffer_t *b0, u32 next_tcp_seq, static_always_inline u32 gso_segment_buffer_inline (vlib_main_t *vm, vnet_interface_per_thread_data_t *ptd, - vlib_buffer_t *b, generic_header_offset_t *gho, - int is_l2, int is_ip6) + vlib_buffer_t *b, int is_l2) { vlib_buffer_t **bufs = 0; u32 n_tx_bytes = 0; + + u8 oflags = vnet_buffer (b)->oflags; + i16 l4_hdr_offset = vnet_buffer (b)->l4_hdr_offset; u16 gso_size = vnet_buffer2 (b)->gso_size; + u16 l4_hdr_sz = vnet_buffer2 (b)->gso_l4_hdr_sz; + u8 tcp_flags = 0, tcp_flags_no_fin_psh = 0; u32 default_bflags = b->flags & ~(VNET_BUFFER_F_GSO | VLIB_BUFFER_NEXT_PRESENT); - u16 hdr_sz = gho->hdr_sz + gho->outer_hdr_sz; + u16 hdr_sz = (l4_hdr_offset - b->current_data) + l4_hdr_sz; u32 next_tcp_seq = 0, tcp_seq = 0; u32 data_size = vlib_buffer_length_in_chain (vm, b) - hdr_sz; u16 size = @@ -200,9 +232,8 @@ gso_segment_buffer_inline (vlib_main_t *vm, vec_validate (bufs, n_bufs - 1); vlib_get_buffers (vm, ptd->split_buffers, bufs, n_bufs); - tcp_header_t *tcp = - (tcp_header_t *) (vlib_buffer_get_current (b) + gho->l4_hdr_offset + - gho->outer_hdr_sz); + tcp_header_t *tcp = (tcp_header_t *) (b->data + l4_hdr_offset); + tcp_seq = next_tcp_seq = clib_net_to_host_u32 (tcp->seq_number); /* store original flags for last packet and reset FIN and PSH */ tcp_flags = tcp->flags; @@ -247,11 +278,11 @@ gso_segment_buffer_inline (vlib_main_t *vm, if (0 == dst_left && data_size) { vlib_prefetch_buffer_header (bufs[i + 1], LOAD); - vlib_prefetch_buffer_data (bufs[i + 1], LOAD); n_tx_bytes += bufs[i]->current_length; - gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, is_ip6, gho, - &c, tcp_flags_no_fin_psh); + gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, oflags, hdr_sz, + l4_hdr_sz, &c, tcp_flags_no_fin_psh, 1, + bufs[i + 1]); i++; dst_left = size; dst_ptr = vlib_buffer_get_current (bufs[i]) + hdr_sz; @@ -264,8 +295,8 @@ gso_segment_buffer_inline (vlib_main_t *vm, ASSERT ((i + 1) == n_alloc); n_tx_bytes += bufs[i]->current_length; - gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, is_ip6, gho, &c, - tcp_flags); + gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, oflags, hdr_sz, + l4_hdr_sz, &c, tcp_flags, 0, NULL); vec_free (bufs); return n_tx_bytes; diff --git a/src/vnet/gso/node.c b/src/vnet/gso/node.c index 910f1585e7f..c1d4459476e 100644 --- a/src/vnet/gso/node.c +++ b/src/vnet/gso/node.c @@ -80,113 +80,108 @@ format_gso_trace (u8 * s, va_list * args) return s; } -static_always_inline u16 -tso_segment_ipip_tunnel_fixup (vlib_main_t * vm, - vnet_interface_per_thread_data_t * ptd, - vlib_buffer_t * sb0, - generic_header_offset_t * gho) +static_always_inline void +tso_segment_ipip_tunnel_fixup (vlib_main_t *vm, + vnet_interface_per_thread_data_t *ptd, + vlib_buffer_t *sb0) { u16 n_tx_bufs = vec_len (ptd->split_buffers); - u16 i = 0, n_tx_bytes = 0; + u16 i = 0; while (i < n_tx_bufs) { vlib_buffer_t *b0 = vlib_get_buffer (vm, ptd->split_buffers[i]); + i16 outer_l3_hdr_offset = vnet_buffer2 (b0)->outer_l3_hdr_offset; + i16 l3_hdr_offset = vnet_buffer (b0)->l3_hdr_offset; - ip4_header_t *ip4 = - (ip4_header_t *) (vlib_buffer_get_current (b0) + - gho->outer_l3_hdr_offset); - ip6_header_t *ip6 = - (ip6_header_t *) (vlib_buffer_get_current (b0) + - gho->outer_l3_hdr_offset); + ip4_header_t *ip4 = (ip4_header_t *) (b0->data + outer_l3_hdr_offset); + ip6_header_t *ip6 = (ip6_header_t *) (b0->data + outer_l3_hdr_offset); - if (gho->gho_flags & GHO_F_OUTER_IP4) + if (vnet_buffer (b0)->oflags & VNET_BUFFER_OFFLOAD_F_OUTER_IP_CKSUM) { - ip4->length = - clib_host_to_net_u16 (b0->current_length - - gho->outer_l3_hdr_offset); + ip4->length = clib_host_to_net_u16 ( + b0->current_length - (outer_l3_hdr_offset - b0->current_data)); ip4->checksum = ip4_header_checksum (ip4); + vnet_buffer_offload_flags_clear ( + b0, VNET_BUFFER_OFFLOAD_F_OUTER_IP_CKSUM | + VNET_BUFFER_OFFLOAD_F_TNL_IPIP); } - else if (gho->gho_flags & GHO_F_OUTER_IP6) + else { - ip6->payload_length = - clib_host_to_net_u16 (b0->current_length - - gho->outer_l4_hdr_offset); + ip6->payload_length = clib_host_to_net_u16 ( + b0->current_length - (l3_hdr_offset - b0->current_data)); + vnet_buffer_offload_flags_clear (b0, VNET_BUFFER_OFFLOAD_F_TNL_IPIP); } - n_tx_bytes += gho->outer_hdr_sz; i++; } - return n_tx_bytes; } static_always_inline void -tso_segment_vxlan_tunnel_headers_fixup (vlib_main_t * vm, vlib_buffer_t * b, - generic_header_offset_t * gho) +tso_segment_vxlan_tunnel_headers_fixup (vlib_main_t *vm, vlib_buffer_t *b) { - u8 proto = 0; ip4_header_t *ip4 = 0; ip6_header_t *ip6 = 0; udp_header_t *udp = 0; + i16 outer_l3_hdr_offset = vnet_buffer2 (b)->outer_l3_hdr_offset; + i16 outer_l4_hdr_offset = vnet_buffer2 (b)->outer_l4_hdr_offset; - ip4 = - (ip4_header_t *) (vlib_buffer_get_current (b) + gho->outer_l3_hdr_offset); - ip6 = - (ip6_header_t *) (vlib_buffer_get_current (b) + gho->outer_l3_hdr_offset); - udp = - (udp_header_t *) (vlib_buffer_get_current (b) + gho->outer_l4_hdr_offset); + ip4 = (ip4_header_t *) (b->data + outer_l3_hdr_offset); + ip6 = (ip6_header_t *) (b->data + outer_l3_hdr_offset); + udp = (udp_header_t *) (b->data + outer_l4_hdr_offset); - if (gho->gho_flags & GHO_F_OUTER_IP4) + if (vnet_buffer (b)->oflags & VNET_BUFFER_OFFLOAD_F_OUTER_IP_CKSUM) { - proto = ip4->protocol; - ip4->length = - clib_host_to_net_u16 (b->current_length - gho->outer_l3_hdr_offset); + ip4->length = clib_host_to_net_u16 ( + b->current_length - (outer_l3_hdr_offset - b->current_data)); ip4->checksum = ip4_header_checksum (ip4); + if (vnet_buffer (b)->oflags & VNET_BUFFER_OFFLOAD_F_OUTER_UDP_CKSUM) + { + udp->length = clib_host_to_net_u16 ( + b->current_length - (outer_l4_hdr_offset - b->current_data)); + // udp checksum is 0, in udp tunnel + udp->checksum = 0; + } + vnet_buffer_offload_flags_clear ( + b, VNET_BUFFER_OFFLOAD_F_OUTER_IP_CKSUM | + VNET_BUFFER_OFFLOAD_F_OUTER_UDP_CKSUM | + VNET_BUFFER_OFFLOAD_F_TNL_VXLAN); } - else if (gho->gho_flags & GHO_F_OUTER_IP6) - { - proto = ip6->protocol; - ip6->payload_length = - clib_host_to_net_u16 (b->current_length - gho->outer_l4_hdr_offset); - } - if (proto == IP_PROTOCOL_UDP) + else { - int bogus; - udp->length = - clib_host_to_net_u16 (b->current_length - gho->outer_l4_hdr_offset); - udp->checksum = 0; - if (gho->gho_flags & GHO_F_OUTER_IP6) + ip6->payload_length = clib_host_to_net_u16 ( + b->current_length - (outer_l4_hdr_offset - b->current_data)); + + if (vnet_buffer (b)->oflags & VNET_BUFFER_OFFLOAD_F_OUTER_UDP_CKSUM) { + int bogus; + udp->length = ip6->payload_length; + // udp checksum is 0, in udp tunnel + udp->checksum = 0; udp->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ip6, &bogus); + vnet_buffer_offload_flags_clear ( + b, VNET_BUFFER_OFFLOAD_F_OUTER_UDP_CKSUM | + VNET_BUFFER_OFFLOAD_F_TNL_VXLAN); } - else if (gho->gho_flags & GHO_F_OUTER_IP4) - { - udp->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4); - } - /* FIXME: it should be OUTER_UDP_CKSUM */ - vnet_buffer_offload_flags_clear (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM); } } -static_always_inline u16 -tso_segment_vxlan_tunnel_fixup (vlib_main_t * vm, - vnet_interface_per_thread_data_t * ptd, - vlib_buffer_t * sb0, - generic_header_offset_t * gho) +static_always_inline void +tso_segment_vxlan_tunnel_fixup (vlib_main_t *vm, + vnet_interface_per_thread_data_t *ptd, + vlib_buffer_t *sb0) { u16 n_tx_bufs = vec_len (ptd->split_buffers); - u16 i = 0, n_tx_bytes = 0; + u16 i = 0; while (i < n_tx_bufs) { vlib_buffer_t *b0 = vlib_get_buffer (vm, ptd->split_buffers[i]); - tso_segment_vxlan_tunnel_headers_fixup (vm, b0, gho); - n_tx_bytes += gho->outer_hdr_sz; + tso_segment_vxlan_tunnel_headers_fixup (vm, b0); i++; } - return n_tx_bytes; } static_always_inline u16 @@ -682,32 +677,10 @@ vnet_gso_node_inline (vlib_main_t * vm, to_next -= 1; n_left_to_next += 1; /* undo the counting. */ - generic_header_offset_t gho = { 0 }; u32 n_tx_bytes = 0; - u32 inner_is_ip6 = is_ip6; - - vnet_generic_header_offset_parser (b[0], &gho, is_l2, - is_ip4, is_ip6); - - if (PREDICT_FALSE (gho.gho_flags & GHO_F_TUNNEL)) - { - if (PREDICT_FALSE - (gho.gho_flags & (GHO_F_GRE_TUNNEL | - GHO_F_GENEVE_TUNNEL))) - { - /* not supported yet */ - drop_one_buffer_and_count (vm, vnm, node, from - 1, - hi->sw_if_index, - GSO_ERROR_UNHANDLED_TYPE); - b += 1; - continue; - } - inner_is_ip6 = (gho.gho_flags & GHO_F_IP6) != 0; - } - - n_tx_bytes = gso_segment_buffer_inline (vm, ptd, b[0], &gho, - is_l2, inner_is_ip6); + n_tx_bytes = + gso_segment_buffer_inline (vm, ptd, b[0], is_l2); if (PREDICT_FALSE (n_tx_bytes == 0)) { @@ -718,19 +691,15 @@ vnet_gso_node_inline (vlib_main_t * vm, continue; } - - if (PREDICT_FALSE (gho.gho_flags & GHO_F_VXLAN_TUNNEL)) + if (PREDICT_FALSE (vnet_buffer (b[0])->oflags & + VNET_BUFFER_OFFLOAD_F_TNL_VXLAN)) { - n_tx_bytes += - tso_segment_vxlan_tunnel_fixup (vm, ptd, b[0], &gho); + tso_segment_vxlan_tunnel_fixup (vm, ptd, b[0]); } - else - if (PREDICT_FALSE - (gho.gho_flags & (GHO_F_IPIP_TUNNEL | - GHO_F_IPIP6_TUNNEL))) + else if (PREDICT_FALSE (vnet_buffer (b[0])->oflags & + VNET_BUFFER_OFFLOAD_F_TNL_IPIP)) { - n_tx_bytes += - tso_segment_ipip_tunnel_fixup (vm, ptd, b[0], &gho); + tso_segment_ipip_tunnel_fixup (vm, ptd, b[0]); } u16 n_tx_bufs = vec_len (ptd->split_buffers); @@ -744,7 +713,6 @@ vnet_gso_node_inline (vlib_main_t * vm, { sbi0 = to_next[0] = from_seg[0]; sb0 = vlib_get_buffer (vm, sbi0); - vnet_buffer_offload_flags_clear (sb0, 0x7F); ASSERT (sb0->current_length > 0); to_next += 1; from_seg += 1; diff --git a/src/vnet/interface_api.c b/src/vnet/interface_api.c index 2995836672d..c727e519138 100644 --- a/src/vnet/interface_api.c +++ b/src/vnet/interface_api.c @@ -1020,21 +1020,19 @@ vl_api_sw_interface_set_interface_name_t_handler ( { vl_api_sw_interface_set_interface_name_reply_t *rmp; vnet_main_t *vnm = vnet_get_main (); - u32 sw_if_index = ntohl (mp->sw_if_index); - vnet_sw_interface_t *si = vnet_get_sw_interface (vnm, sw_if_index); clib_error_t *error; int rv = 0; + VALIDATE_SW_IF_INDEX (mp); + + u32 sw_if_index = ntohl (mp->sw_if_index); + vnet_sw_interface_t *si = vnet_get_sw_interface (vnm, sw_if_index); + if (mp->name[0] == 0) { rv = VNET_API_ERROR_INVALID_VALUE; goto out; } - if (si == 0) - { - rv = VNET_API_ERROR_INVALID_SW_IF_INDEX; - goto out; - } error = vnet_rename_interface (vnm, si->hw_if_index, (char *) mp->name); if (error) @@ -1044,6 +1042,7 @@ vl_api_sw_interface_set_interface_name_t_handler ( } out: + BAD_SW_IF_INDEX_LABEL; REPLY_MACRO (VL_API_SW_INTERFACE_SET_INTERFACE_NAME_REPLY); } diff --git a/src/vnet/ip/icmp4.c b/src/vnet/ip/icmp4.c index 452817dd9c1..fa4a0e12276 100644 --- a/src/vnet/ip/icmp4.c +++ b/src/vnet/ip/icmp4.c @@ -341,7 +341,7 @@ ip4_icmp_error (vlib_main_t * vm, /* Prefer a source address from "offending interface" */ if (!ip4_sas_by_sw_if_index (sw_if_index0, &out_ip0->dst_address, &out_ip0->src_address)) - { /* interface has no IP6 address - should not happen */ + { /* interface has no IP4 address - should not happen */ next0 = IP4_ICMP_ERROR_NEXT_DROP; error0 = ICMP4_ERROR_DROP; } diff --git a/src/vnet/ip/icmp6.c b/src/vnet/ip/icmp6.c index 087ee6b4719..b095f679cc8 100644 --- a/src/vnet/ip/icmp6.c +++ b/src/vnet/ip/icmp6.c @@ -357,7 +357,7 @@ ip6_icmp_error (vlib_main_t * vm, sw_if_index0 = vnet_buffer (p0)->sw_if_index[VLIB_RX]; - vlib_buffer_copy_trace_flag (vm, p0, pi0); + vlib_buffer_copy_trace_flag (vm, org_p0, pi0); /* Add IP header and ICMPv6 header including a 4 byte data field */ vlib_buffer_advance (p0, diff --git a/src/vnet/ip/ip.c b/src/vnet/ip/ip.c index d045c2f37c1..586f7dfbc85 100644 --- a/src/vnet/ip/ip.c +++ b/src/vnet/ip/ip.c @@ -201,7 +201,8 @@ ip_feature_enable_disable (ip_address_family_t af, } int -ip_flow_hash_set (ip_address_family_t af, u32 table_id, u32 flow_hash_config) +ip_flow_hash_set (ip_address_family_t af, u32 table_id, + flow_hash_config_t flow_hash_config) { fib_protocol_t fproto; u32 fib_index; diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c index e85c888f669..ff74b52eb18 100644 --- a/src/vnet/ip/ip4_forward.c +++ b/src/vnet/ip/ip4_forward.c @@ -1190,9 +1190,11 @@ format_ip4_forward_next_trace (u8 * s, va_list * args) CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *); u32 indent = format_get_indent (s); - s = format (s, "%U%U", - format_white_space, indent, - format_ip4_header, t->packet_data, sizeof (t->packet_data)); + + s = format (s, "%Ufib:%d adj:%d flow:0x%08x", format_white_space, indent, + t->fib_index, t->dpo_index, t->flow_hash); + s = format (s, "\n%U%U", format_white_space, indent, format_ip4_header, + t->packet_data, sizeof (t->packet_data)); return s; } #endif diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c index 06c473b1495..48fb633fd32 100644 --- a/src/vnet/ip/ip6_forward.c +++ b/src/vnet/ip/ip6_forward.c @@ -948,8 +948,7 @@ format_ip6_forward_next_trace (u8 * s, va_list * args) ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *); u32 indent = format_get_indent (s); - s = format (s, "%Ufib:%d adj:%d flow:%d", - format_white_space, indent, + s = format (s, "%Ufib:%d adj:%d flow:0x%08x", format_white_space, indent, t->fib_index, t->adj_index, t->flow_hash); s = format (s, "\n%U%U", format_white_space, indent, diff --git a/src/vnet/ipip/ipip_api.c b/src/vnet/ipip/ipip_api.c index dcd3ec875df..2cb7bdf8dae 100644 --- a/src/vnet/ipip/ipip_api.c +++ b/src/vnet/ipip/ipip_api.c @@ -125,22 +125,23 @@ send_ipip_tunnel_details (ipip_tunnel_t * t, vl_api_ipip_tunnel_dump_t * mp) ipip_main_t *im = &ipip_main; vl_api_ipip_tunnel_details_t *rmp; bool is_ipv6 = t->transport == IPIP_TRANSPORT_IP6 ? true : false; + ip46_type_t ip_type = is_ipv6 ? IP46_TYPE_IP6 : IP46_TYPE_IP4; fib_table_t *ft; ft = fib_table_get (t->fib_index, (is_ipv6 ? FIB_PROTOCOL_IP6 : FIB_PROTOCOL_IP4)); - REPLY_MACRO_DETAILS2(VL_API_IPIP_TUNNEL_DETAILS, - ({ - ip_address_encode (&t->tunnel_src, IP46_TYPE_ANY, &rmp->tunnel.src); - ip_address_encode (&t->tunnel_dst, IP46_TYPE_ANY, &rmp->tunnel.dst); - rmp->tunnel.table_id = htonl (ft->ft_table_id); - rmp->tunnel.instance = htonl (t->user_instance); - rmp->tunnel.sw_if_index = htonl (t->sw_if_index); - rmp->tunnel.dscp = ip_dscp_encode(t->dscp); - rmp->tunnel.flags = tunnel_encap_decap_flags_encode(t->flags); - rmp->tunnel.mode = ipip_tunnel_mode_encode (t->mode); - })); + REPLY_MACRO_DETAILS2 ( + VL_API_IPIP_TUNNEL_DETAILS, ({ + ip_address_encode (&t->tunnel_src, ip_type, &rmp->tunnel.src); + ip_address_encode (&t->tunnel_dst, ip_type, &rmp->tunnel.dst); + rmp->tunnel.table_id = htonl (ft->ft_table_id); + rmp->tunnel.instance = htonl (t->user_instance); + rmp->tunnel.sw_if_index = htonl (t->sw_if_index); + rmp->tunnel.dscp = ip_dscp_encode (t->dscp); + rmp->tunnel.flags = tunnel_encap_decap_flags_encode (t->flags); + rmp->tunnel.mode = ipip_tunnel_mode_encode (t->mode); + })); } static void diff --git a/src/vnet/ipsec/esp_decrypt.c b/src/vnet/ipsec/esp_decrypt.c index 94f3204b51f..26d8ca1deee 100644 --- a/src/vnet/ipsec/esp_decrypt.c +++ b/src/vnet/ipsec/esp_decrypt.c @@ -1093,7 +1093,6 @@ esp_decrypt_inline (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs; vlib_buffer_t *sync_bufs[VLIB_FRAME_SIZE]; u16 sync_nexts[VLIB_FRAME_SIZE], *sync_next = sync_nexts, n_sync = 0; - u16 async_nexts[VLIB_FRAME_SIZE], *async_next = async_nexts; u16 noop_nexts[VLIB_FRAME_SIZE], n_noop = 0; u32 sync_bi[VLIB_FRAME_SIZE]; u32 noop_bi[VLIB_FRAME_SIZE]; @@ -1305,8 +1304,6 @@ esp_decrypt_inline (vlib_main_t *vm, vlib_node_runtime_t *node, pd += 1; pd2 += 1; } - else - async_next++; n_left -= 1; b += 1; diff --git a/src/vnet/ipsec/ipsec_itf.c b/src/vnet/ipsec/ipsec_itf.c index 6e66d10660b..b86bf6a110c 100644 --- a/src/vnet/ipsec/ipsec_itf.c +++ b/src/vnet/ipsec/ipsec_itf.c @@ -381,6 +381,7 @@ ipsec_itf_create_cli (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { unformat_input_t _line_input, *line_input = &_line_input; + tunnel_mode_t mode = TUNNEL_MODE_P2P; u32 instance, sw_if_index; clib_error_t *error; mac_address_t mac; @@ -396,6 +397,8 @@ ipsec_itf_create_cli (vlib_main_t * vm, { if (unformat (line_input, "instance %d", &instance)) ; + else if (unformat (line_input, "p2mp")) + mode = TUNNEL_MODE_MP; else { error = clib_error_return (0, "unknown input: %U", @@ -410,7 +413,7 @@ ipsec_itf_create_cli (vlib_main_t * vm, return error; } - rv = ipsec_itf_create (instance, TUNNEL_MODE_P2P, &sw_if_index); + rv = ipsec_itf_create (instance, mode, &sw_if_index); if (rv) return clib_error_return (0, "iPSec interface create failed"); @@ -425,13 +428,13 @@ ipsec_itf_create_cli (vlib_main_t * vm, * * @cliexpar * The following two command syntaxes are equivalent: - * @cliexcmd{ipsec itf create [instance <instance>]} + * @cliexcmd{ipsec itf create [instance <instance>] [p2mp]} * Example of how to create a ipsec interface: * @cliexcmd{ipsec itf create} ?*/ VLIB_CLI_COMMAND (ipsec_itf_create_command, static) = { .path = "ipsec itf create", - .short_help = "ipsec itf create [instance <instance>]", + .short_help = "ipsec itf create [instance <instance>] [p2mp]", .function = ipsec_itf_create_cli, }; diff --git a/src/vnet/l2/l2_classify.h b/src/vnet/l2/l2_classify.h index 68a2bb98e64..3c86fb5ca86 100644 --- a/src/vnet/l2/l2_classify.h +++ b/src/vnet/l2/l2_classify.h @@ -39,7 +39,6 @@ typedef enum L2_INPUT_CLASSIFY_NEXT_ETHERNET_INPUT, L2_INPUT_CLASSIFY_NEXT_IP4_INPUT, L2_INPUT_CLASSIFY_NEXT_IP6_INPUT, - L2_INPUT_CLASSIFY_NEXT_LI, L2_INPUT_CLASSIFY_N_NEXT, } l2_input_classify_next_t; diff --git a/src/vnet/l2/l2_input.h b/src/vnet/l2/l2_input.h index 7d1dc9c1d05..3de1537b45e 100644 --- a/src/vnet/l2/l2_input.h +++ b/src/vnet/l2/l2_input.h @@ -27,6 +27,7 @@ #include <vnet/ethernet/packet.h> #include <vnet/ip/ip4_inlines.h> #include <vnet/ip/ip6_inlines.h> +#include <vnet/mpls/mpls_lookup.h> /* l2 connection type */ typedef enum l2_input_flags_t_ @@ -327,7 +328,7 @@ vnet_update_l2_len (vlib_buffer_t *b) /* * Compute flow hash of an ethernet packet, use 5-tuple hash if L3 packet - * is ip4 or ip6. Otherwise hash on smac/dmac/etype. + * is ip4, ip6, or mpls. Otherwise hash on smac/dmac/etype. * The vlib buffer current pointer is expected to be at ethernet header * and vnet l2.l2_len is expected to be setup already. */ @@ -342,6 +343,9 @@ vnet_l2_compute_flow_hash (vlib_buffer_t * b) return ip4_compute_flow_hash ((ip4_header_t *) l3h, IP_FLOW_HASH_DEFAULT); else if (ethertype == ETHERNET_TYPE_IP6) return ip6_compute_flow_hash ((ip6_header_t *) l3h, IP_FLOW_HASH_DEFAULT); + else if (ethertype == ETHERNET_TYPE_MPLS) + return mpls_compute_flow_hash ((mpls_unicast_header_t *) l3h, + IP_FLOW_HASH_DEFAULT); else { u32 a, b, c; diff --git a/src/vnet/l2/l2_input_classify.c b/src/vnet/l2/l2_input_classify.c index 248158310a1..cc031bd46a5 100644 --- a/src/vnet/l2/l2_input_classify.c +++ b/src/vnet/l2/l2_input_classify.c @@ -461,7 +461,6 @@ VLIB_REGISTER_NODE (l2_input_classify_node) = { [L2_INPUT_CLASSIFY_NEXT_ETHERNET_INPUT] = "ethernet-input-not-l2", [L2_INPUT_CLASSIFY_NEXT_IP4_INPUT] = "ip4-input", [L2_INPUT_CLASSIFY_NEXT_IP6_INPUT] = "ip6-input", - [L2_INPUT_CLASSIFY_NEXT_LI] = "li-hit", }, }; diff --git a/src/vnet/lawful-intercept/lawful_intercept.c b/src/vnet/lawful-intercept/lawful_intercept.c deleted file mode 100644 index 61b1a6165f4..00000000000 --- a/src/vnet/lawful-intercept/lawful_intercept.c +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <vnet/lawful-intercept/lawful_intercept.h> - -li_main_t li_main; - -static clib_error_t * -set_li_command_fn (vlib_main_t * vm, - unformat_input_t * input, vlib_cli_command_t * cmd) -{ - li_main_t *lm = &li_main; - ip4_address_t collector; - u8 collector_set = 0; - ip4_address_t src; - u8 src_set = 0; - u32 tmp; - u16 udp_port = 0; - u8 is_add = 1; - int i; - - while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (input, "collector %U", unformat_ip4_address, &collector)) - collector_set = 1; - if (unformat (input, "src %U", unformat_ip4_address, &src)) - src_set = 1; - else if (unformat (input, "udp-port %d", &tmp)) - udp_port = tmp; - else if (unformat (input, "del")) - is_add = 0; - else - break; - } - - if (collector_set == 0) - return clib_error_return (0, "collector must be set..."); - if (src_set == 0) - return clib_error_return (0, "src must be set..."); - if (udp_port == 0) - return clib_error_return (0, "udp-port must be set..."); - - if (is_add == 1) - { - for (i = 0; i < vec_len (lm->collectors); i++) - { - if (lm->collectors[i].as_u32 == collector.as_u32) - { - if (lm->ports[i] == udp_port) - return clib_error_return ( - 0, "collector %U:%d already configured", format_ip4_address, - &collector, udp_port); - else - return clib_error_return ( - 0, "collector %U already configured with port %d", - format_ip4_address, &collector, (int) (lm->ports[i])); - } - } - vec_add1 (lm->collectors, collector); - vec_add1 (lm->ports, udp_port); - vec_add1 (lm->src_addrs, src); - return 0; - } - else - { - for (i = 0; i < vec_len (lm->collectors); i++) - { - if ((lm->collectors[i].as_u32 == collector.as_u32) - && lm->ports[i] == udp_port) - { - vec_delete (lm->collectors, 1, i); - vec_delete (lm->ports, 1, i); - vec_delete (lm->src_addrs, 1, i); - return 0; - } - } - return clib_error_return (0, "collector %U:%d not configured", - &collector, udp_port); - } - return 0; -} - -VLIB_CLI_COMMAND (set_li_command, static) = { - .path = "set li", - .short_help = - "set li src <ip4-address> collector <ip4-address> udp-port <nnnn>", - .function = set_li_command_fn, -}; - -static clib_error_t * -li_init (vlib_main_t * vm) -{ - li_main_t *lm = &li_main; - - lm->vlib_main = vm; - lm->vnet_main = vnet_get_main (); - lm->hit_node_index = li_hit_node.index; - return 0; -} - -VLIB_INIT_FUNCTION (li_init); - - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/vnet/lawful-intercept/lawful_intercept.h b/src/vnet/lawful-intercept/lawful_intercept.h deleted file mode 100644 index ba74204fb9e..00000000000 --- a/src/vnet/lawful-intercept/lawful_intercept.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef __lawful_intercept_h__ -#define __lawful_intercept_h__ - -#include <vnet/vnet.h> -#include <vnet/ip/ip.h> - -typedef struct -{ - /* LI collector info */ - ip4_address_t *src_addrs; - ip4_address_t *collectors; - u16 *ports; - - /* Hit node index */ - u32 hit_node_index; - - /* convenience */ - vlib_main_t *vlib_main; - vnet_main_t *vnet_main; -} li_main_t; - -extern li_main_t li_main; - -typedef CLIB_PACKED(struct { - ip4_header_t ip4; - udp_header_t udp; -}) ip4_udp_header_t; - -extern vlib_node_registration_t li_hit_node; - -#endif /* __lawful_intercept_h__ */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/vnet/lawful-intercept/node.c b/src/vnet/lawful-intercept/node.c deleted file mode 100644 index 86f135b9ea1..00000000000 --- a/src/vnet/lawful-intercept/node.c +++ /dev/null @@ -1,286 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <vlib/vlib.h> -#include <vnet/vnet.h> -#include <vppinfra/error.h> - -#include <vnet/lawful-intercept/lawful_intercept.h> - -#include <vppinfra/error.h> -#include <vppinfra/elog.h> - -extern vlib_node_registration_t li_hit_node; - -typedef struct -{ - u32 next_index; -} li_hit_trace_t; - -/* packet trace format function */ -static u8 * -format_li_hit_trace (u8 * s, va_list * args) -{ - CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); - CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); - li_hit_trace_t *t = va_arg (*args, li_hit_trace_t *); - - s = format (s, "LI_HIT: next index %d", t->next_index); - - return s; -} - -#define foreach_li_hit_error \ -_(HITS, "LI packets processed") \ -_(NO_COLLECTOR, "No collector configured") \ -_(BUFFER_ALLOCATION_FAILURE, "Buffer allocation failure") - -typedef enum -{ -#define _(sym,str) LI_HIT_ERROR_##sym, - foreach_li_hit_error -#undef _ - LI_HIT_N_ERROR, -} li_hit_error_t; - -static char *li_hit_error_strings[] = { -#define _(sym,string) string, - foreach_li_hit_error -#undef _ -}; - -typedef enum -{ - LI_HIT_NEXT_ETHERNET, - LI_HIT_N_NEXT, -} li_hit_next_t; - -VLIB_NODE_FN (li_hit_node) (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * frame) -{ - u32 n_left_from, *from, *to_next; - li_hit_next_t next_index; - vlib_frame_t *int_frame = 0; - u32 *to_int_next = 0; - li_main_t *lm = &li_main; - - from = vlib_frame_vector_args (frame); - n_left_from = frame->n_vectors; - next_index = node->cached_next_index; - - if (PREDICT_FALSE (vec_len (lm->collectors) == 0)) - { - vlib_node_increment_counter (vm, li_hit_node.index, - LI_HIT_ERROR_NO_COLLECTOR, n_left_from); - } - else - { - /* The intercept frame... */ - int_frame = vlib_get_frame_to_node (vm, ip4_lookup_node.index); - to_int_next = vlib_frame_vector_args (int_frame); - } - - while (n_left_from > 0) - { - u32 n_left_to_next; - - vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); - -#if 0 - while (n_left_from >= 4 && n_left_to_next >= 2) - { - u32 next0 = LI_HIT_NEXT_INTERFACE_OUTPUT; - u32 next1 = LI_HIT_NEXT_INTERFACE_OUTPUT; - u32 sw_if_index0, sw_if_index1; - u8 tmp0[6], tmp1[6]; - ethernet_header_t *en0, *en1; - u32 bi0, bi1; - vlib_buffer_t *b0, *b1; - - /* Prefetch next iteration. */ - { - vlib_buffer_t *p2, *p3; - - p2 = vlib_get_buffer (vm, from[2]); - p3 = vlib_get_buffer (vm, from[3]); - - vlib_prefetch_buffer_header (p2, LOAD); - vlib_prefetch_buffer_header (p3, LOAD); - - clib_prefetch_store (p2->data); - clib_prefetch_store (p3->data); - } - - /* speculatively enqueue b0 and b1 to the current next frame */ - to_next[0] = bi0 = from[0]; - to_next[1] = bi1 = from[1]; - from += 2; - to_next += 2; - n_left_from -= 2; - n_left_to_next -= 2; - - b0 = vlib_get_buffer (vm, bi0); - b1 = vlib_get_buffer (vm, bi1); - - /* $$$$$ Dual loop: process 2 x packets here $$$$$ */ - ASSERT (b0->current_data == 0); - ASSERT (b1->current_data == 0); - - en0 = vlib_buffer_get_current (b0); - en1 = vlib_buffer_get_current (b1); - - sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; - sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX]; - - /* Send pkt back out the RX interface */ - vnet_buffer (b0)->sw_if_index[VLIB_TX] = sw_if_index0; - vnet_buffer (b1)->sw_if_index[VLIB_TX] = sw_if_index1; - - /* $$$$$ End of processing 2 x packets $$$$$ */ - - if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE))) - { - if (b0->flags & VLIB_BUFFER_IS_TRACED) - { - li_hit_trace_t *t = - vlib_add_trace (vm, node, b0, sizeof (*t)); - t->sw_if_index = sw_if_index0; - t->next_index = next0; - } - if (b1->flags & VLIB_BUFFER_IS_TRACED) - { - li_hit_trace_t *t = - vlib_add_trace (vm, node, b1, sizeof (*t)); - t->sw_if_index = sw_if_index1; - t->next_index = next1; - } - } - - /* verify speculative enqueues, maybe switch current next frame */ - vlib_validate_buffer_enqueue_x2 (vm, node, next_index, - to_next, n_left_to_next, - bi0, bi1, next0, next1); - } -#endif /* $$$ dual-loop off */ - - while (n_left_from > 0 && n_left_to_next > 0) - { - u32 bi0; - vlib_buffer_t *b0; - vlib_buffer_t *c0; - ip4_udp_header_t *iu0; - ip4_header_t *ip0; - udp_header_t *udp0; - u32 next0 = LI_HIT_NEXT_ETHERNET; - - /* speculatively enqueue b0 to the current next frame */ - bi0 = from[0]; - to_next[0] = bi0; - from += 1; - to_next += 1; - n_left_from -= 1; - n_left_to_next -= 1; - - b0 = vlib_get_buffer (vm, bi0); - if (PREDICT_TRUE (to_int_next != 0)) - { - /* Make an intercept copy. This can fail. */ - c0 = vlib_buffer_copy (vm, b0); - - if (PREDICT_FALSE (c0 == 0)) - { - vlib_node_increment_counter - (vm, node->node_index, - LI_HIT_ERROR_BUFFER_ALLOCATION_FAILURE, 1); - goto skip; - } - - vlib_buffer_advance (c0, -sizeof (*iu0)); - - iu0 = vlib_buffer_get_current (c0); - ip0 = &iu0->ip4; - - ip0->ip_version_and_header_length = 0x45; - ip0->ttl = 254; - ip0->protocol = IP_PROTOCOL_UDP; - - ip0->src_address.as_u32 = lm->src_addrs[0].as_u32; - ip0->dst_address.as_u32 = lm->collectors[0].as_u32; - ip0->length = vlib_buffer_length_in_chain (vm, c0); - ip0->checksum = ip4_header_checksum (ip0); - - udp0 = &iu0->udp; - udp0->src_port = udp0->dst_port = - clib_host_to_net_u16 (lm->ports[0]); - udp0->checksum = 0; - udp0->length = - clib_net_to_host_u16 (vlib_buffer_length_in_chain (vm, b0)); - - to_int_next[0] = vlib_get_buffer_index (vm, c0); - to_int_next++; - } - - skip: - if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) - && (b0->flags & VLIB_BUFFER_IS_TRACED))) - { - li_hit_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t)); - t->next_index = next0; - } - - /* verify speculative enqueue, maybe switch current next frame */ - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, - to_next, n_left_to_next, - bi0, next0); - } - - vlib_put_next_frame (vm, node, next_index, n_left_to_next); - } - - if (int_frame) - { - int_frame->n_vectors = frame->n_vectors; - vlib_put_frame_to_node (vm, ip4_lookup_node.index, int_frame); - } - - vlib_node_increment_counter (vm, li_hit_node.index, - LI_HIT_ERROR_HITS, frame->n_vectors); - return frame->n_vectors; -} - -VLIB_REGISTER_NODE (li_hit_node) = { - .name = "li-hit", - .vector_size = sizeof (u32), - .format_trace = format_li_hit_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - - .n_errors = ARRAY_LEN(li_hit_error_strings), - .error_strings = li_hit_error_strings, - - .n_next_nodes = LI_HIT_N_NEXT, - - /* edit / add dispositions here */ - .next_nodes = { - [LI_HIT_NEXT_ETHERNET] = "ethernet-input-not-l2", - }, -}; - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/vnet/mpls/mpls_lookup.c b/src/vnet/mpls/mpls_lookup.c index db423392c03..a5ac56534a5 100644 --- a/src/vnet/mpls/mpls_lookup.c +++ b/src/vnet/mpls/mpls_lookup.c @@ -44,13 +44,13 @@ format_mpls_lookup_trace (u8 * s, va_list * args) CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); mpls_lookup_trace_t * t = va_arg (*args, mpls_lookup_trace_t *); - s = format (s, "MPLS: next [%d], lookup fib index %d, LB index %d hash %x " - "label %d eos %d", - t->next_index, t->lfib_index, t->lb_index, t->hash, - vnet_mpls_uc_get_label( - clib_net_to_host_u32(t->label_net_byte_order)), - vnet_mpls_uc_get_s( - clib_net_to_host_u32(t->label_net_byte_order))); + s = format ( + s, + "MPLS: next [%d], lookup fib index %d, LB index %d hash 0x%08x " + "label %d eos %d", + t->next_index, t->lfib_index, t->lb_index, t->hash, + vnet_mpls_uc_get_label (clib_net_to_host_u32 (t->label_net_byte_order)), + vnet_mpls_uc_get_s (clib_net_to_host_u32 (t->label_net_byte_order))); return s; } @@ -482,8 +482,8 @@ format_mpls_load_balance_trace (u8 * s, va_list * args) CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); mpls_load_balance_trace_t * t = va_arg (*args, mpls_load_balance_trace_t *); - s = format (s, "MPLS: next [%d], LB index %d hash %d", - t->next_index, t->lb_index, t->hash); + s = format (s, "MPLS: next [%d], LB index %d hash 0x%08x", t->next_index, + t->lb_index, t->hash); return s; } @@ -553,75 +553,77 @@ VLIB_NODE_FN (mpls_load_balance_node) (vlib_main_t * vm, * We don't want to use the same hash value at each level in the recursion * graph as that would lead to polarisation */ - hc0 = vnet_buffer (p0)->ip.flow_hash = 0; - hc1 = vnet_buffer (p1)->ip.flow_hash = 0; - - if (PREDICT_FALSE (lb0->lb_n_buckets > 1)) - { - if (PREDICT_TRUE (vnet_buffer(p0)->ip.flow_hash)) - { - hc0 = vnet_buffer(p0)->ip.flow_hash = vnet_buffer(p0)->ip.flow_hash >> 1; - } - else - { - hc0 = vnet_buffer(p0)->ip.flow_hash = mpls_compute_flow_hash(mpls0, hc0); - } - dpo0 = load_balance_get_fwd_bucket(lb0, (hc0 & lb0->lb_n_buckets_minus_1)); - } - else - { - dpo0 = load_balance_get_bucket_i (lb0, 0); - } - if (PREDICT_FALSE (lb1->lb_n_buckets > 1)) - { - if (PREDICT_TRUE (vnet_buffer(p1)->ip.flow_hash)) - { - hc1 = vnet_buffer(p1)->ip.flow_hash = vnet_buffer(p1)->ip.flow_hash >> 1; - } - else - { - hc1 = vnet_buffer(p1)->ip.flow_hash = mpls_compute_flow_hash(mpls1, hc1); - } - dpo1 = load_balance_get_fwd_bucket(lb1, (hc1 & lb1->lb_n_buckets_minus_1)); - } - else - { - dpo1 = load_balance_get_bucket_i (lb1, 0); - } - - next0 = dpo0->dpoi_next_node; - next1 = dpo1->dpoi_next_node; - - vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; - vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; - - vlib_increment_combined_counter - (cm, thread_index, lbi0, 1, - vlib_buffer_length_in_chain (vm, p0)); - vlib_increment_combined_counter - (cm, thread_index, lbi1, 1, - vlib_buffer_length_in_chain (vm, p1)); - - if (PREDICT_FALSE(p0->flags & VLIB_BUFFER_IS_TRACED)) - { - mpls_load_balance_trace_t *tr = vlib_add_trace (vm, node, - p0, sizeof (*tr)); - tr->next_index = next0; - tr->lb_index = lbi0; - tr->hash = hc0; - } - if (PREDICT_FALSE(p1->flags & VLIB_BUFFER_IS_TRACED)) - { - mpls_load_balance_trace_t *tr = vlib_add_trace (vm, node, - p1, sizeof (*tr)); - tr->next_index = next1; - tr->lb_index = lbi1; - tr->hash = hc1; - } - - vlib_validate_buffer_enqueue_x2 (vm, node, next, - to_next, n_left_to_next, - pi0, pi1, next0, next1); + hc0 = hc1 = 0; + + if (PREDICT_FALSE (lb0->lb_n_buckets > 1)) + { + if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash)) + { + hc0 = vnet_buffer (p0)->ip.flow_hash = + vnet_buffer (p0)->ip.flow_hash >> 1; + } + else + { + hc0 = vnet_buffer (p0)->ip.flow_hash = + mpls_compute_flow_hash (mpls0, lb0->lb_hash_config); + } + dpo0 = load_balance_get_fwd_bucket ( + lb0, (hc0 & lb0->lb_n_buckets_minus_1)); + } + else + { + dpo0 = load_balance_get_bucket_i (lb0, 0); + } + if (PREDICT_FALSE (lb1->lb_n_buckets > 1)) + { + if (PREDICT_TRUE (vnet_buffer (p1)->ip.flow_hash)) + { + hc1 = vnet_buffer (p1)->ip.flow_hash = + vnet_buffer (p1)->ip.flow_hash >> 1; + } + else + { + hc1 = vnet_buffer (p1)->ip.flow_hash = + mpls_compute_flow_hash (mpls1, lb1->lb_hash_config); + } + dpo1 = load_balance_get_fwd_bucket ( + lb1, (hc1 & lb1->lb_n_buckets_minus_1)); + } + else + { + dpo1 = load_balance_get_bucket_i (lb1, 0); + } + + next0 = dpo0->dpoi_next_node; + next1 = dpo1->dpoi_next_node; + + vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; + vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; + + vlib_increment_combined_counter ( + cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); + vlib_increment_combined_counter ( + cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1)); + + if (PREDICT_FALSE (p0->flags & VLIB_BUFFER_IS_TRACED)) + { + mpls_load_balance_trace_t *tr = + vlib_add_trace (vm, node, p0, sizeof (*tr)); + tr->next_index = next0; + tr->lb_index = lbi0; + tr->hash = hc0; + } + if (PREDICT_FALSE (p1->flags & VLIB_BUFFER_IS_TRACED)) + { + mpls_load_balance_trace_t *tr = + vlib_add_trace (vm, node, p1, sizeof (*tr)); + tr->next_index = next1; + tr->lb_index = lbi1; + tr->hash = hc1; + } + + vlib_validate_buffer_enqueue_x2 ( + vm, node, next, to_next, n_left_to_next, pi0, pi1, next0, next1); } while (n_left_from > 0 && n_left_to_next > 0) @@ -646,44 +648,45 @@ VLIB_NODE_FN (mpls_load_balance_node) (vlib_main_t * vm, lb0 = load_balance_get(lbi0); - hc0 = vnet_buffer (p0)->ip.flow_hash = 0; - if (PREDICT_FALSE (lb0->lb_n_buckets > 1)) - { - if (PREDICT_TRUE (vnet_buffer(p0)->ip.flow_hash)) - { - hc0 = vnet_buffer(p0)->ip.flow_hash = vnet_buffer(p0)->ip.flow_hash >> 1; - } - else - { - hc0 = vnet_buffer(p0)->ip.flow_hash = mpls_compute_flow_hash(mpls0, hc0); - } - dpo0 = load_balance_get_fwd_bucket(lb0, (hc0 & lb0->lb_n_buckets_minus_1)); - } - else - { - dpo0 = load_balance_get_bucket_i (lb0, 0); - } - - next0 = dpo0->dpoi_next_node; - vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; - - if (PREDICT_FALSE(p0->flags & VLIB_BUFFER_IS_TRACED)) - { - mpls_load_balance_trace_t *tr = vlib_add_trace (vm, node, - p0, sizeof (*tr)); - tr->next_index = next0; - tr->lb_index = lbi0; - tr->hash = hc0; - } - - vlib_increment_combined_counter - (cm, thread_index, lbi0, 1, - vlib_buffer_length_in_chain (vm, p0)); - - vlib_validate_buffer_enqueue_x1 (vm, node, next, - to_next, n_left_to_next, - pi0, next0); - } + hc0 = 0; + if (PREDICT_FALSE (lb0->lb_n_buckets > 1)) + { + if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash)) + { + hc0 = vnet_buffer (p0)->ip.flow_hash = + vnet_buffer (p0)->ip.flow_hash >> 1; + } + else + { + hc0 = vnet_buffer (p0)->ip.flow_hash = + mpls_compute_flow_hash (mpls0, lb0->lb_hash_config); + } + dpo0 = load_balance_get_fwd_bucket ( + lb0, (hc0 & lb0->lb_n_buckets_minus_1)); + } + else + { + dpo0 = load_balance_get_bucket_i (lb0, 0); + } + + next0 = dpo0->dpoi_next_node; + vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; + + if (PREDICT_FALSE (p0->flags & VLIB_BUFFER_IS_TRACED)) + { + mpls_load_balance_trace_t *tr = + vlib_add_trace (vm, node, p0, sizeof (*tr)); + tr->next_index = next0; + tr->lb_index = lbi0; + tr->hash = hc0; + } + + vlib_increment_combined_counter ( + cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); + + vlib_validate_buffer_enqueue_x1 (vm, node, next, to_next, + n_left_to_next, pi0, next0); + } vlib_put_next_frame (vm, node, next, n_left_to_next); } diff --git a/src/vnet/pg/input.c b/src/vnet/pg/input.c index f81485de65f..321472c4d85 100644 --- a/src/vnet/pg/input.c +++ b/src/vnet/pg/input.c @@ -1578,7 +1578,7 @@ fill_buffer_offload_flags (vlib_main_t *vm, u32 *buffers, u32 n_buffers, (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID | VNET_BUFFER_F_L3_HDR_OFFSET_VALID | VNET_BUFFER_F_L4_HDR_OFFSET_VALID); - if (buffer_oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM) + if (buffer_oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM || gso_enabled) oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM; } else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6)) @@ -1596,7 +1596,7 @@ fill_buffer_offload_flags (vlib_main_t *vm, u32 *buffers, u32 n_buffers, if (l4_proto == IP_PROTOCOL_TCP) { - if (buffer_oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM) + if (buffer_oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM || gso_enabled) oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; /* only set GSO flag for chained buffers */ diff --git a/src/vnet/session/application.h b/src/vnet/session/application.h index 5fea61bdab4..c68a911230f 100644 --- a/src/vnet/session/application.h +++ b/src/vnet/session/application.h @@ -342,7 +342,7 @@ session_error_t app_worker_start_listen (app_worker_t *app_wrk, int app_worker_stop_listen (app_worker_t * app_wrk, app_listener_t * al); int app_worker_init_accepted (session_t * s); int app_worker_listened_notify (app_worker_t *app_wrk, session_handle_t alsh, - u32 opaque, int err); + u32 opaque, session_error_t err); int app_worker_unlisten_reply (app_worker_t *app_wrk, session_handle_t sh, u32 opaque, session_error_t err); int app_worker_accept_notify (app_worker_t * app_wrk, session_t * s); diff --git a/src/vnet/session/application_interface.c b/src/vnet/session/application_interface.c index 86f3dcdece6..a62f914d43a 100644 --- a/src/vnet/session/application_interface.c +++ b/src/vnet/session/application_interface.c @@ -106,8 +106,8 @@ parse_uri (char *uri, session_endpoint_cfg_t *sep) return 0; } -int -vnet_bind_uri (vnet_listen_args_t * a) +session_error_t +vnet_bind_uri (vnet_listen_args_t *a) { session_endpoint_cfg_t sep = SESSION_ENDPOINT_CFG_NULL; int rv; diff --git a/src/vnet/session/application_interface.h b/src/vnet/session/application_interface.h index 4eee17eeda8..f175e4a58c6 100644 --- a/src/vnet/session/application_interface.h +++ b/src/vnet/session/application_interface.h @@ -126,7 +126,7 @@ typedef struct _vnet_bind_args_t /* * Results */ - u64 handle; + session_handle_t handle; } vnet_listen_args_t; typedef struct _vnet_unlisten_args_t @@ -134,7 +134,7 @@ typedef struct _vnet_unlisten_args_t union { char *uri; - u64 handle; /**< Session handle */ + session_handle_t handle; /**< Session handle */ }; u32 app_index; /**< Owning application index */ u32 wrk_map_index; /**< App's local pool worker index */ @@ -356,7 +356,7 @@ STATIC_ASSERT (sizeof (session_listen_uri_msg_t) <= SESSION_CTRL_MSG_MAX_SIZE, typedef struct session_bound_msg_ { u32 context; - u64 handle; + session_handle_t handle; i32 retval; u8 lcl_is_ip4; u8 lcl_ip[16]; @@ -379,15 +379,15 @@ typedef struct session_unlisten_msg_ typedef struct session_unlisten_reply_msg_ { u32 context; - u64 handle; + session_handle_t handle; i32 retval; } __clib_packed session_unlisten_reply_msg_t; typedef struct session_accepted_msg_ { u32 context; - u64 listener_handle; - u64 handle; + session_handle_t listener_handle; + session_handle_t handle; uword server_rx_fifo; uword server_tx_fifo; u64 segment_handle; @@ -404,7 +404,7 @@ typedef struct session_accepted_reply_msg_ { u32 context; i32 retval; - u64 handle; + session_handle_t handle; } __clib_packed session_accepted_reply_msg_t; typedef struct session_connect_msg_ @@ -444,7 +444,7 @@ typedef struct session_connected_msg_ { u32 context; i32 retval; - u64 handle; + session_handle_t handle; uword server_rx_fifo; uword server_tx_fifo; u64 segment_handle; @@ -474,33 +474,33 @@ typedef struct session_disconnected_msg_ { u32 client_index; u32 context; - u64 handle; + session_handle_t handle; } __clib_packed session_disconnected_msg_t; typedef struct session_disconnected_reply_msg_ { u32 context; i32 retval; - u64 handle; + session_handle_t handle; } __clib_packed session_disconnected_reply_msg_t; typedef struct session_reset_msg_ { u32 client_index; u32 context; - u64 handle; + session_handle_t handle; } __clib_packed session_reset_msg_t; typedef struct session_reset_reply_msg_ { u32 context; i32 retval; - u64 handle; + session_handle_t handle; } __clib_packed session_reset_reply_msg_t; typedef struct session_req_worker_update_msg_ { - u64 session_handle; + session_handle_t session_handle; } __clib_packed session_req_worker_update_msg_t; /* NOTE: using u16 for wrk indices because message needs to fit in 18B */ @@ -509,12 +509,12 @@ typedef struct session_worker_update_msg_ u32 client_index; u16 wrk_index; u16 req_wrk_index; - u64 handle; + session_handle_t handle; } __clib_packed session_worker_update_msg_t; typedef struct session_worker_update_reply_msg_ { - u64 handle; + session_handle_t handle; uword rx_fifo; uword tx_fifo; u64 segment_handle; diff --git a/src/vnet/session/application_worker.c b/src/vnet/session/application_worker.c index c2d0d3b0cf3..befdb7c7002 100644 --- a/src/vnet/session/application_worker.c +++ b/src/vnet/session/application_worker.c @@ -58,10 +58,10 @@ void app_worker_free (app_worker_t * app_wrk) { application_t *app = application_get (app_wrk->app_index); + session_handle_t handle, *handles = 0, *sh; vnet_unlisten_args_t _a, *a = &_a; - u64 handle, *handles = 0, *sm_indices = 0; segment_manager_t *sm; - session_handle_t *sh; + u64 *sm_indices = 0; session_t *ls; u32 sm_index; int i; @@ -853,7 +853,7 @@ u8 * format_app_worker_listener (u8 * s, va_list * args) { app_worker_t *app_wrk = va_arg (*args, app_worker_t *); - u64 handle = va_arg (*args, u64); + session_handle_t handle = va_arg (*args, u64); u32 sm_index = va_arg (*args, u32); int verbose = va_arg (*args, int); session_t *listener; diff --git a/src/vnet/session/session.c b/src/vnet/session/session.c index e2fd1644fbd..67e7ee39001 100644 --- a/src/vnet/session/session.c +++ b/src/vnet/session/session.c @@ -97,6 +97,13 @@ session_send_io_evt_to_thread_custom (void *data, u32 thread_index, } int +session_program_tx_io_evt (session_handle_tu_t sh, session_evt_type_t evt_type) +{ + return session_send_evt_to_thread ((void *) &sh.session_index, 0, + (u32) sh.thread_index, evt_type); +} + +int session_send_ctrl_evt_to_thread (session_t * s, session_evt_type_t evt_type) { /* only events supported are disconnect, shutdown and reset */ @@ -266,7 +273,7 @@ session_cleanup_notify (session_t * s, session_cleanup_ntf_t ntf) app_worker_t *app_wrk; app_wrk = app_worker_get_if_valid (s->app_wrk_index); - if (!app_wrk) + if (PREDICT_FALSE (!app_wrk)) { if (ntf == SESSION_CLEANUP_TRANSPORT) return; @@ -1589,6 +1596,37 @@ session_reset (session_t * s) session_program_transport_ctrl_evt (s, SESSION_CTRL_EVT_RESET); } +void +session_detach_app (session_t *s) +{ + if (s->session_state < SESSION_STATE_TRANSPORT_CLOSING) + { + session_close (s); + } + else if (s->session_state < SESSION_STATE_TRANSPORT_DELETED) + { + transport_connection_t *tc; + + /* Transport is closing but it's not yet deleted. Confirm close and + * subsequently detach transport from session and enqueue a session + * cleanup notification. Transport closed and cleanup notifications are + * going to be dropped by session layer apis */ + transport_close (session_get_transport_proto (s), s->connection_index, + s->thread_index); + tc = session_get_transport (s); + tc->s_index = SESSION_INVALID_INDEX; + session_set_state (s, SESSION_STATE_TRANSPORT_DELETED); + session_cleanup_notify (s, SESSION_CLEANUP_SESSION); + } + else + { + session_cleanup_notify (s, SESSION_CLEANUP_SESSION); + } + + s->flags |= SESSION_F_APP_CLOSED; + s->app_wrk_index = APP_INVALID_INDEX; +} + /** * Notify transport the session can be half-disconnected. * diff --git a/src/vnet/session/session.h b/src/vnet/session/session.h index 78158d5f3ed..a5604bf8725 100644 --- a/src/vnet/session/session.h +++ b/src/vnet/session/session.h @@ -392,44 +392,37 @@ session_get_if_valid (u64 si, u32 thread_index) } always_inline session_t * -session_get_from_handle (session_handle_t handle) +session_get_from_handle (session_handle_tu_t handle) { session_main_t *smm = &session_main; - u32 session_index, thread_index; - session_parse_handle (handle, &session_index, &thread_index); - return pool_elt_at_index (smm->wrk[thread_index].sessions, session_index); + return pool_elt_at_index (smm->wrk[handle.thread_index].sessions, + handle.session_index); } always_inline session_t * -session_get_from_handle_if_valid (session_handle_t handle) +session_get_from_handle_if_valid (session_handle_tu_t handle) { - u32 session_index, thread_index; - session_parse_handle (handle, &session_index, &thread_index); - return session_get_if_valid (session_index, thread_index); + return session_get_if_valid (handle.session_index, handle.thread_index); } -u64 session_segment_handle (session_t * s); - /** * Get session from handle and avoid pool validation if no same thread * * Peekers are fine because pool grows with barrier (see @ref session_alloc) */ always_inline session_t * -session_get_from_handle_safe (u64 handle) +session_get_from_handle_safe (session_handle_tu_t handle) { - u32 thread_index = session_thread_from_handle (handle); - session_worker_t *wrk = &session_main.wrk[thread_index]; + session_worker_t *wrk = &session_main.wrk[handle.thread_index]; - if (thread_index == vlib_get_thread_index ()) + if (handle.thread_index == vlib_get_thread_index ()) { - return pool_elt_at_index (wrk->sessions, - session_index_from_handle (handle)); + return pool_elt_at_index (wrk->sessions, handle.session_index); } else { /* Don't use pool_elt_at index to avoid pool bitmap reallocs */ - return wrk->sessions + session_index_from_handle (handle); + return wrk->sessions + handle.session_index; } } @@ -455,17 +448,19 @@ int session_stop_listen (session_t * s); void session_half_close (session_t *s); void session_close (session_t * s); void session_reset (session_t * s); +void session_detach_app (session_t *s); void session_transport_half_close (session_t *s); void session_transport_close (session_t * s); void session_transport_reset (session_t * s); void session_transport_cleanup (session_t * s); -int session_send_io_evt_to_thread (svm_fifo_t * f, - session_evt_type_t evt_type); int session_enqueue_notify (session_t *s); int session_dequeue_notify (session_t * s); int session_enqueue_notify_cl (session_t *s); +int session_send_io_evt_to_thread (svm_fifo_t *f, session_evt_type_t evt_type); int session_send_io_evt_to_thread_custom (void *data, u32 thread_index, session_evt_type_t evt_type); +int session_program_tx_io_evt (session_handle_tu_t sh, + session_evt_type_t evt_type); void session_send_rpc_evt_to_thread (u32 thread_index, void *fp, void *rpc_args); void session_send_rpc_evt_to_thread_force (u32 thread_index, void *fp, @@ -478,6 +473,7 @@ void session_get_endpoint (session_t * s, transport_endpoint_t * tep, u8 is_lcl); int session_transport_attribute (session_t *s, u8 is_get, transport_endpt_attr_t *attr); +u64 session_segment_handle (session_t *s); u8 *format_session (u8 * s, va_list * args); uword unformat_session (unformat_input_t * input, va_list * args); @@ -654,8 +650,8 @@ session_vlib_thread_is_cl_thread (void) * Listen sessions */ -always_inline u64 -listen_session_get_handle (session_t * s) +always_inline session_handle_t +listen_session_get_handle (session_t *s) { ASSERT (s->session_state == SESSION_STATE_LISTENING || session_get_transport_proto (s) == TRANSPORT_PROTO_QUIC); diff --git a/src/vnet/session/session_api.c b/src/vnet/session/session_api.c index 2ecb464e38c..48eb932a2c9 100644 --- a/src/vnet/session/session_api.c +++ b/src/vnet/session/session_api.c @@ -711,9 +711,10 @@ done: VL_API_APP_WORKER_ADD_DEL_REPLY, ((!rv && mp->is_add) ? vec_len (args.segment->name) : 0), ({ rmp->is_add = mp->is_add; - rmp->wrk_index = clib_host_to_net_u32 (args.wrk_map_index); + rmp->wrk_index = mp->wrk_index; if (!rv && mp->is_add) { + rmp->wrk_index = clib_host_to_net_u32 (args.wrk_map_index); rmp->segment_handle = clib_host_to_net_u64 (args.segment_handle); rmp->app_event_queue_address = fifo_segment_msg_q_offset ((fifo_segment_t *) args.segment, 0); @@ -1465,10 +1466,11 @@ done: rmp = &msg.worker_add_del_reply; rmp->retval = rv; rmp->is_add = mp->is_add; + rmp->wrk_index = mp->wrk_index; rmp->api_client_handle = sapi_handle; - rmp->wrk_index = args.wrk_map_index; if (!rv && mp->is_add) { + rmp->wrk_index = args.wrk_map_index; rmp->segment_handle = args.segment_handle; /* No segment name and size. This supports only memfds */ rmp->app_event_queue_address = diff --git a/src/vnet/session/session_input.c b/src/vnet/session/session_input.c index 41e8beb8abc..73b777127fd 100644 --- a/src/vnet/session/session_input.c +++ b/src/vnet/session/session_input.c @@ -154,8 +154,7 @@ app_worker_flush_events_inline (app_worker_t *app_wrk, u32 thread_index, old_state = s->session_state; if (app->cb_fns.session_accept_callback (s)) { - session_close (s); - s->app_wrk_index = SESSION_INVALID_INDEX; + session_detach_app (s); break; } if (is_builtin) @@ -184,8 +183,7 @@ app_worker_flush_events_inline (app_worker_t *app_wrk, u32 thread_index, break; if (rv) { - session_close (s); - s->app_wrk_index = SESSION_INVALID_INDEX; + session_detach_app (s); break; } if (old_state >= SESSION_STATE_TRANSPORT_CLOSING) @@ -225,13 +223,20 @@ app_worker_flush_events_inline (app_worker_t *app_wrk, u32 thread_index, break; case SESSION_CTRL_EVT_TRANSPORT_CLOSED: s = session_get (evt->session_index, thread_index); + /* Notification enqueued before session was refused by app */ + if (PREDICT_FALSE (s->app_wrk_index == APP_INVALID_INDEX)) + break; if (app->cb_fns.session_transport_closed_callback) app->cb_fns.session_transport_closed_callback (s); break; case SESSION_CTRL_EVT_CLEANUP: s = session_get (evt->as_u64[0] & 0xffffffff, thread_index); - if (app->cb_fns.session_cleanup_callback) - app->cb_fns.session_cleanup_callback (s, evt->as_u64[0] >> 32); + /* Notification enqueued before session was refused by app */ + if (PREDICT_TRUE (s->app_wrk_index != APP_INVALID_INDEX)) + { + if (app->cb_fns.session_cleanup_callback) + app->cb_fns.session_cleanup_callback (s, evt->as_u64[0] >> 32); + } if (evt->as_u64[0] >> 32 != SESSION_CLEANUP_SESSION) break; uword_to_pointer (evt->as_u64[1], void (*) (session_t * s)) (s); diff --git a/src/vnet/session/session_node.c b/src/vnet/session/session_node.c index 0b9a5b4c4c0..0ec158fb429 100644 --- a/src/vnet/session/session_node.c +++ b/src/vnet/session/session_node.c @@ -491,15 +491,13 @@ session_mq_reset_reply_handler (void *data) app_worker_t *app_wrk; session_t *s; application_t *app; - u32 index, thread_index; mp = (session_reset_reply_msg_t *) data; app = application_lookup (mp->context); if (!app) return; - session_parse_handle (mp->handle, &index, &thread_index); - s = session_get_if_valid (index, thread_index); + s = session_get_from_handle_if_valid (mp->handle); /* No session or not the right session */ if (!s || s->session_state < SESSION_STATE_TRANSPORT_CLOSING) diff --git a/src/vnet/session/session_types.h b/src/vnet/session/session_types.h index 8ec972da832..5e650727d61 100644 --- a/src/vnet/session/session_types.h +++ b/src/vnet/session/session_types.h @@ -25,6 +25,19 @@ #define SESSION_CTRL_MSG_TX_MAX_SIZE 160 #define SESSION_NODE_FRAME_SIZE 128 +typedef u8 session_type_t; +typedef u64 session_handle_t; + +typedef union session_handle_tu_ +{ + session_handle_t handle; + struct + { + u32 session_index; + u32 thread_index; + }; +} __attribute__ ((__transparent_union__)) session_handle_tu_t; + #define foreach_session_endpoint_fields \ foreach_transport_endpoint_cfg_fields \ _(u8, transport_proto) \ @@ -125,9 +138,6 @@ session_endpoint_is_zero (session_endpoint_t * sep) return ip_is_zero (&sep->ip, sep->is_ip4); } -typedef u8 session_type_t; -typedef u64 session_handle_t; - typedef enum { SESSION_CLEANUP_TRANSPORT, @@ -144,19 +154,19 @@ typedef enum session_ft_action_ /* * Session states */ -#define foreach_session_state \ - _(CREATED, "created") \ - _(LISTENING, "listening") \ - _(CONNECTING, "connecting") \ - _(ACCEPTING, "accepting") \ - _(READY, "ready") \ - _(OPENED, "opened") \ - _(TRANSPORT_CLOSING, "transport-closing") \ - _(CLOSING, "closing") \ - _(APP_CLOSED, "app-closed") \ - _(TRANSPORT_CLOSED, "transport-closed") \ - _(CLOSED, "closed") \ - _(TRANSPORT_DELETED, "transport-deleted") \ +#define foreach_session_state \ + _ (CREATED, "created") \ + _ (LISTENING, "listening") \ + _ (CONNECTING, "connecting") \ + _ (ACCEPTING, "accepting") \ + _ (READY, "ready") \ + _ (OPENED, "opened") \ + _ (TRANSPORT_CLOSING, "transport-closing") \ + _ (CLOSING, "closing") \ + _ (APP_CLOSED, "app-closed") \ + _ (TRANSPORT_CLOSED, "transport-closed") \ + _ (CLOSED, "closed") \ + _ (TRANSPORT_DELETED, "transport-deleted") typedef enum { @@ -164,7 +174,7 @@ typedef enum foreach_session_state #undef _ SESSION_N_STATES, -} session_state_t; +} __clib_packed session_state_t; #define foreach_session_flag \ _ (RX_EVT, "rx-event") \ @@ -198,23 +208,30 @@ typedef struct session_ svm_fifo_t *rx_fifo; svm_fifo_t *tx_fifo; + union + { + session_handle_t handle; + struct + { + /** Index in thread pool where session was allocated */ + u32 session_index; + + /** Index of the thread that allocated the session */ + u32 thread_index; + }; + }; + /** Type built from transport and network protocol types */ session_type_t session_type; /** State in session layer state machine. See @ref session_state_t */ - volatile u8 session_state; - - /** Index in thread pool where session was allocated */ - u32 session_index; + volatile session_state_t session_state; /** Index of the app worker that owns the session */ u32 app_wrk_index; - /** Index of the thread that allocated the session */ - u8 thread_index; - /** Session flags. See @ref session_flags_t */ - u32 flags; + session_flags_t flags; /** Index of the transport connection associated to the session */ u32 connection_index; @@ -299,45 +316,35 @@ session_tx_is_dgram (session_t * s) always_inline session_handle_t session_handle (session_t * s) { - return ((u64) s->thread_index << 32) | (u64) s->session_index; + return s->handle; } always_inline u32 -session_index_from_handle (session_handle_t handle) +session_index_from_handle (session_handle_tu_t handle) { - return handle & 0xFFFFFFFF; + return handle.session_index; } always_inline u32 -session_thread_from_handle (session_handle_t handle) +session_thread_from_handle (session_handle_tu_t handle) { - return handle >> 32; + return handle.thread_index; } always_inline void -session_parse_handle (session_handle_t handle, u32 * index, - u32 * thread_index) +session_parse_handle (session_handle_tu_t handle, u32 *index, + u32 *thread_index) { - *index = session_index_from_handle (handle); - *thread_index = session_thread_from_handle (handle); + *index = handle.session_index; + *thread_index = handle.thread_index; } static inline session_handle_t session_make_handle (u32 session_index, u32 data) { - return (((u64) data << 32) | (u64) session_index); -} - -always_inline u32 -session_handle_index (session_handle_t ho_handle) -{ - return (ho_handle & 0xffffffff); -} - -always_inline u32 -session_handle_data (session_handle_t ho_handle) -{ - return (ho_handle >> 32); + return ((session_handle_tu_t){ .session_index = session_index, + .thread_index = data }) + .handle; } typedef enum diff --git a/src/vnet/srv6/sr_api.c b/src/vnet/srv6/sr_api.c index da774c4325f..a44c3098112 100644 --- a/src/vnet/srv6/sr_api.c +++ b/src/vnet/srv6/sr_api.c @@ -163,7 +163,7 @@ vl_api_sr_policy_add_v2_t_handler (vl_api_sr_policy_add_v2_t *mp) mp->type, ntohl (mp->fib_table), mp->is_encap, 0, NULL); vec_free (segments); - REPLY_MACRO (VL_API_SR_POLICY_ADD_REPLY); + REPLY_MACRO (VL_API_SR_POLICY_ADD_V2_REPLY); } static void diff --git a/src/vnet/tcp/tcp_format.c b/src/vnet/tcp/tcp_format.c index 751042ce1cd..4674f2cbaed 100644 --- a/src/vnet/tcp/tcp_format.c +++ b/src/vnet/tcp/tcp_format.c @@ -139,7 +139,9 @@ format_tcp_header (u8 * s, va_list * args) clib_net_to_host_u16 (tcp->window), clib_net_to_host_u16 (tcp->checksum)); - if (tcp_options_parse (tcp, &opts, tcp_is_syn (tcp)) < 0) + if (header_bytes > max_header_bytes) + s = format (s, "\n%Uoptions: truncated", format_white_space, indent); + else if (tcp_options_parse (tcp, &opts, tcp_is_syn (tcp)) < 0) s = format (s, "\n%Uoptions: parsing failed", format_white_space, indent); else s = format (s, "\n%U%U", format_white_space, indent, format_tcp_options, diff --git a/src/vnet/tls/tls.c b/src/vnet/tls/tls.c index 6129b09207b..5f00e6e302d 100644 --- a/src/vnet/tls/tls.c +++ b/src/vnet/tls/tls.c @@ -501,7 +501,6 @@ tls_session_accept_callback (session_t *ts) ctx->c_s_index = SESSION_INVALID_INDEX; ctx->c_thread_index = ts->thread_index; ctx->tls_ctx_handle = ctx_handle; - ts->session_state = SESSION_STATE_READY; ts->opaque = ctx_handle; ctx->tls_session_handle = session_handle (ts); ctx->listener_ctx_index = tls_listener->opaque; @@ -518,6 +517,9 @@ tls_session_accept_callback (session_t *ts) tls_disconnect_transport (ctx); } + if (ts->session_state < SESSION_STATE_READY) + ts->session_state = SESSION_STATE_READY; + return 0; } diff --git a/src/vpp-api/client/client.c b/src/vpp-api/client/client.c index 7a30792402c..d59273ed6cb 100644 --- a/src/vpp-api/client/client.c +++ b/src/vpp-api/client/client.c @@ -305,6 +305,8 @@ vac_connect (char * name, char * chroot_prefix, vac_callback_t cb, } /* Start read timeout thread */ + timeout_in_progress = false; + timeout_thread_cancelled = false; rv = pthread_create(&pm->timeout_thread_handle, NULL, vac_timeout_thread_fn, 0); if (rv) { diff --git a/src/vpp-api/vapi/CMakeLists.txt b/src/vpp-api/vapi/CMakeLists.txt index e01692210bb..e53d3e8b238 100644 --- a/src/vpp-api/vapi/CMakeLists.txt +++ b/src/vpp-api/vapi/CMakeLists.txt @@ -45,7 +45,7 @@ install( vapi_json_parser.py vapi_cpp_gen.py DESTINATION - share/vpp + ${CMAKE_INSTALL_DATADIR}/vpp COMPONENT vpp-dev ) diff --git a/src/vpp-api/vapi/vapi.hpp b/src/vpp-api/vapi/vapi.hpp index 109af9fc522..34d8f97ad89 100644 --- a/src/vpp-api/vapi/vapi.hpp +++ b/src/vpp-api/vapi/vapi.hpp @@ -422,7 +422,7 @@ private: void unregister_request (Common_req *request) { std::lock_guard<std::recursive_mutex> lock (requests_mutex); - std::remove (requests.begin (), requests.end (), request); + requests.erase (std::remove (requests.begin (), requests.end (), request)); } template <typename M> void register_event (Event_registration<M> *event) diff --git a/src/vpp-api/vapi/vapi_c_gen.py b/src/vpp-api/vapi/vapi_c_gen.py index 609f4bbd27e..9d1efb5e438 100755 --- a/src/vpp-api/vapi/vapi_c_gen.py +++ b/src/vpp-api/vapi/vapi_c_gen.py @@ -108,9 +108,11 @@ class CField(Field): % ( ".".join([prefix] + path), self.name, - self.get_vla_parameter_name(path) - if is_alloc - else "%s.%s" % (prefix, self.get_vla_field_name(path)), + ( + self.get_vla_parameter_name(path) + if is_alloc + else "%s.%s" % (prefix, self.get_vla_field_name(path)) + ), ) ] else: @@ -593,12 +595,16 @@ class CMessage(Message): ' VAPI_DBG("Swapping `%s\'@%%p to big endian", msg);' % self.get_c_name() ), - " %s(&msg->header);" % self.header.get_swap_to_be_func_name() - if self.header is not None - else "", - " %s(&msg->payload);" % self.get_swap_payload_to_be_func_name() - if self.has_payload() - else "", + ( + " %s(&msg->header);" % self.header.get_swap_to_be_func_name() + if self.header is not None + else "" + ), + ( + " %s(&msg->payload);" % self.get_swap_payload_to_be_func_name() + if self.has_payload() + else "" + ), "}", ] ) @@ -612,12 +618,16 @@ class CMessage(Message): ' VAPI_DBG("Swapping `%s\'@%%p to host byte order", msg);' % self.get_c_name() ), - " %s(&msg->header);" % self.header.get_swap_to_host_func_name() - if self.header is not None - else "", - " %s(&msg->payload);" % self.get_swap_payload_to_host_func_name() - if self.has_payload() - else "", + ( + " %s(&msg->header);" % self.header.get_swap_to_host_func_name() + if self.header is not None + else "" + ), + ( + " %s(&msg->payload);" % self.get_swap_payload_to_host_func_name() + if self.has_payload() + else "" + ), "}", ] ) @@ -791,12 +801,16 @@ class CMessage(Message): " name_with_crc,", " sizeof(name_with_crc) - 1,", " true," if has_context else " false,", - " offsetof(%s, context)," % self.header.get_c_name() - if has_context - else " 0,", - (" offsetof(%s, payload)," % self.get_c_name()) - if self.has_payload() - else " VAPI_INVALID_MSG_ID,", + ( + " offsetof(%s, context)," % self.header.get_c_name() + if has_context + else " 0," + ), + ( + (" offsetof(%s, payload)," % self.get_c_name()) + if self.has_payload() + else " VAPI_INVALID_MSG_ID," + ), " (verify_msg_size_fn_t)%s," % self.get_verify_msg_size_func_name(), " (generic_swap_fn_t)%s," % self.get_swap_to_be_func_name(), " (generic_swap_fn_t)%s," % self.get_swap_to_host_func_name(), diff --git a/src/vpp/api/types.c b/src/vpp/api/types.c index a30736fbb9f..92bbdb30ac9 100644 --- a/src/vpp/api/types.c +++ b/src/vpp/api/types.c @@ -88,8 +88,7 @@ format_vl_api_prefix (u8 * s, va_list * args) { const vl_api_prefix_t *pfx = va_arg (*args, vl_api_prefix_t *); - s = format (s, "%U/%d", format_vl_api_address, - &pfx->address, pfx->len); + s = format (s, "%U/%u", format_vl_api_address, &pfx->address, pfx->len); return s; } @@ -106,7 +105,7 @@ u8 * format_vl_api_version (u8 * s, va_list * args) { vl_api_version_t *ver = va_arg (*args, vl_api_version_t *); - s = format(s, "%d.%d.%d", ver->major, ver->minor, ver->patch); + s = format (s, "%u.%u.%u", ver->major, ver->minor, ver->patch); if (ver->pre_release[0] != 0) { s = format(s, "-%v", ver->pre_release); @@ -176,13 +175,14 @@ unformat_vl_api_ip6_address (unformat_input_t * input, va_list * args) } uword -unformat_vl_api_prefix (unformat_input_t * input, va_list * args) +unformat_vl_api_prefix (unformat_input_t *input, va_list *args) { - vl_api_prefix_t *pfx = va_arg (*args, vl_api_prefix_t *); + vl_api_prefix_t *pfx = va_arg (*args, vl_api_prefix_t *); + + if (unformat (input, "%U/%U", unformat_vl_api_address, &pfx->address, + unformat_u8, &pfx->len)) + return (1); - if (unformat (input, "%U/%d", unformat_vl_api_address, &pfx->address, - &pfx->len)) - return (1); return (0); } @@ -191,14 +191,14 @@ unformat_vl_api_mprefix (unformat_input_t * input, va_list * args) { vl_api_mprefix_t *pfx = va_arg (*args, vl_api_mprefix_t *); - if (unformat (input, "%U/%d", - unformat_vl_api_ip4_address, &pfx->grp_address.ip4, - &pfx->grp_address_length)) - pfx->af = ADDRESS_IP4; - else if (unformat (input, "%U/%d", - unformat_vl_api_ip6_address, &pfx->grp_address.ip6, - &pfx->grp_address_length)) - pfx->af = ADDRESS_IP6; + if (unformat (input, "%U/%U", unformat_vl_api_ip4_address, + &pfx->grp_address.ip4, unformat_u16, + &pfx->grp_address_length)) + pfx->af = ADDRESS_IP4; + else if (unformat (input, "%U/%U", unformat_vl_api_ip6_address, + &pfx->grp_address.ip6, unformat_u16, + &pfx->grp_address_length)) + pfx->af = ADDRESS_IP6; else if (unformat (input, "%U %U", unformat_vl_api_ip4_address, &pfx->src_address.ip4, unformat_vl_api_ip4_address, &pfx->grp_address.ip4)) @@ -235,17 +235,14 @@ unformat_vl_api_mprefix (unformat_input_t * input, va_list * args) uword unformat_vl_api_version (unformat_input_t * input, va_list * args) { -vl_api_version_t *ver = va_arg (*args, vl_api_version_t *); + vl_api_version_t *ver = va_arg (*args, vl_api_version_t *); -if (unformat (input, "%d.%d.%d-%s+%s", ver->major, ver->minor, ver->patch, ver->pre_release, ver->build_metadata - )) - return (1); -else if (unformat (input, "%d.%d.%d-%s", ver->major, ver->minor, ver->patch, ver->pre_release - )) - return (1); -else if (unformat (input, "%d.%d.%d", ver->major, ver->minor, ver->patch - )) - return (1); + if (unformat (input, "%u.%u.%u-%s+%s", ver->major, ver->minor, ver->patch, + ver->pre_release, ver->build_metadata) || + unformat (input, "%u.%u.%u-%s", ver->major, ver->minor, ver->patch, + ver->pre_release) || + unformat (input, "%u.%u.%u", ver->major, ver->minor, ver->patch)) + return (1); return (0); } diff --git a/src/vpp/vnet/main.c b/src/vpp/vnet/main.c index 71434a9c065..c57efd59a62 100644 --- a/src/vpp/vnet/main.c +++ b/src/vpp/vnet/main.c @@ -22,6 +22,8 @@ #include <vppinfra/clib.h> #include <vppinfra/cpu.h> +#include <vppinfra/bitmap.h> +#include <vppinfra/unix.h> #include <vlib/vlib.h> #include <vlib/unix/unix.h> #include <vlib/threads.h> @@ -43,25 +45,26 @@ static void vpp_find_plugin_path () { extern char *vat_plugin_path; - char *p, path[PATH_MAX]; - int rv; - u8 *s; + char *p; + u8 *s, *path; /* find executable path */ - if ((rv = readlink ("/proc/self/exe", path, PATH_MAX - 1)) == -1) + path = os_get_exec_path (); + + if (!path) return; - /* readlink doesn't provide null termination */ - path[rv] = 0; + /* add null termination */ + vec_add1 (path, 0); /* strip filename */ - if ((p = strrchr (path, '/')) == 0) - return; + if ((p = strrchr ((char *) path, '/')) == 0) + goto done; *p = 0; /* strip bin/ */ - if ((p = strrchr (path, '/')) == 0) - return; + if ((p = strrchr ((char *) path, '/')) == 0) + goto done; *p = 0; s = format (0, "%s/" CLIB_LIB_DIR "/vpp_plugins", path, path); @@ -71,6 +74,9 @@ vpp_find_plugin_path () s = format (0, "%s/" CLIB_LIB_DIR "/vpp_api_test_plugins", path, path); vec_add1 (s, 0); vat_plugin_path = (char *) s; + +done: + vec_free (path); } static void diff --git a/src/vppinfra/CMakeLists.txt b/src/vppinfra/CMakeLists.txt index f34ceed9d15..154e3a77cb1 100644 --- a/src/vppinfra/CMakeLists.txt +++ b/src/vppinfra/CMakeLists.txt @@ -229,7 +229,12 @@ elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD") ) endif() -option(VPP_USE_EXTERNAL_LIBEXECINFO "Use external libexecinfo (useful for non-glibc targets)." OFF) +if("${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD") + option(VPP_USE_EXTERNAL_LIBEXECINFO "Use external libexecinfo (useful for non-glibc targets)." ON) +else() + option(VPP_USE_EXTERNAL_LIBEXECINFO "Use external libexecinfo (useful for non-glibc targets)." OFF) +endif() + if(VPP_USE_EXTERNAL_LIBEXECINFO) set(EXECINFO_LIB execinfo) endif() @@ -260,6 +265,7 @@ if(VPP_BUILD_VPPINFRA_TESTS) longjmp macros maplog + mhash pmalloc pool_alloc pool_iterate diff --git a/src/vppinfra/cJSON.c b/src/vppinfra/cJSON.c index 448435de4dc..24e0110ed08 100644 --- a/src/vppinfra/cJSON.c +++ b/src/vppinfra/cJSON.c @@ -20,6 +20,7 @@ THE SOFTWARE. */ /* clang-format off */ + /* cJSON */ /* JSON parser in C. */ @@ -96,9 +97,9 @@ CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void) return (const char*) (global_error.json + global_error.position); } -CJSON_PUBLIC(char *) cJSON_GetStringValue(const cJSON * const item) +CJSON_PUBLIC (char *) cJSON_GetStringValue (const cJSON *const item) { - if (!cJSON_IsString(item)) + if (!cJSON_IsString (item)) { return NULL; } @@ -106,9 +107,9 @@ CJSON_PUBLIC(char *) cJSON_GetStringValue(const cJSON * const item) return item->valuestring; } -CJSON_PUBLIC(double) cJSON_GetNumberValue(const cJSON * const item) +CJSON_PUBLIC (double) cJSON_GetNumberValue (const cJSON *const item) { - if (!cJSON_IsNumber(item)) + if (!cJSON_IsNumber (item)) { return (double) NAN; } @@ -117,8 +118,9 @@ CJSON_PUBLIC(double) cJSON_GetNumberValue(const cJSON * const item) } /* This is a safeguard to prevent copy-pasters from using incompatible C and header files */ -#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 7) || (CJSON_VERSION_PATCH != 14) - #error cJSON.h and cJSON.c have different versions. Make sure that both have the same. +#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 7) || \ + (CJSON_VERSION_PATCH != 17) +#error cJSON.h and cJSON.c have different versions. Make sure that both have the same. #endif CJSON_PUBLIC(const char*) cJSON_Version(void) @@ -157,7 +159,7 @@ typedef struct internal_hooks { void *(CJSON_CDECL *allocate)(size_t size); void (CJSON_CDECL *deallocate)(void *pointer); - void *(CJSON_CDECL *reallocate)(void *pointer, size_t new_size, size_t old_size); + void *(CJSON_CDECL *reallocate) (void *pointer, size_t size); } internal_hooks; #if defined(_MSC_VER) @@ -170,20 +172,17 @@ static void CJSON_CDECL internal_free(void *pointer) { free(pointer); } +static void *CJSON_CDECL +internal_realloc (void *pointer, size_t size) +{ + return realloc (pointer, size); +} #else #define internal_malloc malloc #define internal_free free +#define internal_realloc realloc #endif -static void * CJSON_CDECL internal_realloc(void *pointer, size_t new_size, - size_t old_size) -{ - return realloc(pointer, new_size); -} - -static void * -cjson_realloc_internal (void *ptr, size_t new_size, size_t old_size); - /* strlen of character literals resolved at compile time */ #define static_strlen(string_literal) (sizeof(string_literal) - sizeof("")) @@ -217,8 +216,8 @@ CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks) /* Reset hooks */ global_hooks.allocate = malloc; global_hooks.deallocate = free; - global_hooks.reallocate = internal_realloc; - return; + global_hooks.reallocate = realloc; + return; } global_hooks.allocate = malloc; @@ -233,16 +232,11 @@ CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks) global_hooks.deallocate = hooks->free_fn; } - /* use realloc only if both free and malloc are used */ - global_hooks.reallocate = NULL; - if ((global_hooks.allocate == malloc) && (global_hooks.deallocate == free)) - { - global_hooks.reallocate = internal_realloc; - } - else - { - global_hooks.reallocate = cjson_realloc_internal; - } + global_hooks.reallocate = realloc; + if (hooks->realloc_fn != NULL) + { + global_hooks.reallocate = hooks->realloc_fn; + } } /* Internal constructor. */ @@ -405,14 +399,22 @@ CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number) return object->valuedouble = number; } +/* Note: when passing a NULL valuestring, cJSON_SetValuestring treats this as + * an error and return NULL */ CJSON_PUBLIC(char*) cJSON_SetValuestring(cJSON *object, const char *valuestring) { char *copy = NULL; /* if object's type is not cJSON_String or is cJSON_IsReference, it should not set valuestring */ - if (!(object->type & cJSON_String) || (object->type & cJSON_IsReference)) - { - return NULL; - } + if ((object == NULL) || !(object->type & cJSON_String) || + (object->type & cJSON_IsReference)) + { + return NULL; + } + /* return NULL if the object is corrupted or valuestring is NULL */ + if (object->valuestring == NULL || valuestring == NULL) + { + return NULL; + } if (strlen(valuestring) <= strlen(object->valuestring)) { strcpy(object->valuestring, valuestring); @@ -443,27 +445,6 @@ typedef struct internal_hooks hooks; } printbuffer; -static void * -cjson_realloc_internal (void *ptr, size_t new_size, size_t old_size) -{ - size_t copy_size; - if (old_size < new_size) - copy_size = old_size; - else - copy_size = new_size; - - unsigned char *newbuffer = global_hooks.allocate(new_size); - if (!newbuffer) - { - global_hooks.deallocate(ptr); - return NULL; - } - - memcpy (newbuffer, ptr, copy_size); - global_hooks.deallocate (ptr); - return newbuffer; -} - /* realloc printbuffer if necessary to have at least "needed" bytes more */ static unsigned char* ensure(printbuffer * const p, size_t needed) { @@ -515,14 +496,35 @@ static unsigned char* ensure(printbuffer * const p, size_t needed) newsize = needed * 2; } - newbuffer = p->hooks.reallocate (p->buffer, newsize, p->length); - if (newbuffer == NULL) - { - p->hooks.deallocate(p->buffer); - p->length = 0; - p->buffer = NULL; - return NULL; - } + if (p->hooks.reallocate != NULL) + { + /* reallocate with realloc if available */ + newbuffer = (unsigned char *) p->hooks.reallocate (p->buffer, newsize); + if (newbuffer == NULL) + { + p->hooks.deallocate (p->buffer); + p->length = 0; + p->buffer = NULL; + + return NULL; + } + } + else + { + /* otherwise reallocate manually */ + newbuffer = (unsigned char *) p->hooks.allocate (newsize); + if (!newbuffer) + { + p->hooks.deallocate (p->buffer); + p->length = 0; + p->buffer = NULL; + + return NULL; + } + + memcpy (newbuffer, p->buffer, p->offset + 1); + p->hooks.deallocate (p->buffer); + } p->length = newsize; p->buffer = newbuffer; @@ -570,6 +572,10 @@ static cJSON_bool print_number(const cJSON * const item, printbuffer * const out { length = sprintf((char*)number_buffer, "null"); } + else if (d == (double) item->valueint) + { + length = sprintf ((char *) number_buffer, "%d", item->valueint); + } else { /* Try 15 decimal places of precision to avoid nonsignificant nonzero digits */ @@ -1111,7 +1117,7 @@ CJSON_PUBLIC(cJSON *) cJSON_ParseWithLengthOpts(const char *value, size_t buffer } buffer.content = (const unsigned char*)value; - buffer.length = buffer_length; + buffer.length = buffer_length; buffer.offset = 0; buffer.hooks = global_hooks; @@ -1216,11 +1222,13 @@ static unsigned char *print(const cJSON * const item, cJSON_bool format, const i /* check if reallocate is available */ if (hooks->reallocate != NULL) { - printed = (unsigned char*) hooks->reallocate(buffer->buffer, buffer->offset + 1, default_buffer_size); - if (printed == NULL) { - goto fail; - } - buffer->buffer = NULL; + printed = (unsigned char *) hooks->reallocate (buffer->buffer, + buffer->offset + 1); + if (printed == NULL) + { + goto fail; + } + buffer->buffer = NULL; } else /* otherwise copy the JSON over to a new buffer */ { @@ -1658,8 +1666,13 @@ static cJSON_bool parse_object(cJSON * const item, parse_buffer * const input_bu current_item = new_item; } - /* parse the name of the child */ - input_buffer->offset++; + if (cannot_access_at_index (input_buffer, 1)) + { + goto fail; /* nothing comes after the comma */ + } + + /* parse the name of the child */ + input_buffer->offset++; buffer_skip_whitespace(input_buffer); if (!parse_string(current_item, input_buffer)) { @@ -2268,10 +2281,10 @@ CJSON_PUBLIC(cJSON_bool) cJSON_InsertItemInArray(cJSON *array, int which, cJSON { cJSON *after_inserted = NULL; - if (which < 0) - { - return false; - } + if (which < 0 || newitem == NULL) + { + return false; + } after_inserted = get_array_item(array, (size_t)which); if (after_inserted == NULL) @@ -2279,6 +2292,12 @@ CJSON_PUBLIC(cJSON_bool) cJSON_InsertItemInArray(cJSON *array, int which, cJSON return add_item_to_array(array, newitem); } + if (after_inserted != array->child && after_inserted->prev == NULL) + { + /* return false if after_inserted is a corrupted array item */ + return false; + } + newitem->next = after_inserted; newitem->prev = after_inserted->prev; after_inserted->prev = newitem; @@ -2295,7 +2314,8 @@ CJSON_PUBLIC(cJSON_bool) cJSON_InsertItemInArray(cJSON *array, int which, cJSON CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement) { - if ((parent == NULL) || (replacement == NULL) || (item == NULL)) + if ((parent == NULL) || (parent->child == NULL) || (replacement == NULL) || + (item == NULL)) { return false; } @@ -2365,6 +2385,11 @@ static cJSON_bool replace_item_in_object(cJSON *object, const char *string, cJSO cJSON_free(replacement->string); } replacement->string = (char*)cJSON_strdup((const unsigned char*)string, &global_hooks); + if (replacement->string == NULL) + { + return false; + } + replacement->type &= ~cJSON_StringIsConst; return cJSON_ReplaceItemViaPointer(object, get_object_item(object, string, case_sensitive), replacement); @@ -2639,9 +2664,9 @@ CJSON_PUBLIC(cJSON *) cJSON_CreateDoubleArray(const double *numbers, int count) for (i = 0; a && (i < (size_t) count); i++) { - n = cJSON_CreateNumber(numbers[i]); - if(!n) - { + n = cJSON_CreateNumber (numbers[i]); + if (!n) + { cJSON_Delete(a); return NULL; } @@ -2988,7 +3013,7 @@ CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON * const item) CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive) { - if ((a == NULL) || (b == NULL) || ((a->type & 0xFF) != (b->type & 0xFF)) || cJSON_IsInvalid(a)) + if ((a == NULL) || (b == NULL) || ((a->type & 0xFF) != (b->type & 0xFF))) { return false; } @@ -3121,7 +3146,7 @@ CJSON_PUBLIC(void) cJSON_free(void *object) global_hooks.deallocate(object); } -CJSON_PUBLIC(void *) cJSON_realloc(void *object, size_t new_size, size_t old_size) +CJSON_PUBLIC (void *) cJSON_realloc (void *object, size_t size) { - return global_hooks.reallocate(object, new_size, old_size); + return global_hooks.reallocate (object, size); } diff --git a/src/vppinfra/cJSON.h b/src/vppinfra/cJSON.h index 1474c4e5c49..1c98dfac70e 100644 --- a/src/vppinfra/cJSON.h +++ b/src/vppinfra/cJSON.h @@ -81,7 +81,7 @@ then using the CJSON_API_VISIBILITY flag to "export" the same symbols the way CJ /* project version */ #define CJSON_VERSION_MAJOR 1 #define CJSON_VERSION_MINOR 7 -#define CJSON_VERSION_PATCH 14 +#define CJSON_VERSION_PATCH 17 #include <stddef.h> @@ -127,8 +127,7 @@ typedef struct cJSON_Hooks /* malloc/free are CDECL on Windows regardless of the default calling convention of the compiler, so ensure the hooks allow passing those functions directly. */ void *(CJSON_CDECL *malloc_fn)(size_t sz); void (CJSON_CDECL *free_fn)(void *ptr); - void *(CJSON_CDECL *realloc_fn) (void *ptr, size_t new_size, - size_t old_size); + void *(CJSON_CDECL *realloc_fn) (void *ptr, size_t sz); } cJSON_Hooks; typedef int cJSON_bool; @@ -256,9 +255,10 @@ CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse); * case_sensitive determines if object keys are treated case sensitive (1) or case insensitive (0) */ CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive); -/* Minify a strings, remove blank characters(such as ' ', '\t', '\r', '\n') from strings. - * The input pointer json cannot point to a read-only address area, such as a string constant, - * but should point to a readable and writable adress area. */ +/* Minify a strings, remove blank characters(such as ' ', '\t', '\r', '\n') + * from strings. The input pointer json cannot point to a read-only address + * area, such as a string constant, + * but should point to a readable and writable address area. */ CJSON_PUBLIC(void) cJSON_Minify(char *json); /* Helper functions for creating and adding items to an object at the same time. @@ -281,14 +281,21 @@ CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number); /* Change the valuestring of a cJSON_String object, only takes effect when type of object is cJSON_String */ CJSON_PUBLIC(char*) cJSON_SetValuestring(cJSON *object, const char *valuestring); +/* If the object is not a boolean type this does nothing and returns + * cJSON_Invalid else it returns the new type*/ +#define cJSON_SetBoolValue(object, boolValue) \ + ((object != NULL && ((object)->type & (cJSON_False | cJSON_True))) ? \ + (object)->type = ((object)->type & (~(cJSON_False | cJSON_True))) | \ + ((boolValue) ? cJSON_True : cJSON_False) : \ + cJSON_Invalid) + /* Macro for iterating over an array or object */ #define cJSON_ArrayForEach(element, array) for(element = (array != NULL) ? (array)->child : NULL; element != NULL; element = element->next) /* malloc/free objects using the malloc/free functions that have been set with cJSON_InitHooks */ CJSON_PUBLIC(void *) cJSON_malloc(size_t size); CJSON_PUBLIC(void) cJSON_free(void *object); -CJSON_PUBLIC (void *) -cJSON_realloc (void *object, size_t new_size, size_t old_size); +CJSON_PUBLIC (void *) cJSON_realloc (void *object, size_t size); #ifdef __cplusplus } diff --git a/src/vppinfra/crypto/sha2.h b/src/vppinfra/crypto/sha2.h index 51006151365..69a24a2d087 100644 --- a/src/vppinfra/crypto/sha2.h +++ b/src/vppinfra/crypto/sha2.h @@ -1,16 +1,5 @@ -/* - * Copyright (c) 2019 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2024 Cisco Systems, Inc. */ #ifndef included_sha2_h @@ -18,12 +7,8 @@ #include <vppinfra/clib.h> #include <vppinfra/vector.h> +#include <vppinfra/string.h> -#define SHA224_DIGEST_SIZE 28 -#define SHA224_BLOCK_SIZE 64 - -#define SHA256_DIGEST_SIZE 32 -#define SHA256_BLOCK_SIZE 64 #define SHA256_ROTR(x, y) ((x >> y) | (x << (32 - y))) #define SHA256_CH(a, b, c) ((a & b) ^ (~a & c)) #define SHA256_MAJ(a, b, c) ((a & b) ^ (a & c) ^ (b & c)) @@ -60,17 +45,6 @@ s[0] = t1 + t2; \ } -#define SHA512_224_DIGEST_SIZE 28 -#define SHA512_224_BLOCK_SIZE 128 - -#define SHA512_256_DIGEST_SIZE 32 -#define SHA512_256_BLOCK_SIZE 128 - -#define SHA384_DIGEST_SIZE 48 -#define SHA384_BLOCK_SIZE 128 - -#define SHA512_DIGEST_SIZE 64 -#define SHA512_BLOCK_SIZE 128 #define SHA512_ROTR(x, y) ((x >> y) | (x << (64 - y))) #define SHA512_CH(a, b, c) ((a & b) ^ (~a & c)) #define SHA512_MAJ(a, b, c) ((a & b) ^ (a & c) ^ (b & c)) @@ -125,7 +99,7 @@ static const u32 sha256_h[8] = { 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19 }; -static const u32 sha256_k[64] = { +static const u32 clib_sha2_256_k[64] = { 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, @@ -159,7 +133,7 @@ static const u64 sha512_256_h[8] = { 0x22312194fc2bf72c, 0x9f555fa3c84c64c2, 0x96283ee2a88effe3, 0xbe5e1e2553863992, 0x2b0199fc2c85b8aa, 0x0eb72ddc81c52ca2 }; -static const u64 sha512_k[80] = { +static const u64 clib_sha2_512_k[80] = { 0x428a2f98d728ae22, 0x7137449123ef65cd, 0xb5c0fbcfec4d3b2f, 0xe9b5dba58189dbbc, 0x3956c25bf348b538, 0x59f111f1b605d019, 0x923f82a4af194f9b, 0xab1c5ed5da6d8118, 0xd807aa98a3030242, @@ -199,80 +173,102 @@ typedef enum CLIB_SHA2_512_256, } clib_sha2_type_t; -#define SHA2_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE -#define SHA2_MAX_DIGEST_SIZE SHA512_DIGEST_SIZE +#define CLIB_SHA2_256_BLOCK_SIZE 64 +#define CLIB_SHA2_512_BLOCK_SIZE 128 +#define SHA2_MAX_BLOCK_SIZE CLIB_SHA2_512_BLOCK_SIZE +#define SHA2_MAX_DIGEST_SIZE 64 -typedef struct +static const struct { - u64 total_bytes; - u16 n_pending; u8 block_size; u8 digest_size; - union - { - u32 h32[8]; - u64 h64[8]; + const u32 *h32; + const u64 *h64; +} clib_sha2_variants[] = { + [CLIB_SHA2_224] = { + .block_size = CLIB_SHA2_256_BLOCK_SIZE, + .digest_size = 28, + .h32 = sha224_h, + }, + [CLIB_SHA2_256] = { + .block_size = CLIB_SHA2_256_BLOCK_SIZE, + .digest_size = 32, + .h32 = sha256_h, + }, + [CLIB_SHA2_384] = { + .block_size = CLIB_SHA2_512_BLOCK_SIZE, + .digest_size = 48, + .h64 = sha384_h, + }, + [CLIB_SHA2_512] = { + .block_size = CLIB_SHA2_512_BLOCK_SIZE, + .digest_size = 64, + .h64 = sha512_h, + }, + [CLIB_SHA2_512_224] = { + .block_size = CLIB_SHA2_512_BLOCK_SIZE, + .digest_size = 28, + .h64 = sha512_224_h, + }, + [CLIB_SHA2_512_256] = { + .block_size = CLIB_SHA2_512_BLOCK_SIZE, + .digest_size = 32, + .h64 = sha512_256_h, + }, +}; + +typedef union +{ + u32 h32[8]; + u64 h64[8]; #ifdef CLIB_SHA256_ISA - u32x4 h32x4[2]; + u32x4 h32x4[2]; #endif - }; +} clib_sha2_h_t; + +typedef struct +{ + u64 total_bytes; + u16 n_pending; + clib_sha2_h_t h; union { u8 as_u8[SHA2_MAX_BLOCK_SIZE]; u64 as_u64[SHA2_MAX_BLOCK_SIZE / sizeof (u64)]; uword as_uword[SHA2_MAX_BLOCK_SIZE / sizeof (uword)]; } pending; +} clib_sha2_state_t; + +typedef struct +{ + clib_sha2_type_t type; + u8 block_size; + u8 digest_size; + clib_sha2_state_t state; } clib_sha2_ctx_t; static_always_inline void -clib_sha2_init (clib_sha2_ctx_t *ctx, clib_sha2_type_t type) +clib_sha2_state_init (clib_sha2_state_t *state, clib_sha2_type_t type) { - const u32 *h32 = 0; - const u64 *h64 = 0; + clib_sha2_state_t st = {}; - ctx->total_bytes = 0; - ctx->n_pending = 0; - - switch (type) - { - case CLIB_SHA2_224: - h32 = sha224_h; - ctx->block_size = SHA224_BLOCK_SIZE; - ctx->digest_size = SHA224_DIGEST_SIZE; - break; - case CLIB_SHA2_256: - h32 = sha256_h; - ctx->block_size = SHA256_BLOCK_SIZE; - ctx->digest_size = SHA256_DIGEST_SIZE; - break; - case CLIB_SHA2_384: - h64 = sha384_h; - ctx->block_size = SHA384_BLOCK_SIZE; - ctx->digest_size = SHA384_DIGEST_SIZE; - break; - case CLIB_SHA2_512: - h64 = sha512_h; - ctx->block_size = SHA512_BLOCK_SIZE; - ctx->digest_size = SHA512_DIGEST_SIZE; - break; - case CLIB_SHA2_512_224: - h64 = sha512_224_h; - ctx->block_size = SHA512_224_BLOCK_SIZE; - ctx->digest_size = SHA512_224_DIGEST_SIZE; - break; - case CLIB_SHA2_512_256: - h64 = sha512_256_h; - ctx->block_size = SHA512_256_BLOCK_SIZE; - ctx->digest_size = SHA512_256_DIGEST_SIZE; - break; - } - if (h32) + if (clib_sha2_variants[type].block_size == CLIB_SHA2_256_BLOCK_SIZE) for (int i = 0; i < 8; i++) - ctx->h32[i] = h32[i]; - - if (h64) + st.h.h32[i] = clib_sha2_variants[type].h32[i]; + else for (int i = 0; i < 8; i++) - ctx->h64[i] = h64[i]; + st.h.h64[i] = clib_sha2_variants[type].h64[i]; + + *state = st; +} + +static_always_inline void +clib_sha2_init (clib_sha2_ctx_t *ctx, clib_sha2_type_t type) +{ + clib_sha2_state_init (&ctx->state, type); + ctx->block_size = clib_sha2_variants[type].block_size; + ctx->digest_size = clib_sha2_variants[type].digest_size; + ctx->type = type; } #ifdef CLIB_SHA256_ISA @@ -295,7 +291,7 @@ static inline void clib_sha256_vec_4_rounds (u32x4 w, u8 n, u32x4 s[]) { #ifdef CLIB_SHA256_ISA_INTEL - u32x4 r = *(u32x4 *) (sha256_k + 4 * n) + w; + u32x4 r = *(u32x4 *) (clib_sha2_256_k + 4 * n) + w; s[0] = (u32x4) _mm_sha256rnds2_epu32 ((__m128i) s[0], (__m128i) s[1], (__m128i) r); r = (u32x4) u64x2_interleave_hi ((u64x2) r, (u64x2) r); @@ -303,7 +299,7 @@ clib_sha256_vec_4_rounds (u32x4 w, u8 n, u32x4 s[]) (__m128i) r); #elif defined(CLIB_SHA256_ISA_ARM) u32x4 r0, s0; - const u32x4u *k = (u32x4u *) sha256_k; + const u32x4u *k = (u32x4u *) clib_sha2_256_k; r0 = w + k[n]; s0 = s[0]; @@ -338,14 +334,14 @@ clib_sha256_vec_shuffle (u32x4 d[2]) #endif static inline void -clib_sha256_block (clib_sha2_ctx_t *ctx, const u8 *msg, uword n_blocks) +clib_sha256_block (clib_sha2_state_t *st, const u8 *msg, uword n_blocks) { #if defined(CLIB_SHA256_ISA) u32x4 h[2]; u32x4u *m = (u32x4u *) msg; - h[0] = ctx->h32x4[0]; - h[1] = ctx->h32x4[1]; + h[0] = st->h.h32x4[0]; + h[1] = st->h.h32x4[1]; clib_sha256_vec_shuffle (h); @@ -399,159 +395,176 @@ clib_sha256_block (clib_sha2_ctx_t *ctx, const u8 *msg, uword n_blocks) clib_sha256_vec_shuffle (h); - ctx->h32x4[0] = h[0]; - ctx->h32x4[1] = h[1]; + st->h.h32x4[0] = h[0]; + st->h.h32x4[1] = h[1]; #else u32 w[64], s[8], i; + clib_sha2_h_t h; + + h = st->h; - while (n_blocks) + for (; n_blocks; msg += CLIB_SHA2_256_BLOCK_SIZE, n_blocks--) { for (i = 0; i < 8; i++) - s[i] = ctx->h32[i]; + s[i] = h.h32[i]; for (i = 0; i < 16; i++) { - w[i] = clib_net_to_host_u32 (*((u32 *) msg + i)); - SHA256_TRANSFORM (s, w, i, sha256_k[i]); + w[i] = clib_net_to_host_u32 ((((u32u *) msg)[i])); + SHA256_TRANSFORM (s, w, i, clib_sha2_256_k[i]); } for (i = 16; i < 64; i++) { SHA256_MSG_SCHED (w, i); - SHA256_TRANSFORM (s, w, i, sha256_k[i]); + SHA256_TRANSFORM (s, w, i, clib_sha2_256_k[i]); } for (i = 0; i < 8; i++) - ctx->h32[i] += s[i]; - - /* next */ - msg += SHA256_BLOCK_SIZE; - n_blocks--; + h.h32[i] += s[i]; } + + st->h = h; #endif } static_always_inline void -clib_sha512_block (clib_sha2_ctx_t *ctx, const u8 *msg, uword n_blocks) +clib_sha512_block (clib_sha2_state_t *st, const u8 *msg, uword n_blocks) { u64 w[80], s[8], i; + clib_sha2_h_t h; + + h = st->h; - while (n_blocks) + for (; n_blocks; msg += CLIB_SHA2_512_BLOCK_SIZE, n_blocks--) { for (i = 0; i < 8; i++) - s[i] = ctx->h64[i]; + s[i] = h.h64[i]; for (i = 0; i < 16; i++) { - w[i] = clib_net_to_host_u64 (*((u64 *) msg + i)); - SHA512_TRANSFORM (s, w, i, sha512_k[i]); + w[i] = clib_net_to_host_u64 ((((u64u *) msg)[i])); + SHA512_TRANSFORM (s, w, i, clib_sha2_512_k[i]); } for (i = 16; i < 80; i++) { SHA512_MSG_SCHED (w, i); - SHA512_TRANSFORM (s, w, i, sha512_k[i]); + SHA512_TRANSFORM (s, w, i, clib_sha2_512_k[i]); } for (i = 0; i < 8; i++) - ctx->h64[i] += s[i]; - - /* next */ - msg += SHA512_BLOCK_SIZE; - n_blocks--; + h.h64[i] += s[i]; } + + st->h = h; } static_always_inline void -clib_sha2_update (clib_sha2_ctx_t *ctx, const u8 *msg, uword n_bytes) +clib_sha2_update_internal (clib_sha2_state_t *st, u8 block_size, const u8 *msg, + uword n_bytes) { uword n_blocks; - if (ctx->n_pending) + if (st->n_pending) { - uword n_left = ctx->block_size - ctx->n_pending; + uword n_left = block_size - st->n_pending; if (n_bytes < n_left) { - clib_memcpy_fast (ctx->pending.as_u8 + ctx->n_pending, msg, n_bytes); - ctx->n_pending += n_bytes; + clib_memcpy_fast (st->pending.as_u8 + st->n_pending, msg, n_bytes); + st->n_pending += n_bytes; return; } else { - clib_memcpy_fast (ctx->pending.as_u8 + ctx->n_pending, msg, n_left); - if (ctx->block_size == SHA512_BLOCK_SIZE) - clib_sha512_block (ctx, ctx->pending.as_u8, 1); + clib_memcpy_fast (st->pending.as_u8 + st->n_pending, msg, n_left); + if (block_size == CLIB_SHA2_512_BLOCK_SIZE) + clib_sha512_block (st, st->pending.as_u8, 1); else - clib_sha256_block (ctx, ctx->pending.as_u8, 1); - ctx->n_pending = 0; - ctx->total_bytes += ctx->block_size; + clib_sha256_block (st, st->pending.as_u8, 1); + st->n_pending = 0; + st->total_bytes += block_size; n_bytes -= n_left; msg += n_left; } } - if ((n_blocks = n_bytes / ctx->block_size)) + if ((n_blocks = n_bytes / block_size)) { - if (ctx->block_size == SHA512_BLOCK_SIZE) - clib_sha512_block (ctx, msg, n_blocks); + if (block_size == CLIB_SHA2_512_BLOCK_SIZE) + clib_sha512_block (st, msg, n_blocks); else - clib_sha256_block (ctx, msg, n_blocks); - n_bytes -= n_blocks * ctx->block_size; - msg += n_blocks * ctx->block_size; - ctx->total_bytes += n_blocks * ctx->block_size; + clib_sha256_block (st, msg, n_blocks); + n_bytes -= n_blocks * block_size; + msg += n_blocks * block_size; + st->total_bytes += n_blocks * block_size; } if (n_bytes) { - clib_memset_u8 (ctx->pending.as_u8, 0, ctx->block_size); - clib_memcpy_fast (ctx->pending.as_u8, msg, n_bytes); - ctx->n_pending = n_bytes; + clib_memset_u8 (st->pending.as_u8, 0, block_size); + clib_memcpy_fast (st->pending.as_u8, msg, n_bytes); + st->n_pending = n_bytes; } else - ctx->n_pending = 0; + st->n_pending = 0; } static_always_inline void -clib_sha2_final (clib_sha2_ctx_t *ctx, u8 *digest) +clib_sha2_update (clib_sha2_ctx_t *ctx, const u8 *msg, uword n_bytes) +{ + clib_sha2_update_internal (&ctx->state, ctx->block_size, msg, n_bytes); +} + +static_always_inline void +clib_sha2_final_internal (clib_sha2_state_t *st, u8 block_size, u8 digest_size, + u8 *digest) { int i; - ctx->total_bytes += ctx->n_pending; - if (ctx->n_pending == 0) + st->total_bytes += st->n_pending; + if (st->n_pending == 0) { - clib_memset (ctx->pending.as_u8, 0, ctx->block_size); - ctx->pending.as_u8[0] = 0x80; + clib_memset (st->pending.as_u8, 0, block_size); + st->pending.as_u8[0] = 0x80; } - else if (ctx->n_pending + sizeof (u64) + sizeof (u8) > ctx->block_size) + else if (st->n_pending + sizeof (u64) + sizeof (u8) > block_size) { - ctx->pending.as_u8[ctx->n_pending] = 0x80; - if (ctx->block_size == SHA512_BLOCK_SIZE) - clib_sha512_block (ctx, ctx->pending.as_u8, 1); + st->pending.as_u8[st->n_pending] = 0x80; + if (block_size == CLIB_SHA2_512_BLOCK_SIZE) + clib_sha512_block (st, st->pending.as_u8, 1); else - clib_sha256_block (ctx, ctx->pending.as_u8, 1); - clib_memset (ctx->pending.as_u8, 0, ctx->block_size); + clib_sha256_block (st, st->pending.as_u8, 1); + clib_memset (st->pending.as_u8, 0, block_size); } else - ctx->pending.as_u8[ctx->n_pending] = 0x80; + st->pending.as_u8[st->n_pending] = 0x80; - ctx->pending.as_u64[ctx->block_size / 8 - 1] = - clib_net_to_host_u64 (ctx->total_bytes * 8); - if (ctx->block_size == SHA512_BLOCK_SIZE) - clib_sha512_block (ctx, ctx->pending.as_u8, 1); - else - clib_sha256_block (ctx, ctx->pending.as_u8, 1); + st->pending.as_u64[block_size / 8 - 1] = + clib_net_to_host_u64 (st->total_bytes * 8); - if (ctx->block_size == SHA512_BLOCK_SIZE) + if (block_size == CLIB_SHA2_512_BLOCK_SIZE) { - for (i = 0; i < ctx->digest_size / sizeof (u64); i++) - *((u64 *) digest + i) = clib_net_to_host_u64 (ctx->h64[i]); + clib_sha512_block (st, st->pending.as_u8, 1); + for (i = 0; i < digest_size / sizeof (u64); i++) + ((u64 *) digest)[i] = clib_net_to_host_u64 (st->h.h64[i]); /* sha512-224 case - write half of u64 */ - if (i * sizeof (u64) < ctx->digest_size) - *((u32 *) digest + 2 * i) = clib_net_to_host_u32 (ctx->h64[i] >> 32); + if (i * sizeof (u64) < digest_size) + ((u32 *) digest)[2 * i] = clib_net_to_host_u32 (st->h.h64[i] >> 32); } else - for (i = 0; i < ctx->digest_size / sizeof (u32); i++) - *((u32 *) digest + i) = clib_net_to_host_u32 (ctx->h32[i]); + { + clib_sha256_block (st, st->pending.as_u8, 1); + for (i = 0; i < digest_size / sizeof (u32); i++) + *((u32 *) digest + i) = clib_net_to_host_u32 (st->h.h32[i]); + } +} + +static_always_inline void +clib_sha2_final (clib_sha2_ctx_t *ctx, u8 *digest) +{ + clib_sha2_final_internal (&ctx->state, ctx->block_size, ctx->digest_size, + digest); } static_always_inline void @@ -570,70 +583,133 @@ clib_sha2 (clib_sha2_type_t type, const u8 *msg, uword len, u8 *digest) #define clib_sha512_224(...) clib_sha2 (CLIB_SHA2_512_224, __VA_ARGS__) #define clib_sha512_256(...) clib_sha2 (CLIB_SHA2_512_256, __VA_ARGS__) -static_always_inline void -clib_hmac_sha2 (clib_sha2_type_t type, const u8 *key, uword key_len, - const u8 *msg, uword len, u8 *digest) +/* + * HMAC + */ + +typedef struct { - clib_sha2_ctx_t _ctx, *ctx = &_ctx; - uword key_data[SHA2_MAX_BLOCK_SIZE / sizeof (uword)]; - u8 i_digest[SHA2_MAX_DIGEST_SIZE]; - int i, n_words; + clib_sha2_h_t ipad_h; + clib_sha2_h_t opad_h; +} clib_sha2_hmac_key_data_t; + +typedef struct +{ + clib_sha2_type_t type; + u8 block_size; + u8 digest_size; + clib_sha2_state_t ipad_state; + clib_sha2_state_t opad_state; +} clib_sha2_hmac_ctx_t; - clib_sha2_init (ctx, type); - n_words = ctx->block_size / sizeof (uword); +static_always_inline void +clib_sha2_hmac_key_data (clib_sha2_type_t type, const u8 *key, uword key_len, + clib_sha2_hmac_key_data_t *kd) +{ + u8 block_size = clib_sha2_variants[type].block_size; + u8 data[SHA2_MAX_BLOCK_SIZE] = {}; + u8 ikey[SHA2_MAX_BLOCK_SIZE]; + u8 okey[SHA2_MAX_BLOCK_SIZE]; + clib_sha2_state_t ipad_state; + clib_sha2_state_t opad_state; /* key */ - if (key_len > ctx->block_size) + if (key_len > block_size) { /* key is longer than block, calculate hash of key */ - clib_sha2_update (ctx, key, key_len); - for (i = (ctx->digest_size / sizeof (uword)) / 2; i < n_words; i++) - key_data[i] = 0; - clib_sha2_final (ctx, (u8 *) key_data); - clib_sha2_init (ctx, type); + clib_sha2_ctx_t ctx; + clib_sha2_init (&ctx, type); + clib_sha2_update (&ctx, key, key_len); + clib_sha2_final (&ctx, (u8 *) data); } else + clib_memcpy_fast (data, key, key_len); + + for (int i = 0, w = 0; w < block_size; w += sizeof (uword), i++) { - for (i = 0; i < n_words; i++) - key_data[i] = 0; - clib_memcpy_fast (key_data, key, key_len); + ((uwordu *) ikey)[i] = ((uwordu *) data)[i] ^ 0x3636363636363636UL; + ((uwordu *) okey)[i] = ((uwordu *) data)[i] ^ 0x5c5c5c5c5c5c5c5cUL; } - /* ipad */ - for (i = 0; i < n_words; i++) - ctx->pending.as_uword[i] = key_data[i] ^ (uword) 0x3636363636363636; - if (ctx->block_size == SHA512_BLOCK_SIZE) - clib_sha512_block (ctx, ctx->pending.as_u8, 1); - else - clib_sha256_block (ctx, ctx->pending.as_u8, 1); - ctx->total_bytes += ctx->block_size; - - /* message */ - clib_sha2_update (ctx, msg, len); - clib_sha2_final (ctx, i_digest); - - /* opad */ - clib_sha2_init (ctx, type); - for (i = 0; i < n_words; i++) - ctx->pending.as_uword[i] = key_data[i] ^ (uword) 0x5c5c5c5c5c5c5c5c; - if (ctx->block_size == SHA512_BLOCK_SIZE) - clib_sha512_block (ctx, ctx->pending.as_u8, 1); + clib_sha2_state_init (&ipad_state, type); + clib_sha2_state_init (&opad_state, type); + + if (block_size == CLIB_SHA2_512_BLOCK_SIZE) + { + clib_sha512_block (&ipad_state, ikey, 1); + clib_sha512_block (&opad_state, okey, 1); + } else - clib_sha256_block (ctx, ctx->pending.as_u8, 1); - ctx->total_bytes += ctx->block_size; + { + clib_sha256_block (&ipad_state, ikey, 1); + clib_sha256_block (&opad_state, okey, 1); + } + + kd->ipad_h = ipad_state.h; + kd->opad_h = opad_state.h; +} + +static_always_inline void +clib_sha2_hmac_init (clib_sha2_hmac_ctx_t *ctx, clib_sha2_type_t type, + clib_sha2_hmac_key_data_t *kd) +{ + u8 block_size = clib_sha2_variants[type].block_size; + u8 digest_size = clib_sha2_variants[type].digest_size; + + *ctx = (clib_sha2_hmac_ctx_t) { + .type = type, + .block_size = block_size, + .digest_size = digest_size, + .ipad_state = { + .h = kd->ipad_h, + .total_bytes = block_size, + }, + .opad_state = { + .h = kd->opad_h, + .total_bytes = block_size, + }, + }; +} + +static_always_inline void +clib_sha2_hmac_update (clib_sha2_hmac_ctx_t *ctx, const u8 *msg, uword len) +{ + clib_sha2_update_internal (&ctx->ipad_state, ctx->block_size, msg, len); +} + +static_always_inline void +clib_sha2_hmac_final (clib_sha2_hmac_ctx_t *ctx, u8 *digest) +{ + u8 i_digest[SHA2_MAX_DIGEST_SIZE]; + + clib_sha2_final_internal (&ctx->ipad_state, ctx->block_size, + ctx->digest_size, i_digest); + clib_sha2_update_internal (&ctx->opad_state, ctx->block_size, i_digest, + ctx->digest_size); + clib_sha2_final_internal (&ctx->opad_state, ctx->block_size, + ctx->digest_size, digest); +} + +static_always_inline void +clib_sha2_hmac (clib_sha2_type_t type, const u8 *key, uword key_len, + const u8 *msg, uword len, u8 *digest) +{ + clib_sha2_hmac_ctx_t _ctx, *ctx = &_ctx; + clib_sha2_hmac_key_data_t kd; - /* digest */ - clib_sha2_update (ctx, i_digest, ctx->digest_size); - clib_sha2_final (ctx, digest); + clib_sha2_hmac_key_data (type, key, key_len, &kd); + clib_sha2_hmac_init (ctx, type, &kd); + clib_sha2_hmac_update (ctx, msg, len); + clib_sha2_hmac_final (ctx, digest); } -#define clib_hmac_sha224(...) clib_hmac_sha2 (CLIB_SHA2_224, __VA_ARGS__) -#define clib_hmac_sha256(...) clib_hmac_sha2 (CLIB_SHA2_256, __VA_ARGS__) -#define clib_hmac_sha384(...) clib_hmac_sha2 (CLIB_SHA2_384, __VA_ARGS__) -#define clib_hmac_sha512(...) clib_hmac_sha2 (CLIB_SHA2_512, __VA_ARGS__) +#define clib_hmac_sha224(...) clib_sha2_hmac (CLIB_SHA2_224, __VA_ARGS__) +#define clib_hmac_sha256(...) clib_sha2_hmac (CLIB_SHA2_256, __VA_ARGS__) +#define clib_hmac_sha384(...) clib_sha2_hmac (CLIB_SHA2_384, __VA_ARGS__) +#define clib_hmac_sha512(...) clib_sha2_hmac (CLIB_SHA2_512, __VA_ARGS__) #define clib_hmac_sha512_224(...) \ - clib_hmac_sha2 (CLIB_SHA2_512_224, __VA_ARGS__) + clib_sha2_hmac (CLIB_SHA2_512_224, __VA_ARGS__) #define clib_hmac_sha512_256(...) \ - clib_hmac_sha2 (CLIB_SHA2_512_256, __VA_ARGS__) + clib_sha2_hmac (CLIB_SHA2_512_256, __VA_ARGS__) #endif /* included_sha2_h */ diff --git a/src/vppinfra/format.h b/src/vppinfra/format.h index 24511735a53..a1a70a2d64f 100644 --- a/src/vppinfra/format.h +++ b/src/vppinfra/format.h @@ -276,6 +276,12 @@ unformat_init_cstring (unformat_input_t * input, char *string) /* Setup for unformat of given vector string; vector will be freed by unformat_string. */ void unformat_init_vector (unformat_input_t * input, u8 * vector_string); +/* Unformat u8 */ +unformat_function_t unformat_u8; + +/* Unformat u16 */ +unformat_function_t unformat_u16; + /* Format function for unformat input usable when an unformat error has occurred. */ u8 *format_unformat_error (u8 * s, va_list * va); diff --git a/src/vppinfra/heap.c b/src/vppinfra/heap.c index 7db814200f8..9920528732d 100644 --- a/src/vppinfra/heap.c +++ b/src/vppinfra/heap.c @@ -680,6 +680,7 @@ debug_elt (u8 * s, void *v, word i, word n) i = -n / 2; for (e = e0; 1; e = heap_next (e)) { + s = format (s, " "); if (heap_is_free (e)) s = format (s, "index %4d, free\n", e - h->elts); else if (h->format_elt) diff --git a/src/vppinfra/mhash.c b/src/vppinfra/mhash.c index f0f1aa470d7..babaaeec726 100644 --- a/src/vppinfra/mhash.c +++ b/src/vppinfra/mhash.c @@ -164,6 +164,8 @@ mhash_sanitize_hash_user (mhash_t * mh) h->user = pointer_to_uword (mh); } +static u8 *mhash_format_pair_default (u8 *s, va_list *args); + __clib_export void mhash_init (mhash_t * h, uword n_value_bytes, uword n_key_bytes) { @@ -208,12 +210,12 @@ mhash_init (mhash_t * h, uword n_value_bytes, uword n_key_bytes) vec_validate (h->key_tmps, os_get_nthreads () - 1); ASSERT (n_key_bytes < ARRAY_LEN (t)); - h->hash = hash_create2 ( /* elts */ 0, + h->hash = hash_create2 (/* elts */ 0, /* user */ pointer_to_uword (h), /* value_bytes */ n_value_bytes, t[n_key_bytes].key_sum, t[n_key_bytes].key_equal, /* format pair/arg */ - 0, 0); + mhash_format_pair_default, 0); } static uword @@ -331,8 +333,8 @@ mhash_set_mem (mhash_t * h, void *key, uword * new_value, uword * old_value) { if (key_alloc_from_free_list) { - h->key_vector_free_indices[l] = i; - vec_set_len (h->key_vector_free_indices, l + 1); + vec_set_len (h->key_vector_free_indices, l); + h->key_vector_free_indices[l - 1] = i; } else vec_dec_len (h->key_vector_or_heap, h->n_key_bytes); @@ -371,8 +373,8 @@ mhash_unset (mhash_t * h, void *key, uword * old_value) return 1; } -u8 * -format_mhash_key (u8 * s, va_list * va) +__clib_export u8 * +format_mhash_key (u8 *s, va_list *va) { mhash_t *h = va_arg (*va, mhash_t *); u32 ki = va_arg (*va, u32); @@ -387,7 +389,43 @@ format_mhash_key (u8 * s, va_list * va) else if (h->format_key) s = format (s, "%U", h->format_key, k); else - s = format (s, "%U", format_hex_bytes, k, h->n_key_bytes); + s = format (s, "0x%U", format_hex_bytes, k, h->n_key_bytes); + + return s; +} + +static u8 * +mhash_format_pair_default (u8 *s, va_list *args) +{ + void *CLIB_UNUSED (user_arg) = va_arg (*args, void *); + void *v = va_arg (*args, void *); + hash_pair_t *p = va_arg (*args, hash_pair_t *); + hash_t *h = hash_header (v); + mhash_t *mh = uword_to_pointer (h->user, mhash_t *); + + s = format (s, "%U", format_mhash_key, mh, (u32) p->key); + if (hash_value_bytes (h) > 0) + s = format (s, " -> 0x%8U", format_hex_bytes, &p->value[0], + hash_value_bytes (h)); + return s; +} + +__clib_export u8 * +format_mhash (u8 *s, va_list *va) +{ + mhash_t *h = va_arg (*va, mhash_t *); + int verbose = va_arg (*va, int); + + s = format (s, "mhash %p, %wd elts, \n", h, mhash_elts (h)); + if (mhash_key_vector_is_heap (h)) + s = format (s, " %U", format_heap, h->key_vector_or_heap, verbose); + else + s = format (s, " keys %wd elts, %wd size, %wd free, %wd bytes used\n", + vec_len (h->key_vector_or_heap) / h->n_key_bytes, + h->n_key_bytes, vec_len (h->key_vector_free_indices), + vec_bytes (h->key_vector_or_heap) + + vec_bytes (h->key_vector_free_indices)); + s = format (s, " %U", format_hash, h->hash, verbose); return s; } diff --git a/src/vppinfra/mhash.h b/src/vppinfra/mhash.h index 7eb1918384e..62aee365fa3 100644 --- a/src/vppinfra/mhash.h +++ b/src/vppinfra/mhash.h @@ -166,8 +166,13 @@ do { \ })); \ } while (0) +u8 *format_mhash (u8 *s, va_list *va); + format_function_t format_mhash_key; +/* Main test routine. */ +int test_mhash_main (unformat_input_t *input); + #endif /* included_clib_mhash_h */ /* diff --git a/src/vppinfra/pmalloc.c b/src/vppinfra/pmalloc.c index 2a27379b573..85b9db9d56c 100644 --- a/src/vppinfra/pmalloc.c +++ b/src/vppinfra/pmalloc.c @@ -17,6 +17,9 @@ #include <stdlib.h> #include <sys/types.h> #include <sys/stat.h> +#ifdef __FreeBSD__ +#include <sys/memrange.h> +#endif /* __FreeBSD__ */ #include <fcntl.h> #include <unistd.h> #include <sched.h> @@ -184,8 +187,9 @@ next_chunk: } static void -pmalloc_update_lookup_table (clib_pmalloc_main_t * pm, u32 first, u32 count) +pmalloc_update_lookup_table (clib_pmalloc_main_t *pm, u32 first, u32 count) { +#ifdef __linux uword seek, va, pa, p; int fd; u32 elts_per_page = 1U << (pm->def_log2_page_sz - pm->lookup_log2_page_sz); @@ -223,6 +227,45 @@ pmalloc_update_lookup_table (clib_pmalloc_main_t * pm, u32 first, u32 count) if (fd != -1) close (fd); +#elif defined(__FreeBSD__) + struct mem_extract meme; + uword p; + int fd; + u32 elts_per_page = 1U << (pm->def_log2_page_sz - pm->lookup_log2_page_sz); + + vec_validate_aligned (pm->lookup_table, + vec_len (pm->pages) * elts_per_page - 1, + CLIB_CACHE_LINE_BYTES); + + p = (uword) first * elts_per_page; + if (pm->flags & CLIB_PMALLOC_F_NO_PAGEMAP) + { + while (p < (uword) elts_per_page * count) + { + pm->lookup_table[p] = + pointer_to_uword (pm->base) + (p << pm->lookup_log2_page_sz); + p++; + } + return; + } + + fd = open ((char *) "/dev/mem", O_RDONLY); + if (fd == -1) + return; + + while (p < (uword) elts_per_page * count) + { + meme.me_vaddr = + pointer_to_uword (pm->base) + (p << pm->lookup_log2_page_sz); + if (ioctl (fd, MEM_EXTRACT_PADDR, &meme) == -1) + continue; + pm->lookup_table[p] = meme.me_vaddr - meme.me_paddr; + p++; + } + return; +#else +#error "Unsupported OS" +#endif } static inline clib_pmalloc_page_t * diff --git a/src/vppinfra/test/compress.c b/src/vppinfra/test/compress.c index 7b97f4c31af..083065f9bda 100644 --- a/src/vppinfra/test/compress.c +++ b/src/vppinfra/test/compress.c @@ -51,18 +51,21 @@ static compress_test_t tests[] = { static clib_error_t * test_clib_compress_u64 (clib_error_t *err) { - u64 src[513]; - u64 dst[513]; u32 i, j; - for (i = 0; i < ARRAY_LEN (src); i++) - src[i] = i; - for (i = 0; i < ARRAY_LEN (tests); i++) { compress_test_t *t = tests + i; + u64 src[t->n_elts]; +#ifdef CLIB_SANITIZE_ADDR + u64 dst[t->n_elts]; +#else /* CLIB_SANITIZE_ADDR */ + u64 dst[513]; +#endif /* CLIB_SANITIZE_ADDR */ u64 *dp = dst; u32 r; + for (j = 0; j < t->n_elts; j++) + src[j] = j; for (j = 0; j < ARRAY_LEN (dst); j++) dst[j] = 0xa5a5a5a5a5a5a5a5; @@ -81,8 +84,10 @@ test_clib_compress_u64 (clib_error_t *err) dp++; } +#ifndef CLIB_SANITIZE_ADDR if (dst[dp - dst + 1] != 0xa5a5a5a5a5a5a5a5) return clib_error_return (err, "buffer overrun in testcase %u", i); +#endif /* CLIB_SANITIZE_ADDR */ if (dp - dst != r) return clib_error_return (err, "wrong number of elts in testcase %u", @@ -95,18 +100,21 @@ test_clib_compress_u64 (clib_error_t *err) static clib_error_t * test_clib_compress_u32 (clib_error_t *err) { - u32 src[513]; - u32 dst[513]; u32 i, j; - for (i = 0; i < ARRAY_LEN (src); i++) - src[i] = i; - for (i = 0; i < ARRAY_LEN (tests); i++) { compress_test_t *t = tests + i; + u32 src[t->n_elts]; +#ifdef CLIB_SANITIZE_ADDR + u32 dst[t->n_elts]; +#else /* CLIB_SANITIZE_ADDR */ + u32 dst[513]; +#endif /* CLIB_SANITIZE_ADDR */ u32 *dp = dst; u32 r; + for (j = 0; j < t->n_elts; j++) + src[j] = j; for (j = 0; j < ARRAY_LEN (dst); j++) dst[j] = 0xa5a5a5a5; @@ -126,8 +134,10 @@ test_clib_compress_u32 (clib_error_t *err) dp++; } +#ifndef CLIB_SANITIZE_ADDR if (dst[dp - dst + 1] != 0xa5a5a5a5) return clib_error_return (err, "buffer overrun in testcase %u", i); +#endif /* CLIB_SANITIZE_ADDR */ if (dp - dst != r) return clib_error_return (err, "wrong number of elts in testcase %u", @@ -140,18 +150,21 @@ test_clib_compress_u32 (clib_error_t *err) static clib_error_t * test_clib_compress_u16 (clib_error_t *err) { - u16 src[513]; - u16 dst[513]; u32 i, j; - for (i = 0; i < ARRAY_LEN (src); i++) - src[i] = i; - for (i = 0; i < ARRAY_LEN (tests); i++) { compress_test_t *t = tests + i; + u16 src[t->n_elts]; +#ifdef CLIB_SANITIZE_ADDR + u16 dst[t->n_elts]; +#else /* CLIB_SANITIZE_ADDR */ + u16 dst[513]; +#endif /* CLIB_SANITIZE_ADDR */ u16 *dp = dst; u32 r; + for (j = 0; j < t->n_elts; j++) + src[j] = j; for (j = 0; j < ARRAY_LEN (dst); j++) dst[j] = 0xa5a5; @@ -170,8 +183,10 @@ test_clib_compress_u16 (clib_error_t *err) dp++; } +#ifndef CLIB_SANITIZE_ADDR if (dst[dp - dst + 1] != 0xa5a5) return clib_error_return (err, "buffer overrun in testcase %u", i); +#endif /* CLIB_SANITIZE_ADDR */ if (dp - dst != r) return clib_error_return (err, "wrong number of elts in testcase %u", @@ -184,18 +199,21 @@ test_clib_compress_u16 (clib_error_t *err) static clib_error_t * test_clib_compress_u8 (clib_error_t *err) { - u8 src[513]; - u8 dst[513]; u32 i, j; - for (i = 0; i < ARRAY_LEN (src); i++) - src[i] = i; - for (i = 0; i < ARRAY_LEN (tests); i++) { compress_test_t *t = tests + i; + u8 src[t->n_elts]; +#ifdef CLIB_SANITIZE_ADDR + u8 dst[t->n_elts]; +#else /* CLIB_SANITIZE_ADDR */ + u8 dst[513]; +#endif /* CLIB_SANITIZE_ADDR */ u8 *dp = dst; u32 r; + for (j = 0; j < t->n_elts; j++) + src[j] = j; for (j = 0; j < ARRAY_LEN (dst); j++) dst[j] = 0xa5; @@ -214,8 +232,10 @@ test_clib_compress_u8 (clib_error_t *err) dp++; } +#ifndef CLIB_SANITIZE_ADDR if (dst[dp - dst + 1] != 0xa5) return clib_error_return (err, "buffer overrun in testcase %u", i); +#endif /* CLIB_SANITIZE_ADDR */ if (dp - dst != r) return clib_error_return (err, "wrong number of elts in testcase %u", diff --git a/src/vppinfra/test_mhash.c b/src/vppinfra/test_mhash.c new file mode 100644 index 00000000000..70be2b9b382 --- /dev/null +++ b/src/vppinfra/test_mhash.c @@ -0,0 +1,403 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2023 Yandex LLC. + */ + +#ifdef CLIB_LINUX_KERNEL +#include <linux/unistd.h> +#endif + +#ifdef CLIB_UNIX +#include <unistd.h> +#include <stdlib.h> +#include <stdio.h> +#include <vppinfra/time.h> +#endif + +#include <vppinfra/random.h> +#include <vppinfra/mem.h> +#include <vppinfra/hash.h> +#include <vppinfra/mhash.h> +#include <vppinfra/error.h> +#include <vppinfra/format.h> +#include <vppinfra/bitmap.h> + +static int verbose; +#define if_verbose(format, args...) \ + if (verbose) \ + { \ + clib_warning (format, ##args); \ + } + +typedef struct +{ + int n_iterations; + + int n_iterations_per_print; + + /* Number of pairs to insert into mhash. */ + int n_pairs; + + /* True to validate correctness of mhash functions. */ + int n_iterations_per_validate; + + /* Verbosity level for mhash formats. */ + int verbose; + + /* Random number seed. */ + u32 seed; +} mhash_test_t; + +static clib_error_t * +mhash_next_test (mhash_t *h) +{ + hash_next_t hn = { 0 }; + hash_pair_t *p0, *p1; + clib_error_t *error = 0; + + hash_foreach_pair (p0, h->hash, { + p1 = hash_next (h->hash, &hn); + error = CLIB_ERROR_ASSERT (p0 == p1); + if (error) + break; + }); + + if (!error) + error = CLIB_ERROR_ASSERT (!hash_next (h->hash, &hn)); + + return error; +} + +static clib_error_t * +test_word_key (mhash_test_t *ht) +{ + mhash_t _h = { 0 }, *h = &_h; + word i, j; + + word *keys = 0, *vals = 0; + uword *is_inserted = 0; + + clib_error_t *error = 0; + + vec_resize (keys, ht->n_pairs); + vec_resize (vals, vec_len (keys)); + + mhash_init (h, sizeof (vals[0]), sizeof (keys[0])); + /* borrow 0 elt to make index keys non-zero */ + vec_validate (h->key_vector_or_heap, 0); + + { + uword *unique = 0; + u32 k; + + for (i = 0; i < vec_len (keys); i++) + { + do + { + k = random_u32 (&ht->seed) & 0xfffff; + } + while (clib_bitmap_get (unique, k)); + unique = clib_bitmap_ori (unique, k); + keys[i] = k; + vals[i] = i; + } + + clib_bitmap_free (unique); + } + + for (i = 0; i < ht->n_iterations; i++) + { + u32 vi = random_u32 (&ht->seed) % vec_len (keys); + + if (clib_bitmap_get (is_inserted, vi)) + { + mhash_unset (h, &keys[vi], 0); + mhash_unset (h, &keys[vi], 0); + } + else + { + mhash_set (h, &keys[vi], vals[vi], 0); + mhash_set (h, &keys[vi], vals[vi], 0); + } + + is_inserted = clib_bitmap_xori (is_inserted, vi); + + if (ht->n_iterations_per_print > 0 && + ((i + 1) % ht->n_iterations_per_print) == 0) + if_verbose ("iteration %d\n %U", i + 1, format_mhash, h, ht->verbose); + + if (ht->n_iterations_per_validate == 0 || + (i + 1) % ht->n_iterations_per_validate) + continue; + + { + uword ki, *k, *v; + + mhash_foreach (k, v, h, { + ki = v[0]; + ASSERT (keys[ki] == k[0]); + }); + } + + if ((error = hash_validate (h->hash))) + goto done; + + for (j = 0; j < vec_len (keys); j++) + { + uword *v; + v = mhash_get (h, &keys[j]); + if ((error = CLIB_ERROR_ASSERT (clib_bitmap_get (is_inserted, j) == + (v != 0)))) + goto done; + if (v) + { + if ((error = CLIB_ERROR_ASSERT (v[0] == vals[j]))) + goto done; + } + } + } + + if ((error = mhash_next_test (h))) + goto done; + + if_verbose ("%U", format_mhash, h, ht->verbose); + + for (i = 0; i < vec_len (keys); i++) + { + if (!clib_bitmap_get (is_inserted, i)) + continue; + + mhash_unset (h, &keys[i], 0); + mhash_unset (h, &keys[i], 0); + is_inserted = clib_bitmap_xori (is_inserted, i); + + if (ht->n_iterations_per_validate == 0 || + (i + 1) % ht->n_iterations_per_validate) + continue; + + if ((error = hash_validate (h->hash))) + goto done; + + for (j = 0; j < vec_len (keys); j++) + { + uword *v; + v = mhash_get (h, &keys[j]); + if ((error = CLIB_ERROR_ASSERT (clib_bitmap_get (is_inserted, j) == + (v != 0)))) + goto done; + if (v) + { + if ((error = CLIB_ERROR_ASSERT (v[0] == vals[j]))) + goto done; + } + } + } + +done: + mhash_free (h); + vec_free (keys); + vec_free (vals); + clib_bitmap_free (is_inserted); + + if (verbose) + fformat (stderr, "%U\n", format_clib_mem_usage, /* verbose */ 0); + + return error; +} + +static u8 * +test2_format (u8 *s, va_list *args) +{ + void *CLIB_UNUSED (user_arg) = va_arg (*args, void *); + void *v = va_arg (*args, void *); + hash_pair_t *p = va_arg (*args, hash_pair_t *); + hash_t *h = hash_header (v); + mhash_t *mh = uword_to_pointer (h->user, mhash_t *); + + return format (s, "0x%8U <- %U", format_hex_bytes, &p->value[0], + hash_value_bytes (h), format_mhash_key, mh, (u32) p->key); +} + +static clib_error_t * +test_string_key (mhash_test_t *ht, uword is_c_string) +{ + mhash_t _h = { 0 }, *h = &_h; + word i, j; + + u8 **keys = 0; + word *vals = 0; + uword *is_inserted = 0; + + clib_error_t *error = 0; + + vec_resize (keys, ht->n_pairs); + vec_resize (vals, vec_len (keys)); + + if (is_c_string) + mhash_init_c_string (h, sizeof (vals[0])); + else + mhash_init_vec_string (h, sizeof (vals[0])); + hash_set_pair_format (h->hash, test2_format, 0); + + for (i = 0; i < vec_len (keys); i++) + { + keys[i] = random_string (&ht->seed, 5 + (random_u32 (&ht->seed) & 0xf)); + keys[i] = format (keys[i], "%x", i); + if (is_c_string) + vec_terminate_c_string (keys[i]); + vals[i] = random_u32 (&ht->seed); + } + + for (i = 0; i < ht->n_iterations; i++) + { + u32 vi = random_u32 (&ht->seed) % vec_len (keys); + + if (clib_bitmap_get (is_inserted, vi)) + { + mhash_unset (h, keys[vi], 0); + mhash_unset (h, keys[vi], 0); + } + else + { + mhash_set (h, keys[vi], vals[vi], 0); + mhash_set (h, keys[vi], vals[vi], 0); + } + + is_inserted = clib_bitmap_xori (is_inserted, vi); + + if (ht->n_iterations_per_print > 0 && + ((i + 1) % ht->n_iterations_per_print) == 0) + if_verbose ("iteration %d\n %U", i + 1, format_mhash, h, ht->verbose); + + if (ht->n_iterations_per_validate == 0 || + (i + 1) % ht->n_iterations_per_validate) + continue; + + if ((error = hash_validate (h->hash))) + goto done; + + for (j = 0; j < vec_len (keys); j++) + { + uword *v; + v = mhash_get (h, keys[j]); + if ((error = CLIB_ERROR_ASSERT (clib_bitmap_get (is_inserted, j) == + (v != 0)))) + goto done; + if (v) + { + if ((error = CLIB_ERROR_ASSERT (v[0] == vals[j]))) + goto done; + } + } + } + + if ((error = mhash_next_test (h))) + goto done; + + if_verbose ("%U", format_mhash, h, ht->verbose); + + for (i = 0; i < vec_len (keys); i++) + { + if (!clib_bitmap_get (is_inserted, i)) + continue; + + mhash_unset (h, keys[i], 0); + mhash_unset (h, keys[i], 0); + is_inserted = clib_bitmap_xori (is_inserted, i); + + if (ht->n_iterations_per_validate == 0 || + (i + 1) % ht->n_iterations_per_validate) + continue; + + if ((error = hash_validate (h->hash))) + goto done; + + for (j = 0; j < vec_len (keys); j++) + { + uword *v; + v = mhash_get (h, keys[j]); + if ((error = CLIB_ERROR_ASSERT (clib_bitmap_get (is_inserted, j) == + (v != 0)))) + goto done; + if (v) + { + if ((error = CLIB_ERROR_ASSERT (v[0] == vals[j]))) + goto done; + } + } + } + +done: + mhash_free (h); + vec_free (vals); + clib_bitmap_free (is_inserted); + + for (i = 0; i < vec_len (keys); i++) + vec_free (keys[i]); + vec_free (keys); + + if (verbose) + fformat (stderr, "%U\n", format_clib_mem_usage, /* verbose */ 0); + + return error; +} + +int +test_mhash_main (unformat_input_t *input) +{ + mhash_test_t _ht = { 0 }, *ht = &_ht; + clib_error_t *error; + + ht->n_iterations = 100; + ht->n_pairs = 10; + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (0 == unformat (input, "iter %d", &ht->n_iterations) && + 0 == unformat (input, "print %d", &ht->n_iterations_per_print) && + 0 == unformat (input, "elts %d", &ht->n_pairs) && + 0 == unformat (input, "seed %d", &ht->seed) && + 0 == unformat (input, "verbose %=", &ht->verbose, 1) && + 0 == unformat (input, "valid %d", &ht->n_iterations_per_validate)) + { + clib_warning ("unknown input `%U'", format_unformat_error, input); + return 1; + } + } + + if (!ht->seed) + ht->seed = random_default_seed (); + + if_verbose ("testing %d iterations, seed %d", ht->n_iterations, ht->seed); + + error = test_word_key (ht); + if (error) + clib_error_report (error); + + error = test_string_key (ht, 0); + if (error) + clib_error_report (error); + + error = test_string_key (ht, 1); + if (error) + clib_error_report (error); + + return 0; +} + +#ifdef CLIB_UNIX +int +main (int argc, char *argv[]) +{ + unformat_input_t i; + int ret; + + clib_mem_init (0, 3ULL << 30); + + verbose = (argc > 1); + unformat_init_command_line (&i, argv); + ret = test_mhash_main (&i); + unformat_free (&i); + + return ret; +} +#endif /* CLIB_UNIX */ diff --git a/src/vppinfra/unformat.c b/src/vppinfra/unformat.c index fe1a46e4a12..522517888c3 100644 --- a/src/vppinfra/unformat.c +++ b/src/vppinfra/unformat.c @@ -1185,6 +1185,31 @@ unformat_double_quoted_string (unformat_input_t *input, va_list *va) #endif /* CLIB_UNIX */ +__clib_export uword +unformat_u8 (unformat_input_t *input, va_list *args) +{ + u8 *d = va_arg (*args, u8 *); + + u32 tmp; + if (!unformat (input, "%u", &tmp) || tmp > CLIB_U8_MAX) + return 0; + + *d = tmp; + return 1; +} + +__clib_export uword +unformat_u16 (unformat_input_t *input, va_list *args) +{ + u16 *d = va_arg (*args, u16 *); + + u32 tmp; + if (!unformat (input, "%u", &tmp) || tmp > CLIB_U16_MAX) + return 0; + + *d = tmp; + return 1; +} /* * fd.io coding-style-patch-verification: ON diff --git a/src/vppinfra/unix-misc.c b/src/vppinfra/unix-misc.c index e0591ff4604..31c0a489e8d 100644 --- a/src/vppinfra/unix-misc.c +++ b/src/vppinfra/unix-misc.c @@ -35,6 +35,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + #include <vppinfra/error.h> #include <vppinfra/os.h> #include <vppinfra/bitmap.h> @@ -42,6 +46,15 @@ #include <vppinfra/format.h> #ifdef __linux__ #include <vppinfra/linux/sysfs.h> +#include <sched.h> +#elif defined(__FreeBSD__) +#define _WANT_FREEBSD_BITSET +#include <sys/cdefs.h> +#include <sys/param.h> +#include <sys/types.h> +#include <sys/cpuset.h> +#include <sys/domainset.h> +#include <sys/sysctl.h> #endif #include <sys/stat.h> @@ -275,10 +288,70 @@ os_get_online_cpu_core_bitmap () } __clib_export clib_bitmap_t * +os_get_cpu_affinity_bitmap (int pid) +{ +#if __linux + int index, ret; + cpu_set_t cpuset; + uword *affinity_cpus; + + clib_bitmap_alloc (affinity_cpus, sizeof (cpu_set_t)); + clib_bitmap_zero (affinity_cpus); + + __CPU_ZERO_S (sizeof (cpu_set_t), &cpuset); + + ret = sched_getaffinity (0, sizeof (cpu_set_t), &cpuset); + + if (ret < 0) + { + clib_bitmap_free (affinity_cpus); + return 0; + } + + for (index = 0; index < sizeof (cpu_set_t); index++) + if (__CPU_ISSET_S (index, sizeof (cpu_set_t), &cpuset)) + clib_bitmap_set (affinity_cpus, index, 1); + return affinity_cpus; +#elif defined(__FreeBSD__) + cpuset_t mask; + uword *r = NULL; + + if (cpuset_getaffinity (CPU_LEVEL_CPUSET, CPU_WHICH_CPUSET, -1, + sizeof (mask), &mask) != 0) + { + clib_bitmap_free (r); + return NULL; + } + + for (int bit = 0; bit < CPU_SETSIZE; bit++) + clib_bitmap_set (r, bit, CPU_ISSET (bit, &mask)); + + return r; +#else + return NULL; +#endif +} + +__clib_export clib_bitmap_t * os_get_online_cpu_node_bitmap () { #if __linux__ return clib_sysfs_read_bitmap ("/sys/devices/system/node/online"); +#elif defined(__FreeBSD__) + domainset_t domain; + uword *r = NULL; + int policy; + + if (cpuset_getdomain (CPU_LEVEL_CPUSET, CPU_WHICH_CPUSET, -1, + sizeof (domain), &domain, &policy) != 0) + { + clib_bitmap_free (r); + return NULL; + } + + for (int bit = 0; bit < CPU_SETSIZE; bit++) + clib_bitmap_set (r, bit, CPU_ISSET (bit, &domain)); + return r; #else return 0; #endif @@ -327,6 +400,28 @@ os_get_cpu_phys_core_id (int cpu_id) #endif } +__clib_export u8 * +os_get_exec_path () +{ + u8 *rv = 0; +#ifdef __linux__ + char tmp[PATH_MAX]; + ssize_t sz = readlink ("/proc/self/exe", tmp, sizeof (tmp)); + + if (sz <= 0) + return 0; +#else + char tmp[MAXPATHLEN]; + int mib[4] = { CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1 }; + size_t sz = MAXPATHLEN; + + if (sysctl (mib, 4, tmp, &sz, NULL, 0) == -1) + return 0; +#endif + vec_add (rv, tmp, sz); + return rv; +} + /* * fd.io coding-style-patch-verification: ON * diff --git a/src/vppinfra/unix.h b/src/vppinfra/unix.h index 651f9bb99e0..d0ddb93a46f 100644 --- a/src/vppinfra/unix.h +++ b/src/vppinfra/unix.h @@ -68,6 +68,10 @@ clib_bitmap_t *os_get_cpu_on_node_bitmap (int node); /* Retrieve physical core id of specific cpu, -1 if not available */ int os_get_cpu_phys_core_id (int cpu); +/* Retrieve the path of the current executable as a vector (not + * null-terminated). */ +u8 *os_get_exec_path (); + #endif /* included_clib_unix_h */ /* diff --git a/src/vppinfra/vector/array_mask.h b/src/vppinfra/vector/array_mask.h index ba22d79560f..3d4a82ac01b 100644 --- a/src/vppinfra/vector/array_mask.h +++ b/src/vppinfra/vector/array_mask.h @@ -57,6 +57,7 @@ clib_array_mask_u32 (u32 *src, u32 mask, u32 n_elts) u32x4 mask4 = u32x4_splat (mask); *(u32x4u *) src &= mask4; *(u32x4u *) (src + n_elts - 4) &= mask4; + return; } #endif diff --git a/src/vppinfra/vector/compress.h b/src/vppinfra/vector/compress.h index d2ed716ac8e..5429113984b 100644 --- a/src/vppinfra/vector/compress.h +++ b/src/vppinfra/vector/compress.h @@ -34,6 +34,37 @@ clib_compress_u64_x64 (u64 *dst, u64 *src, u64 mask) return dst; } +static_always_inline u64 * +clib_compress_u64_x64_masked (u64 *dst, u64 *src, u64 mask) +{ +#if defined(CLIB_HAVE_VEC512_COMPRESS) && \ + defined(CLIB_HAVE_VEC512_MASK_LOAD_STORE) + u64x8u *sv = (u64x8u *) src; + for (int i = 0; i < 8; i++) + { + u64x8u s = u64x8_mask_load_zero (&sv[i], mask); + u64x8_compress_store (s, mask, dst); + dst += _popcnt32 ((u8) mask); + mask >>= 8; + } +#elif defined(CLIB_HAVE_VEC256_COMPRESS) && \ + defined(CLIB_HAVE_VEC256_MASK_LOAD_STORE) + u64x4u *sv = (u64x4u *) src; + for (int i = 0; i < 16; i++) + { + u64x4u s = u64x4_mask_load_zero (&sv[i], mask); + u64x4_compress_store (s, mask, dst); + dst += _popcnt32 (((u8) mask) & 0x0f); + mask >>= 4; + } +#else + u32 i; + foreach_set_bit_index (i, mask) + dst++[0] = src[i]; +#endif + return dst; +} + /** \brief Compress array of 64-bit elemments into destination array based on * mask @@ -66,7 +97,9 @@ clib_compress_u64 (u64 *dst, u64 *src, u64 *mask, u32 n_elts) if (PREDICT_TRUE (n_elts == 0)) return dst - dst0; - return clib_compress_u64_x64 (dst, src, mask[0] & pow2_mask (n_elts)) - dst0; + return clib_compress_u64_x64_masked (dst, src, + mask[0] & pow2_mask (n_elts)) - + dst0; } static_always_inline u32 * @@ -97,6 +130,38 @@ clib_compress_u32_x64 (u32 *dst, u32 *src, u64 mask) return dst; } +static_always_inline u32 * +clib_compress_u32_x64_masked (u32 *dst, u32 *src, u64 mask) +{ +#if defined(CLIB_HAVE_VEC512_COMPRESS) && \ + defined(CLIB_HAVE_VEC512_MASK_LOAD_STORE) + u32x16u *sv = (u32x16u *) src; + for (int i = 0; i < 4; i++) + { + u32x16u s = u32x16_mask_load_zero (&sv[i], mask); + u32x16_compress_store (s, mask, dst); + dst += _popcnt32 ((u16) mask); + mask >>= 16; + } + +#elif defined(CLIB_HAVE_VEC256_COMPRESS) && \ + defined(CLIB_HAVE_VEC256_MASK_LOAD_STORE) + u32x8u *sv = (u32x8u *) src; + for (int i = 0; i < 8; i++) + { + u32x8u s = u32x8_mask_load_zero (&sv[i], mask); + u32x8_compress_store (s, mask, dst); + dst += _popcnt32 ((u8) mask); + mask >>= 8; + } +#else + u32 i; + foreach_set_bit_index (i, mask) + dst++[0] = src[i]; +#endif + return dst; +} + /** \brief Compress array of 32-bit elemments into destination array based on * mask @@ -129,7 +194,9 @@ clib_compress_u32 (u32 *dst, u32 *src, u64 *mask, u32 n_elts) if (PREDICT_TRUE (n_elts == 0)) return dst - dst0; - return clib_compress_u32_x64 (dst, src, mask[0] & pow2_mask (n_elts)) - dst0; + return clib_compress_u32_x64_masked (dst, src, + mask[0] & pow2_mask (n_elts)) - + dst0; } static_always_inline u16 * @@ -151,6 +218,27 @@ clib_compress_u16_x64 (u16 *dst, u16 *src, u64 mask) return dst; } +static_always_inline u16 * +clib_compress_u16_x64_masked (u16 *dst, u16 *src, u64 mask) +{ +#if defined(CLIB_HAVE_VEC512_COMPRESS_U8_U16) && \ + defined(CLIB_HAVE_VEC512_MASK_LOAD_STORE) + u16x32u *sv = (u16x32u *) src; + for (int i = 0; i < 2; i++) + { + u16x32u s = u16x32_mask_load_zero (&sv[i], mask); + u16x32_compress_store (s, mask, dst); + dst += _popcnt32 ((u32) mask); + mask >>= 32; + } +#else + u32 i; + foreach_set_bit_index (i, mask) + dst++[0] = src[i]; +#endif + return dst; +} + /** \brief Compress array of 16-bit elemments into destination array based on * mask @@ -183,7 +271,9 @@ clib_compress_u16 (u16 *dst, u16 *src, u64 *mask, u32 n_elts) if (PREDICT_TRUE (n_elts == 0)) return dst - dst0; - return clib_compress_u16_x64 (dst, src, mask[0] & pow2_mask (n_elts)) - dst0; + return clib_compress_u16_x64_masked (dst, src, + mask[0] & pow2_mask (n_elts)) - + dst0; } static_always_inline u8 * @@ -201,6 +291,23 @@ clib_compress_u8_x64 (u8 *dst, u8 *src, u64 mask) return dst; } +static_always_inline u8 * +clib_compress_u8_x64_masked (u8 *dst, u8 *src, u64 mask) +{ +#if defined(CLIB_HAVE_VEC512_COMPRESS_U8_U16) && \ + defined(CLIB_HAVE_VEC512_MASK_LOAD_STORE) + u8x64u *sv = (u8x64u *) src; + u8x64u s = u8x64_mask_load_zero (sv, mask); + u8x64_compress_store (s, mask, dst); + dst += _popcnt64 (mask); +#else + u32 i; + foreach_set_bit_index (i, mask) + dst++[0] = src[i]; +#endif + return dst; +} + /** \brief Compress array of 8-bit elemments into destination array based on * mask @@ -233,7 +340,8 @@ clib_compress_u8 (u8 *dst, u8 *src, u64 *mask, u32 n_elts) if (PREDICT_TRUE (n_elts == 0)) return dst - dst0; - return clib_compress_u8_x64 (dst, src, mask[0] & pow2_mask (n_elts)) - dst0; + return clib_compress_u8_x64_masked (dst, src, mask[0] & pow2_mask (n_elts)) - + dst0; } #endif diff --git a/src/vppinfra/vector/mask_compare.h b/src/vppinfra/vector/mask_compare.h index 92d5ca35474..fc72d7dac35 100644 --- a/src/vppinfra/vector/mask_compare.h +++ b/src/vppinfra/vector/mask_compare.h @@ -8,7 +8,7 @@ #include <vppinfra/memcpy.h> static_always_inline u64 -clib_mask_compare_u16_x64 (u16 v, u16 *a, u32 n_elts) +clib_mask_compare_u16_x64 (u16 v, u16 *a) { u64 mask = 0; #if defined(CLIB_HAVE_VEC512) @@ -47,6 +47,38 @@ clib_mask_compare_u16_x64 (u16 v, u16 *a, u32 n_elts) (u64) i8x16_msb_mask (i8x16_pack (v8 == av[4], v8 == av[5])) << 32 | (u64) i8x16_msb_mask (i8x16_pack (v8 == av[6], v8 == av[7])) << 48); #else + for (int i = 0; i < 64; i++) + if (a[i] == v) + mask |= 1ULL << i; +#endif + return mask; +} + +static_always_inline u64 +clib_mask_compare_u16_x64_n (u16 v, u16 *a, u32 n_elts) +{ + u64 mask = 0; + CLIB_UNUSED (u64 data_mask) = pow2_mask (n_elts); +#if defined(CLIB_HAVE_VEC512) + u16x32 v32 = u16x32_splat (v); + u16x32u *av = (u16x32u *) a; + mask = ((u64) u16x32_is_equal_mask ( + u16x32_mask_load_zero (&av[0], data_mask), v32) | + (u64) u16x32_is_equal_mask ( + u16x32_mask_load_zero (&av[1], data_mask >> 32), v32) + << 32); +#elif defined(CLIB_HAVE_VEC256) && defined(CLIB_HAVE_VEC256_MASK_LOAD_STORE) + u16x16 v16 = u16x16_splat (v); + u16x16u *av = (u16x16u *) a; + i8x32 x; + + x = i8x32_pack (v16 == u16x16_mask_load_zero (&av[0], data_mask), + v16 == u16x16_mask_load_zero (&av[1], data_mask >> 16)); + mask = i8x32_msb_mask ((i8x32) u64x4_permute (x, 0, 2, 1, 3)); + x = i8x32_pack (v16 == u16x16_mask_load_zero (&av[2], data_mask >> 32), + v16 == u16x16_mask_load_zero (&av[3], data_mask >> 48)); + mask |= (u64) i8x32_msb_mask ((i8x32) u64x4_permute (x, 0, 2, 1, 3)) << 32; +#else for (int i = 0; i < n_elts; i++) if (a[i] == v) mask |= 1ULL << i; @@ -68,7 +100,7 @@ clib_mask_compare_u16 (u16 v, u16 *a, u64 *mask, u32 n_elts) { while (n_elts >= 64) { - mask++[0] = clib_mask_compare_u16_x64 (v, a, 64); + mask++[0] = clib_mask_compare_u16_x64 (v, a); n_elts -= 64; a += 64; } @@ -76,11 +108,11 @@ clib_mask_compare_u16 (u16 v, u16 *a, u64 *mask, u32 n_elts) if (PREDICT_TRUE (n_elts == 0)) return; - mask[0] = clib_mask_compare_u16_x64 (v, a, n_elts) & pow2_mask (n_elts); + mask[0] = clib_mask_compare_u16_x64_n (v, a, n_elts) & pow2_mask (n_elts); } static_always_inline u64 -clib_mask_compare_u32_x64 (u32 v, u32 *a, u32 n_elts) +clib_mask_compare_u32_x64 (u32 v, u32 *a) { u64 mask = 0; #if defined(CLIB_HAVE_VEC512) @@ -131,6 +163,57 @@ clib_mask_compare_u32_x64 (u32 v, u32 *a, u32 n_elts) } #else + for (int i = 0; i < 64; i++) + if (a[i] == v) + mask |= 1ULL << i; +#endif + return mask; +} + +static_always_inline u64 +clib_mask_compare_u32_x64_n (u32 v, u32 *a, u32 n_elts) +{ + u64 mask = 0; + CLIB_UNUSED (u64 data_mask) = pow2_mask (n_elts); +#if defined(CLIB_HAVE_VEC512) + u32x16 v16 = u32x16_splat (v); + u32x16u *av = (u32x16u *) a; + mask = ((u64) u32x16_is_equal_mask ( + u32x16_mask_load_zero (&av[0], data_mask), v16) | + (u64) u32x16_is_equal_mask ( + u32x16_mask_load_zero (&av[1], data_mask >> 16), v16) + << 16 | + (u64) u32x16_is_equal_mask ( + u32x16_mask_load_zero (&av[2], data_mask >> 32), v16) + << 32 | + (u64) u32x16_is_equal_mask ( + u32x16_mask_load_zero (&av[3], data_mask >> 48), v16) + << 48); +#elif defined(CLIB_HAVE_VEC256) && defined(CLIB_HAVE_VEC256_MASK_LOAD_STORE) + u32x8 v8 = u32x8_splat (v); + u32x8u *av = (u32x8u *) a; + u32x8 m = { 0, 4, 1, 5, 2, 6, 3, 7 }; + i8x32 c; + + c = i8x32_pack ( + i16x16_pack ( + (i32x8) (v8 == u32x8_mask_load_zero (&av[0], data_mask)), + (i32x8) (v8 == u32x8_mask_load_zero (&av[1], data_mask >> 8))), + i16x16_pack ( + (i32x8) (v8 == u32x8_mask_load_zero (&av[2], data_mask >> 16)), + (i32x8) (v8 == u32x8_mask_load_zero (&av[3], data_mask >> 24)))); + mask = i8x32_msb_mask ((i8x32) u32x8_permute ((u32x8) c, m)); + + c = i8x32_pack ( + i16x16_pack ( + (i32x8) (v8 == u32x8_mask_load_zero (&av[4], data_mask >> 32)), + (i32x8) (v8 == u32x8_mask_load_zero (&av[5], data_mask >> 40))), + i16x16_pack ( + (i32x8) (v8 == u32x8_mask_load_zero (&av[6], data_mask >> 48)), + (i32x8) (v8 == u32x8_mask_load_zero (&av[7], data_mask >> 56)))); + mask |= (u64) i8x32_msb_mask ((i8x32) u32x8_permute ((u32x8) c, m)) << 32; + mask |= (u64) i8x32_msb_mask ((i8x32) u32x8_permute ((u32x8) c, m)) << 32; +#else for (int i = 0; i < n_elts; i++) if (a[i] == v) mask |= 1ULL << i; @@ -152,7 +235,7 @@ clib_mask_compare_u32 (u32 v, u32 *a, u64 *bitmap, u32 n_elts) { while (n_elts >= 64) { - bitmap++[0] = clib_mask_compare_u32_x64 (v, a, 64); + bitmap++[0] = clib_mask_compare_u32_x64 (v, a); n_elts -= 64; a += 64; } @@ -160,11 +243,11 @@ clib_mask_compare_u32 (u32 v, u32 *a, u64 *bitmap, u32 n_elts) if (PREDICT_TRUE (n_elts == 0)) return; - bitmap[0] = clib_mask_compare_u32_x64 (v, a, n_elts) & pow2_mask (n_elts); + bitmap[0] = clib_mask_compare_u32_x64_n (v, a, n_elts) & pow2_mask (n_elts); } static_always_inline u64 -clib_mask_compare_u64_x64 (u64 v, u64 *a, u32 n_elts) +clib_mask_compare_u64_x64 (u64 v, u64 *a) { u64 mask = 0; #if defined(CLIB_HAVE_VEC512) @@ -190,6 +273,59 @@ clib_mask_compare_u64_x64 (u64 v, u64 *a, u32 n_elts) mask |= _pext_u64 (l | h << 32, 0x0101010101010101) << (i * 4); } #else + for (int i = 0; i < 64; i++) + if (a[i] == v) + mask |= 1ULL << i; +#endif + return mask; +} + +static_always_inline u64 +clib_mask_compare_u64_x64_n (u64 v, u64 *a, u32 n_elts) +{ + u64 mask = 0; + CLIB_UNUSED (u64 data_mask) = pow2_mask (n_elts); +#if defined(CLIB_HAVE_VEC512) + u64x8 v8 = u64x8_splat (v); + u64x8u *av = (u64x8u *) a; + mask = + ((u64) u64x8_is_equal_mask (u64x8_mask_load_zero (&av[0], data_mask), v8) | + (u64) u64x8_is_equal_mask (u64x8_mask_load_zero (&av[1], data_mask >> 8), + v8) + << 8 | + (u64) u64x8_is_equal_mask (u64x8_mask_load_zero (&av[2], data_mask >> 16), + v8) + << 16 | + (u64) u64x8_is_equal_mask (u64x8_mask_load_zero (&av[3], data_mask >> 24), + v8) + << 24 | + (u64) u64x8_is_equal_mask (u64x8_mask_load_zero (&av[4], data_mask >> 32), + v8) + << 32 | + (u64) u64x8_is_equal_mask (u64x8_mask_load_zero (&av[5], data_mask >> 40), + v8) + << 40 | + (u64) u64x8_is_equal_mask (u64x8_mask_load_zero (&av[6], data_mask >> 48), + v8) + << 48 | + (u64) u64x8_is_equal_mask (u64x8_mask_load_zero (&av[7], data_mask >> 56), + v8) + << 56); + +#elif defined(CLIB_HAVE_VEC256) && defined(__BMI2__) && \ + defined(CLIB_HAVE_VEC256_MASK_LOAD_STORE) + u64x4 v4 = u64x4_splat (v); + u64x4u *av = (u64x4u *) a; + + for (int i = 0; i < 16; i += 2) + { + u64 l = u8x32_msb_mask (v4 == u64x4_mask_load_zero (&av[i], data_mask)); + u64 h = u8x32_msb_mask ( + v4 == u64x4_mask_load_zero (&av[i + 1], data_mask >> 4)); + mask |= _pext_u64 (l | h << 32, 0x0101010101010101) << (i * 4); + data_mask >>= 8; + } +#else for (int i = 0; i < n_elts; i++) if (a[i] == v) mask |= 1ULL << i; @@ -211,7 +347,7 @@ clib_mask_compare_u64 (u64 v, u64 *a, u64 *bitmap, u32 n_elts) { while (n_elts >= 64) { - bitmap++[0] = clib_mask_compare_u64_x64 (v, a, 64); + bitmap++[0] = clib_mask_compare_u64_x64 (v, a); n_elts -= 64; a += 64; } @@ -219,7 +355,7 @@ clib_mask_compare_u64 (u64 v, u64 *a, u64 *bitmap, u32 n_elts) if (PREDICT_TRUE (n_elts == 0)) return; - bitmap[0] = clib_mask_compare_u64_x64 (v, a, n_elts) & pow2_mask (n_elts); + bitmap[0] = clib_mask_compare_u64_x64_n (v, a, n_elts) & pow2_mask (n_elts); } #endif |