diff options
Diffstat (limited to 'src')
306 files changed, 18583 insertions, 8048 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 68d0a4fe64e..d5c7fd1c718 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -113,19 +113,20 @@ endif() # cross compiling ############################################################################## -if (${CMAKE_SYSTEM_NAME} MATCHES "Linux") - set(COMPILER_SUFFIX "linux-gnu") -elseif (${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD") - set(COMPILER_SUFFIX "freebsd") -endif() - if(CMAKE_CROSSCOMPILING) + if (${CMAKE_SYSTEM_NAME} MATCHES "Linux") + set(COMPILER_SUFFIX "linux-gnu") + elseif (${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD") + set(COMPILER_SUFFIX "freebsd") + endif() + set(CMAKE_IGNORE_PATH /usr/lib/${CMAKE_HOST_SYSTEM_PROCESSOR}-${COMPILER_SUFFIX}/ /usr/lib/${CMAKE_HOST_SYSTEM_PROCESSOR}-${COMPILER_SUFFIX}/lib/ ) -endif() set(CMAKE_C_COMPILER_TARGET ${CMAKE_SYSTEM_PROCESSOR}-${COMPILER_SUFFIX}) +endif() + ############################################################################## # build config ############################################################################## diff --git a/src/cmake/api.cmake b/src/cmake/api.cmake index 10e89d77594..999bbf91be1 100644 --- a/src/cmake/api.cmake +++ b/src/cmake/api.cmake @@ -11,14 +11,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Set the CMP0116 policy +if(POLICY CMP0116) + cmake_policy(SET CMP0116 NEW) +endif() + ############################################################################## # API ############################################################################## function(vpp_generate_api_c_header file) set (output_name ${CMAKE_CURRENT_BINARY_DIR}/${file}.h) + set (dependency_file ${CMAKE_CURRENT_BINARY_DIR}/${file}.d) get_filename_component(output_dir ${output_name} DIRECTORY) if(NOT VPP_APIGEN) - set(VPP_APIGEN ${CMAKE_SOURCE_DIR}/tools/vppapigen/vppapigen) + set(VPP_APIGEN ${CMAKE_SOURCE_DIR}/tools/vppapigen/vppapigen) + set(VPPAPIGEN_SUBMODULES + ${CMAKE_SOURCE_DIR}/tools/vppapigen/vppapigen_c.py + ${CMAKE_SOURCE_DIR}/tools/vppapigen/vppapigen_json.py + ) endif() if (VPP_INCLUDE_DIR) set(includedir "--includedir" ${VPP_INCLUDE_DIR}) @@ -35,16 +45,31 @@ function(vpp_generate_api_c_header file) "${CMAKE_CURRENT_BINARY_DIR}/${file}_test2.c" ) + get_filename_component(barename ${file} NAME) + +# Define a variable for common apigen arguments +set(COMMON_ARGS + OUTPUT ${OUTPUT_HEADERS} + COMMAND mkdir -p ${output_dir} + COMMAND ${PYENV} ${VPP_APIGEN} + ARGS ${includedir} --includedir ${CMAKE_SOURCE_DIR} --input ${CMAKE_CURRENT_SOURCE_DIR}/${file} --outputdir ${output_dir} --output ${output_name} -MF ${dependency_file} + DEPENDS ${VPP_APIGEN} ${CMAKE_CURRENT_SOURCE_DIR}/${file} ${VPPAPIGEN_SUBMODULES} + COMMENT "Generating API header ${output_name}" +) + +if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.20") add_custom_command ( - OUTPUT ${OUTPUT_HEADERS} - COMMAND mkdir -p ${output_dir} - COMMAND ${PYENV} ${VPP_APIGEN} - ARGS ${includedir} --includedir ${CMAKE_SOURCE_DIR} --input ${CMAKE_CURRENT_SOURCE_DIR}/${file} --outputdir ${output_dir} --output ${output_name} - DEPENDS ${VPP_APIGEN} ${CMAKE_CURRENT_SOURCE_DIR}/${file} - COMMENT "Generating API header ${output_name}" + ${COMMON_ARGS} + DEPFILE ${dependency_file} ) - get_filename_component(barename ${file} NAME) +else() + message(WARNING "Your CMake version does not support DEPFILE. Consider upgrading to CMake 3.20 or later for improved dependency handling.") + add_custom_command ( + ${COMMON_ARGS} + ) +endif() set(t ${barename}_deps) + if (NOT TARGET ${t}) add_custom_target(${t} ALL DEPENDS ${OUTPUT_HEADERS}) add_dependencies(api_headers ${t}) @@ -158,6 +183,7 @@ function(vpp_add_api_files name dir component) # "vapi". Both by in-source components (e.g. vpp-api/vapi/vapi.c), and # out-of-tree plugins use #include <vapi/component.api.vapi.h>. # ${file} contains the subdirectory, so strip it here. + file(MAKE_DIRECTORY ${VPP_BINARY_DIR}/vpp-api/vapi) get_filename_component(name ${file} NAME) list(APPEND header_files ${file}.h @@ -175,3 +201,5 @@ endfunction() add_custom_target(api_headers DEPENDS vlibmemory_api_headers vnet_api_headers vpp_api_headers vlib_api_headers ) +add_custom_target(vapi_headers +) diff --git a/src/cmake/platform/cn913x.cmake b/src/cmake/platform/cn913x.cmake new file mode 100644 index 00000000000..8c5cb739f9b --- /dev/null +++ b/src/cmake/platform/cn913x.cmake @@ -0,0 +1,3 @@ + +set(VPP_PLATFORM_CACHE_LINE_SIZE 64) +set(VPP_PLATFORM_MARCH_FLAGS -march=armv8-a+crc+crypto) diff --git a/src/cmake/platform/octeon9.cmake b/src/cmake/platform/octeon9.cmake new file mode 100644 index 00000000000..46ca7dfa64a --- /dev/null +++ b/src/cmake/platform/octeon9.cmake @@ -0,0 +1,4 @@ + +set(VPP_PLATFORM_CACHE_LINE_SIZE 128) +set(VPP_PLATFORM_MARCH_FLAGS -march=armv8.2-a+crc+crypto) +set(VPP_PLATFORM_BUFFER_ALIGN 128) diff --git a/src/plugins/acl/acl.c b/src/plugins/acl/acl.c index e52e82fcf28..fbd94761027 100644 --- a/src/plugins/acl/acl.c +++ b/src/plugins/acl/acl.c @@ -2845,6 +2845,17 @@ acl_set_aclplugin_interface_fn (vlib_main_t * vm, } \ } while (0) +#define vec_validate_macip_acl_rules(v, idx) \ + do \ + { \ + if (vec_len (v) < idx + 1) \ + { \ + vec_validate (v, idx); \ + v[idx].is_permit = 0x1; \ + } \ + } \ + while (0) + static clib_error_t * acl_set_aclplugin_acl_fn (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) @@ -3062,6 +3073,160 @@ acl_show_aclplugin_macip_interface_fn (vlib_main_t * vm, return error; } +static clib_error_t * +acl_set_aclplugin_macip_acl_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + vl_api_macip_acl_rule_t *rules = 0; + int rule_idx = 0; + int rv = 0; + u32 acl_index = ~0; + u32 action = 0; + u8 src_mac[6]; + u8 *tag = 0; + u8 mac_mask_all_1[6] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; + ip_prefix_t src_ip; + + unformat_input_t _line_input, *line_input = &_line_input; + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + vec_validate_macip_acl_rules (rules, rule_idx); + if (unformat (line_input, "permit")) + { + rules[rule_idx].is_permit = 1; + } + else if (unformat (line_input, "deny")) + { + rules[rule_idx].is_permit = 0; + } + else if (unformat (line_input, "action %d", &action)) + { + rules[rule_idx].is_permit = action; + } + else if (unformat (line_input, "ip %U", unformat_ip_prefix, &src_ip)) + { + ip_prefix_encode2 (&src_ip, &rules[rule_idx].src_prefix); + } + else if (unformat (line_input, "src")) + { + /* Everything in MACIP is "source" but allow this verbosity */ + } + else if (unformat (line_input, "mac %U", unformat_mac_address, &src_mac)) + { + memcpy (rules[rule_idx].src_mac, &src_mac, + sizeof (rules[rule_idx].src_mac)); + memcpy (rules[rule_idx].src_mac_mask, &mac_mask_all_1, + sizeof (rules[rule_idx].src_mac_mask)); + } + else if (unformat (line_input, "mask %U", unformat_mac_address, + &src_mac)) + { + memcpy (rules[rule_idx].src_mac_mask, &src_mac, + sizeof (rules[rule_idx].src_mac_mask)); + } + else if (unformat (line_input, "tag %s", &tag)) + ; + else if (unformat (line_input, ",")) + { + rule_idx++; + } + else + break; + } + + if (!tag) + vec_add (tag, "cli", 4); + + rv = macip_acl_add_list (vec_len (rules), rules, &acl_index, tag); + vec_free (rules); + vec_free (tag); + + unformat_free (line_input); + if (rv) + return clib_error_return (0, "Failed to set MACIP ACL rule"); + + vlib_cli_output (vm, "ACL index:%u", acl_index); + return 0; +} + +static clib_error_t * +acl_macip_delete_aclplugin_acl_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + unformat_input_t _line_input, *line_input = &_line_input; + int rv; + u32 macip_acl_index = ~0; + + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "index %u", &macip_acl_index)) + { + /* operate on this acl index (which must exist) */ + } + else + break; + } + + if (macip_acl_index == ~0) + return (clib_error_return (0, "invalid acl index")); + + rv = macip_acl_del_list (macip_acl_index); + + unformat_free (line_input); + if (rv) + return (clib_error_return (0, "Failed to delete ACL index")); + + vlib_cli_output (vm, "Deleted ACL index:%u", macip_acl_index); + return 0; +} + +static clib_error_t * +acl_set_aclplugin_macip_interface_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + int rv = 0; + u32 sw_if_index = ~0; + u32 acl_index = ~0; + u32 is_add = 1; + unformat_input_t _line_input, *line_input = &_line_input; + + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "%U", unformat_vnet_sw_interface, + vnet_get_main (), &sw_if_index)) + ; + else if (unformat (line_input, "add")) + is_add = 1; + else if (unformat (line_input, "del")) + is_add = 0; + else if (unformat (line_input, "acl %u", &acl_index)) + ; + else + break; + } + + if (sw_if_index == ~0) + return (clib_error_return (0, "invalid interface")); + + if (acl_index == ~0) + return (clib_error_return (0, "invalid acl index")); + + rv = macip_acl_interface_add_del_acl (sw_if_index, is_add, acl_index); + + if (rv) + return (clib_error_return (0, "Failed to add acl rule to interface")); + + return 0; +} + static void acl_plugin_show_acl (acl_main_t * am, u32 acl_index) { @@ -3632,6 +3797,38 @@ VLIB_CLI_COMMAND (aclplugin_set_acl_command, static) = { }; /*? + * Create an MACIP Access Control List (ACL) + * A MACIP ACL is used to add L2-L3 ACL rules. + * A MACIP ACL can be added similar to ACL rules by using following command : + * + * @cliexcmd{set acl-plugin macip acl <permit|deny|action N> + * ip <PREFIX> mac <MAC> mask <int> [tag FOO] {use comma + * separated list for multiple rules}} + ?*/ +VLIB_CLI_COMMAND (aclplugin_macip_set_acl_command, static) = { + .path = "set acl-plugin macip acl ", + .short_help = "set acl-plugin macip acl <permit|deny|action N> " + "ip <PREFIX> mac <MAC> mask <int> [tag FOO] {use comma " + "separated list for multiple rules}", + .function = acl_set_aclplugin_macip_acl_fn, +}; + +/*? + * [un]Apply a MACIP ACL to an interface. + * The ACL being applied must already exist. + * + * @cliexpar + * <b><em> set acl-plugin macip interface <interface> <acl INDEX> [del] + </b></em> + * @cliexend + ?*/ +VLIB_CLI_COMMAND (aclplugin_macip_set_interface_command, static) = { + .path = "set acl-plugin macip interface", + .short_help = "set acl-plugin macip interface <interface> <acl INDEX> [del]", + .function = acl_set_aclplugin_macip_interface_fn, +}; + +/*? * Delete an Access Control List (ACL) * Removes an ACL at the specified index, which must exist but not in use by * any interface. @@ -3644,6 +3841,20 @@ VLIB_CLI_COMMAND (aclplugin_delete_acl_command, static) = { .function = acl_delete_aclplugin_acl_fn, }; +/*? + * Delete a MACIP Access Control List (ACL) + * Removes an MACIP ACL at the specified index, which must exist but not in + * use by + * any interface. + * + * @cliexcmd{delete acl-plugin macip acl index <idx>} + ?*/ +VLIB_CLI_COMMAND (aclplugin_macip_delete_acl_command, static) = { + .path = "delete acl-plugin macip acl", + .short_help = "delete acl-plugin macip acl index <idx>", + .function = acl_macip_delete_aclplugin_acl_fn, +}; + static clib_error_t * acl_plugin_config (vlib_main_t * vm, unformat_input_t * input) { diff --git a/src/plugins/avf/avf.h b/src/plugins/avf/avf.h index f6f79cf0e09..774aac0151b 100644 --- a/src/plugins/avf/avf.h +++ b/src/plugins/avf/avf.h @@ -180,6 +180,7 @@ typedef struct u8 int_mode; u8 buffer_pool_index; u32 queue_index; + u64 total_packets; } avf_rxq_t; typedef struct @@ -198,6 +199,8 @@ typedef struct avf_tx_desc_t *tmp_descs; u32 *tmp_bufs; u32 queue_index; + u64 total_packets; + u64 no_free_tx_count; } avf_txq_t; typedef struct diff --git a/src/plugins/avf/device.c b/src/plugins/avf/device.c index 1618800c432..98169f0bcfe 100644 --- a/src/plugins/avf/device.c +++ b/src/plugins/avf/device.c @@ -288,6 +288,7 @@ avf_rxq_init (vlib_main_t * vm, avf_device_t * ad, u16 qid, u16 rxq_size) d->qword[0] = vlib_buffer_get_pa (vm, b); d++; } + rxq->total_packets = 0; return 0; } @@ -337,6 +338,9 @@ avf_txq_init (vlib_main_t * vm, avf_device_t * ad, u16 qid, u16 txq_size) vec_validate_aligned (txq->tmp_descs, txq->size, CLIB_CACHE_LINE_BYTES); vec_validate_aligned (txq->tmp_bufs, txq->size, CLIB_CACHE_LINE_BYTES); + txq->total_packets = 0; + txq->no_free_tx_count = 0; + return 0; } diff --git a/src/plugins/avf/format.c b/src/plugins/avf/format.c index 0a153a093d9..436f5b9fbf2 100644 --- a/src/plugins/avf/format.c +++ b/src/plugins/avf/format.c @@ -104,6 +104,7 @@ format_avf_device (u8 * s, va_list * args) u8 *a = 0; avf_rxq_t *rxq = vec_elt_at_index (ad->rxqs, 0); avf_txq_t *txq = vec_elt_at_index (ad->txqs, 0); + u32 idx = 0; s = format (s, "rx: queues %u, desc %u (min %u max %u)", ad->n_rx_queues, rxq->size, AVF_QUEUE_SZ_MIN, AVF_QUEUE_SZ_MAX); @@ -114,6 +115,22 @@ format_avf_device (u8 * s, va_list * args) format_avf_device_flags, ad); s = format (s, "\n%Ucapability flags: %U", format_white_space, indent, format_avf_vf_cap_flags, ad->cap_flags); + s = + format (s, "\n%U Rx Queue: Total Packets", format_white_space, indent + 4); + for (idx = 0; idx < ad->n_rx_queues; idx++) + { + rxq = vec_elt_at_index (ad->rxqs, idx); + s = format (s, "\n%U %8u : %llu", format_white_space, indent + 4, idx, + rxq->total_packets); + } + s = format (s, "\n%U Tx Queue: Total Packets\t Total Drops", + format_white_space, indent + 4); + for (idx = 0; idx < ad->n_tx_queues; idx++) + { + txq = vec_elt_at_index (ad->txqs, idx); + s = format (s, "\n%U %8u : %llu\t %llu", format_white_space, indent + 4, + idx, txq->total_packets, txq->no_free_tx_count); + } s = format (s, "\n%Unum-queue-pairs %d max-vectors %u max-mtu %u " "rss-key-size %u rss-lut-size %u", format_white_space, indent, diff --git a/src/plugins/avf/input.c b/src/plugins/avf/input.c index 06007db540d..890259c88ab 100644 --- a/src/plugins/avf/input.c +++ b/src/plugins/avf/input.c @@ -539,6 +539,8 @@ done: else avf_rxq_refill (vm, node, rxq, 0 /* use_va_dma */ ); + rxq->total_packets += n_rx_packets; + return n_rx_packets; } diff --git a/src/plugins/avf/output.c b/src/plugins/avf/output.c index daa86ae86b2..0952886aaee 100644 --- a/src/plugins/avf/output.c +++ b/src/plugins/avf/output.c @@ -510,6 +510,7 @@ retry: avf_tail_write (txq->qtx_tail, txq->next); txq->n_enqueued += n_desc; n_left -= n_enq; + txq->total_packets += n_enq; } if (n_left) @@ -522,6 +523,7 @@ retry: vlib_buffer_free (vm, buffers, n_left); vlib_error_count (vm, node->node_index, AVF_TX_ERROR_NO_FREE_SLOTS, n_left); + txq->no_free_tx_count += n_left; } if (tf->shared_queue) diff --git a/src/plugins/builtinurl/FEATURE.yaml b/src/plugins/builtinurl/FEATURE.yaml deleted file mode 100644 index ba8e3c7ea7b..00000000000 --- a/src/plugins/builtinurl/FEATURE.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -name: Builtin URL support for the static http or https server -maintainer: Dave Barach <dave@barachs.net> -features: - - Builtin URLs for the static http/https server -description: "The (builtinurl) plugin adds a set of URLs to the static http/https server. - Current URLs, all of which return data in .json fmt: - <root-url>/version.json - vpp version info - <root-url>/interface_list.json - list of interfaces - <root-url>/interface_stats - single interface via HTTP POST - <root-url>/interface_stats - all intfcs via HTTP GET." -state: development -properties: [API, CLI, MULTITHREAD] diff --git a/src/plugins/builtinurl/builtins.c b/src/plugins/builtinurl/builtins.c deleted file mode 100644 index b04e9dd5c7c..00000000000 --- a/src/plugins/builtinurl/builtins.c +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright (c) 2019 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <vnet/vnet.h> -#include <builtinurl/builtinurl.h> -#include <http_static/http_static.h> -#include <vpp/app/version.h> - -hss_url_handler_rc_t -handle_get_version (hss_url_handler_args_t *args) -{ - u8 *s = 0; - - /* Build some json bullshit */ - s = format (s, "{\"vpp_details\": {"); - s = format (s, " \"version\": \"%s\",", VPP_BUILD_VER); - s = format (s, " \"build_date\": \"%s\"}}\r\n", VPP_BUILD_DATE); - - args->data = s; - args->data_len = vec_len (s); - args->free_vec_data = 1; - return HSS_URL_HANDLER_OK; -} - -void -trim_path_from_request (u8 * s, char *path) -{ - u8 *cp; - int trim_length = strlen (path) + 1 /* remove '?' */ ; - - /* Get rid of the path and question-mark */ - vec_delete (s, trim_length, 0); - - /* Tail trim irrelevant browser info */ - cp = s; - while ((cp - s) < vec_len (s)) - { - if (*cp == ' ') - { - /* - * Makes request a vector which happens to look - * like a c-string. - */ - *cp = 0; - vec_set_len (s, cp - s); - break; - } - cp++; - } -} - -hss_url_handler_rc_t -handle_get_interface_stats (hss_url_handler_args_t *args) -{ - u8 *s = 0, *stats = 0; - uword *p; - u32 *sw_if_indices = 0; - vnet_hw_interface_t *hi; - vnet_sw_interface_t *si; - char *q = "\""; - int i; - int need_comma = 0; - u8 *format_vnet_sw_interface_cntrs (u8 * s, vnet_interface_main_t * im, - vnet_sw_interface_t * si, int json); - vnet_main_t *vnm = vnet_get_main (); - vnet_interface_main_t *im = &vnm->interface_main; - - /* Get stats for a single interface via http POST */ - if (args->reqtype == HTTP_REQ_POST) - { - trim_path_from_request (args->request, "interface_stats.json"); - - /* Find the sw_if_index */ - p = hash_get (im->hw_interface_by_name, args->request); - if (!p) - { - s = format (s, "{\"interface_stats\": {[\n"); - s = format (s, " \"name\": \"%s\",", args->request); - s = format (s, " \"error\": \"%s\"", "UnknownInterface"); - s = format (s, "]}\n"); - goto out; - } - - vec_add1 (sw_if_indices, p[0]); - } - else /* default, HTTP_BUILTIN_METHOD_GET */ - { - pool_foreach (hi, im->hw_interfaces) - { - vec_add1 (sw_if_indices, hi->sw_if_index); - } - } - - s = format (s, "{%sinterface_stats%s: [\n", q, q); - - for (i = 0; i < vec_len (sw_if_indices); i++) - { - si = vnet_get_sw_interface (vnm, sw_if_indices[i]); - if (need_comma) - s = format (s, ",\n"); - - need_comma = 1; - - s = format (s, "{%sname%s: %s%U%s, ", q, q, q, - format_vnet_sw_if_index_name, vnm, sw_if_indices[i], q); - - stats = format_vnet_sw_interface_cntrs (stats, &vnm->interface_main, si, - 1 /* want json */ ); - if (vec_len (stats)) - s = format (s, "%v}", stats); - else - s = format (s, "%snone%s: %strue%s}", q, q, q, q); - vec_reset_length (stats); - } - - s = format (s, "]}\n"); - -out: - args->data = s; - args->data_len = vec_len (s); - args->free_vec_data = 1; - vec_free (sw_if_indices); - vec_free (stats); - return HSS_URL_HANDLER_OK; -} - -hss_url_handler_rc_t -handle_get_interface_list (hss_url_handler_args_t *args) -{ - u8 *s = 0; - int i; - vnet_main_t *vnm = vnet_get_main (); - vnet_interface_main_t *im = &vnm->interface_main; - vnet_hw_interface_t *hi; - u32 *hw_if_indices = 0; - int need_comma = 0; - - /* Construct vector of active hw_if_indexes ... */ - pool_foreach (hi, im->hw_interfaces) - { - /* No point in mentioning "local0"... */ - if (hi - im->hw_interfaces) - vec_add1 (hw_if_indices, hi - im->hw_interfaces); - } - - /* Build answer */ - s = format (s, "{\"interface_list\": [\n"); - for (i = 0; i < vec_len (hw_if_indices); i++) - { - if (need_comma) - s = format (s, ",\n"); - hi = pool_elt_at_index (im->hw_interfaces, hw_if_indices[i]); - s = format (s, "\"%v\"", hi->name); - need_comma = 1; - } - s = format (s, "]}\n"); - vec_free (hw_if_indices); - - args->data = s; - args->data_len = vec_len (s); - args->free_vec_data = 1; - return HSS_URL_HANDLER_OK; -} - -void -builtinurl_handler_init (builtinurl_main_t * bm) -{ - - bm->register_handler (handle_get_version, "version.json", HTTP_REQ_GET); - bm->register_handler (handle_get_interface_list, "interface_list.json", - HTTP_REQ_GET); - bm->register_handler (handle_get_interface_stats, "interface_stats.json", - HTTP_REQ_GET); - bm->register_handler (handle_get_interface_stats, "interface_stats.json", - HTTP_REQ_POST); -} - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/plugins/builtinurl/builtinurl.api b/src/plugins/builtinurl/builtinurl.api deleted file mode 100644 index 80efa73c725..00000000000 --- a/src/plugins/builtinurl/builtinurl.api +++ /dev/null @@ -1,44 +0,0 @@ -/* - * builtinurl.api - binary API skeleton - * - * Copyright (c) <current-year> <your-organization> - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * @file builtinurl.api - * @brief VPP control-plane API messages. - * - * This file defines VPP control-plane binary API messages which are generally - * called through a shared memory interface. - */ - -/* Version and type recitations */ - -option version = "1.0.0"; - -/** @brief API to enable / disable builtinurl on an interface - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param enable_disable - 1 to enable, 0 to disable the feature - @param sw_if_index - interface handle -*/ - -autoreply define builtinurl_enable { - option deprecated="incorporated in http_static plugin"; - /* Client identifier, set from api_main.my_client_index */ - u32 client_index; - - /* Arbitrary context, so client can match reply to request */ - u32 context; -}; diff --git a/src/plugins/builtinurl/builtinurl.c b/src/plugins/builtinurl/builtinurl.c deleted file mode 100644 index 749a2c93b8a..00000000000 --- a/src/plugins/builtinurl/builtinurl.c +++ /dev/null @@ -1,137 +0,0 @@ -/* - * builtinurl.c - skeleton vpp engine plug-in - * - * Copyright (c) 2019 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <vnet/vnet.h> -#include <vnet/plugin/plugin.h> -#include <builtinurl/builtinurl.h> - -#include <vlibapi/api.h> -#include <vlibmemory/api.h> -#include <vpp/app/version.h> -#include <stdbool.h> - -/* define message IDs */ -#include <builtinurl/builtinurl.api_enum.h> -#include <builtinurl/builtinurl.api_types.h> - -#define REPLY_MSG_ID_BASE bmp->msg_id_base -#include <vlibapi/api_helper_macros.h> - -builtinurl_main_t builtinurl_main; - -/* Action function shared between message handler and debug CLI */ - -int -builtinurl_enable (builtinurl_main_t * bmp) -{ - void (*fp) (void *, char *, int); - - if (bmp->initialized) - return 0; - - /* Look up the builtin URL registration handler */ - fp = vlib_get_plugin_symbol - ("http_static_plugin.so", "http_static_server_register_builtin_handler"); - - /* Most likely, the http_static plugin isn't loaded. Done. */ - if (fp == 0) - return VNET_API_ERROR_NO_SUCH_TABLE; - - bmp->register_handler = fp; - builtinurl_handler_init (bmp); - bmp->initialized = 1; - - return 0; -} - -static clib_error_t * -builtinurl_enable_command_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - builtinurl_main_t *bmp = &builtinurl_main; - - int rv; - - rv = builtinurl_enable (bmp); - - switch (rv) - { - case 0: - break; - - case VNET_API_ERROR_NO_SUCH_TABLE: - return clib_error_return - (0, "http_static_server_register_builtin_handler undefined"); - break; - - default: - return clib_error_return (0, "builtinurl_enable returned %d", rv); - } - return 0; -} - -VLIB_CLI_COMMAND (builtinurl_enable_command, static) = -{ - .path = "builtinurl enable", - .short_help = "Turn on builtin http/https GET and POST urls", - .function = builtinurl_enable_command_fn, -}; - -/* API message handler */ -static void vl_api_builtinurl_enable_t_handler - (vl_api_builtinurl_enable_t * mp) -{ - vl_api_builtinurl_enable_reply_t *rmp; - builtinurl_main_t *bmp = &builtinurl_main; - int rv; - - rv = builtinurl_enable (bmp); - - REPLY_MACRO (VL_API_BUILTINURL_ENABLE_REPLY); -} - -#include <builtinurl/builtinurl.api.c> -static clib_error_t * -builtinurl_init (vlib_main_t * vm) -{ - builtinurl_main_t *bmp = &builtinurl_main; - - bmp->vlib_main = vm; - bmp->vnet_main = vnet_get_main (); - - /* Ask for a correctly-sized block of API message decode slots */ - bmp->msg_id_base = setup_message_id_table (); - - return 0; -} - -VLIB_INIT_FUNCTION (builtinurl_init); - -VLIB_PLUGIN_REGISTER () = -{ - .version = VPP_BUILD_VER, - .description = "vpp built-in URL support", -}; - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/plugins/builtinurl/builtinurl.h b/src/plugins/builtinurl/builtinurl.h deleted file mode 100644 index 91302c1eee5..00000000000 --- a/src/plugins/builtinurl/builtinurl.h +++ /dev/null @@ -1,57 +0,0 @@ - -/* - * builtinurl.h - built-in URLs for the http static server - * - * Copyright (c) 2019 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __included_builtinurl_h__ -#define __included_builtinurl_h__ - -#include <vnet/vnet.h> -#include <vnet/ip/ip.h> -#include <vnet/ethernet/ethernet.h> - -#include <vppinfra/hash.h> -#include <vppinfra/error.h> - -typedef struct -{ - /* API message ID base */ - u16 msg_id_base; - - /* GET / POST handler registration function */ - void (*register_handler) (void *, char *, int); - - /* Been there, done that */ - int initialized; - - /* convenience */ - vlib_main_t *vlib_main; - vnet_main_t *vnet_main; - ethernet_main_t *ethernet_main; -} builtinurl_main_t; - -extern builtinurl_main_t builtinurl_main; - -void builtinurl_handler_init (builtinurl_main_t * bm); - -#endif /* __included_builtinurl_h__ */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/plugins/builtinurl/builtinurl_test.c b/src/plugins/builtinurl/builtinurl_test.c deleted file mode 100644 index 9edfb81c525..00000000000 --- a/src/plugins/builtinurl/builtinurl_test.c +++ /dev/null @@ -1,66 +0,0 @@ -/* - * builtinurl.c - skeleton vpp-api-test plug-in - * - * Copyright (c) 2019 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include <vat/vat.h> -#include <vlibapi/api.h> -#include <vlibmemory/api.h> -#include <vppinfra/error.h> -#include <stdbool.h> - -uword unformat_sw_if_index (unformat_input_t * input, va_list * args); - -/* Declare message IDs */ -#include <builtinurl/builtinurl.api_enum.h> -#include <builtinurl/builtinurl.api_types.h> - -typedef struct -{ - /* API message ID base */ - u16 msg_id_base; - vat_main_t *vat_main; -} builtinurl_test_main_t; - -builtinurl_test_main_t builtinurl_test_main; - -#define __plugin_msg_base builtinurl_test_main.msg_id_base -#include <vlibapi/vat_helper_macros.h> - -static int -api_builtinurl_enable (vat_main_t * vam) -{ - vl_api_builtinurl_enable_t *mp; - int ret; - - /* Construct the API message */ - M (BUILTINURL_ENABLE, mp); - - /* send it... */ - S (mp); - - /* Wait for a reply... */ - W (ret); - return ret; -} - -#include <builtinurl/builtinurl.api_test.c> - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/plugins/crypto_native/aes_cbc.c b/src/plugins/crypto_native/aes_cbc.c index dd7ca3f1cf1..c981897783f 100644 --- a/src/plugins/crypto_native/aes_cbc.c +++ b/src/plugins/crypto_native/aes_cbc.c @@ -25,191 +25,40 @@ #pragma GCC optimize ("O3") #endif -#if defined(__VAES__) && defined(__AVX512F__) -#define u8xN u8x64 -#define u32xN u32x16 -#define u32xN_min_scalar u32x16_min_scalar -#define u32xN_is_all_zero u32x16_is_all_zero -#define u32xN_splat u32x16_splat -#elif defined(__VAES__) -#define u8xN u8x32 -#define u32xN u32x8 -#define u32xN_min_scalar u32x8_min_scalar -#define u32xN_is_all_zero u32x8_is_all_zero -#define u32xN_splat u32x8_splat -#else -#define u8xN u8x16 -#define u32xN u32x4 -#define u32xN_min_scalar u32x4_min_scalar -#define u32xN_is_all_zero u32x4_is_all_zero -#define u32xN_splat u32x4_splat -#endif +#define CRYPTO_NATIVE_AES_CBC_ENC_VEC_SIZE 256 static_always_inline u32 aes_ops_enc_aes_cbc (vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops, aes_key_size_t ks) { crypto_native_main_t *cm = &crypto_native_main; - int rounds = AES_KEY_ROUNDS (ks); - u8 placeholder[8192]; - u32 i, j, count, n_left = n_ops; - u32xN placeholder_mask = { }; - u32xN len = { }; - vnet_crypto_key_index_t key_index[4 * N_AES_LANES]; - u8 *src[4 * N_AES_LANES] = {}; - u8 *dst[4 * N_AES_LANES] = {}; - u8xN r[4] = {}; - u8xN k[15][4] = {}; - - for (i = 0; i < 4 * N_AES_LANES; i++) - key_index[i] = ~0; - -more: - for (i = 0; i < 4 * N_AES_LANES; i++) - if (len[i] == 0) - { - if (n_left == 0) - { - /* no more work to enqueue, so we are enqueueing placeholder buffer */ - src[i] = dst[i] = placeholder; - len[i] = sizeof (placeholder); - placeholder_mask[i] = 0; - } - else - { - u8x16 t = aes_block_load (ops[0]->iv); - ((u8x16 *) r)[i] = t; - - src[i] = ops[0]->src; - dst[i] = ops[0]->dst; - len[i] = ops[0]->len; - placeholder_mask[i] = ~0; - if (key_index[i] != ops[0]->key_index) - { - aes_cbc_key_data_t *kd; - key_index[i] = ops[0]->key_index; - kd = (aes_cbc_key_data_t *) cm->key_data[key_index[i]]; - for (j = 0; j < rounds + 1; j++) - ((u8x16 *) k[j])[i] = kd->encrypt_key[j]; - } - ops[0]->status = VNET_CRYPTO_OP_STATUS_COMPLETED; - n_left--; - ops++; - } - } - - count = u32xN_min_scalar (len); - - ASSERT (count % 16 == 0); - - for (i = 0; i < count; i += 16) + u32 i, n_left = n_ops; + uword key_indices[CRYPTO_NATIVE_AES_CBC_ENC_VEC_SIZE] = {}; + u8 *plaintext[CRYPTO_NATIVE_AES_CBC_ENC_VEC_SIZE] = {}; + uword oplen[CRYPTO_NATIVE_AES_CBC_ENC_VEC_SIZE] = {}; + u8 *iv[CRYPTO_NATIVE_AES_CBC_ENC_VEC_SIZE] = {}; + u8 *ciphertext[CRYPTO_NATIVE_AES_CBC_ENC_VEC_SIZE] = {}; + + while (n_left) { -#if defined(__VAES__) && defined(__AVX512F__) - r[0] = u8x64_xor3 (r[0], aes_block_load_x4 (src, i), k[0][0]); - r[1] = u8x64_xor3 (r[1], aes_block_load_x4 (src + 4, i), k[0][1]); - r[2] = u8x64_xor3 (r[2], aes_block_load_x4 (src + 8, i), k[0][2]); - r[3] = u8x64_xor3 (r[3], aes_block_load_x4 (src + 12, i), k[0][3]); - - for (j = 1; j < rounds; j++) + i = 0; + while (n_left && i < CRYPTO_NATIVE_AES_CBC_ENC_VEC_SIZE) { - r[0] = aes_enc_round_x4 (r[0], k[j][0]); - r[1] = aes_enc_round_x4 (r[1], k[j][1]); - r[2] = aes_enc_round_x4 (r[2], k[j][2]); - r[3] = aes_enc_round_x4 (r[3], k[j][3]); + key_indices[i] = ops[0]->key_index; + plaintext[i] = ops[0]->src; + ciphertext[i] = ops[0]->dst; + oplen[i] = ops[0]->len; + iv[i] = ops[0]->iv; + ops[0]->status = VNET_CRYPTO_OP_STATUS_COMPLETED; + + ops++; + n_left--; + i++; } - r[0] = aes_enc_last_round_x4 (r[0], k[j][0]); - r[1] = aes_enc_last_round_x4 (r[1], k[j][1]); - r[2] = aes_enc_last_round_x4 (r[2], k[j][2]); - r[3] = aes_enc_last_round_x4 (r[3], k[j][3]); - - aes_block_store_x4 (dst, i, r[0]); - aes_block_store_x4 (dst + 4, i, r[1]); - aes_block_store_x4 (dst + 8, i, r[2]); - aes_block_store_x4 (dst + 12, i, r[3]); -#elif defined(__VAES__) - r[0] = u8x32_xor3 (r[0], aes_block_load_x2 (src, i), k[0][0]); - r[1] = u8x32_xor3 (r[1], aes_block_load_x2 (src + 2, i), k[0][1]); - r[2] = u8x32_xor3 (r[2], aes_block_load_x2 (src + 4, i), k[0][2]); - r[3] = u8x32_xor3 (r[3], aes_block_load_x2 (src + 6, i), k[0][3]); - - for (j = 1; j < rounds; j++) - { - r[0] = aes_enc_round_x2 (r[0], k[j][0]); - r[1] = aes_enc_round_x2 (r[1], k[j][1]); - r[2] = aes_enc_round_x2 (r[2], k[j][2]); - r[3] = aes_enc_round_x2 (r[3], k[j][3]); - } - r[0] = aes_enc_last_round_x2 (r[0], k[j][0]); - r[1] = aes_enc_last_round_x2 (r[1], k[j][1]); - r[2] = aes_enc_last_round_x2 (r[2], k[j][2]); - r[3] = aes_enc_last_round_x2 (r[3], k[j][3]); - - aes_block_store_x2 (dst, i, r[0]); - aes_block_store_x2 (dst + 2, i, r[1]); - aes_block_store_x2 (dst + 4, i, r[2]); - aes_block_store_x2 (dst + 6, i, r[3]); -#else -#if __x86_64__ - r[0] = u8x16_xor3 (r[0], aes_block_load (src[0] + i), k[0][0]); - r[1] = u8x16_xor3 (r[1], aes_block_load (src[1] + i), k[0][1]); - r[2] = u8x16_xor3 (r[2], aes_block_load (src[2] + i), k[0][2]); - r[3] = u8x16_xor3 (r[3], aes_block_load (src[3] + i), k[0][3]); - - for (j = 1; j < rounds; j++) - { - r[0] = aes_enc_round_x1 (r[0], k[j][0]); - r[1] = aes_enc_round_x1 (r[1], k[j][1]); - r[2] = aes_enc_round_x1 (r[2], k[j][2]); - r[3] = aes_enc_round_x1 (r[3], k[j][3]); - } - - r[0] = aes_enc_last_round_x1 (r[0], k[j][0]); - r[1] = aes_enc_last_round_x1 (r[1], k[j][1]); - r[2] = aes_enc_last_round_x1 (r[2], k[j][2]); - r[3] = aes_enc_last_round_x1 (r[3], k[j][3]); - - aes_block_store (dst[0] + i, r[0]); - aes_block_store (dst[1] + i, r[1]); - aes_block_store (dst[2] + i, r[2]); - aes_block_store (dst[3] + i, r[3]); -#else - r[0] ^= aes_block_load (src[0] + i); - r[1] ^= aes_block_load (src[1] + i); - r[2] ^= aes_block_load (src[2] + i); - r[3] ^= aes_block_load (src[3] + i); - for (j = 0; j < rounds - 1; j++) - { - r[0] = vaesmcq_u8 (vaeseq_u8 (r[0], k[j][0])); - r[1] = vaesmcq_u8 (vaeseq_u8 (r[1], k[j][1])); - r[2] = vaesmcq_u8 (vaeseq_u8 (r[2], k[j][2])); - r[3] = vaesmcq_u8 (vaeseq_u8 (r[3], k[j][3])); - } - r[0] = vaeseq_u8 (r[0], k[j][0]) ^ k[rounds][0]; - r[1] = vaeseq_u8 (r[1], k[j][1]) ^ k[rounds][1]; - r[2] = vaeseq_u8 (r[2], k[j][2]) ^ k[rounds][2]; - r[3] = vaeseq_u8 (r[3], k[j][3]) ^ k[rounds][3]; - aes_block_store (dst[0] + i, r[0]); - aes_block_store (dst[1] + i, r[1]); - aes_block_store (dst[2] + i, r[2]); - aes_block_store (dst[3] + i, r[3]); -#endif -#endif + clib_aes_cbc_encrypt_multi ((aes_cbc_key_data_t **) cm->key_data, + key_indices, plaintext, oplen, iv, ks, + ciphertext, i); } - - len -= u32xN_splat (count); - - for (i = 0; i < 4 * N_AES_LANES; i++) - { - src[i] += count; - dst[i] += count; - } - - if (n_left > 0) - goto more; - - if (!u32xN_is_all_zero (len & placeholder_mask)) - goto more; - return n_ops; } diff --git a/src/plugins/crypto_native/sha2.c b/src/plugins/crypto_native/sha2.c index 459ce6d8e79..6787f629104 100644 --- a/src/plugins/crypto_native/sha2.c +++ b/src/plugins/crypto_native/sha2.c @@ -118,13 +118,25 @@ sha2_key_add (vnet_crypto_key_t *key, clib_sha2_type_t type) static int probe () { -#if defined(__SHA__) && defined(__x86_64__) +#if defined(__x86_64__) + +#if defined(__SHA__) && defined(__AVX512F__) + if (clib_cpu_supports_sha () && clib_cpu_supports_avx512f ()) + return 30; +#elif defined(__SHA__) && defined(__AVX2__) + if (clib_cpu_supports_sha () && clib_cpu_supports_avx2 ()) + return 20; +#elif defined(__SHA__) if (clib_cpu_supports_sha ()) - return 50; -#elif defined(__ARM_FEATURE_SHA2) + return 10; +#endif + +#elif defined(__aarch64__) +#if defined(__ARM_FEATURE_SHA2) if (clib_cpu_supports_sha2 ()) return 10; #endif +#endif return -1; } diff --git a/src/plugins/crypto_openssl/main.c b/src/plugins/crypto_openssl/main.c index b070cf336a5..c59b5d34a29 100644 --- a/src/plugins/crypto_openssl/main.c +++ b/src/plugins/crypto_openssl/main.c @@ -219,6 +219,17 @@ openssl_ops_enc_aead (vlib_main_t *vm, vnet_crypto_op_t *ops[], vnet_crypto_op_t *op = ops[i]; int len = 0; + if (i + 2 < n_ops) + { + CLIB_PREFETCH (ops[i + 1]->src, 4 * CLIB_CACHE_PREFETCH_BYTES, LOAD); + CLIB_PREFETCH (ops[i + 1]->dst, 4 * CLIB_CACHE_PREFETCH_BYTES, + STORE); + + CLIB_PREFETCH (ops[i + 2]->src, 4 * CLIB_CACHE_PREFETCH_BYTES, LOAD); + CLIB_PREFETCH (ops[i + 2]->dst, 4 * CLIB_CACHE_PREFETCH_BYTES, + STORE); + } + ctx = ptd->evp_cipher_enc_ctx[op->key_index]; EVP_EncryptInit_ex (ctx, 0, 0, NULL, op->iv); if (op->aad_len) diff --git a/src/plugins/dev_armada/CMakeLists.txt b/src/plugins/dev_armada/CMakeLists.txt new file mode 100644 index 00000000000..f955a9baa91 --- /dev/null +++ b/src/plugins/dev_armada/CMakeLists.txt @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright(c) 2022 Cisco Systems, Inc. + + +find_path(MUSDK_INCLUDE_DIR NAMES mv_std.h) +find_library(MUSDK_LIB NAMES libmusdk.a) + +if(NOT MUSDK_INCLUDE_DIR OR NOT MUSDK_LIB) + message(WARNING "Marvell MUSDK not found - dev_armada plugin disabled") + return() +endif() + +get_filename_component(MUSDK_LIB_DIR ${MUSDK_LIB} DIRECTORY) +set(MUSDK_LINK_FLAGS "-Wl,--whole-archive,${MUSDK_LIB_DIR}/libmusdk.a,--no-whole-archive") + +add_vpp_plugin(dev_armada + SOURCES + plugin.c + pp2/init.c + pp2/format.c + pp2/port.c + pp2/queue.c + pp2/rx.c + pp2/tx.c + + LINK_FLAGS + ${MUSDK_LINK_FLAGS} +) +include_directories(${MUSDK_INCLUDE_DIR}) + diff --git a/src/plugins/dev_armada/README.rst b/src/plugins/dev_armada/README.rst new file mode 100644 index 00000000000..2c757d04a06 --- /dev/null +++ b/src/plugins/dev_armada/README.rst @@ -0,0 +1,61 @@ +Armada device plugin +===================== + +Overview +-------- + +This plugins provides native device support for Marvell PP2 network +device, found in Marvel Armada family of SOCs. +It uses Marvell Usermode SDK +(`MUSDK <https://github.com/MarvellEmbeddedProcessors/musdk-marvell>`__). + +Prerequisites +------------- + +Plugins depends on installed MUSDK and Marvell provided linux in Marvell SDK. +Following kernel modules from MUSDK must be loaded for plugin to work: +``musdk_cma.ko`` +``mv_pp_uio.ko`` + +Musdk 18.09.3 compilation steps +------------------------------- + +:: + + ./bootstrap + ./configure --prefix=/opt/vpp/external/aarch64/ CFLAGS="-Wno-error=unused-result -g -fPIC" --enable-shared=no + sed -i -e 's/marvell,mv-pp-uio/generic-uio/' modules/pp2/mv_pp_uio.c + sed -i -e 's/O_CREAT/O_CREAT, S_IRUSR | S_IWUSR/' src/lib/file_utils.c + make + sudo make install + +Usage +----- + +Interface Creation and Deletion +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Interfaces are using new vnet dev APIs, CLIs or startup.conf to create and +delete interfaces. + +Sample startup.conf: + +:: + + devices { + dev platform/f2000000.ethernet { + port 1 { name ppio1 } + } + +Device identifier in this example is 'platform/f2000000.ethernet' where +'platform' is bus name and 'f2000000.ethernet' is linux platform bus +identifier for specific PP2. + +Platform identifier can be found in sysfs: + +:: + + $ ls /sys/bus/platform/devices | grep ethernet + f2000000.ethernet + + diff --git a/src/plugins/dev_armada/musdk.h b/src/plugins/dev_armada/musdk.h new file mode 100644 index 00000000000..aad2f4a1cef --- /dev/null +++ b/src/plugins/dev_armada/musdk.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2023 Cisco Systems, Inc. + */ + +#ifndef _MUSDK_H_ +#define _MUSDK_H_ + +#define MVCONF_DBG_LEVEL 0 +#define MVCONF_PP2_BPOOL_COOKIE_SIZE 32 +#define MVCONF_PP2_BPOOL_DMA_ADDR_SIZE 64 +#define MVCONF_DMA_PHYS_ADDR_T_SIZE 64 +#define MVCONF_SYS_DMA_UIO +#define MVCONF_TYPES_PUBLIC +#define MVCONF_DMA_PHYS_ADDR_T_PUBLIC + +#include <mv_std.h> +#include <env/mv_sys_dma.h> +#include <drivers/mv_pp2.h> +#include <drivers/mv_pp2_bpool.h> +#include <drivers/mv_pp2_ppio.h> + +#endif /* _MUSDK_H_ */ diff --git a/src/plugins/dev_armada/plugin.c b/src/plugins/dev_armada/plugin.c new file mode 100644 index 00000000000..1dc465c9a25 --- /dev/null +++ b/src/plugins/dev_armada/plugin.c @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2023 Cisco Systems, Inc. + */ + +#include <vlib/vlib.h> +#include <vnet/plugin/plugin.h> +#include <vpp/app/version.h> + +VLIB_PLUGIN_REGISTER () = { + .version = VPP_BUILD_VER, + .description = "Marvell Armada Drivers", +}; diff --git a/src/plugins/dev_armada/pp2/format.c b/src/plugins/dev_armada/pp2/format.c new file mode 100644 index 00000000000..37d482b5ce8 --- /dev/null +++ b/src/plugins/dev_armada/pp2/format.c @@ -0,0 +1,176 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2023 Cisco Systems, Inc. + */ + +#include <vnet/vnet.h> +#include <vnet/dev/dev.h> +#include <vnet/dev/counters.h> +#include <vnet/dev/bus/platform.h> +#include <dev_armada/musdk.h> +#include <dev_armada/pp2/pp2.h> + +static inline u32 +mrvl_get_u32_bits (void *start, int offset, int first, int last) +{ + u32 value = *(u32 *) (((u8 *) start) + offset); + if ((last == 0) && (first == 31)) + return value; + value >>= last; + value &= (1 << (first - last + 1)) - 1; + return value; +} + +u8 * +format_pp2_ppio_link_info (u8 *s, va_list *args) +{ + struct pp2_ppio_link_info *li = va_arg (*args, struct pp2_ppio_link_info *); + + char *port_duplex[] = { + [MV_NET_LINK_DUPLEX_HALF] = "half", + [MV_NET_LINK_DUPLEX_FULL] = "full", + }; + + u32 port_speeds[] = { + [MV_NET_LINK_SPEED_10] = 10, [MV_NET_LINK_SPEED_100] = 100, + [MV_NET_LINK_SPEED_1000] = 1000, [MV_NET_LINK_SPEED_2500] = 2500, + [MV_NET_LINK_SPEED_10000] = 10000, + }; + + char *port_phy_modes[] = { + [MV_NET_PHY_MODE_NONE] = "NONE", + [MV_NET_PHY_MODE_MII] = "MII", + [MV_NET_PHY_MODE_GMII] = "GMII", + [MV_NET_PHY_MODE_SGMII] = "SGMII", + [MV_NET_PHY_MODE_TBI] = "TBI", + [MV_NET_PHY_MODE_REVMII] = "REVMII", + [MV_NET_PHY_MODE_RMII] = "RMII", + [MV_NET_PHY_MODE_RGMII] = "RGMII", + [MV_NET_PHY_MODE_RGMII_ID] = "RGMII_ID", + [MV_NET_PHY_MODE_RGMII_RXID] = "RGMII_RXID", + [MV_NET_PHY_MODE_RGMII_TXID] = "RGMII_TXID", + [MV_NET_PHY_MODE_RTBI] = "RTBI", + [MV_NET_PHY_MODE_SMII] = "SMII", + [MV_NET_PHY_MODE_XGMII] = "XGMII", + [MV_NET_PHY_MODE_MOCA] = "MOCA", + [MV_NET_PHY_MODE_QSGMII] = "QSGMII", + [MV_NET_PHY_MODE_XAUI] = "XAUI", + [MV_NET_PHY_MODE_RXAUI] = "RXAUI", + [MV_NET_PHY_MODE_KR] = "KR", + }; + + s = + format (s, "duplex %s speed %d up %d phy_mode %s", port_duplex[li->duplex], + port_speeds[li->speed], li->up, port_phy_modes[li->phy_mode]); + + return s; +} + +u8 * +format_mvpp2_port_status (u8 *s, va_list *args) +{ + vnet_dev_format_args_t __clib_unused *a = + va_arg (*args, vnet_dev_format_args_t *); + vnet_dev_port_t *port = va_arg (*args, vnet_dev_port_t *); + mvpp2_port_t *mp = vnet_dev_get_port_data (port); + struct pp2_ppio_link_info li = {}; + + if (mp->ppio == 0 || pp2_ppio_get_link_info (mp->ppio, &li)) + return format (s, "link info not available"); + + return format (s, "%U", format_pp2_ppio_link_info, &li); +} + +u8 * +format_mvpp2_dev_info (u8 *s, va_list *args) +{ + vnet_dev_format_args_t __clib_unused *a = + va_arg (*args, vnet_dev_format_args_t *); + vnet_dev_t *dev = va_arg (*args, vnet_dev_t *); + mvpp2_device_t *md = vnet_dev_get_data (dev); + + format (s, "pp_id is %u", md->pp_id); + return s; +} + +#define foreach_pp2_rx_desc_field \ + _ (0x00, 6, 0, l3_offset) \ + _ (0x00, 12, 8, ip_hdlen) \ + _ (0x00, 14, 13, ec) \ + _ (0x00, 15, 15, es) \ + _ (0x00, 19, 16, pool_id) \ + _ (0x00, 21, 21, hwf_sync) \ + _ (0x00, 22, 22, l4_chk_ok) \ + _ (0x00, 23, 23, ip_frg) \ + _ (0x00, 24, 24, ipv4_hdr_err) \ + _ (0x00, 27, 25, l4_info) \ + _ (0x00, 30, 28, l3_info) \ + _ (0x00, 31, 31, buf_header) \ + _ (0x04, 5, 0, lookup_id) \ + _ (0x04, 8, 6, cpu_code) \ + _ (0x04, 9, 9, pppoe) \ + _ (0x04, 11, 10, l3_cast_info) \ + _ (0x04, 13, 12, l2_cast_info) \ + _ (0x04, 15, 14, vlan_info) \ + _ (0x04, 31, 16, byte_count) \ + _ (0x08, 11, 0, gem_port_id) \ + _ (0x08, 13, 12, color) \ + _ (0x08, 14, 14, gop_sop_u) \ + _ (0x08, 15, 15, key_hash_enable) \ + _ (0x08, 31, 16, l4chk) \ + _ (0x0c, 31, 0, timestamp) \ + _ (0x10, 31, 0, buf_phys_ptr_lo) \ + _ (0x14, 7, 0, buf_phys_ptr_hi) \ + _ (0x14, 31, 8, key_hash) \ + _ (0x18, 31, 0, buf_virt_ptr_lo) \ + _ (0x1c, 7, 0, buf_virt_ptr_hi) \ + _ (0x1c, 14, 8, buf_qset_no) \ + _ (0x1c, 15, 15, buf_type) \ + _ (0x1c, 21, 16, mod_dscp) \ + _ (0x1c, 24, 22, mod_pri) \ + _ (0x1c, 25, 25, mdscp) \ + _ (0x1c, 26, 26, mpri) \ + _ (0x1c, 27, 27, mgpid) \ + _ (0x1c, 31, 29, port_num) + +u8 * +format_mvpp2_rx_desc (u8 *s, va_list *args) + +{ + struct pp2_ppio_desc *d = va_arg (*args, struct pp2_ppio_desc *); + u32 indent = format_get_indent (s); + u32 r32; + +#define _(a, b, c, n) \ + r32 = mrvl_get_u32_bits (d, a, b, c); \ + if (r32 > 9) \ + s = format (s, "%s %u (0x%x)", #n, r32, r32); \ + else \ + s = format (s, "%s %u", #n, r32); \ + if (format_get_indent (s) > 72) \ + s = format (s, "\n%U", format_white_space, indent + 2); \ + else \ + s = format (s, " "); + + foreach_pp2_rx_desc_field; + return s; +} + +u8 * +format_mvpp2_rx_trace (u8 *s, va_list *args) +{ + vlib_main_t *vm = va_arg (*args, vlib_main_t *); + vlib_node_t *node = va_arg (*args, vlib_node_t *); + mvpp2_rx_trace_t *t = va_arg (*args, mvpp2_rx_trace_t *); + vnet_main_t *vnm = vnet_get_main (); + u32 hw_if_index = t->rxq->port->intf.hw_if_index; + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index); + u32 indent = format_get_indent (s); + struct pp2_ppio_desc *d = &t->desc; + + s = format (s, "pp2: %v (%d) next-node %U", hi->name, hw_if_index, + format_vlib_next_node_name, vm, node->index, t->rxq->next_index); + s = format (s, "\n%U%U", format_white_space, indent + 2, + format_mvpp2_rx_desc, d); + + return s; +} diff --git a/src/plugins/dev_armada/pp2/init.c b/src/plugins/dev_armada/pp2/init.c new file mode 100644 index 00000000000..38ff32d8f53 --- /dev/null +++ b/src/plugins/dev_armada/pp2/init.c @@ -0,0 +1,343 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2023 Cisco Systems, Inc. + */ + +#include <vnet/vnet.h> +#include <vnet/dev/dev.h> +#include <vnet/dev/counters.h> +#include <vnet/dev/bus/platform.h> +#include <vppinfra/ring.h> +#include <dev_armada/musdk.h> +#include <dev_armada/pp2/pp2.h> +#include <vnet/ethernet/ethernet.h> +#include <vnet/plugin/plugin.h> +#include <vpp/app/version.h> + +#include <linux/if.h> +#include <sys/ioctl.h> + +#define MV_SYS_DMA_MEM_SZ (2 << 20) + +VLIB_REGISTER_LOG_CLASS (mvpp2_log, static) = { + .class_name = "armada", + .subclass_name = "init", +}; + +static int num_pp2_in_use = 0; +static int dma_mem_initialized = 0; +static int global_pp2_initialized = 0; + +#define _(f, n, s, d) \ + { .name = #n, .desc = d, .severity = VL_COUNTER_SEVERITY_##s }, + +vlib_error_desc_t mvpp2_rx_node_counters[] = { foreach_mvpp2_rx_node_counter }; +vlib_error_desc_t mvpp2_tx_node_counters[] = { foreach_mvpp2_tx_node_counter }; +#undef _ + +vnet_dev_node_t mvpp2_rx_node = { + .error_counters = mvpp2_rx_node_counters, + .n_error_counters = ARRAY_LEN (mvpp2_rx_node_counters), + .format_trace = format_mvpp2_rx_trace, +}; + +vnet_dev_node_t mvpp2_tx_node = { + .error_counters = mvpp2_tx_node_counters, + .n_error_counters = ARRAY_LEN (mvpp2_tx_node_counters), +}; + +static u8 * +mvpp2_probe (vlib_main_t *vm, vnet_dev_bus_index_t bus_index, void *dev_info) +{ + vnet_dev_bus_platform_device_info_t *di = dev_info; + + if (clib_dt_node_is_compatible (di->node, "marvell,armada-7k-pp22")) + return format (0, "Marvell Armada Packet Processor v2.2"); + return 0; +} +static void +mvpp2_global_deinit (vlib_main_t *vm, vnet_dev_t *dev) +{ + mvpp2_device_t *md = vnet_dev_get_data (dev); + log_debug (dev, ""); + if (--num_pp2_in_use == 0) + { + if (global_pp2_initialized) + { + for (u32 i = 0; i < ARRAY_LEN (md->thread); i++) + if (md->thread[i].bpool) + { + pp2_bpool_deinit (md->thread[i].bpool); + md->thread[i].bpool = 0; + } + for (u32 i = 0; i < ARRAY_LEN (md->hif); i++) + if (md->hif[i]) + { + pp2_hif_deinit (md->hif[i]); + md->hif[i] = 0; + } + + pp2_deinit (); + global_pp2_initialized = 0; + } + if (dma_mem_initialized) + { + mv_sys_dma_mem_destroy (); + log_debug (0, "mv_sys_dma_mem_destroy()"); + dma_mem_initialized = 0; + } + } +} + +static void +mvpp2_deinit (vlib_main_t *vm, vnet_dev_t *dev) +{ + log_debug (dev, ""); + mvpp2_global_deinit (vm, dev); +} + +static vnet_dev_rv_t +mvpp2_global_init (vlib_main_t *vm, vnet_dev_t *dev) +{ + mvpp2_device_t *md = vnet_dev_get_data (dev); + vnet_dev_rv_t rv = VNET_DEV_OK; + int mrv; + u16 free_hifs, free_bpools; + u16 n_threads = vlib_get_n_threads (); + + struct pp2_init_params init_params = { + .hif_reserved_map = 0xf, + .bm_pool_reserved_map = 0x7, + }; + + if (num_pp2_in_use++) + return rv; + + mrv = mv_sys_dma_mem_init (MV_SYS_DMA_MEM_SZ); + if (mrv < 0) + { + log_err (0, "mv_sys_dma_mem_init failed, err %d", mrv); + rv = VNET_DEV_ERR_INIT_FAILED; + goto done; + } + + dma_mem_initialized = 1; + log_debug (0, "mv_sys_dma_mem_init(%u) ok", MV_SYS_DMA_MEM_SZ); + + if ((mrv = pp2_init (&init_params))) + { + log_err (dev, "pp2_init failed, err %d", mrv); + rv = VNET_DEV_ERR_INIT_FAILED; + goto done; + } + + log_debug (dev, "pp2_init() ok"); + + free_hifs = pow2_mask (MVPP2_NUM_HIFS) ^ init_params.hif_reserved_map; + free_bpools = + pow2_mask (MVPP2_NUM_BPOOLS) ^ init_params.bm_pool_reserved_map; + + if (n_threads > count_set_bits (free_hifs)) + { + log_err (dev, "no enough HIFs (needed %u available %u)", n_threads, + count_set_bits (free_hifs)); + rv = VNET_DEV_ERR_INIT_FAILED; + goto done; + } + + for (u32 i = 0; i < n_threads; i++) + { + char match[16]; + u8 index; + struct pp2_hif_params hif_params = { + .match = match, + .out_size = 2048, + }; + struct pp2_bpool_params bpool_params = { + .match = match, + .buff_len = vlib_buffer_get_default_data_size (vm), + }; + + index = get_lowest_set_bit_index (free_hifs); + free_hifs ^= 1 << index; + snprintf (match, sizeof (match), "hif-%u", index); + + mrv = pp2_hif_init (&hif_params, md->hif + i); + if (mrv < 0) + { + log_err (dev, "pp2_hif_init failed for hif %u thread %u, err %d", + index, i, mrv); + rv = VNET_DEV_ERR_INIT_FAILED; + goto done; + } + log_debug (dev, "pp2_hif_init(hif %u, thread %u) ok", index, i); + + index = get_lowest_set_bit_index (free_bpools); + free_bpools ^= 1 << index; + snprintf (match, sizeof (match), "pool-%u:%u", md->pp_id, index); + + mrv = pp2_bpool_init (&bpool_params, &md->thread[i].bpool); + if (mrv < 0) + { + log_err (dev, "pp2_bpool_init failed for bpool %u thread %u, err %d", + index, i, mrv); + rv = VNET_DEV_ERR_INIT_FAILED; + goto done; + } + log_debug (dev, "pp2_bpool_init(bpool %u, thread %u) pool-%u:%u ok", + index, i, md->thread[i].bpool->pp2_id, + md->thread[i].bpool->id); + for (u32 j = 0; j < ARRAY_LEN (md->thread[0].bre); j++) + md->thread[i].bre[j].bpool = md->thread[i].bpool; + } + +done: + return rv; +} + +static vnet_dev_rv_t +mvpp2_init (vlib_main_t *vm, vnet_dev_t *dev) +{ + mvpp2_device_t *md = vnet_dev_get_data (dev); + vnet_dev_rv_t rv = VNET_DEV_OK; + vnet_dev_bus_platform_device_data_t *dd = vnet_dev_get_bus_data (dev); + clib_dt_node_t *sc; + int pp_id = -1; + + if (!clib_dt_node_is_compatible (dd->node, "marvell,armada-7k-pp22")) + return VNET_DEV_ERR_NOT_SUPPORTED; + + sc = clib_dt_dereference_node (dd->node, "marvell,system-controller"); + + if (sc && vec_len (sc->path) > strlen ("/cpX/")) + { + if (strncmp ((char *) sc->path, "/cp0/", 4) == 0) + pp_id = 0; + else if (strncmp ((char *) sc->path, "/cp1/", 4) == 0) + pp_id = 1; + } + + if (pp_id < 0) + return VNET_DEV_ERR_UNKNOWN_DEVICE; + + if ((mvpp2_global_init (vm, dev)) != VNET_DEV_OK) + return rv; + + md->pp_id = pp_id; + + vec_foreach_pointer (cn, dd->node->child_nodes) + { + clib_dt_property_t *p; + char netdev_name[IFNAMSIZ]; + struct ifreq s = {}; + u8 ppio_id; + int fd, srv; + + p = clib_dt_get_node_property_by_name (cn, "port-id"); + + if (!clib_dt_proprerty_is_u32 (p)) + continue; + + ppio_id = clib_dt_proprerty_get_u32 (p); + log_debug (dev, "found port with ppio id %u", ppio_id); + + if (pp2_ppio_available (md->pp_id, ppio_id) == 0) + continue; + + if (pp2_netdev_get_ifname (md->pp_id, ppio_id, netdev_name) < 0) + { + log_warn (dev, "failed to get ifname, skipping port %u ", ppio_id); + continue; + } + + srv = -1; + if ((fd = socket (PF_INET, SOCK_DGRAM, IPPROTO_IP)) >= 0) + { + strcpy (s.ifr_name, netdev_name); + srv = ioctl (fd, SIOCGIFHWADDR, &s); + close (fd); + } + + if (srv < 0) + { + log_warn (dev, "unable to get hw address, skipping port %u", + ppio_id); + continue; + } + + log_debug (dev, "adding ppio %u (netdev name %s, hwaddr %U)", ppio_id, + netdev_name, format_ethernet_address, s.ifr_addr.sa_data); + + mvpp2_port_t mvpp2_port = { + .ppio_id = ppio_id, + }; + + vnet_dev_port_add_args_t port_add_args = { + .port = { + .attr = { + .type = VNET_DEV_PORT_TYPE_ETHERNET, + .max_rx_queues = PP2_PPIO_MAX_NUM_INQS, + .max_tx_queues = PP2_PPIO_MAX_NUM_OUTQS, + .max_supported_rx_frame_size = 9216, + }, + .ops = { + .init = mvpp2_port_init, + .deinit = mvpp2_port_deinit, + .start = mvpp2_port_start, + .stop = mvpp2_port_stop, + .config_change = mvpp2_port_cfg_change, + .config_change_validate = mvpp2_port_cfg_change_validate, + .format_status = format_mvpp2_port_status, + }, + .data_size = sizeof (mvpp2_port_t), + .initial_data = &mvpp2_port, + }, + .rx_node = &mvpp2_rx_node, + .tx_node = &mvpp2_tx_node, + .rx_queue = { + .config = { + .data_size = sizeof (mvpp2_rxq_t), + .default_size = 512, + .multiplier = 32, + .min_size = 32, + .max_size = 4096, + .size_is_power_of_two = 1, + }, + }, + .tx_queue = { + .config = { + .data_size = sizeof (mvpp2_txq_t), + .default_size = 512, + .multiplier = 32, + .min_size = 32, + .max_size = 4096, + .size_is_power_of_two = 1, + }, + .ops = { + .alloc = mvpp2_txq_alloc, + .free = mvpp2_txq_free, + }, + }, + }; + + vnet_dev_set_hw_addr_eth_mac (&port_add_args.port.attr.hw_addr, + (u8 *) s.ifr_addr.sa_data); + + vnet_dev_port_add (vm, dev, ppio_id, &port_add_args); + } + + if (rv != VNET_DEV_OK) + mvpp2_deinit (vm, dev); + return rv; +} + +VNET_DEV_REGISTER_DRIVER (pp2) = { + .name = "mvpp2", + .bus = PLATFORM_BUS_NAME, + .device_data_sz = sizeof (mvpp2_device_t), + .ops = { + .init = mvpp2_init, + .deinit = mvpp2_deinit, + .probe = mvpp2_probe, + .format_info = format_mvpp2_dev_info, + }, +}; diff --git a/src/plugins/dev_armada/pp2/port.c b/src/plugins/dev_armada/pp2/port.c new file mode 100644 index 00000000000..8e785e5e0e4 --- /dev/null +++ b/src/plugins/dev_armada/pp2/port.c @@ -0,0 +1,280 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2023 Cisco Systems, Inc. + */ + +#include <vnet/vnet.h> +#include <vnet/ethernet/ethernet.h> +#include <vnet/dev/dev.h> +#include <vnet/dev/counters.h> +#include <vnet/dev/bus/platform.h> +#include <vppinfra/ring.h> +#include <dev_armada/musdk.h> +#include <dev_armada/pp2/pp2.h> + +VLIB_REGISTER_LOG_CLASS (mvpp2_log, static) = { + .class_name = "armada", + .subclass_name = "pp2-port", +}; + +vnet_dev_rv_t +mvpp2_port_init (vlib_main_t *vm, vnet_dev_port_t *port) +{ + vnet_dev_t *dev = port->dev; + mvpp2_device_t *md = vnet_dev_get_data (dev); + mvpp2_port_t *mp = vnet_dev_get_port_data (port); + vnet_dev_rv_t rv = VNET_DEV_OK; + struct pp2_ppio_link_info li; + char match[16]; + int mrv; + + log_debug (port->dev, ""); + + snprintf (match, sizeof (match), "ppio-%d:%d", md->pp_id, port->port_id); + + struct pp2_ppio_params ppio_params = { + .match = match, + .type = PP2_PPIO_T_NIC, + .eth_start_hdr = mp->is_dsa ? PP2_PPIO_HDR_ETH_DSA : PP2_PPIO_HDR_ETH, + .inqs_params = { + .num_tcs = 1, + .tcs_params[0] = { + .pkt_offset = 0, + .num_in_qs = 1, + .inqs_params = &(struct pp2_ppio_inq_params) { .size = 512 }, + .pools[0][0] = md->thread[0].bpool, + }, + }, + }; + + foreach_vnet_dev_port_rx_queue (q, port) + { + struct pp2_ppio_outqs_params *oqs = &ppio_params.outqs_params; + oqs->outqs_params[0].weight = 1; + oqs->outqs_params[0].size = q->size; + oqs->num_outqs++; + } + + mrv = pp2_ppio_init (&ppio_params, &mp->ppio); + if (mrv) + { + rv = VNET_DEV_ERR_INIT_FAILED; + log_err (dev, "port %u ppio '%s' init failed, rv %d", port->port_id, + match, mrv); + goto done; + } + log_debug (dev, "port %u ppio '%s' init ok", port->port_id, match); + + mrv = pp2_ppio_get_link_info (mp->ppio, &li); + if (mrv) + { + rv = VNET_DEV_ERR_INIT_FAILED; + log_err (dev, "failed to get link info for port %u, rv %d", + port->port_id, mrv); + goto done; + } + + log_debug (dev, "port %u %U", port->port_id, format_pp2_ppio_link_info, &li); + +done: + if (rv != VNET_DEV_OK) + mvpp2_port_stop (vm, port); + return rv; +} + +void +mvpp2_port_deinit (vlib_main_t *vm, vnet_dev_port_t *port) +{ + mvpp2_port_t *mp = vnet_dev_get_port_data (port); + + log_debug (port->dev, ""); + + if (mp->ppio) + { + pp2_ppio_deinit (mp->ppio); + mp->ppio = 0; + } +} + +void +mvpp2_port_poll (vlib_main_t *vm, vnet_dev_port_t *port) +{ + mvpp2_port_t *mp = vnet_dev_get_port_data (port); + vnet_dev_t *dev = port->dev; + vnet_dev_port_state_changes_t changes = {}; + struct pp2_ppio_link_info li; + int mrv; + + mrv = pp2_ppio_get_link_info (mp->ppio, &li); + + if (mrv) + { + log_debug (dev, "pp2_ppio_get_link_info: failed, rv %d", mrv); + return; + } + + if (mp->last_link_info.up != li.up) + { + changes.change.link_state = 1; + changes.link_state = li.up != 0; + log_debug (dev, "link state changed to %u", changes.link_state); + } + + if (mp->last_link_info.duplex != li.duplex) + { + changes.change.link_duplex = 1; + changes.full_duplex = li.duplex != 0; + log_debug (dev, "link full duplex changed to %u", changes.full_duplex); + } + + if (mp->last_link_info.speed != li.speed) + { + u32 speeds[] = { + [MV_NET_LINK_SPEED_AN] = 0, + [MV_NET_LINK_SPEED_10] = 10000, + [MV_NET_LINK_SPEED_100] = 100000, + [MV_NET_LINK_SPEED_1000] = 1000000, + [MV_NET_LINK_SPEED_2500] = 2500000, + [MV_NET_LINK_SPEED_10000] = 10000000, + }; + + if (li.speed < ARRAY_LEN (speeds)) + { + changes.change.link_speed = 1; + changes.link_speed = speeds[li.speed]; + log_debug (dev, "link speed changed to %u", changes.link_speed); + } + } + + if (changes.change.any == 0) + return; + + mp->last_link_info = li; + + vnet_dev_port_state_change (vm, port, changes); +} + +vnet_dev_rv_t +mvpp2_port_start (vlib_main_t *vm, vnet_dev_port_t *port) +{ + mvpp2_port_t *mp = vnet_dev_get_port_data (port); + int mrv; + + log_debug (port->dev, ""); + + mrv = pp2_ppio_enable (mp->ppio); + if (mrv) + { + log_err (port->dev, "pp2_ppio_enable() failed, rv %d", mrv); + return VNET_DEV_ERR_NOT_READY; + } + + mp->is_enabled = 1; + + vnet_dev_poll_port_add (vm, port, 0.5, mvpp2_port_poll); + + return VNET_DEV_OK; +} + +void +mvpp2_port_stop (vlib_main_t *vm, vnet_dev_port_t *port) +{ + int rv; + mvpp2_port_t *mp = vnet_dev_get_port_data (port); + + log_debug (port->dev, ""); + + if (mp->is_enabled) + { + vnet_dev_poll_port_remove (vm, port, mvpp2_port_poll); + + rv = pp2_ppio_disable (mp->ppio); + if (rv) + log_err (port->dev, "pp2_ppio_disable() failed, rv %d", rv); + + vnet_dev_port_state_change (vm, port, + (vnet_dev_port_state_changes_t){ + .change.link_state = 1, + .change.link_speed = 1, + .link_speed = 0, + .link_state = 0, + }); + mp->is_enabled = 0; + } +} + +vnet_dev_rv_t +mvpp2_port_cfg_change_validate (vlib_main_t *vm, vnet_dev_port_t *port, + vnet_dev_port_cfg_change_req_t *req) +{ + vnet_dev_rv_t rv = VNET_DEV_OK; + + switch (req->type) + { + case VNET_DEV_PORT_CFG_PROMISC_MODE: + case VNET_DEV_PORT_CFG_ADD_SECONDARY_HW_ADDR: + case VNET_DEV_PORT_CFG_REMOVE_SECONDARY_HW_ADDR: + break; + + default: + rv = VNET_DEV_ERR_NOT_SUPPORTED; + }; + + return rv; +} + +vnet_dev_rv_t +mvpp2_port_cfg_change (vlib_main_t *vm, vnet_dev_port_t *port, + vnet_dev_port_cfg_change_req_t *req) +{ + mvpp2_port_t *mp = vnet_dev_get_port_data (port); + vnet_dev_rv_t rv = VNET_DEV_OK; + eth_addr_t addr; + int mrv; + + switch (req->type) + { + + case VNET_DEV_PORT_CFG_PROMISC_MODE: + mrv = pp2_ppio_set_promisc (mp->ppio, req->promisc); + if (mrv) + { + log_err (port->dev, "pp2_ppio_set_promisc: failed, rv %d", mrv); + rv = VNET_DEV_ERR_INTERNAL; + } + else + log_debug (port->dev, "pp2_ppio_set_promisc: promisc %u", + req->promisc); + break; + + case VNET_DEV_PORT_CFG_ADD_SECONDARY_HW_ADDR: + clib_memcpy (&addr, req->addr.eth_mac, sizeof (addr)); + mrv = pp2_ppio_add_mac_addr (mp->ppio, addr); + if (mrv) + { + log_err (port->dev, "pp2_ppio_add_mac_addr: failed, rv %d", mrv); + rv = VNET_DEV_ERR_INTERNAL; + } + else + log_debug (port->dev, "pp2_ppio_add_mac_addr: %U added", + format_ethernet_address, &addr); + break; + + case VNET_DEV_PORT_CFG_REMOVE_SECONDARY_HW_ADDR: + clib_memcpy (&addr, req->addr.eth_mac, sizeof (addr)); + mrv = pp2_ppio_remove_mac_addr (mp->ppio, addr); + if (mrv) + { + log_err (port->dev, "pp2_ppio_remove_mac_addr: failed, rv %d", mrv); + rv = VNET_DEV_ERR_INTERNAL; + } + else + log_debug (port->dev, "pp2_ppio_remove_mac_addr: %U added", + format_ethernet_address, &addr); + break; + + default: + return VNET_DEV_ERR_NOT_SUPPORTED; + }; + + return rv; +} diff --git a/src/plugins/dev_armada/pp2/pp2.h b/src/plugins/dev_armada/pp2/pp2.h new file mode 100644 index 00000000000..6b12dc737a7 --- /dev/null +++ b/src/plugins/dev_armada/pp2/pp2.h @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2023 Cisco Systems, Inc. + */ + +#ifndef _PP2_H_ +#define _PP2_H_ + +#include <vppinfra/clib.h> +#include <vppinfra/error_bootstrap.h> +#include <vppinfra/format.h> +#include <vnet/vnet.h> +#include <vnet/dev/dev.h> + +#define MVCONF_DBG_LEVEL 0 +#define MVCONF_PP2_BPOOL_COOKIE_SIZE 32 +#define MVCONF_PP2_BPOOL_DMA_ADDR_SIZE 64 +#define MVCONF_DMA_PHYS_ADDR_T_SIZE 64 +#define MVCONF_SYS_DMA_UIO +#define MVCONF_TYPES_PUBLIC +#define MVCONF_DMA_PHYS_ADDR_T_PUBLIC + +#include "mv_std.h" +#include "env/mv_sys_dma.h" +#include "drivers/mv_pp2.h" +#include <drivers/mv_pp2_bpool.h> +#include <drivers/mv_pp2_ppio.h> + +#define MVPP2_NUM_HIFS 9 +#define MVPP2_NUM_BPOOLS 16 +#define MVPP2_MAX_THREADS 4 +#define MRVL_PP2_BUFF_BATCH_SZ 32 + +typedef struct +{ + u8 pp_id; + struct pp2_hif *hif[MVPP2_NUM_HIFS]; + struct + { + struct pp2_bpool *bpool; + struct buff_release_entry bre[MRVL_PP2_BUFF_BATCH_SZ]; + } thread[MVPP2_NUM_BPOOLS]; + +} mvpp2_device_t; + +typedef struct +{ + u8 is_enabled : 1; + u8 is_dsa : 1; + struct pp2_ppio *ppio; + u8 ppio_id; + struct pp2_ppio_link_info last_link_info; +} mvpp2_port_t; + +typedef struct +{ + u16 next; + u16 n_enq; + u32 *buffers; +} mvpp2_txq_t; + +typedef struct +{ +} mvpp2_rxq_t; + +typedef struct +{ + struct pp2_ppio_desc desc; + vnet_dev_rx_queue_t *rxq; +} mvpp2_rx_trace_t; + +/* format.c */ +format_function_t format_pp2_ppio_link_info; +format_function_t format_mvpp2_port_status; +format_function_t format_mvpp2_dev_info; +format_function_t format_mvpp2_rx_trace; +format_function_t format_mvpp2_rx_desc; + +/* port.c */ +vnet_dev_port_op_t mvpp2_port_init; +vnet_dev_port_op_no_rv_t mvpp2_port_deinit; +vnet_dev_port_op_t mvpp2_port_start; +vnet_dev_port_op_no_rv_t mvpp2_port_stop; +vnet_dev_rv_t mvpp2_port_cfg_change (vlib_main_t *, vnet_dev_port_t *, + vnet_dev_port_cfg_change_req_t *); +vnet_dev_rv_t +mvpp2_port_cfg_change_validate (vlib_main_t *, vnet_dev_port_t *, + vnet_dev_port_cfg_change_req_t *); + +/* queue.c */ +vnet_dev_tx_queue_op_t mvpp2_txq_alloc; +vnet_dev_tx_queue_op_no_rv_t mvpp2_txq_free; + +/* inline funcs */ + +#define log_debug(dev, f, ...) \ + vlib_log (VLIB_LOG_LEVEL_DEBUG, mvpp2_log.class, "%U" f, \ + format_vnet_dev_log, (dev), \ + clib_string_skip_prefix (__func__, "mvpp2_"), ##__VA_ARGS__) +#define log_info(dev, f, ...) \ + vlib_log (VLIB_LOG_LEVEL_INFO, mvpp2_log.class, "%U" f, \ + format_vnet_dev_log, (dev), 0, ##__VA_ARGS__) +#define log_notice(dev, f, ...) \ + vlib_log (VLIB_LOG_LEVEL_NOTICE, mvpp2_log.class, "%U" f, \ + format_vnet_dev_log, (dev), 0, ##__VA_ARGS__) +#define log_warn(dev, f, ...) \ + vlib_log (VLIB_LOG_LEVEL_WARNING, mvpp2_log.class, "%U" f, \ + format_vnet_dev_log, (dev), 0, ##__VA_ARGS__) +#define log_err(dev, f, ...) \ + vlib_log (VLIB_LOG_LEVEL_ERR, mvpp2_log.class, "%U" f, format_vnet_dev_log, \ + (dev), 0, ##__VA_ARGS__) + +#define foreach_mvpp2_tx_node_counter \ + _ (NO_FREE_SLOTS, no_free_slots, ERROR, "no free tx slots") \ + _ (PPIO_SEND, ppio_semd, ERROR, "pp2_ppio_send errors") \ + _ (PPIO_GET_NUM_OUTQ_DONE, ppio_get_num_outq_done, ERROR, \ + "pp2_ppio_get_num_outq_done errors") + +typedef enum +{ +#define _(f, n, s, d) MVPP2_TX_NODE_CTR_##f, + foreach_mvpp2_tx_node_counter +#undef _ +} mvpp2_tx_node_counter_t; + +#define foreach_mvpp2_rx_node_counter \ + _ (PPIO_RECV, ppio_recv, ERROR, "pp2_ppio_recv error") \ + _ (BPOOL_GET_NUM_BUFFS, bpool_get_num_bufs, ERROR, \ + "pp2_bpool_get_num_buffs error") \ + _ (BPOOL_PUT_BUFFS, bpool_put_buffs, ERROR, "pp2_bpool_put_buffs error") \ + _ (BUFFER_ALLOC, buffer_alloc, ERROR, "buffer alloc error") \ + _ (MAC_CE, mac_ce, ERROR, "MAC error (CRC error)") \ + _ (MAC_OR, mac_or, ERROR, "overrun error") \ + _ (MAC_RSVD, mac_rsvd, ERROR, "unknown MAC error") \ + _ (MAC_RE, mac_re, ERROR, "resource error") \ + _ (IP_HDR, ip_hdr, ERROR, "ip4 header error") + +typedef enum +{ +#define _(f, n, s, d) MVPP2_RX_NODE_CTR_##f, + foreach_mvpp2_rx_node_counter +#undef _ +} mvpp2_rx_node_counter_t; + +#endif /* _PP2_H_ */ diff --git a/src/plugins/dev_armada/pp2/queue.c b/src/plugins/dev_armada/pp2/queue.c new file mode 100644 index 00000000000..05015414816 --- /dev/null +++ b/src/plugins/dev_armada/pp2/queue.c @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2023 Cisco Systems, Inc. + */ + +#include <vnet/vnet.h> +#include <vnet/ethernet/ethernet.h> +#include <vnet/dev/dev.h> +#include <vnet/dev/counters.h> +#include <vnet/dev/bus/platform.h> +#include <vppinfra/ring.h> +#include <dev_armada/musdk.h> +#include <dev_armada/pp2/pp2.h> + +VLIB_REGISTER_LOG_CLASS (mvpp2_log, static) = { + .class_name = "armada", + .subclass_name = "pp2-queue", +}; + +vnet_dev_rv_t +mvpp2_txq_alloc (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) +{ + vnet_dev_rv_t rv = VNET_DEV_OK; + mvpp2_txq_t *mtq = vnet_dev_get_tx_queue_data (txq); + log_debug (txq->port->dev, ""); + + ASSERT (mtq->buffers == 0); + if (mtq->buffers == 0) + { + u32 sz = sizeof (u32) * txq->size; + mtq->buffers = clib_mem_alloc_aligned (sz, CLIB_CACHE_LINE_BYTES); + clib_memset (mtq->buffers, 0, sz); + } + + return rv; +} + +void +mvpp2_txq_free (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) +{ + mvpp2_txq_t *mtq = vnet_dev_get_tx_queue_data (txq); + + log_debug (txq->port->dev, ""); + if (mtq->buffers) + { + clib_mem_free (mtq->buffers); + mtq->buffers = 0; + } +} diff --git a/src/plugins/dev_armada/pp2/rx.c b/src/plugins/dev_armada/pp2/rx.c new file mode 100644 index 00000000000..81101ef9313 --- /dev/null +++ b/src/plugins/dev_armada/pp2/rx.c @@ -0,0 +1,158 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Cisco Systems, Inc. + */ + +#include <vlib/vlib.h> +#include <vnet/dev/dev.h> +#include <vnet/ethernet/ethernet.h> + +#include <dev_armada/pp2/pp2.h> + +static_always_inline void +mvpp2_rx_trace (vlib_main_t *vm, vlib_node_runtime_t *node, + vnet_dev_rx_queue_t *rxq, vlib_buffer_t *b0, uword *n_trace, + struct pp2_ppio_desc *d) +{ + if (PREDICT_TRUE (vlib_trace_buffer (vm, node, rxq->next_index, b0, + /* follow_chain */ 0))) + { + mvpp2_rx_trace_t *tr; + vlib_set_trace_count (vm, node, --(*n_trace)); + tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); + tr->desc = *d; + tr->rxq = rxq; + } +} + +static_always_inline uword +mrvl_pp2_rx_inline (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_frame_t *frame, vnet_dev_rx_queue_t *rxq) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_dev_port_t *port = rxq->port; + vnet_dev_t *dev = port->dev; + mvpp2_device_t *md = vnet_dev_get_data (dev); + mvpp2_port_t *mp = vnet_dev_get_port_data (port); + vlib_buffer_template_t bt = rxq->buffer_template; + u32 thread_index = vm->thread_index; + uword n_trace = vlib_get_trace_count (vm, node); + u32 next_index = rxq->next_index; + u32 n_rx_packets = 0, n_rx_bytes = 0; + struct pp2_hif *hif = md->hif[thread_index]; + struct pp2_ppio_desc descs[VLIB_FRAME_SIZE], *d; + struct pp2_bpool *bpool = md->thread[thread_index].bpool; + struct buff_release_entry *bre = md->thread[thread_index].bre; + u16 n_desc = VLIB_FRAME_SIZE; + u32 buffers[VLIB_FRAME_SIZE]; + u32 n_bufs, *bi, i; + vlib_buffer_t *b0, *b1; + + if (PREDICT_FALSE ( + pp2_ppio_recv (mp->ppio, 0, rxq->queue_id, descs, &n_desc))) + { + vlib_error_count (vm, node->node_index, MVPP2_RX_NODE_CTR_PPIO_RECV, 1); + n_desc = 0; + } + + n_rx_packets = n_desc; + + for (i = 0; i < n_desc; i++) + buffers[i] = pp2_ppio_inq_desc_get_cookie (descs + i); + + bt.current_data = 2; + + for (d = descs, bi = buffers; n_desc >= 4; d += 2, bi += 2, n_desc -= 2) + { + /* prefetch */ + b0 = vlib_get_buffer (vm, bi[0]); + b1 = vlib_get_buffer (vm, bi[1]); + b0->template = bt; + b1->template = bt; + + n_rx_bytes += b0->current_length = pp2_ppio_inq_desc_get_pkt_len (d); + n_rx_bytes += b1->current_length = pp2_ppio_inq_desc_get_pkt_len (d + 1); + + if (PREDICT_FALSE (n_trace > 0)) + { + mvpp2_rx_trace (vm, node, rxq, b0, &n_trace, d); + if (n_trace > 0) + mvpp2_rx_trace (vm, node, rxq, b1, &n_trace, d + 1); + } + } + + for (; n_desc; d++, bi++, n_desc--) + { + b0 = vlib_get_buffer (vm, bi[0]); + b0->template = bt; + + n_rx_bytes += b0->current_length = pp2_ppio_inq_desc_get_pkt_len (d); + + if (PREDICT_FALSE (n_trace > 0)) + mvpp2_rx_trace (vm, node, rxq, b0, &n_trace, d); + } + + vlib_buffer_enqueue_to_single_next (vm, node, buffers, next_index, + n_rx_packets); + + vlib_increment_combined_counter ( + vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, + thread_index, port->intf.sw_if_index, n_rx_packets, n_rx_bytes); + + if (PREDICT_FALSE (pp2_bpool_get_num_buffs (bpool, &n_bufs))) + { + vlib_error_count (vm, node->node_index, + MVPP2_RX_NODE_CTR_BPOOL_GET_NUM_BUFFS, 1); + goto done; + } + + n_bufs = rxq->size - n_bufs; + while (n_bufs >= MRVL_PP2_BUFF_BATCH_SZ) + { + u16 n_alloc, i; + struct buff_release_entry *e = bre; + + n_alloc = vlib_buffer_alloc (vm, buffers, MRVL_PP2_BUFF_BATCH_SZ); + i = n_alloc; + + if (PREDICT_FALSE (n_alloc == 0)) + { + vlib_error_count (vm, node->node_index, + MVPP2_RX_NODE_CTR_BUFFER_ALLOC, 1); + goto done; + } + + for (bi = buffers; i--; e++, bi++) + { + + vlib_buffer_t *b = vlib_get_buffer (vm, bi[0]); + e->buff.addr = vlib_buffer_get_pa (vm, b) - 64; + e->buff.cookie = bi[0]; + } + + i = n_alloc; + if (PREDICT_FALSE (pp2_bpool_put_buffs (hif, bre, &i))) + { + vlib_error_count (vm, node->node_index, + MVPP2_RX_NODE_CTR_BPOOL_PUT_BUFFS, 1); + vlib_buffer_free (vm, buffers, n_alloc); + goto done; + } + + if (PREDICT_FALSE (i != n_alloc)) + vlib_buffer_free (vm, buffers + i, n_alloc - i); + + n_bufs -= i; + } + +done: + return n_rx_packets; +} + +VNET_DEV_NODE_FN (mvpp2_rx_node) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) +{ + u32 n_rx = 0; + foreach_vnet_dev_rx_queue_runtime (rxq, node) + n_rx += mrvl_pp2_rx_inline (vm, node, frame, rxq); + return n_rx; +} diff --git a/src/plugins/dev_armada/pp2/tx.c b/src/plugins/dev_armada/pp2/tx.c new file mode 100644 index 00000000000..1e6675c9746 --- /dev/null +++ b/src/plugins/dev_armada/pp2/tx.c @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Cisco Systems, Inc. + */ + +#include <vlib/vlib.h> +#include <vnet/dev/dev.h> +#include <vnet/ethernet/ethernet.h> + +#include <dev_armada/pp2/pp2.h> + +VNET_DEV_NODE_FN (mvpp2_tx_node) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) +{ + vnet_dev_tx_node_runtime_t *rt = vnet_dev_get_tx_node_runtime (node); + vnet_dev_tx_queue_t *txq = rt->tx_queue; + vnet_dev_port_t *port = txq->port; + vnet_dev_t *dev = port->dev; + mvpp2_txq_t *mtq = vnet_dev_get_tx_queue_data (txq); + mvpp2_port_t *mp = vnet_dev_get_port_data (port); + mvpp2_device_t *md = vnet_dev_get_data (dev); + u8 qid = txq->queue_id; + u32 *buffers = vlib_frame_vector_args (frame); + u32 n_vectors = frame->n_vectors, n_left; + u16 n_sent; + struct pp2_ppio *ppio = mp->ppio; + struct pp2_hif *hif = md->hif[vm->thread_index]; + struct pp2_ppio_desc descs[VLIB_FRAME_SIZE], *d = descs; + u16 sz = txq->size; + u16 mask = sz - 1; + + if (mtq->n_enq) + { + u16 n_done = 0; + if (PREDICT_FALSE (pp2_ppio_get_num_outq_done (ppio, hif, qid, &n_done))) + vlib_error_count (vm, node->node_index, + MVPP2_TX_NODE_CTR_PPIO_GET_NUM_OUTQ_DONE, 1); + + if (n_done) + { + vlib_buffer_free_from_ring ( + vm, mtq->buffers, (mtq->next - mtq->n_enq) & mask, sz, n_done); + mtq->n_enq -= n_done; + } + } + + n_sent = clib_min (n_vectors, sz - mtq->n_enq); + + for (d = descs, n_left = n_sent; n_left; d++, buffers++, n_left--) + { + vlib_buffer_t *b0 = vlib_get_buffer (vm, buffers[0]); + u64 paddr = vlib_buffer_get_pa (vm, b0); + + pp2_ppio_outq_desc_reset (d); + pp2_ppio_outq_desc_set_phys_addr (d, paddr + b0->current_data); + pp2_ppio_outq_desc_set_pkt_offset (d, 0); + pp2_ppio_outq_desc_set_pkt_len (d, b0->current_length); + } + + buffers = vlib_frame_vector_args (frame); + + if (pp2_ppio_send (ppio, hif, qid, descs, &n_sent)) + { + n_sent = 0; + vlib_error_count (vm, node->node_index, MVPP2_TX_NODE_CTR_PPIO_SEND, 1); + } + else if (n_sent) + { + vlib_buffer_copy_indices_to_ring (mtq->buffers, buffers, + mtq->next & mask, sz, n_sent); + mtq->next += n_sent; + mtq->n_enq += n_sent; + } + + /* free unsent buffers */ + if (PREDICT_FALSE (n_sent != n_vectors)) + { + vlib_buffer_free (vm, buffers + n_sent, n_vectors - n_sent); + vlib_error_count (vm, node->node_index, MVPP2_TX_NODE_CTR_NO_FREE_SLOTS, + n_vectors - n_sent); + } + + return n_sent; +} diff --git a/src/plugins/dev_ena/ena.c b/src/plugins/dev_ena/ena.c index ead090839c7..ed5c47ed505 100644 --- a/src/plugins/dev_ena/ena.c +++ b/src/plugins/dev_ena/ena.c @@ -4,7 +4,7 @@ #include <vnet/vnet.h> #include <vnet/dev/dev.h> -#include <vnet/dev/pci.h> +#include <vnet/dev/bus/pci.h> #include <dev_ena/ena.h> #include <dev_ena/ena_inlines.h> #include <vnet/ethernet/ethernet.h> diff --git a/src/plugins/dev_ena/port.c b/src/plugins/dev_ena/port.c index 2b26fefc5e3..95d8ff3a08c 100644 --- a/src/plugins/dev_ena/port.c +++ b/src/plugins/dev_ena/port.c @@ -4,7 +4,6 @@ #include <vnet/vnet.h> #include <vnet/dev/dev.h> -#include <vnet/dev/pci.h> #include <dev_ena/ena.h> #include <dev_ena/ena_inlines.h> #include <vnet/ethernet/ethernet.h> diff --git a/src/plugins/dev_iavf/adminq.c b/src/plugins/dev_iavf/adminq.c index c12dc8aa2f6..2072c697033 100644 --- a/src/plugins/dev_iavf/adminq.c +++ b/src/plugins/dev_iavf/adminq.c @@ -5,7 +5,7 @@ #include <ctype.h> #include <vnet/vnet.h> #include <vnet/dev/dev.h> -#include <vnet/dev/pci.h> +#include <vnet/dev/bus/pci.h> #include <vnet/dev/counters.h> #include <dev_iavf/iavf.h> #include <dev_iavf/iavf_regs.h> diff --git a/src/plugins/dev_iavf/counters.c b/src/plugins/dev_iavf/counters.c index 6dcd01141f0..3ab463edb9a 100644 --- a/src/plugins/dev_iavf/counters.c +++ b/src/plugins/dev_iavf/counters.c @@ -4,7 +4,6 @@ #include <vnet/vnet.h> #include <vnet/dev/dev.h> -#include <vnet/dev/pci.h> #include <vnet/dev/counters.h> #include <dev_iavf/iavf.h> #include <dev_iavf/virtchnl.h> diff --git a/src/plugins/dev_iavf/format.c b/src/plugins/dev_iavf/format.c index 9a3dde47ee9..b4a29e4e20a 100644 --- a/src/plugins/dev_iavf/format.c +++ b/src/plugins/dev_iavf/format.c @@ -4,7 +4,6 @@ #include <vnet/vnet.h> #include <vnet/dev/dev.h> -#include <vnet/dev/pci.h> #include <vnet/dev/counters.h> #include <dev_iavf/iavf.h> #include <dev_iavf/virtchnl.h> diff --git a/src/plugins/dev_iavf/iavf.c b/src/plugins/dev_iavf/iavf.c index d1c2b9edc63..f13440f4161 100644 --- a/src/plugins/dev_iavf/iavf.c +++ b/src/plugins/dev_iavf/iavf.c @@ -4,7 +4,7 @@ #include <vnet/vnet.h> #include <vnet/dev/dev.h> -#include <vnet/dev/pci.h> +#include <vnet/dev/bus/pci.h> #include <vnet/dev/counters.h> #include <vppinfra/ring.h> #include <dev_iavf/iavf.h> diff --git a/src/plugins/dev_iavf/port.c b/src/plugins/dev_iavf/port.c index 90e81e960c4..f1578fccb59 100644 --- a/src/plugins/dev_iavf/port.c +++ b/src/plugins/dev_iavf/port.c @@ -4,7 +4,7 @@ #include <vnet/vnet.h> #include <vnet/dev/dev.h> -#include <vnet/dev/pci.h> +#include <vnet/dev/bus/pci.h> #include <vnet/dev/counters.h> #include <dev_iavf/iavf.h> #include <dev_iavf/iavf_regs.h> @@ -91,7 +91,7 @@ iavf_port_init_rss (vlib_main_t *vm, vnet_dev_port_t *port) .key_len = keylen, }; - clib_memcpy (key->key, default_rss_key, sizeof (default_rss_key)); + clib_memcpy (key->key, default_rss_key, keylen); return iavf_vc_op_config_rss_key (vm, dev, key); } @@ -425,17 +425,20 @@ iavf_port_add_del_eth_addr (vlib_main_t *vm, vnet_dev_port_t *port, int is_primary) { iavf_port_t *ap = vnet_dev_get_port_data (port); - virtchnl_ether_addr_list_t al = { + u8 buffer[VIRTCHNL_MSG_SZ (virtchnl_ether_addr_list_t, list, 1)]; + virtchnl_ether_addr_list_t *al = (virtchnl_ether_addr_list_t *) buffer; + + *al = (virtchnl_ether_addr_list_t){ .vsi_id = ap->vsi_id, .num_elements = 1, .list[0].primary = is_primary ? 1 : 0, .list[0].extra = is_primary ? 0 : 1, }; - clib_memcpy (al.list[0].addr, addr, sizeof (al.list[0].addr)); + clib_memcpy (al->list[0].addr, addr, sizeof (al->list[0].addr)); - return is_add ? iavf_vc_op_add_eth_addr (vm, port->dev, &al) : - iavf_vc_op_del_eth_addr (vm, port->dev, &al); + return is_add ? iavf_vc_op_add_eth_addr (vm, port->dev, al) : + iavf_vc_op_del_eth_addr (vm, port->dev, al); } static vnet_dev_rv_t diff --git a/src/plugins/dev_iavf/queue.c b/src/plugins/dev_iavf/queue.c index 113c0dbdfc7..51bf69a458a 100644 --- a/src/plugins/dev_iavf/queue.c +++ b/src/plugins/dev_iavf/queue.c @@ -4,7 +4,6 @@ #include <vnet/vnet.h> #include <vnet/dev/dev.h> -#include <vnet/dev/pci.h> #include <vnet/dev/counters.h> #include <vppinfra/ring.h> #include <dev_iavf/iavf.h> diff --git a/src/plugins/dev_iavf/virtchnl.c b/src/plugins/dev_iavf/virtchnl.c index eca48106ce3..7e7715262c2 100644 --- a/src/plugins/dev_iavf/virtchnl.c +++ b/src/plugins/dev_iavf/virtchnl.c @@ -4,7 +4,6 @@ #include <vnet/vnet.h> #include <vnet/dev/dev.h> -#include <vnet/dev/pci.h> #include <vnet/dev/counters.h> #include <dev_iavf/iavf.h> #include <dev_iavf/virtchnl.h> diff --git a/src/plugins/dev_iavf/virtchnl_funcs.h b/src/plugins/dev_iavf/virtchnl_funcs.h index e7f3901e0ee..0d4ab2835f4 100644 --- a/src/plugins/dev_iavf/virtchnl_funcs.h +++ b/src/plugins/dev_iavf/virtchnl_funcs.h @@ -9,6 +9,10 @@ #include <vnet/dev/dev.h> #include <dev_iavf/iavf.h> +/* The "+ 1" fakes a trailing element, but the driver requires that. + * Using this "wrong" macro is the easiest solution, as long as + * port.c uses buffer sized by the same macro as the functions here. + */ #define VIRTCHNL_MSG_SZ(s, e, n) STRUCT_OFFSET_OF (s, e[(n) + 1]) typedef struct diff --git a/src/plugins/dev_octeon/CMakeLists.txt b/src/plugins/dev_octeon/CMakeLists.txt index e8abf1a3389..c6271ecdfba 100644 --- a/src/plugins/dev_octeon/CMakeLists.txt +++ b/src/plugins/dev_octeon/CMakeLists.txt @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright(c) 2022 Cisco Systems, Inc. -if (NOT VPP_PLATFORM_NAME STREQUAL "octeon10") +if (NOT VPP_PLATFORM_NAME STREQUAL "octeon10" AND NOT VPP_PLATFORM_NAME STREQUAL "octeon9") return() endif() @@ -21,6 +21,10 @@ endif() include_directories (${OCTEON_ROC_DIR}/) +if (VPP_PLATFORM_NAME STREQUAL "octeon9") + add_compile_definitions(PLATFORM_OCTEON9) +endif() + add_vpp_plugin(dev_octeon SOURCES init.c @@ -31,6 +35,7 @@ add_vpp_plugin(dev_octeon rx_node.c tx_node.c flow.c + counter.c MULTIARCH_SOURCES rx_node.c diff --git a/src/plugins/dev_octeon/counter.c b/src/plugins/dev_octeon/counter.c new file mode 100644 index 00000000000..6f57c1ee649 --- /dev/null +++ b/src/plugins/dev_octeon/counter.c @@ -0,0 +1,337 @@ +/* + * Copyright (c) 2024 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#include <vnet/vnet.h> +#include <vnet/dev/dev.h> +#include <vnet/dev/counters.h> +#include <dev_octeon/octeon.h> +#include <dev_octeon/common.h> + +VLIB_REGISTER_LOG_CLASS (oct_log, static) = { + .class_name = "oct", + .subclass_name = "counters", +}; + +typedef enum +{ + OCT_PORT_CTR_RX_BYTES, + OCT_PORT_CTR_TX_BYTES, + OCT_PORT_CTR_RX_PACKETS, + OCT_PORT_CTR_TX_PACKETS, + OCT_PORT_CTR_RX_DROPS, + OCT_PORT_CTR_TX_DROPS, + OCT_PORT_CTR_RX_DROP_BYTES, + OCT_PORT_CTR_RX_UCAST, + OCT_PORT_CTR_TX_UCAST, + OCT_PORT_CTR_RX_MCAST, + OCT_PORT_CTR_TX_MCAST, + OCT_PORT_CTR_RX_BCAST, + OCT_PORT_CTR_TX_BCAST, + OCT_PORT_CTR_RX_FCS, + OCT_PORT_CTR_RX_ERR, + OCT_PORT_CTR_RX_DROP_MCAST, + OCT_PORT_CTR_RX_DROP_BCAST, + OCT_PORT_CTR_RX_DROP_L3_MCAST, + OCT_PORT_CTR_RX_DROP_L3_BCAST, +} oct_port_counter_id_t; + +vnet_dev_counter_t oct_port_counters[] = { + VNET_DEV_CTR_RX_BYTES (OCT_PORT_CTR_RX_BYTES), + VNET_DEV_CTR_RX_PACKETS (OCT_PORT_CTR_RX_PACKETS), + VNET_DEV_CTR_RX_DROPS (OCT_PORT_CTR_RX_DROPS), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_DROP_BYTES, RX, BYTES, "drop bytes"), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_UCAST, RX, PACKETS, "unicast"), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_MCAST, RX, PACKETS, "multicast"), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_BCAST, RX, PACKETS, "broadcast"), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_FCS, RX, PACKETS, "fcs"), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_ERR, RX, PACKETS, "error"), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_DROP_MCAST, RX, PACKETS, + "drop multicast"), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_DROP_BCAST, RX, PACKETS, + "drop broadcast"), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_DROP_L3_MCAST, RX, PACKETS, + "drop L3 multicast"), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_DROP_L3_BCAST, RX, PACKETS, + "drop L3 broadcast"), + + VNET_DEV_CTR_TX_BYTES (OCT_PORT_CTR_TX_BYTES), + VNET_DEV_CTR_TX_PACKETS (OCT_PORT_CTR_TX_PACKETS), + VNET_DEV_CTR_TX_DROPS (OCT_PORT_CTR_TX_DROPS), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_TX_UCAST, TX, PACKETS, "unicast"), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_TX_MCAST, TX, PACKETS, "multicast"), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_TX_BCAST, TX, PACKETS, "broadcast"), +}; + +typedef enum +{ + OCT_RXQ_CTR_BYTES, + OCT_RXQ_CTR_PKTS, + OCT_RXQ_CTR_DROPS, + OCT_RXQ_CTR_DROP_BYTES, + OCT_RXQ_CTR_ERR, +} oct_rxq_counter_id_t; + +vnet_dev_counter_t oct_rxq_counters[] = { + VNET_DEV_CTR_RX_BYTES (OCT_RXQ_CTR_BYTES), + VNET_DEV_CTR_RX_PACKETS (OCT_RXQ_CTR_PKTS), + VNET_DEV_CTR_RX_DROPS (OCT_RXQ_CTR_DROPS), + VNET_DEV_CTR_VENDOR (OCT_RXQ_CTR_DROP_BYTES, RX, BYTES, "drop bytes"), + VNET_DEV_CTR_VENDOR (OCT_RXQ_CTR_ERR, RX, PACKETS, "error"), +}; + +typedef enum +{ + OCT_TXQ_CTR_BYTES, + OCT_TXQ_CTR_PKTS, + OCT_TXQ_CTR_DROPS, + OCT_TXQ_CTR_DROP_BYTES, +} oct_txq_counter_id_t; + +vnet_dev_counter_t oct_txq_counters[] = { + VNET_DEV_CTR_TX_BYTES (OCT_TXQ_CTR_BYTES), + VNET_DEV_CTR_TX_PACKETS (OCT_TXQ_CTR_PKTS), + VNET_DEV_CTR_TX_DROPS (OCT_TXQ_CTR_DROPS), + VNET_DEV_CTR_VENDOR (OCT_TXQ_CTR_DROP_BYTES, TX, BYTES, "drop bytes"), +}; + +static vnet_dev_rv_t +oct_roc_err (vnet_dev_t *dev, int rv, char *fmt, ...) +{ + u8 *s = 0; + va_list va; + + va_start (va, fmt); + s = va_format (s, fmt, &va); + va_end (va); + + log_err (dev, "%v - ROC error %s (%d)", s, roc_error_msg_get (rv), rv); + + vec_free (s); + return VNET_DEV_ERR_INTERNAL; +} + +void +oct_port_add_counters (vlib_main_t *vm, vnet_dev_port_t *port) +{ + vnet_dev_port_add_counters (vm, port, oct_port_counters, + ARRAY_LEN (oct_port_counters)); + + foreach_vnet_dev_port_rx_queue (rxq, port) + { + vnet_dev_rx_queue_add_counters (vm, rxq, oct_rxq_counters, + ARRAY_LEN (oct_rxq_counters)); + } + + foreach_vnet_dev_port_tx_queue (txq, port) + { + vnet_dev_tx_queue_add_counters (vm, txq, oct_txq_counters, + ARRAY_LEN (oct_txq_counters)); + } +} + +vnet_dev_rv_t +oct_port_get_stats (vlib_main_t *vm, vnet_dev_port_t *port) +{ + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + int rrv; + struct roc_nix_stats stats; + + if ((rrv = roc_nix_stats_get (nix, &stats))) + return oct_roc_err (dev, rrv, "roc_nix_stats_get() failed"); + + foreach_vnet_dev_counter (c, port->counter_main) + { + switch (c->user_data) + { + case OCT_PORT_CTR_RX_BYTES: + vnet_dev_counter_value_update (vm, c, stats.rx_octs); + break; + case OCT_PORT_CTR_TX_BYTES: + vnet_dev_counter_value_update (vm, c, stats.tx_octs); + break; + case OCT_PORT_CTR_RX_PACKETS: + vnet_dev_counter_value_update ( + vm, c, stats.rx_ucast + stats.rx_bcast + stats.rx_mcast); + break; + case OCT_PORT_CTR_TX_PACKETS: + vnet_dev_counter_value_update ( + vm, c, stats.tx_ucast + stats.tx_bcast + stats.tx_mcast); + break; + case OCT_PORT_CTR_RX_DROPS: + vnet_dev_counter_value_update (vm, c, stats.rx_drop); + break; + case OCT_PORT_CTR_TX_DROPS: + vnet_dev_counter_value_update (vm, c, stats.tx_drop); + break; + case OCT_PORT_CTR_RX_DROP_BYTES: + vnet_dev_counter_value_update (vm, c, stats.rx_drop_octs); + break; + case OCT_PORT_CTR_RX_UCAST: + vnet_dev_counter_value_update (vm, c, stats.rx_ucast); + break; + case OCT_PORT_CTR_TX_UCAST: + vnet_dev_counter_value_update (vm, c, stats.tx_ucast); + break; + case OCT_PORT_CTR_RX_MCAST: + vnet_dev_counter_value_update (vm, c, stats.rx_mcast); + break; + case OCT_PORT_CTR_TX_MCAST: + vnet_dev_counter_value_update (vm, c, stats.tx_mcast); + break; + case OCT_PORT_CTR_RX_BCAST: + vnet_dev_counter_value_update (vm, c, stats.rx_bcast); + break; + case OCT_PORT_CTR_TX_BCAST: + vnet_dev_counter_value_update (vm, c, stats.tx_bcast); + break; + case OCT_PORT_CTR_RX_FCS: + vnet_dev_counter_value_update (vm, c, stats.rx_fcs); + break; + case OCT_PORT_CTR_RX_ERR: + vnet_dev_counter_value_update (vm, c, stats.rx_err); + break; + case OCT_PORT_CTR_RX_DROP_MCAST: + vnet_dev_counter_value_update (vm, c, stats.rx_drop_mcast); + break; + case OCT_PORT_CTR_RX_DROP_BCAST: + vnet_dev_counter_value_update (vm, c, stats.rx_drop_bcast); + break; + case OCT_PORT_CTR_RX_DROP_L3_MCAST: + vnet_dev_counter_value_update (vm, c, stats.rx_drop_l3_mcast); + break; + case OCT_PORT_CTR_RX_DROP_L3_BCAST: + vnet_dev_counter_value_update (vm, c, stats.rx_drop_l3_bcast); + break; + default: + ASSERT (0); + } + } + + return VNET_DEV_OK; +} + +vnet_dev_rv_t +oct_rxq_get_stats (vlib_main_t *vm, vnet_dev_port_t *port, + vnet_dev_rx_queue_t *rxq) +{ + oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); + struct roc_nix_stats_queue qstats; + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + int rrv; + + if ((rrv = roc_nix_stats_queue_get (nix, crq->rq.qid, 1, &qstats))) + return oct_roc_err (dev, rrv, "roc_nix_stats_queue_get() failed"); + + foreach_vnet_dev_counter (c, rxq->counter_main) + { + switch (c->user_data) + { + case OCT_RXQ_CTR_BYTES: + vnet_dev_counter_value_update (vm, c, qstats.rx_octs); + break; + case OCT_RXQ_CTR_PKTS: + vnet_dev_counter_value_update (vm, c, qstats.rx_pkts); + break; + case OCT_RXQ_CTR_DROPS: + vnet_dev_counter_value_update (vm, c, qstats.rx_drop_pkts); + break; + case OCT_RXQ_CTR_DROP_BYTES: + vnet_dev_counter_value_update (vm, c, qstats.rx_drop_octs); + break; + case OCT_RXQ_CTR_ERR: + vnet_dev_counter_value_update (vm, c, qstats.rx_error_pkts); + break; + default: + ASSERT (0); + } + } + + return VNET_DEV_OK; +} + +vnet_dev_rv_t +oct_txq_get_stats (vlib_main_t *vm, vnet_dev_port_t *port, + vnet_dev_tx_queue_t *txq) +{ + oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + struct roc_nix_stats_queue qstats; + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + int rrv; + + if ((rrv = roc_nix_stats_queue_get (nix, ctq->sq.qid, 0, &qstats))) + return oct_roc_err (dev, rrv, "roc_nix_stats_queue_get() failed"); + + foreach_vnet_dev_counter (c, txq->counter_main) + { + switch (c->user_data) + { + case OCT_TXQ_CTR_BYTES: + vnet_dev_counter_value_update (vm, c, qstats.tx_octs); + break; + case OCT_TXQ_CTR_PKTS: + vnet_dev_counter_value_update (vm, c, qstats.tx_pkts); + break; + case OCT_TXQ_CTR_DROPS: + vnet_dev_counter_value_update (vm, c, qstats.tx_drop_pkts); + break; + case OCT_TXQ_CTR_DROP_BYTES: + vnet_dev_counter_value_update (vm, c, qstats.tx_drop_octs); + break; + default: + ASSERT (0); + } + } + + return VNET_DEV_OK; +} + +void +oct_port_clear_counters (vlib_main_t *vm, vnet_dev_port_t *port) +{ + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + int rrv; + + if ((rrv = roc_nix_stats_reset (nix))) + oct_roc_err (dev, rrv, "roc_nix_stats_reset() failed"); +} + +void +oct_rxq_clear_counters (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq) +{ + oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); + vnet_dev_t *dev = rxq->port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + int rrv; + + if ((rrv = roc_nix_stats_queue_reset (nix, crq->rq.qid, 1))) + oct_roc_err (dev, rrv, + "roc_nix_stats_queue_reset() failed for rx queue %u", + rxq->queue_id); +} + +void +oct_txq_clear_counters (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) +{ + oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + vnet_dev_t *dev = txq->port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + int rrv; + + if ((rrv = roc_nix_stats_queue_reset (nix, ctq->sq.qid, 0))) + oct_roc_err (dev, rrv, + "roc_nix_stats_queue_reset() failed for tx queue %u", + txq->queue_id); +} diff --git a/src/plugins/dev_octeon/flow.c b/src/plugins/dev_octeon/flow.c index 1c367a036ab..5bef25f5369 100644 --- a/src/plugins/dev_octeon/flow.c +++ b/src/plugins/dev_octeon/flow.c @@ -46,6 +46,8 @@ VLIB_REGISTER_LOG_CLASS (oct_log, static) = { (f->type == VNET_FLOW_TYPE_IP4_GTPC) || \ (f->type == VNET_FLOW_TYPE_IP4_GTPU)) +#define FLOW_IS_GENERIC_TYPE(f) (f->type == VNET_FLOW_TYPE_GENERIC) + #define OCT_FLOW_UNSUPPORTED_ACTIONS(f) \ ((f->actions == VNET_FLOW_ACTION_BUFFER_ADVANCE) || \ (f->actions == VNET_FLOW_ACTION_REDIRECT_TO_NODE)) @@ -71,6 +73,9 @@ VLIB_REGISTER_LOG_CLASS (oct_log, static) = { _ (62, FLOW_KEY_TYPE_L3_DST, "l3-dst-only") \ _ (63, FLOW_KEY_TYPE_L3_SRC, "l3-src-only") +#define GTPU_PORT 2152 +#define VXLAN_PORT 4789 + typedef struct { u16 src_port; @@ -87,6 +92,27 @@ typedef struct u32 teid; } gtpu_header_t; +typedef struct +{ + u8 layer; + u16 nxt_proto; + vnet_dev_port_t *port; + struct roc_npc_item_info *items; + struct + { + u8 *spec; + u8 *mask; + u16 off; + } oct_drv; + struct + { + u8 *spec; + u8 *mask; + u16 off; + u16 len; + } generic; +} oct_flow_parse_state; + static void oct_flow_convert_rss_types (u64 *key, u64 rss_types) { @@ -163,6 +189,14 @@ oct_flow_rule_create (vnet_dev_port_t *port, struct roc_npc_action *actions, npc = &oct_port->npc; + for (int i = 0; item_info[i].type != ROC_NPC_ITEM_TYPE_END; i++) + { + log_debug (port->dev, "Flow[%d] Item[%d] type %d spec 0x%U mask 0x%U", + flow->index, i, item_info[i].type, format_hex_bytes, + item_info[i].spec, item_info[i].size, format_hex_bytes, + item_info[i].mask, item_info[i].size); + } + npc_flow = roc_npc_flow_create (npc, &attr, item_info, actions, npc->pf_func, &rv); if (rv) @@ -183,6 +217,320 @@ oct_flow_rule_create (vnet_dev_port_t *port, struct roc_npc_action *actions, return VNET_DEV_OK; } +static int +oct_parse_l2 (oct_flow_parse_state *pst) +{ + struct roc_npc_flow_item_eth *eth_spec = + (struct roc_npc_flow_item_eth *) &pst->oct_drv.spec[pst->oct_drv.off]; + struct roc_npc_flow_item_eth *eth_mask = + (struct roc_npc_flow_item_eth *) &pst->oct_drv.mask[pst->oct_drv.off]; + ethernet_header_t *eth_hdr_mask = + (ethernet_header_t *) &pst->generic.mask[pst->generic.off]; + ethernet_header_t *eth_hdr = + (ethernet_header_t *) &pst->generic.spec[pst->generic.off]; + u16 tpid, etype; + + tpid = etype = clib_net_to_host_u16 (eth_hdr->type); + clib_memcpy_fast (eth_spec, eth_hdr, sizeof (ethernet_header_t)); + clib_memcpy_fast (eth_mask, eth_hdr_mask, sizeof (ethernet_header_t)); + eth_spec->has_vlan = 0; + + pst->items[pst->layer].spec = (void *) eth_spec; + pst->items[pst->layer].mask = (void *) eth_mask; + pst->items[pst->layer].size = sizeof (ethernet_header_t); + pst->items[pst->layer].type = ROC_NPC_ITEM_TYPE_ETH; + pst->generic.off += sizeof (ethernet_header_t); + pst->oct_drv.off += sizeof (struct roc_npc_flow_item_eth); + pst->layer++; + + /* Parse VLAN Tags if any */ + struct roc_npc_flow_item_vlan *vlan_spec = + (struct roc_npc_flow_item_vlan *) &pst->oct_drv.spec[pst->oct_drv.off]; + struct roc_npc_flow_item_vlan *vlan_mask = + (struct roc_npc_flow_item_vlan *) &pst->oct_drv.mask[pst->oct_drv.off]; + ethernet_vlan_header_t *vlan_hdr, *vlan_hdr_mask; + u8 vlan_cnt = 0; + + while (tpid == ETHERNET_TYPE_DOT1AD || tpid == ETHERNET_TYPE_VLAN) + { + if (pst->generic.off >= pst->generic.len) + break; + + vlan_hdr = + (ethernet_vlan_header_t *) &pst->generic.spec[pst->generic.off]; + vlan_hdr_mask = + (ethernet_vlan_header_t *) &pst->generic.mask[pst->generic.off]; + tpid = etype = clib_net_to_host_u16 (vlan_hdr->type); + clib_memcpy (&vlan_spec[vlan_cnt], vlan_hdr, + sizeof (ethernet_vlan_header_t)); + clib_memcpy (&vlan_mask[vlan_cnt], vlan_hdr_mask, + sizeof (ethernet_vlan_header_t)); + pst->items[pst->layer].spec = (void *) &vlan_spec[vlan_cnt]; + pst->items[pst->layer].mask = (void *) &vlan_mask[vlan_cnt]; + pst->items[pst->layer].size = sizeof (ethernet_vlan_header_t); + pst->items[pst->layer].type = ROC_NPC_ITEM_TYPE_VLAN; + pst->generic.off += sizeof (ethernet_vlan_header_t); + pst->oct_drv.off += sizeof (struct roc_npc_flow_item_vlan); + pst->layer++; + vlan_cnt++; + } + + /* Inner most vlan tag */ + if (vlan_cnt) + vlan_spec[vlan_cnt - 1].has_more_vlan = 0; + + pst->nxt_proto = etype; + return 0; +} + +static int +oct_parse_l3 (oct_flow_parse_state *pst) +{ + + if (pst->generic.off >= pst->generic.len || pst->nxt_proto == 0) + return 0; + + if (pst->nxt_proto == ETHERNET_TYPE_MPLS) + { + int label_stack_bottom = 0; + do + { + + u8 *mpls_spec = &pst->generic.spec[pst->generic.off]; + u8 *mpls_mask = &pst->generic.mask[pst->generic.off]; + + label_stack_bottom = mpls_spec[2] & 1; + pst->items[pst->layer].spec = (void *) mpls_spec; + pst->items[pst->layer].mask = (void *) mpls_mask; + pst->items[pst->layer].size = sizeof (u32); + pst->items[pst->layer].type = ROC_NPC_ITEM_TYPE_MPLS; + pst->generic.off += sizeof (u32); + pst->layer++; + } + while (label_stack_bottom); + + pst->nxt_proto = 0; + return 0; + } + else if (pst->nxt_proto == ETHERNET_TYPE_IP4) + { + ip4_header_t *ip4_spec = + (ip4_header_t *) &pst->generic.spec[pst->generic.off]; + ip4_header_t *ip4_mask = + (ip4_header_t *) &pst->generic.mask[pst->generic.off]; + pst->items[pst->layer].spec = (void *) ip4_spec; + pst->items[pst->layer].mask = (void *) ip4_mask; + pst->items[pst->layer].size = sizeof (ip4_header_t); + pst->items[pst->layer].type = ROC_NPC_ITEM_TYPE_IPV4; + pst->generic.off += sizeof (ip4_header_t); + pst->layer++; + pst->nxt_proto = ip4_spec->protocol; + } + else if (pst->nxt_proto == ETHERNET_TYPE_IP6) + { + struct roc_npc_flow_item_ipv6 *ip6_spec = + (struct roc_npc_flow_item_ipv6 *) &pst->oct_drv.spec[pst->oct_drv.off]; + struct roc_npc_flow_item_ipv6 *ip6_mask = + (struct roc_npc_flow_item_ipv6 *) &pst->oct_drv.mask[pst->oct_drv.off]; + ip6_header_t *ip6_hdr_mask = + (ip6_header_t *) &pst->generic.mask[pst->generic.off]; + ip6_header_t *ip6_hdr = + (ip6_header_t *) &pst->generic.spec[pst->generic.off]; + u8 nxt_hdr = ip6_hdr->protocol; + + clib_memcpy (ip6_spec, ip6_hdr, sizeof (ip6_header_t)); + clib_memcpy (ip6_mask, ip6_hdr_mask, sizeof (ip6_header_t)); + pst->items[pst->layer].spec = (void *) ip6_spec; + pst->items[pst->layer].mask = (void *) ip6_mask; + pst->items[pst->layer].size = sizeof (ip6_header_t); + pst->items[pst->layer].type = ROC_NPC_ITEM_TYPE_IPV6; + pst->generic.off += sizeof (ip6_header_t); + pst->oct_drv.off += sizeof (struct roc_npc_flow_item_ipv6); + pst->layer++; + + while (nxt_hdr == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS || + nxt_hdr == IP_PROTOCOL_IP6_DESTINATION_OPTIONS || + nxt_hdr == IP_PROTOCOL_IPV6_ROUTE) + { + if (pst->generic.off >= pst->generic.len) + return 0; + + ip6_ext_header_t *ip6_ext_spec = + (ip6_ext_header_t *) &pst->generic.spec[pst->generic.off]; + ip6_ext_header_t *ip6_ext_mask = + (ip6_ext_header_t *) &pst->generic.mask[pst->generic.off]; + nxt_hdr = ip6_ext_spec->next_hdr; + + pst->items[pst->layer].spec = (void *) ip6_ext_spec; + pst->items[pst->layer].mask = (void *) ip6_ext_mask; + pst->items[pst->layer].type = ROC_NPC_ITEM_TYPE_IPV6_EXT; + pst->generic.off += ip6_ext_header_len (ip6_ext_spec); + pst->layer++; + } + + if (pst->generic.off >= pst->generic.len) + return 0; + + if (nxt_hdr == IP_PROTOCOL_IPV6_FRAGMENTATION) + { + ip6_frag_hdr_t *ip6_ext_frag_spec = + (ip6_frag_hdr_t *) &pst->generic.spec[pst->generic.off]; + ip6_frag_hdr_t *ip6_ext_frag_mask = + (ip6_frag_hdr_t *) &pst->generic.mask[pst->generic.off]; + + pst->items[pst->layer].spec = (void *) ip6_ext_frag_spec; + pst->items[pst->layer].mask = (void *) ip6_ext_frag_mask; + pst->items[pst->layer].size = sizeof (ip6_frag_hdr_t); + pst->items[pst->layer].type = ROC_NPC_ITEM_TYPE_IPV6_FRAG_EXT; + pst->generic.off += sizeof (ip6_frag_hdr_t); + pst->layer++; + } + + pst->nxt_proto = nxt_hdr; + } + /* Unsupported L3. */ + else + return -1; + + return 0; +} + +static int +oct_parse_l4 (oct_flow_parse_state *pst) +{ + + if (pst->generic.off >= pst->generic.len || pst->nxt_proto == 0) + return 0; + +#define _(protocol_t, protocol_value, ltype) \ + if (pst->nxt_proto == protocol_value) \ + \ + { \ + \ + protocol_t *spec = (protocol_t *) &pst->generic.spec[pst->generic.off]; \ + protocol_t *mask = (protocol_t *) &pst->generic.mask[pst->generic.off]; \ + pst->items[pst->layer].spec = spec; \ + pst->items[pst->layer].mask = mask; \ + \ + pst->items[pst->layer].size = sizeof (protocol_t); \ + \ + pst->items[pst->layer].type = ROC_NPC_ITEM_TYPE_##ltype; \ + pst->generic.off += sizeof (protocol_t); \ + pst->layer++; \ + return 0; \ + } + + _ (esp_header_t, IP_PROTOCOL_IPSEC_ESP, ESP) + _ (udp_header_t, IP_PROTOCOL_UDP, UDP) + _ (tcp_header_t, IP_PROTOCOL_TCP, TCP) + _ (sctp_header_t, IP_PROTOCOL_SCTP, SCTP) + _ (icmp46_header_t, IP_PROTOCOL_ICMP, ICMP) + _ (icmp46_header_t, IP_PROTOCOL_ICMP6, ICMP) + _ (igmp_header_t, IP_PROTOCOL_IGMP, IGMP) + _ (gre_header_t, IP_PROTOCOL_GRE, GRE) + + /* Unsupported L4. */ + return -1; +} + +static int +oct_parse_tunnel (oct_flow_parse_state *pst) +{ + if (pst->generic.off >= pst->generic.len) + return 0; + + if (pst->items[pst->layer - 1].type == ROC_NPC_ITEM_TYPE_GRE) + { + gre_header_t *gre_hdr = (gre_header_t *) pst->items[pst->layer - 1].spec; + pst->nxt_proto = clib_net_to_host_u16 (gre_hdr->protocol); + goto parse_l3; + } + + else if (pst->items[pst->layer - 1].type == ROC_NPC_ITEM_TYPE_UDP) + { + udp_header_t *udp_h = (udp_header_t *) pst->items[pst->layer - 1].spec; + u16 dport = clib_net_to_host_u16 (udp_h->dst_port); + + if (dport == GTPU_PORT) + { + gtpu_header_t *gtpu_spec = + (gtpu_header_t *) &pst->generic.spec[pst->generic.off]; + gtpu_header_t *gtpu_mask = + (gtpu_header_t *) &pst->generic.mask[pst->generic.off]; + pst->items[pst->layer].spec = (void *) gtpu_spec; + pst->items[pst->layer].mask = (void *) gtpu_mask; + pst->items[pst->layer].size = sizeof (gtpu_header_t); + pst->items[pst->layer].type = ROC_NPC_ITEM_TYPE_GTPU; + pst->generic.off += sizeof (gtpu_header_t); + pst->layer++; + pst->nxt_proto = 0; + return 0; + } + else if (dport == VXLAN_PORT) + { + vxlan_header_t *vxlan_spec = + (vxlan_header_t *) &pst->generic.spec[pst->generic.off]; + vxlan_header_t *vxlan_mask = + (vxlan_header_t *) &pst->generic.spec[pst->generic.off]; + pst->items[pst->layer].spec = (void *) vxlan_spec; + pst->items[pst->layer].mask = (void *) vxlan_mask; + pst->items[pst->layer].size = sizeof (vxlan_header_t); + pst->items[pst->layer].type = ROC_NPC_ITEM_TYPE_VXLAN; + pst->generic.off += sizeof (vxlan_header_t); + pst->layer++; + pst->nxt_proto = 0; + goto parse_l2; + } + } + /* No supported Tunnel detected. */ + else + { + log_err (pst->port->dev, + "Partially parsed till offset %u, not able to parse further", + pst->generic.off); + return 0; + } +parse_l2: + if (oct_parse_l2 (pst)) + return -1; +parse_l3: + if (oct_parse_l3 (pst)) + return -1; + + return oct_parse_l4 (pst); +} + +static vnet_dev_rv_t +oct_flow_generic_pattern_parse (oct_flow_parse_state *pst) +{ + + if (oct_parse_l2 (pst)) + goto err; + + if (oct_parse_l3 (pst)) + goto err; + + if (oct_parse_l4 (pst)) + goto err; + + if (oct_parse_tunnel (pst)) + goto err; + + if (pst->generic.off < pst->generic.len) + { + log_err (pst->port->dev, + "Partially parsed till offset %u, not able to parse further", + pst->generic.off); + goto err; + } + + pst->items[pst->layer].type = ROC_NPC_ITEM_TYPE_END; + return VNET_DEV_OK; + +err: + return VNET_DEV_ERR_NOT_SUPPORTED; +} + static vnet_dev_rv_t oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, uword *private_data) @@ -190,12 +538,22 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, struct roc_npc_item_info item_info[ROC_NPC_ITEM_TYPE_END] = {}; struct roc_npc_action actions[ROC_NPC_ITEM_TYPE_END] = {}; oct_port_t *oct_port = vnet_dev_get_port_data (port); + ethernet_header_t eth_spec = {}, eth_mask = {}; + sctp_header_t sctp_spec = {}, sctp_mask = {}; + gtpu_header_t gtpu_spec = {}, gtpu_mask = {}; + ip4_header_t ip4_spec = {}, ip4_mask = {}; + ip6_header_t ip6_spec = {}, ip6_mask = {}; + udp_header_t udp_spec = {}, udp_mask = {}; + tcp_header_t tcp_spec = {}, tcp_mask = {}; + esp_header_t esp_spec = {}, esp_mask = {}; u16 l4_src_port = 0, l4_dst_port = 0; u16 l4_src_mask = 0, l4_dst_mask = 0; struct roc_npc_action_rss rss_conf = {}; struct roc_npc_action_queue conf = {}; struct roc_npc_action_mark mark = {}; struct roc_npc *npc = &oct_port->npc; + u8 *flow_spec = 0, *flow_mask = 0; + u8 *drv_spec = 0, *drv_mask = 0; vnet_dev_rv_t rv = VNET_DEV_OK; int layer = 0, index = 0; u16 *queues = NULL; @@ -203,11 +561,52 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, u8 proto = 0; u16 action = 0; + if (FLOW_IS_GENERIC_TYPE (flow)) + { + unformat_input_t input; + int rc; + + unformat_init_string ( + &input, (const char *) flow->generic.pattern.spec, + strlen ((const char *) flow->generic.pattern.spec)); + unformat_user (&input, unformat_hex_string, &flow_spec); + unformat_free (&input); + + unformat_init_string ( + &input, (const char *) flow->generic.pattern.mask, + strlen ((const char *) flow->generic.pattern.mask)); + unformat_user (&input, unformat_hex_string, &flow_mask); + unformat_free (&input); + + vec_validate (drv_spec, 1024); + vec_validate (drv_mask, 1024); + oct_flow_parse_state pst = { + .nxt_proto = 0, + .port = port, + .items = item_info, + .oct_drv = { .spec = drv_spec, .mask = drv_mask }, + .generic = { .spec = flow_spec, + .mask = flow_mask, + .len = vec_len (flow_spec) }, + }; + + rc = oct_flow_generic_pattern_parse (&pst); + if (rc) + { + vec_free (flow_spec); + vec_free (flow_mask); + vec_free (drv_spec); + vec_free (drv_mask); + return VNET_DEV_ERR_NOT_SUPPORTED; + } + + goto parse_flow_actions; + } + if (FLOW_IS_ETHERNET_CLASS (flow)) { - ethernet_header_t eth_spec = { .type = clib_host_to_net_u16 ( - flow->ethernet.eth_hdr.type) }, - eth_mask = { .type = 0xFFFF }; + eth_spec.type = clib_host_to_net_u16 (flow->ethernet.eth_hdr.type); + eth_mask.type = 0xFFFF; item_info[layer].spec = (void *) ð_spec; item_info[layer].mask = (void *) ð_mask; @@ -220,10 +619,11 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, { vnet_flow_ip4_t *ip4_hdr = &flow->ip4; proto = ip4_hdr->protocol.prot; - ip4_header_t ip4_spec = { .src_address = ip4_hdr->src_addr.addr, - .dst_address = ip4_hdr->dst_addr.addr }, - ip4_mask = { .src_address = ip4_hdr->src_addr.mask, - .dst_address = ip4_hdr->dst_addr.mask }; + + ip4_spec.src_address = ip4_hdr->src_addr.addr; + ip4_spec.dst_address = ip4_hdr->dst_addr.addr; + ip4_mask.src_address = ip4_hdr->src_addr.mask; + ip4_mask.dst_address = ip4_hdr->dst_addr.mask; item_info[layer].spec = (void *) &ip4_spec; item_info[layer].mask = (void *) &ip4_mask; @@ -245,10 +645,11 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, { vnet_flow_ip6_t *ip6_hdr = &flow->ip6; proto = ip6_hdr->protocol.prot; - ip6_header_t ip6_spec = { .src_address = ip6_hdr->src_addr.addr, - .dst_address = ip6_hdr->dst_addr.addr }, - ip6_mask = { .src_address = ip6_hdr->src_addr.mask, - .dst_address = ip6_hdr->dst_addr.mask }; + + ip6_spec.src_address = ip6_hdr->src_addr.addr; + ip6_spec.dst_address = ip6_hdr->dst_addr.addr; + ip6_mask.src_address = ip6_hdr->src_addr.mask; + ip6_mask.dst_address = ip6_hdr->dst_addr.mask; item_info[layer].spec = (void *) &ip6_spec; item_info[layer].mask = (void *) &ip6_mask; @@ -273,16 +674,15 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, switch (proto) { case IP_PROTOCOL_UDP: - item_info[layer].type = ROC_NPC_ITEM_TYPE_UDP; - - udp_header_t udp_spec = { .src_port = l4_src_port, - .dst_port = l4_dst_port }, - udp_mask = { .src_port = l4_src_mask, - .dst_port = l4_dst_mask }; + udp_spec.src_port = l4_src_port; + udp_spec.dst_port = l4_dst_port; + udp_mask.src_port = l4_src_mask; + udp_mask.dst_port = l4_dst_mask; item_info[layer].spec = (void *) &udp_spec; item_info[layer].mask = (void *) &udp_mask; item_info[layer].size = sizeof (udp_header_t); + item_info[layer].type = ROC_NPC_ITEM_TYPE_UDP; layer++; if (FLOW_IS_L4_TUNNEL_TYPE (flow)) @@ -290,14 +690,13 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, switch (flow->type) { case VNET_FLOW_TYPE_IP4_GTPU: - item_info[layer].type = ROC_NPC_ITEM_TYPE_GTPU; - gtpu_header_t gtpu_spec = { .teid = clib_host_to_net_u32 ( - flow->ip4_gtpu.teid) }, - gtpu_mask = { .teid = 0XFFFFFFFF }; + gtpu_spec.teid = clib_host_to_net_u32 (flow->ip4_gtpu.teid); + gtpu_mask.teid = 0XFFFFFFFF; item_info[layer].spec = (void *) >pu_spec; item_info[layer].mask = (void *) >pu_mask; item_info[layer].size = sizeof (gtpu_header_t); + item_info[layer].type = ROC_NPC_ITEM_TYPE_GTPU; layer++; break; @@ -309,42 +708,39 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, break; case IP_PROTOCOL_TCP: - item_info[layer].type = ROC_NPC_ITEM_TYPE_TCP; - - tcp_header_t tcp_spec = { .src_port = l4_src_port, - .dst_port = l4_dst_port }, - tcp_mask = { .src_port = l4_src_mask, - .dst_port = l4_dst_mask }; + tcp_spec.src_port = l4_src_port; + tcp_spec.dst_port = l4_dst_port; + tcp_mask.src_port = l4_src_mask; + tcp_mask.dst_port = l4_dst_mask; item_info[layer].spec = (void *) &tcp_spec; item_info[layer].mask = (void *) &tcp_mask; item_info[layer].size = sizeof (tcp_header_t); + item_info[layer].type = ROC_NPC_ITEM_TYPE_TCP; layer++; break; case IP_PROTOCOL_SCTP: - item_info[layer].type = ROC_NPC_ITEM_TYPE_SCTP; - - sctp_header_t sctp_spec = { .src_port = l4_src_port, - .dst_port = l4_dst_port }, - sctp_mask = { .src_port = l4_src_mask, - .dst_port = l4_dst_mask }; + sctp_spec.src_port = l4_src_port; + sctp_spec.dst_port = l4_dst_port; + sctp_mask.src_port = l4_src_mask; + sctp_mask.dst_port = l4_dst_mask; item_info[layer].spec = (void *) &sctp_spec; item_info[layer].mask = (void *) &sctp_mask; item_info[layer].size = sizeof (sctp_header_t); + item_info[layer].type = ROC_NPC_ITEM_TYPE_SCTP; layer++; break; case IP_PROTOCOL_IPSEC_ESP: - item_info[layer].type = ROC_NPC_ITEM_TYPE_ESP; - esp_header_t esp_spec = { .spi = clib_host_to_net_u32 ( - flow->ip4_ipsec_esp.spi) }, - esp_mask = { .spi = 0xFFFFFFFF }; + esp_spec.spi = clib_host_to_net_u32 (flow->ip4_ipsec_esp.spi); + esp_mask.spi = 0xFFFFFFFF; item_info[layer].spec = (void *) &esp_spec; item_info[layer].mask = (void *) &esp_mask; item_info[layer].size = sizeof (u32); + item_info[layer].type = ROC_NPC_ITEM_TYPE_ESP; layer++; break; @@ -357,6 +753,7 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, end_item_info: item_info[layer].type = ROC_NPC_ITEM_TYPE_END; +parse_flow_actions: if (flow->actions & VNET_FLOW_ACTION_REDIRECT_TO_QUEUE) { conf.index = flow->redirect_queue; @@ -422,6 +819,11 @@ end_item_info: if (queues) clib_mem_free (queues); + vec_free (flow_spec); + vec_free (flow_mask); + vec_free (drv_spec); + vec_free (drv_mask); + return rv; } diff --git a/src/plugins/dev_octeon/format.c b/src/plugins/dev_octeon/format.c index e624b84f54e..d0f53013d99 100644 --- a/src/plugins/dev_octeon/format.c +++ b/src/plugins/dev_octeon/format.c @@ -25,7 +25,7 @@ format_oct_nix_rx_cqe_desc (u8 *s, va_list *args) typeof (d->sg0) *sg0 = &d->sg0; typeof (d->sg0) *sg1 = &d->sg1; - s = format (s, "hdr: cqe_type %u nude %u q %u tag 0x%x", h->cqe_type, + s = format (s, "hdr: cqe_type %u nude %u qid %u tag 0x%x", h->cqe_type, h->node, h->q, h->tag); s = format (s, "\n%Uparse:", format_white_space, indent); #define _(n, f) s = format (s, " " #n " " f, p->n) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 97a11e0d0d7..2f0c82c1c01 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -4,7 +4,7 @@ #include <vnet/vnet.h> #include <vnet/dev/dev.h> -#include <vnet/dev/pci.h> +#include <vnet/dev/bus/pci.h> #include <vnet/dev/counters.h> #include <vnet/ethernet/ethernet.h> #include <vnet/plugin/plugin.h> @@ -110,6 +110,7 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) .reta_sz = ROC_NIX_RSS_RETA_SZ_256, .max_sqb_count = 512, .pci_dev = &cd->plt_pci_dev, + .hw_vlan_ins = true, }; if ((rrv = roc_nix_dev_init (cd->nix))) @@ -131,6 +132,9 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) .rx_offloads = { .ip4_cksum = 1, }, + .tx_offloads = { + .ip4_cksum = 1, + }, }, .ops = { .init = oct_port_init, @@ -141,6 +145,7 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) .config_change_validate = oct_port_cfg_change_validate, .format_status = format_oct_port_status, .format_flow = format_oct_port_flow, + .clear_counters = oct_port_clear_counters, }, .data_size = sizeof (oct_port_t), .initial_data = &oct_port, @@ -159,6 +164,7 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) .alloc = oct_rx_queue_alloc, .free = oct_rx_queue_free, .format_info = format_oct_rxq_info, + .clear_counters = oct_rxq_clear_counters, }, }, .tx_queue = { @@ -173,6 +179,7 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) .alloc = oct_tx_queue_alloc, .free = oct_tx_queue_free, .format_info = format_oct_txq_info, + .clear_counters = oct_txq_clear_counters, }, }, }; @@ -245,6 +252,7 @@ oct_init (vlib_main_t *vm, vnet_dev_t *dev) { case OCT_DEVICE_TYPE_RVU_PF: case OCT_DEVICE_TYPE_RVU_VF: + case OCT_DEVICE_TYPE_LBK_VF: case OCT_DEVICE_TYPE_SDP_VF: return oct_init_nix (vm, dev); diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index e43cde0a35f..a87a5e3e1ed 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -12,6 +12,12 @@ #include <vnet/flow/flow.h> #include <vnet/udp/udp.h> #include <vnet/ipsec/esp.h> +#include <vnet/ethernet/packet.h> +#include <vnet/ip/ip_packet.h> +#include <vnet/ip/icmp46_packet.h> +#include <vnet/ip/igmp_packet.h> +#include <vnet/gre/packet.h> +#include <vxlan/vxlan.h> #include <base/roc_api.h> #include <dev_octeon/hw_defs.h> @@ -141,6 +147,17 @@ vnet_dev_rv_t oct_flow_validate_params (vlib_main_t *, vnet_dev_port_t *, vnet_dev_rv_t oct_flow_query (vlib_main_t *, vnet_dev_port_t *, u32, uword, u64 *); +/* counter.c */ +void oct_port_add_counters (vlib_main_t *, vnet_dev_port_t *); +void oct_port_clear_counters (vlib_main_t *, vnet_dev_port_t *); +void oct_rxq_clear_counters (vlib_main_t *, vnet_dev_rx_queue_t *); +void oct_txq_clear_counters (vlib_main_t *, vnet_dev_tx_queue_t *); +vnet_dev_rv_t oct_port_get_stats (vlib_main_t *, vnet_dev_port_t *); +vnet_dev_rv_t oct_rxq_get_stats (vlib_main_t *, vnet_dev_port_t *, + vnet_dev_rx_queue_t *); +vnet_dev_rv_t oct_txq_get_stats (vlib_main_t *, vnet_dev_port_t *, + vnet_dev_tx_queue_t *); + #define log_debug(dev, f, ...) \ vlib_log (VLIB_LOG_LEVEL_DEBUG, oct_log.class, "%U: " f, \ format_vnet_dev_addr, (dev), ##__VA_ARGS__) diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 98a4c28b37d..528683fa3c7 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -4,7 +4,6 @@ #include <vnet/vnet.h> #include <vnet/dev/dev.h> -#include <vnet/dev/pci.h> #include <vnet/dev/counters.h> #include <dev_octeon/octeon.h> #include <dev_octeon/common.h> @@ -54,11 +53,83 @@ oct_roc_err (vnet_dev_t *dev, int rv, char *fmt, ...) } vnet_dev_rv_t +oct_port_pause_flow_control_init (vlib_main_t *vm, vnet_dev_port_t *port) +{ + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + struct roc_nix_fc_cfg fc_cfg; + struct roc_nix_sq *sq; + struct roc_nix_cq *cq; + struct roc_nix_rq *rq; + int rrv; + + /* pause flow control is not supported on SDP/LBK devices */ + if (roc_nix_is_sdp (nix) || roc_nix_is_lbk (nix)) + { + log_notice (dev, + "pause flow control is not supported on SDP/LBK devices"); + return VNET_DEV_OK; + } + + fc_cfg.type = ROC_NIX_FC_RXCHAN_CFG; + fc_cfg.rxchan_cfg.enable = true; + rrv = roc_nix_fc_config_set (nix, &fc_cfg); + if (rrv) + return oct_roc_err (dev, rrv, "roc_nix_fc_config_set failed"); + + memset (&fc_cfg, 0, sizeof (struct roc_nix_fc_cfg)); + fc_cfg.type = ROC_NIX_FC_RQ_CFG; + fc_cfg.rq_cfg.enable = true; + fc_cfg.rq_cfg.tc = 0; + + foreach_vnet_dev_port_rx_queue (rxq, port) + { + oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); + + rq = &crq->rq; + cq = &crq->cq; + + fc_cfg.rq_cfg.rq = rq->qid; + fc_cfg.rq_cfg.cq_drop = cq->drop_thresh; + + rrv = roc_nix_fc_config_set (nix, &fc_cfg); + if (rrv) + return oct_roc_err (dev, rrv, "roc_nix_fc_config_set failed"); + } + + memset (&fc_cfg, 0, sizeof (struct roc_nix_fc_cfg)); + fc_cfg.type = ROC_NIX_FC_TM_CFG; + fc_cfg.tm_cfg.tc = 0; + fc_cfg.tm_cfg.enable = true; + + foreach_vnet_dev_port_tx_queue (txq, port) + { + oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + + sq = &ctq->sq; + + fc_cfg.tm_cfg.sq = sq->qid; + rrv = roc_nix_fc_config_set (nix, &fc_cfg); + if (rrv) + return oct_roc_err (dev, rrv, "roc_nix_fc_config_set failed"); + } + + /* By default, enable pause flow control */ + rrv = roc_nix_fc_mode_set (nix, ROC_NIX_FC_FULL); + if (rrv) + return oct_roc_err (dev, rrv, "roc_nix_fc_mode_set failed"); + + return VNET_DEV_OK; +} + +vnet_dev_rv_t oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) { vnet_dev_t *dev = port->dev; oct_device_t *cd = vnet_dev_get_data (dev); oct_port_t *cp = vnet_dev_get_port_data (port); + u8 mac_addr[PLT_ETHER_ADDR_LEN]; struct roc_nix *nix = cd->nix; vnet_dev_rv_t rv; int rrv; @@ -76,6 +147,22 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) } cp->lf_allocated = 1; + if (!roc_nix_is_vf_or_sdp (nix)) + { + if ((rrv = roc_nix_npc_mac_addr_get (nix, mac_addr))) + { + oct_port_deinit (vm, port); + return oct_roc_err (dev, rrv, "roc_nix_npc_mac_addr_get failed"); + } + + /* Sync MAC address to CGX/RPM table */ + if ((rrv = roc_nix_mac_addr_set (nix, mac_addr))) + { + oct_port_deinit (vm, port); + return oct_roc_err (dev, rrv, "roc_nix_mac_addr_set failed"); + } + } + if ((rrv = roc_nix_tm_init (nix))) { oct_port_deinit (vm, port); @@ -124,6 +211,21 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) return rv; } + oct_port_add_counters (vm, port); + + if ((rrv = roc_nix_mac_mtu_set (nix, port->max_rx_frame_size))) + { + rv = oct_roc_err (dev, rrv, "roc_nix_mac_mtu_set() failed"); + return rv; + } + + /* Configure pause frame flow control*/ + if ((rv = oct_port_pause_flow_control_init (vm, port))) + { + oct_port_deinit (vm, port); + return rv; + } + return VNET_DEV_OK; } @@ -172,7 +274,22 @@ oct_port_poll (vlib_main_t *vm, vnet_dev_port_t *port) vnet_dev_port_state_changes_t changes = {}; int rrv; - if (roc_nix_is_lbk (nix)) + if (oct_port_get_stats (vm, port)) + return; + + foreach_vnet_dev_port_rx_queue (q, port) + { + if (oct_rxq_get_stats (vm, port, q)) + return; + } + + foreach_vnet_dev_port_tx_queue (q, port) + { + if (oct_txq_get_stats (vm, port, q)) + return; + } + + if (roc_nix_is_lbk (nix) || roc_nix_is_sdp (nix)) { link_info.status = 1; link_info.full_duplex = 1; @@ -203,7 +320,8 @@ oct_port_poll (vlib_main_t *vm, vnet_dev_port_t *port) if (cd->speed != link_info.speed) { changes.change.link_speed = 1; - changes.link_speed = link_info.speed; + /* Convert to Kbps */ + changes.link_speed = link_info.speed * 1000; cd->speed = link_info.speed; } @@ -327,12 +445,6 @@ oct_port_start (vlib_main_t *vm, vnet_dev_port_t *port) ctq->n_enq = 0; } - if ((rrv = roc_nix_mac_mtu_set (nix, 9200))) - { - rv = oct_roc_err (dev, rrv, "roc_nix_mac_mtu_set() failed"); - goto done; - } - if ((rrv = roc_nix_npc_rx_ena_dis (nix, true))) { rv = oct_roc_err (dev, rrv, "roc_nix_npc_rx_ena_dis() failed"); @@ -376,6 +488,18 @@ oct_port_stop (vlib_main_t *vm, vnet_dev_port_t *port) foreach_vnet_dev_port_tx_queue (q, port) oct_txq_stop (vm, q); + + vnet_dev_port_state_change (vm, port, + (vnet_dev_port_state_changes_t){ + .change.link_state = 1, + .change.link_speed = 1, + .link_speed = 0, + .link_state = 0, + }); + + /* Update the device status */ + cd->status = 0; + cd->speed = 0; } vnet_dev_rv_t @@ -385,7 +509,7 @@ oct_validate_config_promisc_mode (vnet_dev_port_t *port, int enable) oct_device_t *cd = vnet_dev_get_data (dev); struct roc_nix *nix = cd->nix; - if (roc_nix_is_vf_or_sdp (nix)) + if (roc_nix_is_sdp (nix) || roc_nix_is_lbk (nix)) return VNET_DEV_ERR_UNSUPPORTED_DEVICE; return VNET_DEV_OK; @@ -405,6 +529,9 @@ oct_op_config_promisc_mode (vlib_main_t *vm, vnet_dev_port_t *port, int enable) return oct_roc_err (dev, rv, "roc_nix_npc_promisc_ena_dis failed"); } + if (!roc_nix_is_pf (nix)) + return VNET_DEV_OK; + rv = roc_nix_mac_promisc_mode_enable (nix, enable); if (rv) { @@ -425,7 +552,6 @@ oct_port_add_del_eth_addr (vlib_main_t *vm, vnet_dev_port_t *port, oct_device_t *cd = vnet_dev_get_data (dev); struct roc_nix *nix = cd->nix; vnet_dev_rv_t rv = VNET_DEV_OK; - i32 rrv; if (is_primary) @@ -451,6 +577,24 @@ oct_port_add_del_eth_addr (vlib_main_t *vm, vnet_dev_port_t *port, } } } + + return rv; +} + +vnet_dev_rv_t +oct_op_config_max_rx_len (vlib_main_t *vm, vnet_dev_port_t *port, + u32 rx_frame_size) +{ + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + vnet_dev_rv_t rv = VNET_DEV_OK; + i32 rrv; + + rrv = roc_nix_mac_max_rx_len_set (nix, rx_frame_size); + if (rrv) + rv = oct_roc_err (dev, rrv, "roc_nix_mac_max_rx_len_set() failed"); + return rv; } @@ -515,6 +659,7 @@ oct_port_cfg_change (vlib_main_t *vm, vnet_dev_port_t *port, break; case VNET_DEV_PORT_CFG_MAX_RX_FRAME_SIZE: + rv = oct_op_config_max_rx_len (vm, port, req->max_rx_frame_size); break; case VNET_DEV_PORT_CFG_ADD_RX_FLOW: diff --git a/src/plugins/dev_octeon/queue.c b/src/plugins/dev_octeon/queue.c index d6ae794fb8d..58d391b8508 100644 --- a/src/plugins/dev_octeon/queue.c +++ b/src/plugins/dev_octeon/queue.c @@ -4,7 +4,6 @@ #include <vnet/vnet.h> #include <vnet/dev/dev.h> -#include <vnet/dev/pci.h> #include <vnet/dev/counters.h> #include <dev_octeon/octeon.h> #include <vnet/ethernet/ethernet.h> diff --git a/src/plugins/dev_octeon/roc_helper.c b/src/plugins/dev_octeon/roc_helper.c index f10c2cb578b..16e0a871a9d 100644 --- a/src/plugins/dev_octeon/roc_helper.c +++ b/src/plugins/dev_octeon/roc_helper.c @@ -49,6 +49,12 @@ oct_plt_get_thread_index (void) return __os_thread_index; } +static u64 +oct_plt_get_cache_line_size (void) +{ + return CLIB_CACHE_LINE_BYTES; +} + static void oct_drv_physmem_free (vlib_main_t *vm, void *mem) { @@ -178,4 +184,5 @@ oct_plt_init_param_t oct_plt_init_param = { .oct_plt_spinlock_unlock = oct_plt_spinlock_unlock, .oct_plt_spinlock_trylock = oct_plt_spinlock_trylock, .oct_plt_get_thread_index = oct_plt_get_thread_index, + .oct_plt_get_cache_line_size = oct_plt_get_cache_line_size, }; diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index 997f1356199..b057c4d7047 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -104,7 +104,9 @@ oct_rx_batch (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, { oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); vlib_buffer_template_t bt = rxq->buffer_template; - u32 n_left; + u32 b0_err_flags = 0, b1_err_flags = 0; + u32 b2_err_flags = 0, b3_err_flags = 0; + u32 n_left, err_flags = 0; oct_nix_rx_cqe_desc_t *d = ctx->next_desc; vlib_buffer_t *b[4]; @@ -145,6 +147,13 @@ oct_rx_batch (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, oct_rx_attach_tail (vm, ctx, b[2], d + 2); oct_rx_attach_tail (vm, ctx, b[3], d + 3); } + + b0_err_flags = (d[0].parse.w[0] >> 20) & 0xFFF; + b1_err_flags = (d[1].parse.w[0] >> 20) & 0xFFF; + b2_err_flags = (d[2].parse.w[0] >> 20) & 0xFFF; + b3_err_flags = (d[3].parse.w[0] >> 20) & 0xFFF; + + err_flags |= b0_err_flags | b1_err_flags | b2_err_flags | b3_err_flags; } for (; n_left; d += 1, n_left -= 1, ctx->to_next += 1) @@ -157,14 +166,51 @@ oct_rx_batch (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, ctx->n_segs += 1; if (d[0].sg0.segs > 1) oct_rx_attach_tail (vm, ctx, b[0], d + 0); + + err_flags |= ((d[0].parse.w[0] >> 20) & 0xFFF); } plt_write64 ((crq->cq.wdata | n), crq->cq.door); ctx->n_rx_pkts += n; ctx->n_left_to_next -= n; + if (err_flags) + ctx->parse_w0_or = (err_flags << 20); + return n; } +#ifdef PLATFORM_OCTEON9 +static_always_inline u32 +oct_rxq_refill (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u16 n_refill) +{ + u32 n_alloc, n_free; + u32 buffer_indices[n_refill]; + vlib_buffer_t *buffers[n_refill]; + u8 bpi = vnet_dev_get_rx_queue_buffer_pool_index (rxq); + oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); + u64 aura = roc_npa_aura_handle_to_aura (crq->aura_handle); + const uint64_t addr = + roc_npa_aura_handle_to_base (crq->aura_handle) + NPA_LF_AURA_OP_FREE0; + + if (n_refill < 256) + return 0; + + n_alloc = vlib_buffer_alloc (vm, buffer_indices, n_refill); + if (PREDICT_FALSE (n_alloc < n_refill)) + goto alloc_fail; + + vlib_get_buffers (vm, buffer_indices, (vlib_buffer_t **) buffers, n_alloc); + + for (n_free = 0; n_free < n_alloc; n_free++) + roc_store_pair ((u64) buffers[n_free], aura, addr); + + return n_alloc; + +alloc_fail: + vlib_buffer_unalloc_to_pool (vm, buffer_indices, n_alloc, bpi); + return 0; +} +#else static_always_inline void oct_rxq_refill_batch (vlib_main_t *vm, u64 lmt_id, u64 addr, oct_npa_lf_aura_batch_free_line_t *lines, u32 *bi, @@ -260,6 +306,7 @@ oct_rxq_refill (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u16 n_refill) return n_enq; } +#endif static_always_inline void oct_rx_trace (vlib_main_t *vm, vlib_node_runtime_t *node, diff --git a/src/plugins/dev_octeon/tx_node.c b/src/plugins/dev_octeon/tx_node.c index a2e4b07de8a..f42f18d989b 100644 --- a/src/plugins/dev_octeon/tx_node.c +++ b/src/plugins/dev_octeon/tx_node.c @@ -32,6 +32,44 @@ typedef struct lmt_line_t *lmt_lines; } oct_tx_ctx_t; +#ifdef PLATFORM_OCTEON9 +static_always_inline u32 +oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq) +{ + oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + u16 off = ctq->hdr_off; + u64 ah = ctq->aura_handle; + u32 n_freed = 0, n; + + ah = ctq->aura_handle; + + if ((n = roc_npa_aura_op_available (ah)) >= 32) + { + u64 buffers[n]; + u32 bi[n]; + + n_freed = roc_npa_aura_op_bulk_alloc (ah, buffers, n, 0, 1); + vlib_get_buffer_indices_with_offset (vm, (void **) &buffers, bi, n_freed, + off); + vlib_buffer_free_no_next (vm, bi, n_freed); + } + + return n_freed; +} + +static_always_inline void +oct_lmt_copy (void *lmt_addr, u64 io_addr, void *desc, u64 dwords) +{ + u64 lmt_status; + + do + { + roc_lmt_mov_seg (lmt_addr, desc, dwords); + lmt_status = roc_lmt_submit_ldeor (io_addr); + } + while (lmt_status == 0); +} +#else static_always_inline u32 oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq) { @@ -133,6 +171,7 @@ oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq) return n_freed; } +#endif static_always_inline u8 oct_tx_enq1 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vlib_buffer_t *b, @@ -158,6 +197,11 @@ oct_tx_enq1 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vlib_buffer_t *b, return 0; } +#ifdef PLATFORM_OCTEON9 + /* Override line for Octeon9 */ + line = ctx->lmt_lines; +#endif + if (!simple && flags & VLIB_BUFFER_NEXT_PRESENT) { u8 n_tail_segs = 0; @@ -211,19 +255,18 @@ oct_tx_enq1 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vlib_buffer_t *b, if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM) { d.hdr_w1.ol3type = NIX_SENDL3TYPE_IP4_CKSUM; - d.hdr_w1.ol3ptr = vnet_buffer (b)->l3_hdr_offset; - d.hdr_w1.ol4ptr = - vnet_buffer (b)->l3_hdr_offset + sizeof (ip4_header_t); + d.hdr_w1.ol3ptr = vnet_buffer (b)->l3_hdr_offset - b->current_data; + d.hdr_w1.ol4ptr = d.hdr_w1.ol3ptr + sizeof (ip4_header_t); } if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM) { d.hdr_w1.ol4type = NIX_SENDL4TYPE_UDP_CKSUM; - d.hdr_w1.ol4ptr = vnet_buffer (b)->l4_hdr_offset; + d.hdr_w1.ol4ptr = vnet_buffer (b)->l4_hdr_offset - b->current_data; } else if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM) { d.hdr_w1.ol4type = NIX_SENDL4TYPE_TCP_CKSUM; - d.hdr_w1.ol4ptr = vnet_buffer (b)->l4_hdr_offset; + d.hdr_w1.ol4ptr = vnet_buffer (b)->l4_hdr_offset - b->current_data; } } @@ -238,8 +281,12 @@ oct_tx_enq1 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vlib_buffer_t *b, t->sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_TX]; } +#ifdef PLATFORM_OCTEON9 + oct_lmt_copy (line, ctx->lmt_ioaddr, &d, n_dwords); +#else for (u32 i = 0; i < n_dwords; i++) line->dwords[i] = d.as_u128[i]; +#endif *dpl = n_dwords; *n = *n + 1; @@ -252,8 +299,9 @@ oct_tx_enq16 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq, vlib_buffer_t **b, u32 n_pkts, int trace) { u8 dwords_per_line[16], *dpl = dwords_per_line; - u64 lmt_arg, ioaddr, n_lines; - u32 n_left, or_flags_16 = 0, n = 0; + u64 __attribute__ ((unused)) lmt_arg, ioaddr, n_lines; + u32 __attribute__ ((unused)) or_flags_16 = 0; + u32 n_left, n = 0; const u32 not_simple_flags = VLIB_BUFFER_NEXT_PRESENT | VNET_BUFFER_F_OFFLOAD; lmt_line_t *l = ctx->lmt_lines; @@ -331,6 +379,7 @@ oct_tx_enq16 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq, if (PREDICT_FALSE (!n_lines)) return n_pkts; +#ifndef PLATFORM_OCTEON9 if (PREDICT_FALSE (or_flags_16 & VLIB_BUFFER_NEXT_PRESENT)) { dpl = dwords_per_line; @@ -359,6 +408,7 @@ oct_tx_enq16 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq, } roc_lmt_submit_steorl (lmt_arg, ioaddr); +#endif return n_pkts; } @@ -375,7 +425,11 @@ VNET_DEV_NODE_FN (oct_tx_node) u32 *from = vlib_frame_vector_args (frame); u32 n, n_enq, n_left, n_pkts = frame->n_vectors; vlib_buffer_t *buffers[VLIB_FRAME_SIZE + 8], **b = buffers; +#ifdef PLATFORM_OCTEON9 + u64 lmt_id = 0; +#else u64 lmt_id = vm->thread_index << ROC_LMT_LINES_PER_CORE_LOG2; +#endif oct_tx_ctx_t ctx = { .node = node, diff --git a/src/plugins/dpdk/device/dpdk.h b/src/plugins/dpdk/device/dpdk.h index 88a4d9ff618..a069fbe3818 100644 --- a/src/plugins/dpdk/device/dpdk.h +++ b/src/plugins/dpdk/device/dpdk.h @@ -131,6 +131,7 @@ typedef struct u32 interface_number_from_port_id : 1; u32 use_intel_phdr_cksum : 1; u32 int_unmaskable : 1; + vlib_simple_counter_main_t *xstats_counters; } dpdk_driver_t; dpdk_driver_t *dpdk_driver_find (const char *name, const char **desc); @@ -240,6 +241,7 @@ typedef struct _ (num_rx_desc) \ _ (num_tx_desc) \ _ (max_lro_pkt_size) \ + _ (disable_rxq_int) \ _ (rss_fn) typedef enum diff --git a/src/plugins/dpdk/device/dpdk_priv.h b/src/plugins/dpdk/device/dpdk_priv.h index cb7b185c112..e5b5a35df80 100644 --- a/src/plugins/dpdk/device/dpdk_priv.h +++ b/src/plugins/dpdk/device/dpdk_priv.h @@ -47,14 +47,18 @@ dpdk_device_flag_set (dpdk_device_t *xd, __typeof__ (xd->flags) flag, int val) xd->flags = val ? xd->flags | flag : xd->flags & ~flag; } +void dpdk_counters_xstats_init (dpdk_device_t *xd); + static inline void -dpdk_get_xstats (dpdk_device_t * xd) +dpdk_get_xstats (dpdk_device_t *xd, u32 thread_index) { - int len, ret; - + int ret; + int i; + int len; if (!(xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP)) return; - + if (xd->driver == 0) + return; len = rte_eth_xstats_get (xd->port_id, NULL, 0); if (len < 0) return; @@ -64,11 +68,26 @@ dpdk_get_xstats (dpdk_device_t * xd) ret = rte_eth_xstats_get (xd->port_id, xd->xstats, len); if (ret < 0 || ret > len) { + /* Failed, expand vector and try again on next time around the track. */ + vec_validate (xd->xstats, ret - 1); vec_set_len (xd->xstats, 0); + dpdk_log_warn ("rte_eth_xstats_get(%d) failed: %d", xd->port_id, ret); return; } - - vec_set_len (xd->xstats, len); + if (len == vec_len (xd->driver->xstats_counters)) + { + vec_foreach_index (i, xd->xstats) + { + vlib_set_simple_counter (&xd->driver->xstats_counters[i], + thread_index, xd->sw_if_index, + xd->xstats[i].value); + } + } + else + { + dpdk_log_warn ("rte_eth_xstats_get vector size mismatch (%d/%d", len, + vec_len (xd->driver->xstats_counters)); + } } #define DPDK_UPDATE_COUNTER(vnm, tidx, xd, stat, cnt) \ @@ -107,7 +126,7 @@ dpdk_update_counters (dpdk_device_t * xd, f64 now) DPDK_UPDATE_COUNTER (vnm, thread_index, xd, ierrors, VNET_INTERFACE_COUNTER_RX_ERROR); - dpdk_get_xstats (xd); + dpdk_get_xstats (xd, thread_index); } #if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0) diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c index e416efe2e4d..fa1b234874d 100644 --- a/src/plugins/dpdk/device/init.c +++ b/src/plugins/dpdk/device/init.c @@ -30,7 +30,7 @@ #include <dpdk/cryptodev/cryptodev.h> #include <vlib/pci/pci.h> #include <vlib/vmbus/vmbus.h> - +#include <vlib/stats/stats.h> #include <rte_ring.h> #include <rte_vect.h> @@ -226,6 +226,75 @@ dpdk_find_startup_config (struct rte_eth_dev_info *di) return &dm->conf->default_devconf; } +/* + * Initialise or refresh the xstats counters for a device + */ +void +dpdk_counters_xstats_init (dpdk_device_t *xd) +{ + int len, ret, i; + struct rte_eth_xstat_name *xstats_names = 0; + char *name; + dpdk_driver_t *dr = xd->driver; + + /* Only support xstats for supported drivers */ + if (!dr) + return; + + len = rte_eth_xstats_get_names (xd->port_id, 0, 0); + if (len < 0) + { + dpdk_log_err ("[%u] rte_eth_xstats_get_names failed: %d", xd->port_id, + len); + return; + } + /* Counters for this driver is already initialised */ + if (vec_len (dr->xstats_counters) == len) + { + vec_foreach_index (i, dr->xstats_counters) + { + vlib_validate_simple_counter (&dr->xstats_counters[i], + xd->sw_if_index); + vlib_zero_simple_counter (&dr->xstats_counters[i], xd->sw_if_index); + } + return; + } + + /* Same driver, different interface, different length of counter array. */ + ASSERT (vec_len (dr->xstats_counters) == 0); + + vec_validate (xstats_names, len - 1); + + ret = rte_eth_xstats_get_names (xd->port_id, xstats_names, len); + if (ret >= 0 && ret <= len) + { + vec_validate (dr->xstats_counters, len - 1); + vec_foreach_index (i, xstats_names) + { + name = (char *) format (0, "/if/%s/%s%c", dr->drivers->name, + xstats_names[i].name, 0); + + /* There is a bug in the ENA driver where the xstats names are not + * unique. */ + if (vlib_stats_find_entry_index (name) != STAT_SEGMENT_INDEX_INVALID) + { + vec_free (name); + name = (char *) format (0, "/if/%s/%s_%d%c", dr->drivers->name, + xstats_names[i].name, i, 0); + } + + dr->xstats_counters[i].name = name; + dr->xstats_counters[i].stat_segment_name = name; + dr->xstats_counters[i].counters = 0; + vlib_validate_simple_counter (&dr->xstats_counters[i], + xd->sw_if_index); + vlib_zero_simple_counter (&dr->xstats_counters[i], xd->sw_if_index); + vec_free (name); + } + } + vec_free (xstats_names); +} + static clib_error_t * dpdk_lib_init (dpdk_main_t * dm) { @@ -519,6 +588,9 @@ dpdk_lib_init (dpdk_main_t * dm) if (devconf->max_lro_pkt_size) xd->conf.max_lro_pkt_size = devconf->max_lro_pkt_size; + if (devconf->disable_rxq_int) + xd->conf.enable_rxq_int = 0; + dpdk_device_setup (xd); /* rss queues should be configured after dpdk_device_setup() */ @@ -532,6 +604,7 @@ dpdk_lib_init (dpdk_main_t * dm) if (vec_len (xd->errors)) dpdk_log_err ("[%u] setup failed Errors:\n %U", port_id, format_dpdk_device_errors, xd); + dpdk_counters_xstats_init (xd); } for (int i = 0; i < vec_len (dm->devices); i++) @@ -659,7 +732,8 @@ dpdk_bind_devices_to_uio (dpdk_config_main_t * conf) ; /* Cisco VIC */ else if (d->vendor_id == 0x1137 && - (d->device_id == 0x0043 || d->device_id == 0x0071)) + (d->device_id == 0x0043 || d->device_id == 0x0071 || + d->device_id == 0x02b7)) ; /* Chelsio T4/T5 */ else if (d->vendor_id == 0x1425 && (d->device_id & 0xe000) == 0x4000) @@ -936,6 +1010,10 @@ dpdk_device_config (dpdk_config_main_t *conf, void *addr, if (error) break; } + else if (unformat (input, "no-rx-interrupts")) + { + devconf->disable_rxq_int = 1; + } else if (unformat (input, "tso on")) { devconf->tso = DPDK_DEVICE_TSO_ON; @@ -1052,6 +1130,7 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) #ifdef __linux vlib_thread_main_t *tm = vlib_get_thread_main (); uword default_hugepage_sz, x; + u8 file_prefix = 0; #endif /* __linux__ */ u8 *s, *tmp = 0; int ret, i; @@ -1059,7 +1138,6 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) int eal_no_hugetlb = 0; u8 no_pci = 0; u8 no_vmbus = 0; - u8 file_prefix = 0; u8 *socket_mem = 0; u32 vendor, device, domain, bus, func; void *fmt_func; @@ -1219,6 +1297,7 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) } foreach_eal_double_hyphen_predicate_arg #undef _ +#ifdef __linux__ #define _(a) \ else if (unformat(input, #a " %s", &s)) \ { \ @@ -1234,6 +1313,7 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) } foreach_eal_double_hyphen_arg #undef _ +#endif /* __linux__ */ #define _(a,b) \ else if (unformat(input, #a " %s", &s)) \ { \ diff --git a/src/plugins/dpdk/main.c b/src/plugins/dpdk/main.c index 9781d0ed7f0..437cfbd230e 100644 --- a/src/plugins/dpdk/main.c +++ b/src/plugins/dpdk/main.c @@ -50,7 +50,7 @@ rte_delay_us_override (unsigned us) { /* Only suspend for the admin_down_process */ vlib_process_t *proc = vlib_get_current_process (vm); - if (!(proc->flags & VLIB_PROCESS_IS_RUNNING) || + if (proc->state != VLIB_PROCESS_STATE_RUNNING || (proc->node_runtime.node_index != admin_up_down_process_node.index)) return 0; diff --git a/src/plugins/flowprobe/flowprobe.c b/src/plugins/flowprobe/flowprobe.c index 58a7cfe22f1..ee0a8eb8a31 100644 --- a/src/plugins/flowprobe/flowprobe.c +++ b/src/plugins/flowprobe/flowprobe.c @@ -48,7 +48,7 @@ uword flowprobe_walker_process (vlib_main_t * vm, vlib_node_runtime_t * rt, VNET_FEATURE_INIT (flowprobe_input_ip4_unicast, static) = { .arc_name = "ip4-unicast", .node_name = "flowprobe-input-ip4", - .runs_before = VNET_FEATURES ("ip4-lookup"), + .runs_before = VNET_FEATURES ("ip4-lookup", "ip4-inacl"), }; VNET_FEATURE_INIT (flowprobe_input_ip4_multicast, static) = { .arc_name = "ip4-multicast", @@ -58,7 +58,7 @@ VNET_FEATURE_INIT (flowprobe_input_ip4_multicast, static) = { VNET_FEATURE_INIT (flowprobe_input_ip6_unicast, static) = { .arc_name = "ip6-unicast", .node_name = "flowprobe-input-ip6", - .runs_before = VNET_FEATURES ("ip6-lookup"), + .runs_before = VNET_FEATURES ("ip6-lookup", "ip6-inacl"), }; VNET_FEATURE_INIT (flowprobe_input_ip6_multicast, static) = { .arc_name = "ip6-multicast", diff --git a/src/plugins/hs_apps/CMakeLists.txt b/src/plugins/hs_apps/CMakeLists.txt index 179c9c7a4c4..ba03e393f44 100644 --- a/src/plugins/hs_apps/CMakeLists.txt +++ b/src/plugins/hs_apps/CMakeLists.txt @@ -21,8 +21,10 @@ add_vpp_plugin(hs_apps hs_apps.c http_cli.c http_client_cli.c + http_simple_post.c http_tps.c proxy.c + test_builtins.c ) ############################################################################## diff --git a/src/plugins/hs_apps/echo_client.c b/src/plugins/hs_apps/echo_client.c index d1443e75e80..8dec5d86824 100644 --- a/src/plugins/hs_apps/echo_client.c +++ b/src/plugins/hs_apps/echo_client.c @@ -429,8 +429,11 @@ ec_init (vlib_main_t *vm) ecm->app_is_init = 1; + session_enable_disable_args_t args = { .is_en = 1, + .rt_engine_type = + RT_BACKEND_ENGINE_RULE_TABLE }; vlib_worker_thread_barrier_sync (vm); - vnet_session_enable_disable (vm, 1 /* turn on session and transports */); + vnet_session_enable_disable (vm, &args); /* Turn on the builtin client input nodes */ foreach_vlib_main () diff --git a/src/plugins/hs_apps/echo_server.c b/src/plugins/hs_apps/echo_server.c index 0243252434a..756a1cc3451 100644 --- a/src/plugins/hs_apps/echo_server.c +++ b/src/plugins/hs_apps/echo_server.c @@ -736,7 +736,10 @@ echo_server_create_command_fn (vlib_main_t * vm, unformat_input_t * input, goto cleanup; } - vnet_session_enable_disable (vm, 1 /* turn on TCP, etc. */ ); + session_enable_disable_args_t args = { .is_en = 1, + .rt_engine_type = + RT_BACKEND_ENGINE_RULE_TABLE }; + vnet_session_enable_disable (vm, &args); if (!server_uri_set) { diff --git a/src/plugins/hs_apps/http_cli.c b/src/plugins/hs_apps/http_cli.c index f42f65342c3..e0bb9afba4a 100644 --- a/src/plugins/hs_apps/http_cli.c +++ b/src/plugins/hs_apps/http_cli.c @@ -17,12 +17,23 @@ #include <vnet/session/application_interface.h> #include <vnet/session/session.h> #include <http/http.h> +#include <http/http_header_names.h> +#include <http/http_content_types.h> + +#define HCS_DEBUG 0 + +#if HCS_DEBUG +#define HCS_DBG(_fmt, _args...) clib_warning (_fmt, ##_args) +#else +#define HCS_DBG(_fmt, _args...) +#endif typedef struct { u32 hs_index; u32 thread_index; u64 node_index; + u8 plain_text; u8 *buf; } hcs_cli_args_t; @@ -34,6 +45,7 @@ typedef struct u8 *tx_buf; u32 tx_offset; u32 vpp_session_index; + http_header_t *resp_headers; } hcs_session_t; typedef struct @@ -143,26 +155,48 @@ start_send_data (hcs_session_t *hs, http_status_code_t status) { http_msg_t msg; session_t *ts; + u8 *headers_buf = 0; int rv; + if (vec_len (hs->resp_headers)) + { + headers_buf = http_serialize_headers (hs->resp_headers); + vec_free (hs->resp_headers); + msg.data.headers_offset = 0; + msg.data.headers_len = vec_len (headers_buf); + } + else + { + msg.data.headers_offset = 0; + msg.data.headers_len = 0; + } + msg.type = HTTP_MSG_REPLY; msg.code = status; - msg.content_type = HTTP_CONTENT_TEXT_HTML; msg.data.type = HTTP_MSG_DATA_INLINE; - msg.data.len = vec_len (hs->tx_buf); + msg.data.body_len = vec_len (hs->tx_buf); + msg.data.body_offset = msg.data.headers_len; + msg.data.len = msg.data.body_len + msg.data.headers_len; ts = session_get (hs->vpp_session_index, hs->thread_index); rv = svm_fifo_enqueue (ts->tx_fifo, sizeof (msg), (u8 *) &msg); ASSERT (rv == sizeof (msg)); - if (!msg.data.len) + if (msg.data.headers_len) + { + rv = svm_fifo_enqueue (ts->tx_fifo, vec_len (headers_buf), headers_buf); + ASSERT (rv == msg.data.headers_len); + vec_free (headers_buf); + } + + if (!msg.data.body_len) goto done; rv = svm_fifo_enqueue (ts->tx_fifo, vec_len (hs->tx_buf), hs->tx_buf); if (rv != vec_len (hs->tx_buf)) { - hs->tx_offset = rv; + hs->tx_offset = (rv > 0) ? rv : 0; svm_fifo_add_want_deq_ntf (ts->tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); } else @@ -173,7 +207,7 @@ start_send_data (hcs_session_t *hs, http_status_code_t status) done: if (svm_fifo_set_event (ts->tx_fifo)) - session_send_io_evt_to_thread (ts->tx_fifo, SESSION_IO_EVT_TX); + session_program_tx_io_evt (ts->handle, SESSION_IO_EVT_TX); } static void @@ -181,6 +215,7 @@ send_data_to_http (void *rpc_args) { hcs_cli_args_t *args = (hcs_cli_args_t *) rpc_args; hcs_session_t *hs; + http_content_type_t type = HTTP_CONTENT_TEXT_HTML; hs = hcs_session_get (args->thread_index, args->hs_index); if (!hs) @@ -190,6 +225,13 @@ send_data_to_http (void *rpc_args) } hs->tx_buf = args->buf; + if (args->plain_text) + type = HTTP_CONTENT_TEXT_PLAIN; + + http_add_header (&hs->resp_headers, + http_header_name_token (HTTP_HEADER_CONTENT_TYPE), + http_content_type_token (type)); + start_send_data (hs, HTTP_STATUS_OK); cleanup: @@ -218,17 +260,9 @@ hcs_cli_process (vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f) { if (request[i] == '/') request[i] = ' '; - else if (request[i] == ' ') - { - /* vlib_cli_input is vector-based, no need for a NULL */ - vec_set_len (request, i); - break; - } i++; } - - /* Generate the html header */ - html = format (0, html_header_template, request /* title */ ); + HCS_DBG ("%v", request); /* Run the command */ unformat_init_vector (&input, vec_dup (request)); @@ -236,9 +270,17 @@ hcs_cli_process (vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f) unformat_free (&input); request = 0; - /* Generate the html page */ - html = format (html, "%v", reply); - html = format (html, html_footer); + if (args->plain_text) + { + html = format (0, "%v", reply); + } + else + { + /* Generate the html page */ + html = format (0, html_header_template, request /* title */); + html = format (html, "%v", reply); + html = format (html, html_footer); + } /* Send it */ rpc_args = clib_mem_alloc (sizeof (*args)); @@ -308,9 +350,11 @@ hcs_ts_rx_callback (session_t *ts) hcs_cli_args_t args = {}; hcs_session_t *hs; http_msg_t msg; - int rv; + int rv, is_encoded = 0; hs = hcs_session_get (ts->thread_index, ts->opaque); + hs->tx_buf = 0; + hs->resp_headers = 0; /* Read the http message header */ rv = svm_fifo_dequeue (ts->rx_fifo, sizeof (msg), (u8 *) &msg); @@ -318,23 +362,66 @@ hcs_ts_rx_callback (session_t *ts) if (msg.type != HTTP_MSG_REQUEST || msg.method_type != HTTP_REQ_GET) { - hs->tx_buf = 0; + http_add_header (&hs->resp_headers, + http_header_name_token (HTTP_HEADER_ALLOW), + http_token_lit ("GET")); start_send_data (hs, HTTP_STATUS_METHOD_NOT_ALLOWED); - return 0; + goto done; } - if (msg.data.len == 0) + if (msg.data.target_path_len == 0 || + msg.data.target_form != HTTP_TARGET_ORIGIN_FORM) { - hs->tx_buf = 0; start_send_data (hs, HTTP_STATUS_BAD_REQUEST); - return 0; + goto done; } /* send the command to a new/recycled vlib process */ - vec_validate (args.buf, msg.data.len - 1); - rv = svm_fifo_dequeue (ts->rx_fifo, msg.data.len, args.buf); - ASSERT (rv == msg.data.len); - vec_set_len (args.buf, rv); + vec_validate (args.buf, msg.data.target_path_len - 1); + rv = svm_fifo_peek (ts->rx_fifo, msg.data.target_path_offset, + msg.data.target_path_len, args.buf); + ASSERT (rv == msg.data.target_path_len); + HCS_DBG ("%v", args.buf); + if (http_validate_abs_path_syntax (args.buf, &is_encoded)) + { + start_send_data (hs, HTTP_STATUS_BAD_REQUEST); + vec_free (args.buf); + goto done; + } + if (is_encoded) + { + u8 *decoded = http_percent_decode (args.buf); + vec_free (args.buf); + args.buf = decoded; + } + + if (msg.data.headers_len) + { + u8 *headers = 0; + http_header_table_t *ht; + vec_validate (headers, msg.data.headers_len - 1); + rv = svm_fifo_peek (ts->rx_fifo, msg.data.headers_offset, + msg.data.headers_len, headers); + ASSERT (rv == msg.data.headers_len); + if (http_parse_headers (headers, &ht)) + { + start_send_data (hs, HTTP_STATUS_BAD_REQUEST); + vec_free (args.buf); + vec_free (headers); + goto done; + } + const char *accept_value = + http_get_header (ht, http_header_name_str (HTTP_HEADER_ACCEPT)); + if (accept_value) + { + HCS_DBG ("client accept: %s", accept_value); + /* just for testing purpose, we don't care about precedence */ + if (strstr (accept_value, "text/plain")) + args.plain_text = 1; + } + http_free_header_table (ht); + vec_free (headers); + } args.hs_index = hs->session_index; args.thread_index = ts->thread_index; @@ -345,6 +432,9 @@ hcs_ts_rx_callback (session_t *ts) sizeof (args)); else alloc_cli_process (&args); + +done: + svm_fifo_dequeue_drop (ts->rx_fifo, msg.data.len); return 0; } @@ -379,7 +469,7 @@ hcs_ts_tx_callback (session_t *ts) } if (svm_fifo_set_event (ts->tx_fifo)) - session_send_io_evt_to_thread (ts->tx_fifo, SESSION_IO_EVT_TX); + session_program_tx_io_evt (ts->handle, SESSION_IO_EVT_TX); return 0; } @@ -641,7 +731,10 @@ start_server: if (hcm->app_index != (u32) ~0) return clib_error_return (0, "test http server is already running"); - vnet_session_enable_disable (vm, 1 /* turn on TCP, etc. */ ); + session_enable_disable_args_t args = { .is_en = 1, + .rt_engine_type = + RT_BACKEND_ENGINE_RULE_TABLE }; + vnet_session_enable_disable (vm, &args); rv = hcs_create (vm); switch (rv) diff --git a/src/plugins/hs_apps/http_client_cli.c b/src/plugins/hs_apps/http_client_cli.c index a99169bafea..4b8ef173bd9 100644 --- a/src/plugins/hs_apps/http_client_cli.c +++ b/src/plugins/hs_apps/http_client_cli.c @@ -16,6 +16,9 @@ #include <vnet/session/application_interface.h> #include <vnet/session/session.h> #include <http/http.h> +#include <http/http_header_names.h> +#include <http/http_content_types.h> +#include <http/http_status_codes.h> #define HCC_DEBUG 0 @@ -32,14 +35,14 @@ typedef struct u32 thread_index; u32 rx_offset; u32 vpp_session_index; - u32 to_recv; + u64 to_recv; u8 is_closed; + http_header_t *req_headers; } hcc_session_t; typedef struct { hcc_session_t *sessions; - u8 *rx_buf; u32 thread_index; } hcc_worker_t; @@ -95,13 +98,6 @@ hcc_session_get (u32 hs_index, u32 thread_index) return pool_elt_at_index (wrk->sessions, hs_index); } -static void -hcc_session_free (u32 thread_index, hcc_session_t *hs) -{ - hcc_worker_t *wrk = hcc_worker_get (thread_index); - pool_put (wrk->sessions, hs); -} - static int hcc_ts_accept_callback (session_t *ts) { @@ -128,6 +124,7 @@ hcc_ts_connected_callback (u32 app_index, u32 hc_index, session_t *as, hcc_session_t *hs, *new_hs; hcc_worker_t *wrk; http_msg_t msg; + u8 *headers_buf; int rv; HCC_DBG ("hc_index: %d", hc_index); @@ -149,24 +146,42 @@ hcc_ts_connected_callback (u32 app_index, u32 hc_index, session_t *as, hs->vpp_session_index = as->session_index; + http_add_header (&hs->req_headers, + http_header_name_token (HTTP_HEADER_ACCEPT), + http_content_type_token (HTTP_CONTENT_TEXT_HTML)); + headers_buf = http_serialize_headers (hs->req_headers); + vec_free (hs->req_headers); + msg.type = HTTP_MSG_REQUEST; msg.method_type = HTTP_REQ_GET; - msg.content_type = HTTP_CONTENT_TEXT_HTML; + /* request target */ + msg.data.target_form = HTTP_TARGET_ORIGIN_FORM; + msg.data.target_path_offset = 0; + msg.data.target_path_len = vec_len (hcm->http_query); + /* custom headers */ + msg.data.headers_offset = msg.data.target_path_len; + msg.data.headers_len = vec_len (headers_buf); + /* request body */ + msg.data.body_len = 0; + /* data type and total length */ msg.data.type = HTTP_MSG_DATA_INLINE; - msg.data.len = vec_len (hcm->http_query); + msg.data.len = + msg.data.target_path_len + msg.data.headers_len + msg.data.body_len; - svm_fifo_seg_t segs[2] = { { (u8 *) &msg, sizeof (msg) }, - { hcm->http_query, vec_len (hcm->http_query) } }; + svm_fifo_seg_t segs[3] = { { (u8 *) &msg, sizeof (msg) }, + { hcm->http_query, vec_len (hcm->http_query) }, + { headers_buf, vec_len (headers_buf) } }; - rv = svm_fifo_enqueue_segments (as->tx_fifo, segs, 2, 0 /* allow partial */); - if (rv < 0 || rv != sizeof (msg) + vec_len (hcm->http_query)) + rv = svm_fifo_enqueue_segments (as->tx_fifo, segs, 3, 0 /* allow partial */); + vec_free (headers_buf); + if (rv < 0 || rv != sizeof (msg) + msg.data.len) { clib_warning ("failed app enqueue"); return -1; } if (svm_fifo_set_event (as->tx_fifo)) - session_send_io_evt_to_thread (as->tx_fifo, SESSION_IO_EVT_TX); + session_program_tx_io_evt (as->handle, SESSION_IO_EVT_TX); return 0; } @@ -221,23 +236,32 @@ hcc_ts_rx_callback (session_t *ts) if (hs->to_recv == 0) { + /* read the http message header */ rv = svm_fifo_dequeue (ts->rx_fifo, sizeof (msg), (u8 *) &msg); ASSERT (rv == sizeof (msg)); - if (msg.type != HTTP_MSG_REPLY || msg.code != HTTP_STATUS_OK) + if (msg.type != HTTP_MSG_REPLY) { clib_warning ("unexpected msg type %d", msg.type); return 0; } - vec_validate (hcm->http_response, msg.data.len - 1); + /* drop everything up to body */ + svm_fifo_dequeue_drop (ts->rx_fifo, msg.data.body_offset); + hs->to_recv = msg.data.body_len; + if (msg.code != HTTP_STATUS_OK && hs->to_recv == 0) + { + hcm->http_response = format (0, "request failed, response code: %U", + format_http_status_code, msg.code); + goto done; + } + vec_validate (hcm->http_response, msg.data.body_len - 1); vec_reset_length (hcm->http_response); - hs->to_recv = msg.data.len; } u32 max_deq = svm_fifo_max_dequeue (ts->rx_fifo); u32 n_deq = clib_min (hs->to_recv, max_deq); - u32 curr = vec_len (hcm->http_response); + u64 curr = vec_len (hcm->http_response); rv = svm_fifo_dequeue (ts->rx_fifo, n_deq, hcm->http_response + curr); if (rv < 0) { @@ -251,10 +275,12 @@ hcc_ts_rx_callback (session_t *ts) vec_set_len (hcm->http_response, curr + n_deq); ASSERT (hs->to_recv >= rv); hs->to_recv -= rv; - HCC_DBG ("app rcvd %d, remains %d", rv, hs->to_recv); + HCC_DBG ("app rcvd %d, remains %llu", rv, hs->to_recv); +done: if (hs->to_recv == 0) { + HCC_DBG ("all data received, going to disconnect"); hcc_session_disconnect (ts); vlib_process_signal_event_mt (hcm->vlib_main, hcm->cli_node_index, HCC_REPLY_RECEIVED, 0); @@ -264,18 +290,6 @@ hcc_ts_rx_callback (session_t *ts) } static void -hcc_ts_cleanup_callback (session_t *s, session_cleanup_ntf_t ntf) -{ - hcc_session_t *hs; - - hs = hcc_session_get (s->thread_index, s->opaque); - if (!hs) - return; - - hcc_session_free (s->thread_index, hs); -} - -static void hcc_ts_transport_closed (session_t *s) { hcc_main_t *hcm = &hcc_main; @@ -293,7 +307,6 @@ static session_cb_vft_t hcc_session_cb_vft = { .builtin_app_rx_callback = hcc_ts_rx_callback, .builtin_app_tx_callback = hcc_ts_tx_callback, .session_reset_callback = hcc_ts_reset_callback, - .session_cleanup_callback = hcc_ts_cleanup_callback, .session_transport_closed_callback = hcc_ts_transport_closed, }; @@ -423,7 +436,6 @@ hcc_run (vlib_main_t *vm, int print_output) case HCC_REPLY_RECEIVED: if (print_output) vlib_cli_output (vm, "%v", hcm->http_response); - vec_free (hcm->http_response); break; case HCC_TRANSPORT_CLOSED: err = clib_error_return (0, "error, transport closed"); @@ -460,6 +472,28 @@ hcc_detach () return rv; } +static void +hcc_worker_cleanup (hcc_worker_t *wrk) +{ + pool_free (wrk->sessions); +} + +static void +hcc_cleanup () +{ + hcc_main_t *hcm = &hcc_main; + hcc_worker_t *wrk; + + vec_foreach (wrk, hcm->wrk) + hcc_worker_cleanup (wrk); + + vec_free (hcm->uri); + vec_free (hcm->http_query); + vec_free (hcm->http_response); + vec_free (hcm->appns_id); + vec_free (hcm->wrk); +} + static clib_error_t * hcc_command_fn (vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd) @@ -509,7 +543,6 @@ hcc_command_fn (vlib_main_t *vm, unformat_input_t *input, } } - vec_free (hcm->appns_id); hcm->appns_id = appns_id; hcm->cli_node_index = vlib_get_current_process (vm)->node_runtime.node_index; @@ -525,8 +558,11 @@ hcc_command_fn (vlib_main_t *vm, unformat_input_t *input, goto done; } + session_enable_disable_args_t args = { .is_en = 1, + .rt_engine_type = + RT_BACKEND_ENGINE_RULE_TABLE }; vlib_worker_thread_barrier_sync (vm); - vnet_session_enable_disable (vm, 1 /* turn on TCP, etc. */); + vnet_session_enable_disable (vm, &args); vlib_worker_thread_barrier_release (vm); err = hcc_run (vm, print_output); @@ -540,8 +576,7 @@ hcc_command_fn (vlib_main_t *vm, unformat_input_t *input, } done: - vec_free (hcm->uri); - vec_free (hcm->http_query); + hcc_cleanup (); unformat_free (line_input); return err; } diff --git a/src/plugins/hs_apps/http_simple_post.c b/src/plugins/hs_apps/http_simple_post.c new file mode 100644 index 00000000000..6212eac1c97 --- /dev/null +++ b/src/plugins/hs_apps/http_simple_post.c @@ -0,0 +1,581 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2024 Cisco Systems, Inc. + */ + +#include <vnet/session/application.h> +#include <vnet/session/application_interface.h> +#include <vnet/session/session.h> +#include <http/http.h> +#include <http/http_header_names.h> +#include <http/http_content_types.h> +#include <vppinfra/unix.h> + +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + u32 session_index; + u32 thread_index; + u32 vpp_session_index; + u8 is_closed; +} hsp_session_t; + +typedef struct +{ + hsp_session_t *sessions; + u32 thread_index; +} hsp_worker_t; + +typedef struct +{ + u32 app_index; + vlib_main_t *vlib_main; + u32 cli_node_index; + u8 attached; + u8 *uri; + session_endpoint_cfg_t connect_sep; + u8 *target; + u8 *headers_buf; + u8 *data; + u64 data_offset; + hsp_worker_t *wrk; + u8 *http_response; + u8 is_file; + u8 use_ptr; +} hsp_main_t; + +typedef enum +{ + HSP_CONNECT_FAILED = 1, + HSP_TRANSPORT_CLOSED, + HSP_REPLY_RECEIVED, +} hsp_cli_signal_t; + +static hsp_main_t hsp_main; + +static inline hsp_worker_t * +hsp_worker_get (u32 thread_index) +{ + return &hsp_main.wrk[thread_index]; +} + +static inline hsp_session_t * +hsp_session_get (u32 session_index, u32 thread_index) +{ + hsp_worker_t *wrk = hsp_worker_get (thread_index); + return pool_elt_at_index (wrk->sessions, session_index); +} + +static hsp_session_t * +hsp_session_alloc (hsp_worker_t *wrk) +{ + hsp_session_t *s; + + pool_get_zero (wrk->sessions, s); + s->session_index = s - wrk->sessions; + s->thread_index = wrk->thread_index; + + return s; +} + +static int +hsp_session_connected_callback (u32 app_index, u32 hsp_session_index, + session_t *s, session_error_t err) +{ + hsp_main_t *hspm = &hsp_main; + hsp_session_t *hsp_session, *new_hsp_session; + hsp_worker_t *wrk; + http_header_t *headers = 0; + http_msg_t msg; + u64 to_send; + u32 n_enq; + int rv; + + if (err) + { + clib_warning ("hsp_session_index[%d] connected error: %U", + hsp_session_index, format_session_error, err); + vlib_process_signal_event_mt (hspm->vlib_main, hspm->cli_node_index, + HSP_CONNECT_FAILED, 0); + return -1; + } + + hsp_session = hsp_session_get (hsp_session_index, 0); + wrk = hsp_worker_get (s->thread_index); + new_hsp_session = hsp_session_alloc (wrk); + clib_memcpy_fast (new_hsp_session, hsp_session, sizeof (*hsp_session)); + hsp_session->vpp_session_index = s->session_index; + + if (hspm->is_file) + { + http_add_header ( + &headers, http_header_name_token (HTTP_HEADER_CONTENT_TYPE), + http_content_type_token (HTTP_CONTENT_APP_OCTET_STREAM)); + } + else + { + http_add_header ( + &headers, http_header_name_token (HTTP_HEADER_CONTENT_TYPE), + http_content_type_token (HTTP_CONTENT_APP_X_WWW_FORM_URLENCODED)); + } + hspm->headers_buf = http_serialize_headers (headers); + vec_free (headers); + + msg.type = HTTP_MSG_REQUEST; + msg.method_type = HTTP_REQ_POST; + /* request target */ + msg.data.target_form = HTTP_TARGET_ORIGIN_FORM; + msg.data.target_path_len = vec_len (hspm->target); + /* custom headers */ + msg.data.headers_len = vec_len (hspm->headers_buf); + /* request body */ + msg.data.body_len = vec_len (hspm->data); + /* total length */ + msg.data.len = + msg.data.target_path_len + msg.data.headers_len + msg.data.body_len; + + if (hspm->use_ptr) + { + uword target = pointer_to_uword (hspm->target); + uword headers = pointer_to_uword (hspm->headers_buf); + uword body = pointer_to_uword (hspm->data); + msg.data.type = HTTP_MSG_DATA_PTR; + svm_fifo_seg_t segs[4] = { + { (u8 *) &msg, sizeof (msg) }, + { (u8 *) &target, sizeof (target) }, + { (u8 *) &headers, sizeof (headers) }, + { (u8 *) &body, sizeof (body) }, + }; + + rv = + svm_fifo_enqueue_segments (s->tx_fifo, segs, 4, 0 /* allow partial */); + ASSERT (rv == (sizeof (msg) + sizeof (target) + sizeof (headers) + + sizeof (body))); + goto done; + } + + msg.data.type = HTTP_MSG_DATA_INLINE; + msg.data.target_path_offset = 0; + msg.data.headers_offset = msg.data.target_path_len; + msg.data.body_offset = msg.data.headers_offset + msg.data.headers_len; + + rv = svm_fifo_enqueue (s->tx_fifo, sizeof (msg), (u8 *) &msg); + ASSERT (rv == sizeof (msg)); + + rv = svm_fifo_enqueue (s->tx_fifo, vec_len (hspm->target), hspm->target); + ASSERT (rv == vec_len (hspm->target)); + + rv = svm_fifo_enqueue (s->tx_fifo, vec_len (hspm->headers_buf), + hspm->headers_buf); + ASSERT (rv == msg.data.headers_len); + + to_send = vec_len (hspm->data); + n_enq = clib_min (svm_fifo_size (s->tx_fifo), to_send); + + rv = svm_fifo_enqueue (s->tx_fifo, n_enq, hspm->data); + + if (rv < to_send) + { + hspm->data_offset = (rv > 0) ? rv : 0; + svm_fifo_add_want_deq_ntf (s->tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); + } + +done: + if (svm_fifo_set_event (s->tx_fifo)) + session_program_tx_io_evt (s->handle, SESSION_IO_EVT_TX); + + return 0; +} + +static void +hsp_session_disconnect_callback (session_t *s) +{ + hsp_main_t *hspm = &hsp_main; + vnet_disconnect_args_t _a = { 0 }, *a = &_a; + int rv; + + a->handle = session_handle (s); + a->app_index = hspm->app_index; + if ((rv = vnet_disconnect_session (a))) + clib_warning ("warning: disconnect returned: %U", format_session_error, + rv); +} + +static void +hsp_session_transport_closed_callback (session_t *s) +{ + hsp_main_t *hspm = &hsp_main; + + vlib_process_signal_event_mt (hspm->vlib_main, hspm->cli_node_index, + HSP_TRANSPORT_CLOSED, 0); +} + +static void +hsp_session_reset_callback (session_t *s) +{ + hsp_main_t *hspm = &hsp_main; + hsp_session_t *hsp_session; + vnet_disconnect_args_t _a = { 0 }, *a = &_a; + int rv; + + hsp_session = hsp_session_get (s->opaque, s->thread_index); + hsp_session->is_closed = 1; + + a->handle = session_handle (s); + a->app_index = hspm->app_index; + if ((rv = vnet_disconnect_session (a))) + clib_warning ("warning: disconnect returned: %U", format_session_error, + rv); +} + +static int +hsp_rx_callback (session_t *s) +{ + hsp_main_t *hspm = &hsp_main; + hsp_session_t *hsp_session; + http_msg_t msg; + int rv; + + hsp_session = hsp_session_get (s->opaque, s->thread_index); + + if (hsp_session->is_closed) + { + clib_warning ("hsp_session_index[%d] is closed", s->opaque); + return -1; + } + + rv = svm_fifo_dequeue (s->rx_fifo, sizeof (msg), (u8 *) &msg); + ASSERT (rv == sizeof (msg)); + + if (msg.type != HTTP_MSG_REPLY) + { + clib_warning ("unexpected msg type %d", msg.type); + return -1; + } + + svm_fifo_dequeue_drop_all (s->rx_fifo); + + if (msg.code == HTTP_STATUS_OK) + hspm->http_response = format (0, "request success"); + else + hspm->http_response = format (0, "request failed"); + + hsp_session_disconnect_callback (s); + vlib_process_signal_event_mt (hspm->vlib_main, hspm->cli_node_index, + HSP_REPLY_RECEIVED, 0); + return 0; +} + +static int +hsp_tx_callback (session_t *s) +{ + hsp_main_t *hspm = &hsp_main; + u64 to_send; + u32 n_enq; + int rv; + + to_send = vec_len (hspm->data) - hspm->data_offset; + n_enq = clib_min (svm_fifo_size (s->tx_fifo), to_send); + + rv = svm_fifo_enqueue (s->tx_fifo, n_enq, hspm->data + hspm->data_offset); + + if (rv <= 0) + { + svm_fifo_add_want_deq_ntf (s->tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); + return 0; + } + + if (rv < to_send) + { + hspm->data_offset += rv; + svm_fifo_add_want_deq_ntf (s->tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); + } + + if (svm_fifo_set_event (s->tx_fifo)) + session_program_tx_io_evt (s->handle, SESSION_IO_EVT_TX); + + return 0; +} + +static session_cb_vft_t hsp_session_cb_vft = { + .session_connected_callback = hsp_session_connected_callback, + .session_disconnect_callback = hsp_session_disconnect_callback, + .session_transport_closed_callback = hsp_session_transport_closed_callback, + .session_reset_callback = hsp_session_reset_callback, + .builtin_app_rx_callback = hsp_rx_callback, + .builtin_app_tx_callback = hsp_tx_callback, +}; + +static clib_error_t * +hsp_attach () +{ + hsp_main_t *hspm = &hsp_main; + vnet_app_attach_args_t _a, *a = &_a; + u64 options[18]; + int rv; + + clib_memset (a, 0, sizeof (*a)); + clib_memset (options, 0, sizeof (options)); + + a->api_client_index = APP_INVALID_INDEX; + a->name = format (0, "http_simple_post"); + a->session_cb_vft = &hsp_session_cb_vft; + a->options = options; + a->options[APP_OPTIONS_FLAGS] = APP_OPTIONS_FLAGS_IS_BUILTIN; + + if ((rv = vnet_application_attach (a))) + return clib_error_return (0, "attach returned: %U", format_session_error, + rv); + + hspm->app_index = a->app_index; + vec_free (a->name); + hspm->attached = 1; + + return 0; +} + +static int +hsp_connect_rpc (void *rpc_args) +{ + vnet_connect_args_t *a = rpc_args; + int rv; + + rv = vnet_connect (a); + if (rv) + clib_warning (0, "connect returned: %U", format_session_error, rv); + + vec_free (a); + return rv; +} + +static void +hsp_connect () +{ + hsp_main_t *hspm = &hsp_main; + vnet_connect_args_t *a = 0; + hsp_worker_t *wrk; + hsp_session_t *hsp_session; + + vec_validate (a, 0); + clib_memset (a, 0, sizeof (a[0])); + + clib_memcpy (&a->sep_ext, &hspm->connect_sep, sizeof (hspm->connect_sep)); + a->app_index = hspm->app_index; + + /* allocate http session on main thread */ + wrk = hsp_worker_get (0); + hsp_session = hsp_session_alloc (wrk); + a->api_context = hsp_session->session_index; + + session_send_rpc_evt_to_thread_force (transport_cl_thread (), + hsp_connect_rpc, a); +} + +static clib_error_t * +hsp_run (vlib_main_t *vm) +{ + hsp_main_t *hspm = &hsp_main; + vlib_thread_main_t *vtm = vlib_get_thread_main (); + u32 num_threads; + hsp_worker_t *wrk; + uword event_type, *event_data = 0; + clib_error_t *err; + + num_threads = 1 /* main thread */ + vtm->n_threads; + vec_validate (hspm->wrk, num_threads); + vec_foreach (wrk, hspm->wrk) + wrk->thread_index = wrk - hspm->wrk; + + if ((err = hsp_attach ())) + return clib_error_return (0, "http simple post attach: %U", + format_clib_error, err); + + hsp_connect (); + + vlib_process_wait_for_event_or_clock (vm, 10); + event_type = vlib_process_get_events (vm, &event_data); + switch (event_type) + { + case ~0: + err = clib_error_return (0, "error: timeout"); + break; + case HSP_CONNECT_FAILED: + err = clib_error_return (0, "error: failed to connect"); + break; + case HSP_TRANSPORT_CLOSED: + err = clib_error_return (0, "error: transport closed"); + break; + case HSP_REPLY_RECEIVED: + vlib_cli_output (vm, "%v", hspm->http_response); + break; + default: + err = clib_error_return (0, "error: unexpected event %d", event_type); + break; + } + + vec_free (event_data); + return err; +} + +static int +hsp_detach () +{ + hsp_main_t *hspm = &hsp_main; + vnet_app_detach_args_t _da, *da = &_da; + int rv; + + if (!hspm->attached) + return 0; + + da->app_index = hspm->app_index; + da->api_client_index = APP_INVALID_INDEX; + rv = vnet_application_detach (da); + hspm->attached = 0; + hspm->app_index = APP_INVALID_INDEX; + + return rv; +} + +static void +hcc_worker_cleanup (hsp_worker_t *wrk) +{ + pool_free (wrk->sessions); +} + +static void +hsp_cleanup () +{ + hsp_main_t *hspm = &hsp_main; + hsp_worker_t *wrk; + + vec_foreach (wrk, hspm->wrk) + hcc_worker_cleanup (wrk); + + vec_free (hspm->uri); + vec_free (hspm->target); + vec_free (hspm->headers_buf); + vec_free (hspm->data); + vec_free (hspm->http_response); + vec_free (hspm->wrk); +} + +static clib_error_t * +hsp_command_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + hsp_main_t *hspm = &hsp_main; + clib_error_t *err = 0; + unformat_input_t _line_input, *line_input = &_line_input; + u8 *path = 0; + u8 *file_data; + int rv; + + if (hspm->attached) + return clib_error_return (0, "failed: already running!"); + + hspm->use_ptr = 0; + + /* Get a line of input. */ + if (!unformat_user (input, unformat_line_input, line_input)) + return clib_error_return (0, "expected required arguments"); + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "uri %s", &hspm->uri)) + ; + else if (unformat (line_input, "data %v", &hspm->data)) + hspm->is_file = 0; + else if (unformat (line_input, "target %s", &hspm->target)) + ; + else if (unformat (line_input, "file %s", &path)) + hspm->is_file = 1; + else if (unformat (line_input, "use-ptr")) + hspm->use_ptr = 1; + else + { + err = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } + } + + if (!hspm->uri) + { + err = clib_error_return (0, "URI not defined"); + goto done; + } + if (!hspm->target) + { + err = clib_error_return (0, "target not defined"); + goto done; + } + if (!hspm->data) + { + if (path) + { + err = clib_file_contents ((char *) path, &file_data); + if (err) + goto done; + hspm->data = file_data; + } + else + { + err = clib_error_return (0, "data not defined"); + goto done; + } + } + + if ((rv = parse_uri ((char *) hspm->uri, &hspm->connect_sep))) + { + err = + clib_error_return (0, "URI parse error: %U", format_session_error, rv); + goto done; + } + + session_enable_disable_args_t args = { .is_en = 1, + .rt_engine_type = + RT_BACKEND_ENGINE_RULE_TABLE }; + vlib_worker_thread_barrier_sync (vm); + vnet_session_enable_disable (vm, &args); + vlib_worker_thread_barrier_release (vm); + + hspm->cli_node_index = + vlib_get_current_process (vm)->node_runtime.node_index; + + err = hsp_run (vm); + + if ((rv = hsp_detach ())) + { + /* don't override last error */ + if (!err) + err = clib_error_return (0, "detach returned: %U", + format_session_error, rv); + else + clib_warning ("warning: detach returned: %U", format_session_error, + rv); + } + +done: + hsp_cleanup (); + unformat_free (line_input); + return err; +} + +VLIB_CLI_COMMAND (hsp_command, static) = { + .path = "http post", + .short_help = "uri http://<ip-addr> target <origin-form> " + "[data <form-urlencoded> | file <file-path>] [use-ptr]", + .function = hsp_command_fn, + .is_mp_safe = 1, +}; + +static clib_error_t * +hsp_main_init (vlib_main_t *vm) +{ + hsp_main_t *hspm = &hsp_main; + + hspm->app_index = APP_INVALID_INDEX; + hspm->vlib_main = vm; + return 0; +} + +VLIB_INIT_FUNCTION (hsp_main_init); diff --git a/src/plugins/hs_apps/http_tps.c b/src/plugins/hs_apps/http_tps.c index 920f7ea731f..cdeafa5d54a 100644 --- a/src/plugins/hs_apps/http_tps.c +++ b/src/plugins/hs_apps/http_tps.c @@ -17,6 +17,10 @@ #include <vnet/session/application_interface.h> #include <vnet/session/session.h> #include <http/http.h> +#include <http/http_header_names.h> +#include <http/http_content_types.h> + +#define HTS_RX_BUF_SIZE (64 << 10) typedef struct { @@ -26,6 +30,8 @@ typedef struct u64 data_len; u64 data_offset; u32 vpp_session_index; + u64 left_recv; + u64 total_recv; union { /** threshold after which connection is closed */ @@ -34,6 +40,8 @@ typedef struct u32 close_rate; }; u8 *uri; + u8 *rx_buf; + http_header_t *resp_headers; } hts_session_t; typedef struct hts_listen_cfg_ @@ -102,6 +110,8 @@ hts_session_free (hts_session_t *hs) if (htm->debug_level > 0) clib_warning ("Freeing session %u", hs->session_index); + vec_free (hs->rx_buf); + if (CLIB_DEBUG) clib_memset (hs, 0xfa, sizeof (*hs)); @@ -151,7 +161,7 @@ hts_session_tx_zc (hts_session_t *hs, session_t *ts) svm_fifo_add_want_deq_ntf (ts->tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); if (svm_fifo_set_event (ts->tx_fifo)) - session_send_io_evt_to_thread (ts->tx_fifo, SESSION_IO_EVT_TX); + session_program_tx_io_evt (ts->handle, SESSION_IO_EVT_TX); } static void @@ -198,7 +208,7 @@ hts_session_tx_no_zc (hts_session_t *hs, session_t *ts) svm_fifo_add_want_deq_ntf (ts->tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); if (svm_fifo_set_event (ts->tx_fifo)) - session_send_io_evt_to_thread (ts->tx_fifo, SESSION_IO_EVT_TX); + session_program_tx_io_evt (ts->handle, SESSION_IO_EVT_TX); } static inline void @@ -223,22 +233,46 @@ hts_start_send_data (hts_session_t *hs, http_status_code_t status) { http_msg_t msg; session_t *ts; + u8 *headers_buf = 0; + u32 n_segs = 1; + svm_fifo_seg_t seg[2]; int rv; + if (vec_len (hs->resp_headers)) + { + headers_buf = http_serialize_headers (hs->resp_headers); + vec_free (hs->resp_headers); + msg.data.headers_offset = 0; + msg.data.headers_len = vec_len (headers_buf); + seg[1].data = headers_buf; + seg[1].len = msg.data.headers_len; + n_segs = 2; + } + else + { + msg.data.headers_offset = 0; + msg.data.headers_len = 0; + } + msg.type = HTTP_MSG_REPLY; msg.code = status; - msg.content_type = HTTP_CONTENT_APP_OCTET_STREAM; msg.data.type = HTTP_MSG_DATA_INLINE; - msg.data.len = hs->data_len; + msg.data.body_len = hs->data_len; + msg.data.body_offset = msg.data.headers_len; + msg.data.len = msg.data.body_len + msg.data.headers_len; + seg[0].data = (u8 *) &msg; + seg[0].len = sizeof (msg); ts = session_get (hs->vpp_session_index, hs->thread_index); - rv = svm_fifo_enqueue (ts->tx_fifo, sizeof (msg), (u8 *) &msg); - ASSERT (rv == sizeof (msg)); + rv = svm_fifo_enqueue_segments (ts->tx_fifo, seg, n_segs, + 0 /* allow partial */); + vec_free (headers_buf); + ASSERT (rv == (sizeof (msg) + msg.data.headers_len)); - if (!msg.data.len) + if (!msg.data.body_len) { if (svm_fifo_set_event (ts->tx_fifo)) - session_send_io_evt_to_thread (ts->tx_fifo, SESSION_IO_EVT_TX); + session_program_tx_io_evt (ts->handle, SESSION_IO_EVT_TX); return; } @@ -246,7 +280,7 @@ hts_start_send_data (hts_session_t *hs, http_status_code_t status) } static int -try_test_file (hts_session_t *hs, u8 *request) +try_test_file (hts_session_t *hs, u8 *target) { char *test_str = "test_file"; hts_main_t *htm = &hts_main; @@ -254,10 +288,10 @@ try_test_file (hts_session_t *hs, u8 *request) uword file_size; int rc = 0; - if (memcmp (request, test_str, clib_strnlen (test_str, 9))) + if (memcmp (target, test_str, clib_strnlen (test_str, 9))) return -1; - unformat_init_vector (&input, vec_dup (request)); + unformat_init_vector (&input, vec_dup (target)); if (!unformat (&input, "test_file_%U", unformat_memory_size, &file_size)) { rc = -1; @@ -286,6 +320,10 @@ try_test_file (hts_session_t *hs, u8 *request) } } + http_add_header (&hs->resp_headers, + http_header_name_token (HTTP_HEADER_CONTENT_TYPE), + http_content_type_token (HTTP_CONTENT_APP_OCTET_STREAM)); + hts_start_send_data (hs, HTTP_STATUS_OK); done: @@ -294,39 +332,121 @@ done: return rc; } +static inline void +hts_session_rx_body (hts_session_t *hs, session_t *ts) +{ + hts_main_t *htm = &hts_main; + u32 n_deq; + int rv; + + n_deq = svm_fifo_max_dequeue (ts->rx_fifo); + if (!htm->no_zc) + { + svm_fifo_dequeue_drop_all (ts->rx_fifo); + } + else + { + n_deq = clib_min (n_deq, HTS_RX_BUF_SIZE); + rv = svm_fifo_dequeue (ts->rx_fifo, n_deq, hs->rx_buf); + ASSERT (rv == n_deq); + } + hs->left_recv -= n_deq; + + if (hs->close_threshold > 0) + { + if ((f64) (hs->total_recv - hs->left_recv) / hs->total_recv > + hs->close_threshold) + hts_disconnect_transport (hs); + } + + if (hs->left_recv == 0) + { + hts_start_send_data (hs, HTTP_STATUS_OK); + vec_free (hs->rx_buf); + } +} + static int hts_ts_rx_callback (session_t *ts) { + hts_main_t *htm = &hts_main; hts_session_t *hs; - u8 *request = 0; + u8 *target = 0; http_msg_t msg; int rv; hs = hts_session_get (ts->thread_index, ts->opaque); - /* Read the http message header */ - rv = svm_fifo_dequeue (ts->rx_fifo, sizeof (msg), (u8 *) &msg); - ASSERT (rv == sizeof (msg)); - - if (msg.type != HTTP_MSG_REQUEST || msg.method_type != HTTP_REQ_GET) + if (hs->left_recv == 0) { - hts_start_send_data (hs, HTTP_STATUS_METHOD_NOT_ALLOWED); - goto done; - } + hs->data_len = 0; + hs->resp_headers = 0; + hs->rx_buf = 0; - if (!msg.data.len) - { - hts_start_send_data (hs, HTTP_STATUS_BAD_REQUEST); - goto done; - } + /* Read the http message header */ + rv = svm_fifo_dequeue (ts->rx_fifo, sizeof (msg), (u8 *) &msg); + ASSERT (rv == sizeof (msg)); + + if (msg.type != HTTP_MSG_REQUEST) + { + hts_start_send_data (hs, HTTP_STATUS_INTERNAL_ERROR); + goto done; + } + if (msg.method_type != HTTP_REQ_GET && msg.method_type != HTTP_REQ_POST) + { + http_add_header (&hs->resp_headers, + http_header_name_token (HTTP_HEADER_ALLOW), + http_token_lit ("GET, POST")); + hts_start_send_data (hs, HTTP_STATUS_METHOD_NOT_ALLOWED); + goto done; + } - vec_validate (request, msg.data.len - 1); - rv = svm_fifo_dequeue (ts->rx_fifo, msg.data.len, request); + if (msg.data.target_path_len == 0 || + msg.data.target_form != HTTP_TARGET_ORIGIN_FORM) + { + hts_start_send_data (hs, HTTP_STATUS_BAD_REQUEST); + goto done; + } - if (try_test_file (hs, request)) - hts_start_send_data (hs, HTTP_STATUS_NOT_FOUND); + vec_validate (target, msg.data.target_path_len - 1); + rv = svm_fifo_peek (ts->rx_fifo, msg.data.target_path_offset, + msg.data.target_path_len, target); + ASSERT (rv == msg.data.target_path_len); -done: + if (htm->debug_level) + clib_warning ("%s request target: %v", + msg.method_type == HTTP_REQ_GET ? "GET" : "POST", + target); + + if (msg.method_type == HTTP_REQ_GET) + { + if (try_test_file (hs, target)) + hts_start_send_data (hs, HTTP_STATUS_NOT_FOUND); + vec_free (target); + } + else + { + vec_free (target); + if (!msg.data.body_len) + { + hts_start_send_data (hs, HTTP_STATUS_BAD_REQUEST); + goto done; + } + /* drop everything up to body */ + svm_fifo_dequeue_drop (ts->rx_fifo, msg.data.body_offset); + hs->left_recv = msg.data.body_len; + hs->total_recv = msg.data.body_len; + if (htm->no_zc) + vec_validate (hs->rx_buf, HTS_RX_BUF_SIZE - 1); + hts_session_rx_body (hs, ts); + return 0; + } + + done: + svm_fifo_dequeue_drop (ts->rx_fifo, msg.data.len); + } + else + hts_session_rx_body (hs, ts); return 0; } @@ -354,6 +474,7 @@ hts_ts_accept_callback (session_t *ts) hs = hts_session_alloc (ts->thread_index); hs->vpp_session_index = ts->session_index; + hs->left_recv = 0; ts->opaque = hs->session_index; ts->session_state = SESSION_STATE_READY; @@ -717,7 +838,10 @@ start_server: if (htm->app_index == (u32) ~0) { - vnet_session_enable_disable (vm, 1 /* is_enable */); + session_enable_disable_args_t args = { .is_en = 1, + .rt_engine_type = + RT_BACKEND_ENGINE_RULE_TABLE }; + vnet_session_enable_disable (vm, &args); if (hts_create (vm)) { diff --git a/src/plugins/hs_apps/proxy.c b/src/plugins/hs_apps/proxy.c index e8fedf921a5..c7e7b2a653c 100644 --- a/src/plugins/hs_apps/proxy.c +++ b/src/plugins/hs_apps/proxy.c @@ -368,6 +368,13 @@ proxy_rx_callback (session_t * s) u32 max_dequeue, ps_index; int actual_transfer __attribute__ ((unused)); + /* maybe we were already here */ + if (ps->active_open_establishing) + { + clib_spinlock_unlock_if_init (&pm->sessions_lock); + return 0; + } + rx_fifo = s->rx_fifo; tx_fifo = s->tx_fifo; @@ -418,12 +425,12 @@ static void proxy_force_ack (void *handlep) { transport_connection_t *tc; - session_t *ao_s; + session_t *s; - ao_s = session_get_from_handle (pointer_to_uword (handlep)); - if (session_get_transport_proto (ao_s) != TRANSPORT_PROTO_TCP) + s = session_get_from_handle (pointer_to_uword (handlep)); + if (session_get_transport_proto (s) != TRANSPORT_PROTO_TCP) return; - tc = session_get_transport (ao_s); + tc = session_get_transport (s); tcp_send_ack ((tcp_connection_t *) tc); } @@ -619,9 +626,7 @@ static int active_open_tx_callback (session_t * ao_s) { proxy_main_t *pm = &proxy_main; - transport_connection_t *tc; proxy_session_t *ps; - session_t *proxy_s; u32 min_free; min_free = clib_min (svm_fifo_size (ao_s->tx_fifo) >> 3, 128 << 10); @@ -637,14 +642,13 @@ active_open_tx_callback (session_t * ao_s) if (!ps) goto unlock; - if (ps->vpp_server_handle == ~0) + if (ps->vpp_server_handle == SESSION_INVALID_HANDLE) goto unlock; - proxy_s = session_get_from_handle (ps->vpp_server_handle); - /* Force ack on proxy side to update rcv wnd */ - tc = session_get_transport (proxy_s); - tcp_send_ack ((tcp_connection_t *) tc); + void *arg = uword_to_pointer (ps->vpp_server_handle, void *); + session_send_rpc_evt_to_thread ( + session_thread_from_handle (ps->vpp_server_handle), proxy_force_ack, arg); unlock: clib_spinlock_unlock_if_init (&pm->sessions_lock); @@ -915,7 +919,10 @@ proxy_server_create_command_fn (vlib_main_t * vm, unformat_input_t * input, goto done; } - vnet_session_enable_disable (vm, 1 /* turn on session and transport */ ); + session_enable_disable_args_t args = { .is_en = 1, + .rt_engine_type = + RT_BACKEND_ENGINE_RULE_TABLE }; + vnet_session_enable_disable (vm, &args); rv = proxy_server_create (vm); switch (rv) diff --git a/src/plugins/hs_apps/test_builtins.c b/src/plugins/hs_apps/test_builtins.c new file mode 100644 index 00000000000..631c1f1a8a2 --- /dev/null +++ b/src/plugins/hs_apps/test_builtins.c @@ -0,0 +1,179 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2024 Cisco Systems, Inc. + */ + +#include <http_static/http_static.h> +#include <vppinfra/tw_timer_2t_1w_2048sl.h> + +typedef struct +{ + u32 stop_timer_handle; + hss_session_handle_t sh; +} tw_timer_elt_t; + +typedef struct tb_main_ +{ + tw_timer_elt_t *delayed_resps; + tw_timer_wheel_2t_1w_2048sl_t tw; + hss_session_send_fn send_data; +} tb_main_t; + +static tb_main_t tb_main; + +static uword +test_builtins_timer_process (vlib_main_t *vm, vlib_node_runtime_t *rt, + vlib_frame_t *f) +{ + tb_main_t *tbm = &tb_main; + f64 now, timeout = 1.0; + uword *event_data = 0; + uword __clib_unused event_type; + + while (1) + { + vlib_process_wait_for_event_or_clock (vm, timeout); + now = vlib_time_now (vm); + event_type = vlib_process_get_events (vm, (uword **) &event_data); + + /* expire timers */ + tw_timer_expire_timers_2t_1w_2048sl (&tbm->tw, now); + + vec_reset_length (event_data); + } + return 0; +} + +VLIB_REGISTER_NODE (test_builtins_timer_process_node) = { + .function = test_builtins_timer_process, + .type = VLIB_NODE_TYPE_PROCESS, + .name = "test-builtins-timer-process", + .state = VLIB_NODE_STATE_DISABLED, +}; + +static void +send_data_to_hss (hss_session_handle_t sh, u8 *data) +{ + tb_main_t *tbm = &tb_main; + hss_url_handler_args_t args = {}; + + args.sh = sh; + args.data = data; + args.data_len = vec_len (data); + args.ct = HTTP_CONTENT_TEXT_PLAIN; + args.sc = HTTP_STATUS_OK; + args.free_vec_data = 1; + + tbm->send_data (&args); +} + +static hss_url_handler_rc_t +handle_get_test1 (hss_url_handler_args_t *args) +{ + u8 *data; + + clib_warning ("get request on test1"); + data = format (0, "hello"); + send_data_to_hss (args->sh, data); + + return HSS_URL_HANDLER_ASYNC; +} + +static hss_url_handler_rc_t +handle_get_test2 (hss_url_handler_args_t *args) +{ + u8 *data; + + clib_warning ("get request on test2"); + data = format (0, "some data"); + send_data_to_hss (args->sh, data); + + return HSS_URL_HANDLER_ASYNC; +} + +static void +delayed_resp_cb (u32 *expired_timers) +{ + tb_main_t *tbm = &tb_main; + int i; + u32 pool_index; + tw_timer_elt_t *e; + u8 *data; + + for (i = 0; i < vec_len (expired_timers); i++) + { + pool_index = expired_timers[i] & 0x7FFFFFFF; + e = pool_elt_at_index (tbm->delayed_resps, pool_index); + clib_warning ("sending delayed data"); + data = format (0, "delayed data"); + send_data_to_hss (e->sh, data); + pool_put (tbm->delayed_resps, e); + } +} + +static hss_url_handler_rc_t +handle_get_test_delayed (hss_url_handler_args_t *args) +{ + tb_main_t *tbm = &tb_main; + tw_timer_elt_t *e; + + clib_warning ("get request on test_delayed"); + pool_get (tbm->delayed_resps, e); + e->sh = args->sh; + e->stop_timer_handle = + tw_timer_start_2t_1w_2048sl (&tbm->tw, e - tbm->delayed_resps, 0, 5); + + return HSS_URL_HANDLER_ASYNC; +} + +static hss_url_handler_rc_t +handle_post_test3 (hss_url_handler_args_t *args) +{ + send_data_to_hss (args->sh, 0); + return HSS_URL_HANDLER_ASYNC; +} + +static void +test_builtins_init (vlib_main_t *vm) +{ + tb_main_t *tbm = &tb_main; + hss_register_url_fn fp; + vlib_node_t *n; + + fp = vlib_get_plugin_symbol ("http_static_plugin.so", + "hss_register_url_handler"); + + if (fp == 0) + { + clib_warning ("http_static_plugin.so not loaded..."); + return; + } + + (*fp) (handle_get_test1, "test1", HTTP_REQ_GET); + (*fp) (handle_get_test2, "test2", HTTP_REQ_GET); + (*fp) (handle_get_test_delayed, "test_delayed", HTTP_REQ_GET); + (*fp) (handle_post_test3, "test3", HTTP_REQ_POST); + + tbm->send_data = + vlib_get_plugin_symbol ("http_static_plugin.so", "hss_session_send_data"); + + tw_timer_wheel_init_2t_1w_2048sl (&tbm->tw, delayed_resp_cb, 1.0, ~0); + + vlib_node_set_state (vm, test_builtins_timer_process_node.index, + VLIB_NODE_STATE_POLLING); + n = vlib_get_node (vm, test_builtins_timer_process_node.index); + vlib_start_process (vm, n->runtime_index); +} + +static clib_error_t * +test_builtins_enable_command_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + test_builtins_init (vm); + return 0; +} + +VLIB_CLI_COMMAND (test_builtins_enable_command, static) = { + .path = "test-url-handler enable", + .short_help = "test-url-handler enable", + .function = test_builtins_enable_command_fn, +}; diff --git a/src/plugins/hs_apps/vcl/vcl_test.h b/src/plugins/hs_apps/vcl/vcl_test.h index 0ce27ef84e2..11667fb144a 100644 --- a/src/plugins/hs_apps/vcl/vcl_test.h +++ b/src/plugins/hs_apps/vcl/vcl_test.h @@ -124,7 +124,7 @@ typedef struct typedef struct { - const vcl_test_proto_vft_t *protos[VPPCOM_PROTO_SRTP + 1]; + const vcl_test_proto_vft_t *protos[VPPCOM_PROTO_HTTP + 1]; uint32_t ckpair_index; hs_test_cfg_t cfg; vcl_test_wrk_t *wrk; @@ -420,6 +420,39 @@ vcl_test_write (vcl_test_session_t *ts, void *buf, uint32_t nbytes) return (tx_bytes); } +static inline int +vcl_test_write_ds (vcl_test_session_t *ts) +{ + vcl_test_stats_t *stats = &ts->stats; + int tx_bytes; + + do + { + stats->tx_xacts++; + if (ts->ds[1].len) + tx_bytes = vppcom_session_write_segments (ts->fd, ts->ds, 2); + else + tx_bytes = vppcom_session_write_segments (ts->fd, ts->ds, 1); + + if (tx_bytes < 0) + errno = -tx_bytes; + if ((tx_bytes == 0) || + ((tx_bytes < 0) && ((errno == EAGAIN) || (errno == EWOULDBLOCK)))) + stats->rx_eagain++; + } + while ((tx_bytes == 0) || + ((tx_bytes < 0) && ((errno == EAGAIN) || (errno == EWOULDBLOCK)))); + + if (tx_bytes < 0) + { + vterr ("vppcom_session_write_segments()", -errno); + } + else + stats->tx_bytes += tx_bytes; + + return (tx_bytes); +} + static inline void dump_help (void) { diff --git a/src/plugins/hs_apps/vcl/vcl_test_client.c b/src/plugins/hs_apps/vcl/vcl_test_client.c index a4a10b562ff..8bac1f00b9d 100644 --- a/src/plugins/hs_apps/vcl/vcl_test_client.c +++ b/src/plugins/hs_apps/vcl/vcl_test_client.c @@ -419,13 +419,8 @@ vtc_worker_run_select (vcl_test_client_worker_t *wrk) if (vcm->incremental_stats) vtc_inc_stats_check (ts); } - if ((!check_rx && ts->stats.tx_bytes >= ts->cfg.total_bytes) || - (check_rx && ts->stats.rx_bytes >= ts->cfg.total_bytes)) - { - clock_gettime (CLOCK_REALTIME, &ts->stats.stop); - ts->is_done = 1; - n_active_sessions--; - } + if (vtc_session_check_is_done (ts, check_rx)) + n_active_sessions -= 1; } } diff --git a/src/plugins/hs_apps/vcl/vcl_test_protos.c b/src/plugins/hs_apps/vcl/vcl_test_protos.c index cd1ac2b24f4..9c81c5f17a1 100644 --- a/src/plugins/hs_apps/vcl/vcl_test_protos.c +++ b/src/plugins/hs_apps/vcl/vcl_test_protos.c @@ -14,6 +14,23 @@ */ #include <hs_apps/vcl/vcl_test.h> +#include <http/http.h> +#include <http/http_header_names.h> +#include <http/http_content_types.h> + +typedef enum vcl_test_http_state_ +{ + VCL_TEST_HTTP_IDLE = 0, + VCL_TEST_HTTP_IN_PROGRESS, + VCL_TEST_HTTP_COMPLETED, +} vcl_test_http_state_t; + +typedef struct vcl_test_http_ctx_t +{ + u8 is_server; + vcl_test_http_state_t test_state; + u64 rem_data; +} vcl_test_http_ctx_t; static int vt_tcp_connect (vcl_test_session_t *ts, vppcom_endpt_t *endpt) @@ -978,6 +995,418 @@ static const vcl_test_proto_vft_t vcl_test_srtp = { VCL_TEST_REGISTER_PROTO (VPPCOM_PROTO_SRTP, vcl_test_srtp); +static void +vt_http_session_init (vcl_test_session_t *ts, u8 is_server) +{ + vcl_test_http_ctx_t *http_ctx; + + http_ctx = malloc (sizeof (vcl_test_http_ctx_t)); + memset (http_ctx, 0, sizeof (*http_ctx)); + http_ctx->is_server = is_server; + ts->opaque = http_ctx; +} + +static inline void +vt_http_send_reply_msg (vcl_test_session_t *ts, http_status_code_t status) +{ + http_msg_t msg; + int rv = 0; + + memset (&msg, 0, sizeof (http_msg_t)); + msg.type = HTTP_MSG_REPLY; + msg.code = status; + + vppcom_data_segment_t segs[1] = { { (u8 *) &msg, sizeof (msg) } }; + + do + { + rv = vppcom_session_write_segments (ts->fd, segs, 1); + + if (rv < 0) + { + errno = -rv; + if (errno == EAGAIN || errno == EWOULDBLOCK) + continue; + + vterr ("vppcom_session_write()", -errno); + break; + } + } + while (rv <= 0); +} + +static inline int +vt_process_http_server_read_msg (vcl_test_session_t *ts, void *buf, + uint32_t nbytes) +{ + http_msg_t msg; + u8 *target_path = 0; + vcl_test_http_ctx_t *vcl_test_http_ctx = (vcl_test_http_ctx_t *) ts->opaque; + vcl_test_stats_t *stats = &ts->stats; + int rv = 0; + + do + { + stats->rx_xacts++; + rv = vppcom_session_read (ts->fd, buf, nbytes); + + if (rv <= 0) + { + errno = -rv; + if (errno == EAGAIN || errno == EWOULDBLOCK) + { + stats->rx_eagain++; + continue; + } + + vterr ("vppcom_session_read()", -errno); + return 0; + } + + if (PREDICT_TRUE (vcl_test_http_ctx->test_state == + VCL_TEST_HTTP_IN_PROGRESS)) + { + vcl_test_http_ctx->rem_data -= rv; + + if (vcl_test_http_ctx->rem_data == 0) + { + vcl_test_http_ctx->test_state = VCL_TEST_HTTP_COMPLETED; + vt_http_send_reply_msg (ts, HTTP_STATUS_OK); + } + } + else if (PREDICT_FALSE (vcl_test_http_ctx->test_state == + VCL_TEST_HTTP_IDLE)) + { + msg = *(http_msg_t *) buf; + + /* verify that we have received http post request from client */ + if (msg.type != HTTP_MSG_REQUEST || msg.method_type != HTTP_REQ_POST) + { + vt_http_send_reply_msg (ts, HTTP_STATUS_METHOD_NOT_ALLOWED); + vterr ("error! only POST requests allowed from client", 0); + return 0; + } + + if (msg.data.target_form != HTTP_TARGET_ORIGIN_FORM) + { + vt_http_send_reply_msg (ts, HTTP_STATUS_BAD_REQUEST); + vterr ("error! http target not in origin form", 0); + return 0; + } + + /* validate target path syntax */ + if (msg.data.target_path_len) + { + vec_validate (target_path, msg.data.target_path_len - 1); + memcpy (target_path, + buf + sizeof (msg) + msg.data.target_path_offset - 1, + msg.data.target_path_len + 1); + if (http_validate_abs_path_syntax (target_path, 0)) + { + vt_http_send_reply_msg (ts, HTTP_STATUS_BAD_REQUEST); + vterr ("error! target path is not absolute", 0); + vec_free (target_path); + return 0; + } + vec_free (target_path); + } + + /* read body */ + if (msg.data.body_len) + { + vcl_test_http_ctx->rem_data = msg.data.body_len; + /* | <http_msg_t> | <target> | <headers> | <body> | */ + vcl_test_http_ctx->rem_data -= + (rv - sizeof (msg) - msg.data.body_offset); + vcl_test_http_ctx->test_state = VCL_TEST_HTTP_IN_PROGRESS; + } + } + + if (rv < nbytes) + stats->rx_incomp++; + } + while (rv <= 0); + + stats->rx_bytes += rv; + return (rv); +} + +static inline int +vt_process_http_client_read_msg (vcl_test_session_t *ts, void *buf, + uint32_t nbytes) +{ + http_msg_t msg; + int rv = 0; + + do + { + rv = vppcom_session_read (ts->fd, buf, nbytes); + + if (rv < 0) + { + errno = -rv; + if (errno == EAGAIN || errno == EWOULDBLOCK) + continue; + + vterr ("vppcom_session_read()", -errno); + break; + } + } + while (!rv); + + msg = *(http_msg_t *) buf; + + if (msg.type == HTTP_MSG_REPLY && msg.code == HTTP_STATUS_OK) + vtinf ("received 200 OK from server"); + else + vterr ("received unexpected reply from server", 0); + + return (rv); +} + +static inline int +vt_process_http_client_write_msg (vcl_test_session_t *ts, void *buf, + uint32_t nbytes) +{ + http_msg_t msg; + http_header_t *req_headers = 0; + u8 *headers_buf = 0; + u8 *target; + vcl_test_http_ctx_t *vcl_test_http_ctx = (vcl_test_http_ctx_t *) ts->opaque; + vcl_test_stats_t *stats = &ts->stats; + int rv = 0; + + if (PREDICT_TRUE (vcl_test_http_ctx->test_state == + VCL_TEST_HTTP_IN_PROGRESS)) + { + do + { + rv = vppcom_session_write ( + ts->fd, buf, clib_min (nbytes, vcl_test_http_ctx->rem_data)); + + if (rv <= 0) + { + errno = -rv; + if (errno == EAGAIN || errno == EWOULDBLOCK) + { + stats->tx_eagain++; + continue; + } + + vterr ("vppcom_session_write()", -errno); + return 0; + } + + vcl_test_http_ctx->rem_data -= rv; + + if (vcl_test_http_ctx->rem_data == 0) + { + vcl_test_http_ctx->test_state = VCL_TEST_HTTP_COMPLETED; + vtinf ("client finished sending %ld bytes of data", + ts->cfg.total_bytes); + } + + if (rv < nbytes) + stats->tx_incomp++; + } + while (rv <= 0); + } + + else if (PREDICT_FALSE (vcl_test_http_ctx->test_state == VCL_TEST_HTTP_IDLE)) + { + http_add_header ( + &req_headers, http_header_name_token (HTTP_HEADER_CONTENT_TYPE), + http_content_type_token (HTTP_CONTENT_APP_OCTET_STREAM)); + headers_buf = http_serialize_headers (req_headers); + vec_free (req_headers); + + memset (&msg, 0, sizeof (http_msg_t)); + msg.type = HTTP_MSG_REQUEST; + msg.method_type = HTTP_REQ_POST; + + /* target */ + msg.data.target_form = HTTP_TARGET_ORIGIN_FORM; + target = (u8 *) "/vcl_test_http\0"; + msg.data.target_path_len = strlen ((char *) target); + + /* headers */ + msg.data.headers_offset = msg.data.target_path_len; + msg.data.headers_len = vec_len (headers_buf); + + /* body */ + msg.data.body_offset = msg.data.headers_offset + msg.data.headers_len; + msg.data.body_len = ts->cfg.total_bytes; + + msg.data.len = + msg.data.target_path_len + msg.data.headers_len + msg.data.body_len; + msg.data.type = HTTP_MSG_DATA_INLINE; + + vppcom_data_segment_t segs[3] = { { (u8 *) &msg, sizeof (msg) }, + { target, strlen ((char *) target) }, + { headers_buf, + vec_len (headers_buf) } }; + + do + { + rv = vppcom_session_write_segments (ts->fd, segs, 3); + + if (rv <= 0) + { + errno = -rv; + if (errno == EAGAIN || errno == EWOULDBLOCK) + { + stats->tx_eagain++; + continue; + } + + vterr ("vppcom_session_write_segments()", -errno); + vec_free (headers_buf); + return 0; + } + } + while (rv <= 0); + + vcl_test_http_ctx->test_state = VCL_TEST_HTTP_IN_PROGRESS; + vcl_test_http_ctx->rem_data = ts->cfg.total_bytes; + vec_free (headers_buf); + } + + stats->tx_bytes += rv; + return (rv); +} + +static inline int +vt_process_http_server_write_msg (vcl_test_session_t *ts, void *buf, + uint32_t nbytes) +{ + return 0; +} + +static inline int +vt_http_read (vcl_test_session_t *ts, void *buf, uint32_t nbytes) +{ + vcl_test_http_ctx_t *vcl_test_http_ctx = (vcl_test_http_ctx_t *) ts->opaque; + + if (vcl_test_http_ctx->is_server) + return vt_process_http_server_read_msg (ts, buf, nbytes); + else + return vt_process_http_client_read_msg (ts, buf, nbytes); +} + +static inline int +vt_http_write (vcl_test_session_t *ts, void *buf, uint32_t nbytes) +{ + vcl_test_http_ctx_t *vcl_test_http_ctx = (vcl_test_http_ctx_t *) ts->opaque; + + if (vcl_test_http_ctx->is_server) + return vt_process_http_server_write_msg (ts, buf, nbytes); + else + return vt_process_http_client_write_msg (ts, buf, nbytes); +} + +static int +vt_http_connect (vcl_test_session_t *ts, vppcom_endpt_t *endpt) +{ + uint32_t flags, flen; + int rv; + + ts->fd = vppcom_session_create (VPPCOM_PROTO_HTTP, ts->noblk_connect); + if (ts->fd < 0) + { + vterr ("vppcom_session_create()", ts->fd); + return ts->fd; + } + + rv = vppcom_session_connect (ts->fd, endpt); + if (rv < 0 && rv != VPPCOM_EINPROGRESS) + { + vterr ("vppcom_session_connect()", rv); + return rv; + } + + ts->read = vt_http_read; + ts->write = vt_http_write; + + if (!ts->noblk_connect) + { + flags = O_NONBLOCK; + flen = sizeof (flags); + vppcom_session_attr (ts->fd, VPPCOM_ATTR_SET_FLAGS, &flags, &flen); + vtinf ("Test session %d (fd %d) connected.", ts->session_index, ts->fd); + } + + vt_http_session_init (ts, 0 /* is_server */); + + return 0; +} + +static int +vt_http_listen (vcl_test_session_t *ts, vppcom_endpt_t *endpt) +{ + int rv; + + ts->fd = vppcom_session_create (VPPCOM_PROTO_HTTP, 1 /* is_nonblocking */); + if (ts->fd < 0) + { + vterr ("vppcom_session_create()", ts->fd); + return ts->fd; + } + + rv = vppcom_session_bind (ts->fd, endpt); + if (rv < 0) + { + vterr ("vppcom_session_bind()", rv); + return rv; + } + + rv = vppcom_session_listen (ts->fd, 10); + if (rv < 0) + { + vterr ("vppcom_session_listen()", rv); + return rv; + } + + return 0; +} + +static int +vt_http_accept (int listen_fd, vcl_test_session_t *ts) +{ + int client_fd; + + client_fd = vppcom_session_accept (listen_fd, &ts->endpt, 0); + if (client_fd < 0) + { + vterr ("vppcom_session_accept()", client_fd); + return client_fd; + } + + ts->fd = client_fd; + ts->is_open = 1; + ts->read = vt_http_read; + ts->write = vt_http_write; + + vt_http_session_init (ts, 1 /* is_server */); + + return 0; +} + +static int +vt_http_close (vcl_test_session_t *ts) +{ + free (ts->opaque); + return 0; +} + +static const vcl_test_proto_vft_t vcl_test_http = { + .open = vt_http_connect, + .listen = vt_http_listen, + .accept = vt_http_accept, + .close = vt_http_close, +}; + +VCL_TEST_REGISTER_PROTO (VPPCOM_PROTO_HTTP, vcl_test_http); + /* * fd.io coding-style-patch-verification: ON * diff --git a/src/plugins/hs_apps/vcl/vcl_test_server.c b/src/plugins/hs_apps/vcl/vcl_test_server.c index d17a2089ba7..5de53173784 100644 --- a/src/plugins/hs_apps/vcl/vcl_test_server.c +++ b/src/plugins/hs_apps/vcl/vcl_test_server.c @@ -282,11 +282,7 @@ vts_server_process_rx (vcl_test_session_t *conn, int rx_bytes) if (conn->cfg.test == HS_TEST_TYPE_BI) { if (vsm->use_ds) - { - (void) vcl_test_write (conn, conn->ds[0].data, conn->ds[0].len); - if (conn->ds[1].len) - (void) vcl_test_write (conn, conn->ds[1].data, conn->ds[1].len); - } + (void) vcl_test_write_ds (conn); else (void) vcl_test_write (conn, conn->rxbuf, rx_bytes); } diff --git a/src/plugins/http/CMakeLists.txt b/src/plugins/http/CMakeLists.txt index d9cd84a3955..c51a7dce36d 100644 --- a/src/plugins/http/CMakeLists.txt +++ b/src/plugins/http/CMakeLists.txt @@ -16,4 +16,5 @@ add_vpp_plugin(http http.c http_buffer.c http_timer.c + http_test.c ) diff --git a/src/plugins/http/http.c b/src/plugins/http/http.c index 893dd877c29..4f741c2e6b4 100644 --- a/src/plugins/http/http.c +++ b/src/plugins/http/http.c @@ -16,11 +16,11 @@ #include <http/http.h> #include <vnet/session/session.h> #include <http/http_timer.h> +#include <http/http_status_codes.h> static http_main_t http_main; #define HTTP_FIFO_THRESH (16 << 10) -#define CONTENT_LEN_STR "Content-Length: " /* HTTP state machine result */ typedef enum http_sm_result_t_ @@ -30,24 +30,12 @@ typedef enum http_sm_result_t_ HTTP_SM_ERROR = -1, } http_sm_result_t; -const char *http_status_code_str[] = { -#define _(c, s, str) str, - foreach_http_status_code -#undef _ -}; - -const char *http_content_type_str[] = { -#define _(s, ext, str) str, - foreach_http_content_type -#undef _ -}; - const http_buffer_type_t msg_to_buf_type[] = { [HTTP_MSG_DATA_INLINE] = HTTP_BUFFER_FIFO, [HTTP_MSG_DATA_PTR] = HTTP_BUFFER_PTR, }; -u8 * +static u8 * format_http_state (u8 *s, va_list *va) { http_state_t state = va_arg (*va, http_state_t); @@ -83,6 +71,24 @@ format_http_state (u8 *s, va_list *va) } \ while (0) +static inline int +http_state_is_tx_valid (http_conn_t *hc) +{ + http_state_t state = hc->http_state; + return (state == HTTP_STATE_APP_IO_MORE_DATA || + state == HTTP_STATE_WAIT_APP_REPLY || + state == HTTP_STATE_WAIT_APP_METHOD); +} + +static inline int +http_state_is_rx_valid (http_conn_t *hc) +{ + http_state_t state = hc->http_state; + return (state == HTTP_STATE_WAIT_SERVER_REPLY || + state == HTTP_STATE_CLIENT_IO_MORE_DATA || + state == HTTP_STATE_WAIT_CLIENT_METHOD); +} + static inline http_worker_t * http_worker_get (u32 thread_index) { @@ -364,47 +370,54 @@ http_ts_reset_callback (session_t *ts) */ static const char *http_error_template = "HTTP/1.1 %s\r\n" "Date: %U GMT\r\n" - "Content-Type: text/html\r\n" "Connection: close\r\n" - "Pragma: no-cache\r\n" "Content-Length: 0\r\n\r\n"; -static const char *http_redirect_template = "HTTP/1.1 %s\r\n"; - /** * http response boilerplate */ static const char *http_response_template = "HTTP/1.1 %s\r\n" "Date: %U GMT\r\n" - "Expires: %U GMT\r\n" - "Server: %s\r\n" - "Content-Type: %s\r\n" - "Content-Length: %lu\r\n\r\n"; + "Server: %v\r\n" + "Content-Length: %llu\r\n" + "%s"; + +/** + * http request boilerplate + */ +static const char *http_get_request_template = "GET %s HTTP/1.1\r\n" + "Host: %v\r\n" + "User-Agent: %v\r\n" + "%s"; -static const char *http_request_template = "GET %s HTTP/1.1\r\n" - "User-Agent: %s\r\n" - "Accept: */*\r\n"; +static const char *http_post_request_template = "POST %s HTTP/1.1\r\n" + "Host: %v\r\n" + "User-Agent: %v\r\n" + "Content-Length: %llu\r\n" + "%s"; static u32 -http_send_data (http_conn_t *hc, u8 *data, u32 length, u32 offset) +http_send_data (http_conn_t *hc, u8 *data, u32 length) { const u32 max_burst = 64 << 10; session_t *ts; u32 to_send; - int sent; + int rv; ts = session_get_from_handle (hc->h_tc_session_handle); - to_send = clib_min (length - offset, max_burst); - sent = svm_fifo_enqueue (ts->tx_fifo, to_send, data + offset); - - if (sent <= 0) - return offset; + to_send = clib_min (length, max_burst); + rv = svm_fifo_enqueue (ts->tx_fifo, to_send, data); + if (rv <= 0) + { + clib_warning ("svm_fifo_enqueue failed, rv %d", rv); + return 0; + } if (svm_fifo_set_event (ts->tx_fifo)) - session_send_io_evt_to_thread (ts->tx_fifo, SESSION_IO_EVT_TX); + session_program_tx_io_evt (ts->handle, SESSION_IO_EVT_TX); - return (offset + sent); + return rv; } static void @@ -420,37 +433,70 @@ http_send_error (http_conn_t *hc, http_status_code_t ec) now = clib_timebase_now (&hm->timebase); data = format (0, http_error_template, http_status_code_str[ec], format_clib_timebase_time, now); - http_send_data (hc, data, vec_len (data), 0); + HTTP_DBG (1, "%v", data); + http_send_data (hc, data, vec_len (data)); vec_free (data); } static int http_read_message (http_conn_t *hc) { - u32 max_deq, cursize; + u32 max_deq; session_t *ts; int n_read; ts = session_get_from_handle (hc->h_tc_session_handle); - cursize = vec_len (hc->rx_buf); max_deq = svm_fifo_max_dequeue (ts->rx_fifo); if (PREDICT_FALSE (max_deq == 0)) return -1; - vec_validate (hc->rx_buf, cursize + max_deq - 1); - n_read = svm_fifo_dequeue (ts->rx_fifo, max_deq, hc->rx_buf + cursize); + vec_validate (hc->rx_buf, max_deq - 1); + n_read = svm_fifo_peek (ts->rx_fifo, 0, max_deq, hc->rx_buf); ASSERT (n_read == max_deq); + HTTP_DBG (1, "read %u bytes from rx_fifo", n_read); + + return 0; +} + +static void +http_read_message_drop (http_conn_t *hc, u32 len) +{ + session_t *ts; + + ts = session_get_from_handle (hc->h_tc_session_handle); + svm_fifo_dequeue_drop (ts->rx_fifo, len); + vec_reset_length (hc->rx_buf); if (svm_fifo_is_empty (ts->rx_fifo)) svm_fifo_unset_event (ts->rx_fifo); +} - vec_set_len (hc->rx_buf, cursize + n_read); - return 0; +static void +http_read_message_drop_all (http_conn_t *hc) +{ + session_t *ts; + + ts = session_get_from_handle (hc->h_tc_session_handle); + svm_fifo_dequeue_drop_all (ts->rx_fifo); + vec_reset_length (hc->rx_buf); + + if (svm_fifo_is_empty (ts->rx_fifo)) + svm_fifo_unset_event (ts->rx_fifo); } -static int -v_find_index (u8 *vec, u32 offset, char *str) +/** + * @brief Find the first occurrence of the string in the vector. + * + * @param vec The vector to be scanned. + * @param offset Search offset in the vector. + * @param num Maximum number of characters to be searched if non-zero. + * @param str The string to be searched. + * + * @return @c -1 if the string is not found within the vector; index otherwise. + */ +static inline int +v_find_index (u8 *vec, u32 offset, u32 num, char *str) { int start_index = offset; u32 slen = (u32) strnlen_s_inline (str, 16); @@ -461,7 +507,15 @@ v_find_index (u8 *vec, u32 offset, char *str) if (vlen <= slen) return -1; - for (; start_index < (vlen - slen); start_index++) + int end_index = vlen - slen; + if (num) + { + if (num < slen) + return -1; + end_index = clib_min (end_index, offset + num - slen); + } + + for (; start_index <= end_index; start_index++) { if (!memcmp (vec + start_index, str, slen)) return start_index; @@ -470,50 +524,389 @@ v_find_index (u8 *vec, u32 offset, char *str) return -1; } +static void +http_identify_optional_query (http_conn_t *hc) +{ + int i; + for (i = hc->target_path_offset; + i < (hc->target_path_offset + hc->target_path_len); i++) + { + if (hc->rx_buf[i] == '?') + { + hc->target_query_offset = i + 1; + hc->target_query_len = hc->target_path_offset + hc->target_path_len - + hc->target_query_offset; + hc->target_path_len = hc->target_path_len - hc->target_query_len - 1; + break; + } + } +} + static int -http_parse_header (http_conn_t *hc, int *content_length) +http_get_target_form (http_conn_t *hc) { - unformat_input_t input; - int i, len; - u8 *line; + int i; + + /* "*" */ + if ((hc->rx_buf[hc->target_path_offset] == '*') && + (hc->target_path_len == 1)) + { + hc->target_form = HTTP_TARGET_ASTERISK_FORM; + return 0; + } + + /* 1*( "/" segment ) [ "?" query ] */ + if (hc->rx_buf[hc->target_path_offset] == '/') + { + /* drop leading slash */ + hc->target_path_len--; + hc->target_path_offset++; + hc->target_form = HTTP_TARGET_ORIGIN_FORM; + http_identify_optional_query (hc); + return 0; + } - i = v_find_index (hc->rx_buf, hc->rx_buf_offset, CONTENT_LEN_STR); + /* scheme "://" host [ ":" port ] *( "/" segment ) [ "?" query ] */ + i = v_find_index (hc->rx_buf, hc->target_path_offset, hc->target_path_len, + "://"); + if (i > 0) + { + hc->target_form = HTTP_TARGET_ABSOLUTE_FORM; + http_identify_optional_query (hc); + return 0; + } + + /* host ":" port */ + for (i = hc->target_path_offset; + i < (hc->target_path_offset + hc->target_path_len); i++) + { + if ((hc->rx_buf[i] == ':') && (isdigit (hc->rx_buf[i + 1]))) + { + hc->target_form = HTTP_TARGET_AUTHORITY_FORM; + return 0; + } + } + + return -1; +} + +static int +http_parse_request_line (http_conn_t *hc, http_status_code_t *ec) +{ + int i, target_len; + u32 next_line_offset, method_offset; + + /* request-line = method SP request-target SP HTTP-version CRLF */ + i = v_find_index (hc->rx_buf, 8, 0, "\r\n"); if (i < 0) { - clib_warning ("cannot find '%s' in the header!", CONTENT_LEN_STR); + clib_warning ("request line incomplete"); + *ec = HTTP_STATUS_BAD_REQUEST; return -1; } + HTTP_DBG (0, "request line length: %d", i); + hc->control_data_len = i + 2; + next_line_offset = hc->control_data_len; - hc->rx_buf_offset = i; + /* there should be at least one more CRLF */ + if (vec_len (hc->rx_buf) < (next_line_offset + 2)) + { + clib_warning ("malformed message, too short"); + *ec = HTTP_STATUS_BAD_REQUEST; + return -1; + } - i = v_find_index (hc->rx_buf, hc->rx_buf_offset, "\n"); + /* + * RFC9112 2.2: + * In the interest of robustness, a server that is expecting to receive and + * parse a request-line SHOULD ignore at least one empty line (CRLF) + * received prior to the request-line. + */ + method_offset = hc->rx_buf[0] == '\r' && hc->rx_buf[1] == '\n' ? 2 : 0; + /* parse method */ + if (!memcmp (hc->rx_buf + method_offset, "GET ", 4)) + { + HTTP_DBG (0, "GET method"); + hc->method = HTTP_REQ_GET; + hc->target_path_offset = method_offset + 4; + } + else if (!memcmp (hc->rx_buf + method_offset, "POST ", 5)) + { + HTTP_DBG (0, "POST method"); + hc->method = HTTP_REQ_POST; + hc->target_path_offset = method_offset + 5; + } + else + { + if (hc->rx_buf[method_offset] - 'A' <= 'Z' - hc->rx_buf[method_offset]) + { + clib_warning ("not method name: %8v", hc->rx_buf); + *ec = HTTP_STATUS_BAD_REQUEST; + return -1; + } + else + { + clib_warning ("method not implemented: %8v", hc->rx_buf); + *ec = HTTP_STATUS_NOT_IMPLEMENTED; + return -1; + } + } + + /* find version */ + i = v_find_index (hc->rx_buf, next_line_offset - 11, 11, " HTTP/"); + if (i < 0) + { + clib_warning ("HTTP version not present"); + *ec = HTTP_STATUS_BAD_REQUEST; + return -1; + } + /* verify major version */ + if (isdigit (hc->rx_buf[i + 6])) + { + if (hc->rx_buf[i + 6] != '1') + { + clib_warning ("HTTP major version '%c' not supported", + hc->rx_buf[i + 6]); + *ec = HTTP_STATUS_HTTP_VERSION_NOT_SUPPORTED; + return -1; + } + } + else + { + clib_warning ("HTTP major version '%c' is not digit", hc->rx_buf[i + 6]); + *ec = HTTP_STATUS_BAD_REQUEST; + return -1; + } + + /* parse request-target */ + HTTP_DBG (0, "http at %d", i); + target_len = i - hc->target_path_offset; + HTTP_DBG (0, "target_len %d", target_len); + if (target_len < 1) + { + clib_warning ("request-target not present"); + *ec = HTTP_STATUS_BAD_REQUEST; + return -1; + } + hc->target_path_len = target_len; + hc->target_query_offset = 0; + hc->target_query_len = 0; + if (http_get_target_form (hc)) + { + clib_warning ("invalid target"); + *ec = HTTP_STATUS_BAD_REQUEST; + return -1; + } + HTTP_DBG (0, "request-target path length: %u", hc->target_path_len); + HTTP_DBG (0, "request-target path offset: %u", hc->target_path_offset); + HTTP_DBG (0, "request-target query length: %u", hc->target_query_len); + HTTP_DBG (0, "request-target query offset: %u", hc->target_query_offset); + + /* set buffer offset to nex line start */ + hc->rx_buf_offset = next_line_offset; + + return 0; +} + +#define expect_char(c) \ + if (*p++ != c) \ + { \ + clib_warning ("unexpected character"); \ + return -1; \ + } + +#define parse_int(val, mul) \ + do \ + { \ + if (!isdigit (*p)) \ + { \ + clib_warning ("expected digit"); \ + return -1; \ + } \ + val += mul * (*p++ - '0'); \ + } \ + while (0) + +static int +http_parse_status_line (http_conn_t *hc) +{ + int i; + u32 next_line_offset; + u8 *p, *end; + u16 status_code = 0; + + i = v_find_index (hc->rx_buf, 0, 0, "\r\n"); + /* status-line = HTTP-version SP status-code SP [ reason-phrase ] CRLF */ if (i < 0) { - clib_warning ("end of line missing; incomplete data"); + clib_warning ("status line incomplete"); + return -1; + } + HTTP_DBG (0, "status line length: %d", i); + if (i < 12) + { + clib_warning ("status line too short (%d)", i); return -1; } + hc->control_data_len = i + 2; + next_line_offset = hc->control_data_len; + p = hc->rx_buf; + end = hc->rx_buf + i; + + /* there should be at least one more CRLF */ + if (vec_len (hc->rx_buf) < (next_line_offset + 2)) + { + clib_warning ("malformed message, too short"); + return -1; + } + + /* parse version */ + expect_char ('H'); + expect_char ('T'); + expect_char ('T'); + expect_char ('P'); + expect_char ('/'); + expect_char ('1'); + expect_char ('.'); + if (!isdigit (*p++)) + { + clib_warning ("invalid HTTP minor version"); + return -1; + } + + /* skip space(s) */ + if (*p != ' ') + { + clib_warning ("no space after HTTP version"); + return -1; + } + do + { + p++; + if (p == end) + { + clib_warning ("no status code"); + return -1; + } + } + while (*p == ' '); + /* parse status code */ + if ((end - p) < 3) + { + clib_warning ("not enough characters for status code"); + return -1; + } + parse_int (status_code, 100); + parse_int (status_code, 10); + parse_int (status_code, 1); + if (status_code < 100 || status_code > 599) + { + clib_warning ("invalid status code %d", status_code); + return -1; + } + hc->status_code = status_code; + HTTP_DBG (0, "status code: %d", hc->status_code); + + /* set buffer offset to nex line start */ + hc->rx_buf_offset = next_line_offset; + + return 0; +} + +static int +http_identify_headers (http_conn_t *hc, http_status_code_t *ec) +{ + int i; + + /* check if we have any header */ + if ((hc->rx_buf[hc->rx_buf_offset] == '\r') && + (hc->rx_buf[hc->rx_buf_offset + 1] == '\n')) + { + /* just another CRLF -> no headers */ + HTTP_DBG (0, "no headers"); + hc->headers_len = 0; + hc->control_data_len += 2; + return 0; + } + + /* find empty line indicating end of header section */ + i = v_find_index (hc->rx_buf, hc->rx_buf_offset, 0, "\r\n\r\n"); + if (i < 0) + { + clib_warning ("cannot find header section end"); + *ec = HTTP_STATUS_BAD_REQUEST; + return -1; + } + hc->headers_offset = hc->rx_buf_offset; + hc->headers_len = i - hc->rx_buf_offset + 2; + hc->control_data_len += (hc->headers_len + 2); + HTTP_DBG (0, "headers length: %u", hc->headers_len); + HTTP_DBG (0, "headers offset: %u", hc->headers_offset); + + return 0; +} + +static int +http_identify_message_body (http_conn_t *hc, http_status_code_t *ec) +{ + unformat_input_t input; + int i, len; + u8 *line; + u64 body_len; + + hc->body_len = 0; + + if (hc->headers_len == 0) + { + HTTP_DBG (0, "no header, no message-body"); + return 0; + } + + /* TODO check for chunked transfer coding */ + + /* try to find Content-Length header */ + i = v_find_index (hc->rx_buf, hc->headers_offset, hc->headers_len, + "Content-Length:"); + if (i < 0) + { + HTTP_DBG (0, "Content-Length header not present, no message-body"); + return 0; + } + hc->rx_buf_offset = i + 15; + + i = v_find_index (hc->rx_buf, hc->rx_buf_offset, hc->headers_len, "\r\n"); + if (i < 0) + { + clib_warning ("end of line missing"); + *ec = HTTP_STATUS_BAD_REQUEST; + return -1; + } len = i - hc->rx_buf_offset; + if (len < 1) + { + clib_warning ("invalid header, content length value missing"); + *ec = HTTP_STATUS_BAD_REQUEST; + return -1; + } + line = vec_new (u8, len); clib_memcpy (line, hc->rx_buf + hc->rx_buf_offset, len); + HTTP_DBG (0, "%v", line); unformat_init_vector (&input, line); - if (!unformat (&input, CONTENT_LEN_STR "%d", content_length)) + if (!unformat (&input, "%llu", &body_len)) { - clib_warning ("failed to unformat content length!"); + clib_warning ("failed to unformat content length value"); + *ec = HTTP_STATUS_BAD_REQUEST; return -1; } unformat_free (&input); + hc->body_len = body_len; - /* skip rest of the header */ - hc->rx_buf_offset += len; - i = v_find_index (hc->rx_buf, hc->rx_buf_offset, "<html>"); - if (i < 0) - { - clib_warning ("<html> tag not found"); - return -1; - } - hc->rx_buf_offset = i; + hc->body_offset = hc->headers_offset + hc->headers_len + 2; + HTTP_DBG (0, "body length: %llu", hc->body_len); + HTTP_DBG (0, "body offset: %u", hc->body_offset); return 0; } @@ -521,10 +914,13 @@ http_parse_header (http_conn_t *hc, int *content_length) static http_sm_result_t http_state_wait_server_reply (http_conn_t *hc, transport_send_params_t *sp) { - int i, rv, content_length; + int rv; http_msg_t msg = {}; app_worker_t *app_wrk; session_t *as; + u32 len, max_enq, body_sent; + http_status_code_t ec; + http_main_t *hm = &http_main; rv = http_read_message (hc); @@ -535,72 +931,74 @@ http_state_wait_server_reply (http_conn_t *hc, transport_send_params_t *sp) return HTTP_SM_STOP; } + HTTP_DBG (0, "%v", hc->rx_buf); + if (vec_len (hc->rx_buf) < 8) { clib_warning ("response buffer too short"); goto error; } - if ((i = v_find_index (hc->rx_buf, 0, "200 OK")) >= 0) - { - msg.type = HTTP_MSG_REPLY; - msg.content_type = HTTP_CONTENT_TEXT_HTML; - msg.code = HTTP_STATUS_OK; - msg.data.type = HTTP_MSG_DATA_INLINE; - msg.data.len = 0; + rv = http_parse_status_line (hc); + if (rv) + goto error; - rv = http_parse_header (hc, &content_length); - if (rv) - { - clib_warning ("failed to parse http reply"); - goto error; - } - msg.data.len = content_length; - u32 dlen = vec_len (hc->rx_buf) - hc->rx_buf_offset; - as = session_get_from_handle (hc->h_pa_session_handle); - svm_fifo_seg_t segs[2] = { { (u8 *) &msg, sizeof (msg) }, - { &hc->rx_buf[hc->rx_buf_offset], dlen } }; - - rv = svm_fifo_enqueue_segments (as->rx_fifo, segs, 2, - 0 /* allow partial */); - if (rv < 0) - { - clib_warning ("error enqueue"); - return HTTP_SM_ERROR; - } + rv = http_identify_headers (hc, &ec); + if (rv) + goto error; - hc->rx_buf_offset += dlen; - hc->to_recv = content_length - dlen; + rv = http_identify_message_body (hc, &ec); + if (rv) + goto error; - if (hc->rx_buf_offset == vec_len (hc->rx_buf)) - { - vec_reset_length (hc->rx_buf); - hc->rx_buf_offset = 0; - } + /* send at least "control data" which is necessary minimum, + * if there is some space send also portion of body */ + as = session_get_from_handle (hc->h_pa_session_handle); + max_enq = svm_fifo_max_enqueue (as->rx_fifo); + if (max_enq < hc->control_data_len) + { + clib_warning ("not enough room for control data in app's rx fifo"); + goto error; + } + len = clib_min (max_enq, vec_len (hc->rx_buf)); + + msg.type = HTTP_MSG_REPLY; + msg.code = hm->sc_by_u16[hc->status_code]; + msg.data.headers_offset = hc->headers_offset; + msg.data.headers_len = hc->headers_len; + msg.data.body_offset = hc->body_offset; + msg.data.body_len = hc->body_len; + msg.data.type = HTTP_MSG_DATA_INLINE; + msg.data.len = len; - if (hc->to_recv == 0) - { - hc->rx_buf_offset = 0; - vec_reset_length (hc->rx_buf); - http_state_change (hc, HTTP_STATE_WAIT_APP_METHOD); - } - else - { - http_state_change (hc, HTTP_STATE_CLIENT_IO_MORE_DATA); - } + svm_fifo_seg_t segs[2] = { { (u8 *) &msg, sizeof (msg) }, + { hc->rx_buf, len } }; - app_wrk = app_worker_get_if_valid (as->app_wrk_index); - if (app_wrk) - app_worker_rx_notify (app_wrk, as); - return HTTP_SM_STOP; + rv = svm_fifo_enqueue_segments (as->rx_fifo, segs, 2, 0 /* allow partial */); + ASSERT (rv == (sizeof (msg) + len)); + + http_read_message_drop (hc, len); + + body_sent = len - hc->control_data_len; + hc->to_recv = hc->body_len - body_sent; + if (hc->to_recv == 0) + { + /* all sent, we are done */ + http_state_change (hc, HTTP_STATE_WAIT_APP_METHOD); } else { - clib_warning ("Unknown http method %v", hc->rx_buf); - goto error; + /* stream rest of the response body */ + http_state_change (hc, HTTP_STATE_CLIENT_IO_MORE_DATA); } + app_wrk = app_worker_get_if_valid (as->app_wrk_index); + if (app_wrk) + app_worker_rx_notify (app_wrk, as); + return HTTP_SM_STOP; + error: + http_read_message_drop_all (hc); session_transport_closing_notify (&hc->connection); session_transport_closed_notify (&hc->connection); http_disconnect_transport (hc); @@ -614,9 +1012,9 @@ http_state_wait_client_method (http_conn_t *hc, transport_send_params_t *sp) app_worker_t *app_wrk; http_msg_t msg; session_t *as; - int i, rv; - u32 len; - u8 *buf; + int rv; + u32 len, max_enq, body_sent; + u64 max_deq; rv = http_read_message (hc); @@ -624,64 +1022,76 @@ http_state_wait_client_method (http_conn_t *hc, transport_send_params_t *sp) if (rv) return HTTP_SM_STOP; + HTTP_DBG (0, "%v", hc->rx_buf); + if (vec_len (hc->rx_buf) < 8) { ec = HTTP_STATUS_BAD_REQUEST; goto error; } - if ((i = v_find_index (hc->rx_buf, 0, "GET ")) >= 0) - { - hc->method = HTTP_REQ_GET; - hc->rx_buf_offset = i + 5; + rv = http_parse_request_line (hc, &ec); + if (rv) + goto error; - i = v_find_index (hc->rx_buf, hc->rx_buf_offset, "HTTP"); - if (i < 0) - { - ec = HTTP_STATUS_BAD_REQUEST; - goto error; - } + rv = http_identify_headers (hc, &ec); + if (rv) + goto error; - HTTP_DBG (0, "GET method %v", hc->rx_buf); - len = i - hc->rx_buf_offset - 1; - } - else if ((i = v_find_index (hc->rx_buf, 0, "POST ")) >= 0) - { - hc->method = HTTP_REQ_POST; - hc->rx_buf_offset = i + 6; - len = vec_len (hc->rx_buf) - hc->rx_buf_offset - 1; - HTTP_DBG (0, "POST method %v", hc->rx_buf); - } - else + rv = http_identify_message_body (hc, &ec); + if (rv) + goto error; + + /* send at least "control data" which is necessary minimum, + * if there is some space send also portion of body */ + as = session_get_from_handle (hc->h_pa_session_handle); + max_enq = svm_fifo_max_enqueue (as->rx_fifo); + if (max_enq < hc->control_data_len) { - HTTP_DBG (0, "Unknown http method %v", hc->rx_buf); - ec = HTTP_STATUS_METHOD_NOT_ALLOWED; + clib_warning ("not enough room for control data in app's rx fifo"); + ec = HTTP_STATUS_INTERNAL_ERROR; goto error; } - - buf = &hc->rx_buf[hc->rx_buf_offset]; + /* do not dequeue more than one HTTP request, we do not support pipelining */ + max_deq = + clib_min (hc->control_data_len + hc->body_len, vec_len (hc->rx_buf)); + len = clib_min (max_enq, max_deq); msg.type = HTTP_MSG_REQUEST; msg.method_type = hc->method; - msg.content_type = HTTP_CONTENT_TEXT_HTML; msg.data.type = HTTP_MSG_DATA_INLINE; msg.data.len = len; + msg.data.target_form = hc->target_form; + msg.data.target_path_offset = hc->target_path_offset; + msg.data.target_path_len = hc->target_path_len; + msg.data.target_query_offset = hc->target_query_offset; + msg.data.target_query_len = hc->target_query_len; + msg.data.headers_offset = hc->headers_offset; + msg.data.headers_len = hc->headers_len; + msg.data.body_offset = hc->body_offset; + msg.data.body_len = hc->body_len; + + svm_fifo_seg_t segs[2] = { { (u8 *) &msg, sizeof (msg) }, + { hc->rx_buf, len } }; - svm_fifo_seg_t segs[2] = { { (u8 *) &msg, sizeof (msg) }, { buf, len } }; - - as = session_get_from_handle (hc->h_pa_session_handle); rv = svm_fifo_enqueue_segments (as->rx_fifo, segs, 2, 0 /* allow partial */); - if (rv < 0 || rv != sizeof (msg) + len) + ASSERT (rv == (sizeof (msg) + len)); + + body_sent = len - hc->control_data_len; + hc->to_recv = hc->body_len - body_sent; + if (hc->to_recv == 0) { - clib_warning ("failed app enqueue"); - /* This should not happen as we only handle 1 request per session, - * and fifo is allocated, but going forward we should consider - * rescheduling */ - return HTTP_SM_ERROR; + /* drop everything, we do not support pipelining */ + http_read_message_drop_all (hc); + /* all sent, we are done */ + http_state_change (hc, HTTP_STATE_WAIT_APP_REPLY); + } + else + { + http_read_message_drop (hc, len); + /* stream rest of the response body */ + http_state_change (hc, HTTP_STATE_CLIENT_IO_MORE_DATA); } - - vec_free (hc->rx_buf); - http_state_change (hc, HTTP_STATE_WAIT_APP_REPLY); app_wrk = app_worker_get_if_valid (as->app_wrk_index); if (app_wrk) @@ -690,7 +1100,7 @@ http_state_wait_client_method (http_conn_t *hc, transport_send_params_t *sp) return HTTP_SM_STOP; error: - + http_read_message_drop_all (hc); http_send_error (hc, ec); session_transport_closing_notify (&hc->connection); http_disconnect_transport (hc); @@ -702,13 +1112,14 @@ static http_sm_result_t http_state_wait_app_reply (http_conn_t *hc, transport_send_params_t *sp) { http_main_t *hm = &http_main; - u8 *header; - u32 offset; + u8 *response; + u32 sent; f64 now; session_t *as; http_status_code_t sc; http_msg_t msg; int rv; + http_sm_result_t sm_result = HTTP_SM_ERROR; as = session_get_from_handle (hc->h_pa_session_handle); @@ -729,68 +1140,82 @@ http_state_wait_app_reply (http_conn_t *hc, transport_send_params_t *sp) goto error; } - http_buffer_init (&hc->tx_buf, msg_to_buf_type[msg.data.type], as->tx_fifo, - msg.data.len); + if (msg.code >= HTTP_N_STATUS) + { + clib_warning ("unsupported status code: %d", msg.code); + return HTTP_SM_ERROR; + } /* - * Add headers. For now: + * Add "protocol layer" headers: * - current time - * - expiration time * - server name - * - content type * - data length */ now = clib_timebase_now (&hm->timebase); - - switch (msg.code) - { - case HTTP_STATUS_NOT_FOUND: - case HTTP_STATUS_METHOD_NOT_ALLOWED: - case HTTP_STATUS_BAD_REQUEST: - case HTTP_STATUS_INTERNAL_ERROR: - case HTTP_STATUS_OK: - header = - format (0, http_response_template, http_status_code_str[msg.code], - /* Date */ - format_clib_timebase_time, now, - /* Expires */ - format_clib_timebase_time, now + 600.0, - /* Server */ - hc->app_name, - /* Content type */ - http_content_type_str[msg.content_type], - /* Length */ - msg.data.len); - break; - case HTTP_STATUS_MOVED: - header = - format (0, http_redirect_template, http_status_code_str[msg.code]); - /* Location: http(s)://new-place already queued up as data */ - break; - default: - clib_warning ("unsupported status code: %d", msg.code); - return HTTP_SM_ERROR; + response = format (0, http_response_template, http_status_code_str[msg.code], + /* Date */ + format_clib_timebase_time, now, + /* Server */ + hc->app_name, + /* Length */ + msg.data.body_len, + /* Any headers from app? */ + msg.data.headers_len ? "" : "\r\n"); + + /* Add headers from app (if any) */ + if (msg.data.headers_len) + { + HTTP_DBG (0, "got headers from app, len %d", msg.data.headers_len); + if (msg.data.type == HTTP_MSG_DATA_PTR) + { + uword app_headers_ptr; + rv = svm_fifo_dequeue (as->tx_fifo, sizeof (app_headers_ptr), + (u8 *) &app_headers_ptr); + ASSERT (rv == sizeof (app_headers_ptr)); + vec_append (response, uword_to_pointer (app_headers_ptr, u8 *)); + } + else + { + u32 orig_len = vec_len (response); + vec_resize (response, msg.data.headers_len); + u8 *p = response + orig_len; + rv = svm_fifo_dequeue (as->tx_fifo, msg.data.headers_len, p); + ASSERT (rv == msg.data.headers_len); + } } + HTTP_DBG (0, "%v", response); - offset = http_send_data (hc, header, vec_len (header), 0); - if (offset != vec_len (header)) + sent = http_send_data (hc, response, vec_len (response)); + if (sent != vec_len (response)) { - clib_warning ("couldn't send response header!"); + clib_warning ("sending status-line and headers failed!"); sc = HTTP_STATUS_INTERNAL_ERROR; - vec_free (header); + vec_free (response); goto error; } - vec_free (header); + vec_free (response); - /* Start sending the actual data */ - http_state_change (hc, HTTP_STATE_APP_IO_MORE_DATA); + if (msg.data.body_len) + { + /* Start sending the actual data */ + http_buffer_init (&hc->tx_buf, msg_to_buf_type[msg.data.type], + as->tx_fifo, msg.data.body_len); + http_state_change (hc, HTTP_STATE_APP_IO_MORE_DATA); + sm_result = HTTP_SM_CONTINUE; + } + else + { + /* No response body, we are done */ + http_state_change (hc, HTTP_STATE_WAIT_CLIENT_METHOD); + sm_result = HTTP_SM_STOP; + } - ASSERT (sp->max_burst_size >= offset); - sp->max_burst_size -= offset; - return HTTP_SM_CONTINUE; + ASSERT (sp->max_burst_size >= sent); + sp->max_burst_size -= sent; + return sm_result; error: - clib_warning ("unexpected msg type from app %u", msg.type); http_send_error (hc, sc); http_state_change (hc, HTTP_STATE_WAIT_CLIENT_METHOD); session_transport_closing_notify (&hc->connection); @@ -803,9 +1228,11 @@ http_state_wait_app_method (http_conn_t *hc, transport_send_params_t *sp) { http_msg_t msg; session_t *as; - u8 *buf = 0, *request; - u32 offset; + u8 *target_buff = 0, *request = 0, *target; + u32 sent; int rv; + http_sm_result_t sm_result = HTTP_SM_ERROR; + http_state_t next_state; as = session_get_from_handle (hc->h_pa_session_handle); @@ -824,38 +1251,131 @@ http_state_wait_app_method (http_conn_t *hc, transport_send_params_t *sp) goto error; } - /* currently we support only GET method */ - if (msg.method_type != HTTP_REQ_GET) + /* read request target */ + if (msg.data.type == HTTP_MSG_DATA_PTR) + { + uword target_ptr; + rv = svm_fifo_dequeue (as->tx_fifo, sizeof (target_ptr), + (u8 *) &target_ptr); + ASSERT (rv == sizeof (target_ptr)); + target = uword_to_pointer (target_ptr, u8 *); + } + else + { + vec_validate (target_buff, msg.data.target_path_len - 1); + rv = + svm_fifo_dequeue (as->tx_fifo, msg.data.target_path_len, target_buff); + ASSERT (rv == msg.data.target_path_len); + target = target_buff; + } + + /* currently we support only GET and POST method */ + if (msg.method_type == HTTP_REQ_GET) + { + if (msg.data.body_len) + { + clib_warning ("GET request shouldn't include data"); + goto error; + } + /* + * Add "protocol layer" headers: + * - host + * - user agent + */ + request = format (0, http_get_request_template, + /* target */ + target, + /* Host */ + hc->host, + /* User-Agent */ + hc->app_name, + /* Any headers from app? */ + msg.data.headers_len ? "" : "\r\n"); + + next_state = HTTP_STATE_WAIT_SERVER_REPLY; + sm_result = HTTP_SM_STOP; + } + else if (msg.method_type == HTTP_REQ_POST) + { + if (!msg.data.body_len) + { + clib_warning ("POST request should include data"); + goto error; + } + /* + * Add "protocol layer" headers: + * - host + * - user agent + * - content length + */ + request = format (0, http_post_request_template, + /* target */ + target, + /* Host */ + hc->host, + /* User-Agent */ + hc->app_name, + /* Content-Length */ + msg.data.body_len, + /* Any headers from app? */ + msg.data.headers_len ? "" : "\r\n"); + + http_buffer_init (&hc->tx_buf, msg_to_buf_type[msg.data.type], + as->tx_fifo, msg.data.body_len); + + next_state = HTTP_STATE_APP_IO_MORE_DATA; + sm_result = HTTP_SM_CONTINUE; + } + else { clib_warning ("unsupported method %d", msg.method_type); goto error; } - vec_validate (buf, msg.data.len - 1); - rv = svm_fifo_dequeue (as->tx_fifo, msg.data.len, buf); - ASSERT (rv == msg.data.len); + /* Add headers from app (if any) */ + if (msg.data.headers_len) + { + HTTP_DBG (0, "got headers from app, len %d", msg.data.headers_len); + if (msg.data.type == HTTP_MSG_DATA_PTR) + { + uword app_headers_ptr; + rv = svm_fifo_dequeue (as->tx_fifo, sizeof (app_headers_ptr), + (u8 *) &app_headers_ptr); + ASSERT (rv == sizeof (app_headers_ptr)); + vec_append (request, uword_to_pointer (app_headers_ptr, u8 *)); + } + else + { + u32 orig_len = vec_len (request); + vec_resize (request, msg.data.headers_len); + u8 *p = request + orig_len; + rv = svm_fifo_dequeue (as->tx_fifo, msg.data.headers_len, p); + ASSERT (rv == msg.data.headers_len); + } + } + HTTP_DBG (0, "%v", request); - request = format (0, http_request_template, buf, hc->app_name); - offset = http_send_data (hc, request, vec_len (request), 0); - if (offset != vec_len (request)) + sent = http_send_data (hc, request, vec_len (request)); + if (sent != vec_len (request)) { - clib_warning ("sending request failed!"); + clib_warning ("sending request-line and headers failed!"); + sm_result = HTTP_SM_ERROR; goto error; } - http_state_change (hc, HTTP_STATE_WAIT_SERVER_REPLY); - - vec_free (buf); - vec_free (request); - - return HTTP_SM_STOP; + http_state_change (hc, next_state); + goto done; error: svm_fifo_dequeue_drop_all (as->tx_fifo); session_transport_closing_notify (&hc->connection); session_transport_closed_notify (&hc->connection); http_disconnect_transport (hc); - return HTTP_SM_ERROR; + +done: + vec_free (target_buff); + vec_free (request); + return sm_result; } static http_sm_result_t @@ -910,14 +1430,14 @@ http_state_client_io_more_data (http_conn_t *hc, transport_send_params_t *sp) return HTTP_SM_ERROR; } hc->to_recv -= rv; - HTTP_DBG (1, "drained %d from ts; remains %d", rv, hc->to_recv); + HTTP_DBG (1, "drained %d from ts; remains %lu", rv, hc->to_recv); + /* Finished transaction: + * server back to HTTP_STATE_WAIT_APP_REPLY + * client to HTTP_STATE_WAIT_APP_METHOD */ if (hc->to_recv == 0) - { - hc->rx_buf_offset = 0; - vec_reset_length (hc->rx_buf); - http_state_change (hc, HTTP_STATE_WAIT_APP_METHOD); - } + http_state_change (hc, hc->is_server ? HTTP_STATE_WAIT_APP_REPLY : + HTTP_STATE_WAIT_APP_METHOD); app_wrk = app_worker_get_if_valid (as->app_wrk_index); if (app_wrk) @@ -955,7 +1475,7 @@ http_state_app_io_more_data (http_conn_t *hc, transport_send_params_t *sp) if (!http_buffer_is_drained (hb)) { if (sent && svm_fifo_set_event (ts->tx_fifo)) - session_send_io_evt_to_thread (ts->tx_fifo, SESSION_IO_EVT_TX); + session_program_tx_io_evt (ts->handle, SESSION_IO_EVT_TX); if (svm_fifo_max_enqueue (ts->tx_fifo) < HTTP_FIFO_THRESH) { @@ -969,10 +1489,13 @@ http_state_app_io_more_data (http_conn_t *hc, transport_send_params_t *sp) else { if (sent && svm_fifo_set_event (ts->tx_fifo)) - session_send_io_evt_to_thread (ts->tx_fifo, SESSION_IO_EVT_TX_FLUSH); + session_program_tx_io_evt (ts->handle, SESSION_IO_EVT_TX_FLUSH); - /* Finished transaction, back to HTTP_STATE_WAIT_METHOD */ - http_state_change (hc, HTTP_STATE_WAIT_CLIENT_METHOD); + /* Finished transaction: + * server back to HTTP_STATE_WAIT_METHOD + * client to HTTP_STATE_WAIT_SERVER_REPLY */ + http_state_change (hc, hc->is_server ? HTTP_STATE_WAIT_CLIENT_METHOD : + HTTP_STATE_WAIT_SERVER_REPLY); http_buffer_free (&hc->tx_buf); } @@ -1024,12 +1547,16 @@ http_ts_rx_callback (session_t *ts) return -1; } - if (hc->state == HTTP_CONN_STATE_CLOSED) + if (!http_state_is_rx_valid (hc)) { + if (hc->state != HTTP_CONN_STATE_CLOSED) + clib_warning ("app data req state '%U' session state %u", + format_http_state, hc->http_state, hc->state); svm_fifo_dequeue_drop_all (ts->tx_fifo); return 0; } + HTTP_DBG (1, "run state machine"); http_req_run_state_machine (hc, 0); if (hc->state == HTTP_CONN_STATE_TRANSPORT_CLOSED) @@ -1065,6 +1592,7 @@ http_ts_cleanup_callback (session_t *ts, session_cleanup_ntf_t ntf) clib_warning ("no http connection for %u", ts->session_index); return; } + HTTP_DBG (1, "going to free session %x", ts->opaque); vec_free (hc->rx_buf); @@ -1072,6 +1600,12 @@ http_ts_cleanup_callback (session_t *ts, session_cleanup_ntf_t ntf) http_conn_timer_stop (hc); session_transport_delete_notify (&hc->connection); + + if (!hc->is_server) + { + vec_free (hc->app_name); + vec_free (hc->host); + } http_conn_free (hc); } @@ -1116,8 +1650,6 @@ http_transport_enable (vlib_main_t *vm, u8 is_en) return 0; } - vec_validate (hm->wrk, vlib_num_workers ()); - clib_memset (a, 0, sizeof (*a)); clib_memset (options, 0, sizeof (options)); @@ -1139,10 +1671,16 @@ http_transport_enable (vlib_main_t *vm, u8 is_en) hm->app_index = a->app_index; vec_free (a->name); + if (hm->is_init) + return 0; + + vec_validate (hm->wrk, vlib_num_workers ()); + clib_timebase_init (&hm->timebase, 0 /* GMT */, CLIB_TIMEBASE_DAYLIGHT_NONE, &vm->clib_time /* share the system clock */); http_timers_init (vm, http_conn_timeout_cb); + hm->is_init = 1; return 0; } @@ -1173,11 +1711,20 @@ http_transport_connect (transport_endpoint_cfg_t *tep) hc->state = HTTP_CONN_STATE_CONNECTING; cargs->api_context = hc_index; + hc->is_server = 0; + if (vec_len (app->name)) hc->app_name = vec_dup (app->name); else hc->app_name = format (0, "VPP HTTP client"); + if (sep->is_ip4) + hc->host = format (0, "%U:%d", format_ip4_address, &sep->ip.ip4, + clib_net_to_host_u16 (sep->port)); + else + hc->host = format (0, "%U:%d", format_ip6_address, &sep->ip.ip6, + clib_net_to_host_u16 (sep->port)); + HTTP_DBG (1, "hc ho_index %x", hc_index); if ((error = vnet_connect (cargs))) @@ -1230,6 +1777,8 @@ http_start_listen (u32 app_listener_index, transport_endpoint_cfg_t *tep) lhc->c_s_index = app_listener_index; lhc->c_flags |= TRANSPORT_CONNECTION_F_NO_LOOKUP; + lhc->is_server = 1; + if (vec_len (app->name)) lhc->app_name = vec_dup (app->name); else @@ -1331,6 +1880,7 @@ http_app_tx_callback (void *session, transport_send_params_t *sp) max_burst_sz = sp->max_burst_size * TRANSPORT_PACER_MIN_MSS; sp->max_burst_size = max_burst_sz; + HTTP_DBG (1, "run state machine"); http_req_run_state_machine (hc, sp); if (hc->state == HTTP_CONN_STATE_APP_CLOSED) @@ -1476,6 +2026,7 @@ static clib_error_t * http_transport_init (vlib_main_t *vm) { http_main_t *hm = &http_main; + int i; transport_register_protocol (TRANSPORT_PROTO_HTTP, &http_proto, FIB_PROTOCOL_IP4, ~0); @@ -1487,7 +2038,26 @@ http_transport_init (vlib_main_t *vm) hm->first_seg_size = 32 << 20; hm->fifo_size = 512 << 10; - return 0; + /* Setup u16 to http_status_code_t map */ + /* Unrecognized status code is equivalent to the x00 status */ + vec_validate (hm->sc_by_u16, 599); + for (i = 100; i < 200; i++) + hm->sc_by_u16[i] = HTTP_STATUS_CONTINUE; + for (i = 200; i < 300; i++) + hm->sc_by_u16[i] = HTTP_STATUS_OK; + for (i = 300; i < 400; i++) + hm->sc_by_u16[i] = HTTP_STATUS_MULTIPLE_CHOICES; + for (i = 400; i < 500; i++) + hm->sc_by_u16[i] = HTTP_STATUS_BAD_REQUEST; + for (i = 500; i < 600; i++) + hm->sc_by_u16[i] = HTTP_STATUS_INTERNAL_ERROR; + + /* Registered status codes */ +#define _(c, s, str) hm->sc_by_u16[c] = HTTP_STATUS_##s; + foreach_http_status_code +#undef _ + + return 0; } VLIB_INIT_FUNCTION (http_transport_init); diff --git a/src/plugins/http/http.h b/src/plugins/http/http.h index c9912dd6db8..5f74edb5e47 100644 --- a/src/plugins/http/http.h +++ b/src/plugins/http/http.h @@ -16,6 +16,8 @@ #ifndef SRC_PLUGINS_HTTP_HTTP_H_ #define SRC_PLUGINS_HTTP_HTTP_H_ +#include <ctype.h> + #include <vnet/plugin/plugin.h> #include <vpp/app/version.h> @@ -49,6 +51,14 @@ typedef struct http_conn_id_ STATIC_ASSERT (sizeof (http_conn_id_t) <= TRANSPORT_CONN_ID_LEN, "ctx id must be less than TRANSPORT_CONN_ID_LEN"); +typedef struct +{ + char *base; + uword len; +} http_token_t; + +#define http_token_lit(s) (s), sizeof (s) - 1 + typedef enum http_conn_state_ { HTTP_CONN_STATE_LISTEN, @@ -83,86 +93,96 @@ typedef enum http_msg_type_ HTTP_MSG_REPLY } http_msg_type_t; +typedef enum http_target_form_ +{ + HTTP_TARGET_ORIGIN_FORM, + HTTP_TARGET_ABSOLUTE_FORM, + HTTP_TARGET_AUTHORITY_FORM, + HTTP_TARGET_ASTERISK_FORM +} http_target_form_t; + #define foreach_http_content_type \ - _ (APP_7Z, ".7z", "application / x - 7z - compressed") \ - _ (APP_DOC, ".doc", "application / msword") \ + _ (APP_7Z, ".7z", "application/x-7z-compressed") \ + _ (APP_DOC, ".doc", "application/msword") \ _ (APP_DOCX, ".docx", \ - "application / vnd.openxmlformats - " \ + "application/vnd.openxmlformats-" \ "officedocument.wordprocessingml.document") \ - _ (APP_EPUB, ".epub", "application / epub + zip") \ - _ (APP_FONT, ".eot", "application / vnd.ms - fontobject") \ - _ (APP_JAR, ".jar", "application / java - archive") \ - _ (APP_JSON, ".json", "application / json") \ - _ (APP_JSON_LD, ".jsonld", "application / ld + json") \ - _ (APP_MPKG, ".mpkg", "application / vnd.apple.installer + xml") \ - _ (APP_ODP, ".odp", "application / vnd.oasis.opendocument.presentation") \ - _ (APP_ODS, ".ods", "application / vnd.oasis.opendocument.spreadsheet") \ - _ (APP_ODT, ".odt", "application / vnd.oasis.opendocument.text") \ - _ (APP_OGX, ".ogx", "application / ogg") \ - _ (APP_PDF, ".pdf", "application / pdf") \ - _ (APP_PHP, ".php", "application / x - httpd - php") \ - _ (APP_PPT, ".ppt", "application / vnd.ms - powerpoint") \ - _ (APP_PPTX, ".pptx", "application / vnd.ms - powerpoint") \ - _ (APP_RAR, ".rar", "application / vnd.rar") \ - _ (APP_RTF, ".rtf", "application / rtf") \ - _ (APP_SH, ".sh", "application / x - sh") \ - _ (APP_TAR, ".tar", "application / x - tar") \ - _ (APP_VSD, ".vsd", "application / vnd.visio") \ - _ (APP_XHTML, ".xhtml", "application / xhtml + xml") \ - _ (APP_XLS, ".xls", "application / vnd.ms - excel") \ - _ (APP_XML, ".xml", "application / xml") \ + _ (APP_EPUB, ".epub", "application/epub+zip") \ + _ (APP_FONT, ".eot", "application/vnd.ms-fontobject") \ + _ (APP_JAR, ".jar", "application/java-archive") \ + _ (APP_JSON, ".json", "application/json") \ + _ (APP_JSON_LD, ".jsonld", "application/ld+json") \ + _ (APP_MPKG, ".mpkg", "application/vnd.apple.installer+xml") \ + _ (APP_ODP, ".odp", "application/vnd.oasis.opendocument.presentation") \ + _ (APP_ODS, ".ods", "application/vnd.oasis.opendocument.spreadsheet") \ + _ (APP_ODT, ".odt", "application/vnd.oasis.opendocument.text") \ + _ (APP_OGX, ".ogx", "application/ogg") \ + _ (APP_PDF, ".pdf", "application/pdf") \ + _ (APP_PHP, ".php", "application/x-httpd-php") \ + _ (APP_PPT, ".ppt", "application/vnd.ms-powerpoint") \ + _ (APP_PPTX, ".pptx", "application/vnd.ms-powerpoint") \ + _ (APP_RAR, ".rar", "application/vnd.rar") \ + _ (APP_RTF, ".rtf", "application/rtf") \ + _ (APP_SH, ".sh", "application/x-sh") \ + _ (APP_TAR, ".tar", "application/x-tar") \ + _ (APP_VSD, ".vsd", "application/vnd.visio") \ + _ (APP_XHTML, ".xhtml", "application/xhtml+xml") \ + _ (APP_XLS, ".xls", "application/vnd.ms-excel") \ + _ (APP_XML, ".xml", "application/xml") \ _ (APP_XSLX, ".xlsx", \ - "application / vnd.openxmlformats - officedocument.spreadsheetml.sheet") \ - _ (APP_XUL, ".xul", "application / vnd.mozilla.xul + xml") \ - _ (APP_ZIP, ".zip", "application / zip") \ - _ (AUDIO_AAC, ".aac", "audio / aac") \ - _ (AUDIO_CD, ".cda", "application / x - cdf") \ - _ (AUDIO_WAV, ".wav", "audio / wav") \ - _ (AUDIO_WEBA, ".weba", "audio / webm") \ - _ (AUDO_MIDI, ".midi", "audio / midi") \ - _ (AUDO_MID, ".mid", "audo / midi") \ - _ (AUDO_MP3, ".mp3", "audio / mpeg") \ - _ (AUDO_OGA, ".oga", "audio / ogg") \ - _ (AUDO_OPUS, ".opus", "audio / opus") \ - _ (APP_OCTET_STREAM, ".bin", "application / octet - stream") \ - _ (BZIP2, ".bz2", "application / x - bzip2") \ - _ (BZIP, ".bz", "application / x - bzip") \ - _ (FONT_OTF, ".otf", "font / otf") \ - _ (FONT_TTF, ".ttf", "font / ttf") \ - _ (FONT_WOFF2, ".woff2", "font / woff2") \ - _ (FONT_WOFF, ".woff", "font / woff") \ - _ (GZIP, ".gz", "application / gzip") \ - _ (IMAGE_AVIF, ".avif", "image / avif") \ - _ (IMAGE_BMP, ".bmp", "image / bmp") \ - _ (IMAGE_GIF, ".gif", "image / gif") \ - _ (IMAGE_ICON, ".ico", "image / vnd.microsoft.icon") \ - _ (IMAGE_JPEG, ".jpeg", "image / jpeg") \ - _ (IMAGE_JPG, ".jpg", "image / jpeg") \ - _ (IMAGE_PNG, ".png", "image / png") \ - _ (IMAGE_SVG, ".svg", "image / svg + xml") \ - _ (IMAGE_TIFF, ".tiff", "image / tiff") \ - _ (IMAGE_TIF, ".tif", "image / tiff") \ - _ (IMAGE_WEBP, ".webp", "image / webp") \ - _ (SCRIPT_CSH, ".csh", "application / x - csh") \ - _ (TEXT_ABIWORD, ".abw", "application / x - abiword") \ - _ (TEXT_ARCHIVE, ".arc", "application / x - freearc") \ - _ (TEXT_AZW, ".azw", "application / vnd.amazon.ebook") \ - _ (TEXT_CALENDAR, ".ics", "text / calendar") \ - _ (TEXT_CSS, ".css", "text / css") \ - _ (TEXT_CSV, ".csv", "text / csv") \ - _ (TEXT_HTM, ".htm", "text / html") \ - _ (TEXT_HTML, ".html", "text / html") \ - _ (TEXT_JS, ".js", "text / javascript") \ - _ (TEXT_MJS, ".mjs", "text / javascript") \ - _ (TEXT_PLAIN, ".txt", "text / plain") \ - _ (VIDEO_3GP2, ".3g2", "video / 3gpp2") \ - _ (VIDEO_3GP, ".3gp", "video / 3gpp") \ - _ (VIDEO_AVI, ".avi", "video / x - msvideo") \ - _ (VIDEO_MP4, ".mp4", "video / mp4") \ - _ (VIDEO_MPEG, ".mpeg", "video / mpeg") \ - _ (VIDEO_OGG, ".ogv", "video / ogg") \ - _ (VIDEO_TS, ".ts", "video / mp2t") \ - _ (VIDEO_WEBM, ".webm", "video / webm") + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") \ + _ (APP_XUL, ".xul", "application/vnd.mozilla.xul+xml") \ + _ (APP_X_WWW_FORM_URLENCODED, ".invalid", \ + "application/x-www-form-urlencoded") \ + _ (APP_ZIP, ".zip", "application/zip") \ + _ (AUDIO_AAC, ".aac", "audio/aac") \ + _ (AUDIO_CD, ".cda", "application/x-cdf") \ + _ (AUDIO_WAV, ".wav", "audio/wav") \ + _ (AUDIO_WEBA, ".weba", "audio/webm") \ + _ (AUDO_MIDI, ".midi", "audio/midi") \ + _ (AUDO_MID, ".mid", "audo/midi") \ + _ (AUDO_MP3, ".mp3", "audio/mpeg") \ + _ (AUDO_OGA, ".oga", "audio/ogg") \ + _ (AUDO_OPUS, ".opus", "audio/opus") \ + _ (APP_OCTET_STREAM, ".bin", "application/octet-stream") \ + _ (BZIP2, ".bz2", "application/x-bzip2") \ + _ (BZIP, ".bz", "application/x-bzip") \ + _ (FONT_OTF, ".otf", "font/otf") \ + _ (FONT_TTF, ".ttf", "font/ttf") \ + _ (FONT_WOFF2, ".woff2", "font/woff2") \ + _ (FONT_WOFF, ".woff", "font/woff") \ + _ (GZIP, ".gz", "application/gzip") \ + _ (IMAGE_AVIF, ".avif", "image/avif") \ + _ (IMAGE_BMP, ".bmp", "image/bmp") \ + _ (IMAGE_GIF, ".gif", "image/gif") \ + _ (IMAGE_ICON, ".ico", "image/vnd.microsoft.icon") \ + _ (IMAGE_JPEG, ".jpeg", "image/jpeg") \ + _ (IMAGE_JPG, ".jpg", "image/jpeg") \ + _ (IMAGE_PNG, ".png", "image/png") \ + _ (IMAGE_SVG, ".svg", "image/svg+xml") \ + _ (IMAGE_TIFF, ".tiff", "image/tiff") \ + _ (IMAGE_TIF, ".tif", "image/tiff") \ + _ (IMAGE_WEBP, ".webp", "image/webp") \ + _ (SCRIPT_CSH, ".csh", "application/x-csh") \ + _ (TEXT_ABIWORD, ".abw", "application/x-abiword") \ + _ (TEXT_ARCHIVE, ".arc", "application/x-freearc") \ + _ (TEXT_AZW, ".azw", "application/vnd.amazon.ebook") \ + _ (TEXT_CALENDAR, ".ics", "text/calendar") \ + _ (TEXT_CSS, ".css", "text/css") \ + _ (TEXT_CSV, ".csv", "text/csv") \ + _ (TEXT_HTM, ".htm", "text/html") \ + _ (TEXT_HTML, ".html", "text/html") \ + _ (TEXT_JS, ".js", "text/javascript") \ + _ (TEXT_MJS, ".mjs", "text/javascript") \ + _ (TEXT_PLAIN, ".txt", "text/plain") \ + _ (VIDEO_3GP2, ".3g2", "video/3gpp2") \ + _ (VIDEO_3GP, ".3gp", "video/3gpp") \ + _ (VIDEO_AVI, ".avi", "video/x-msvideo") \ + _ (VIDEO_MP4, ".mp4", "video/mp4") \ + _ (VIDEO_MPEG, ".mpeg", "video/mpeg") \ + _ (VIDEO_OGG, ".ogv", "video/ogg") \ + _ (VIDEO_TS, ".ts", "video/mp2t") \ + _ (VIDEO_WEBM, ".webm", "video/webm") typedef enum http_content_type_ { @@ -172,12 +192,50 @@ typedef enum http_content_type_ } http_content_type_t; #define foreach_http_status_code \ + _ (100, CONTINUE, "100 Continue") \ + _ (101, SWITCHING_PROTOCOLS, "101 Switching Protocols") \ _ (200, OK, "200 OK") \ + _ (201, CREATED, "201 Created") \ + _ (202, ACCEPTED, "202 Accepted") \ + _ (203, NON_UTHORITATIVE_INFORMATION, "203 Non-Authoritative Information") \ + _ (204, NO_CONTENT, "204 No Content") \ + _ (205, RESET_CONTENT, "205 Reset Content") \ + _ (206, PARTIAL_CONTENT, "206 Partial Content") \ + _ (300, MULTIPLE_CHOICES, "300 Multiple Choices") \ _ (301, MOVED, "301 Moved Permanently") \ + _ (302, FOUND, "302 Found") \ + _ (303, SEE_OTHER, "303 See Other") \ + _ (304, NOT_MODIFIED, "304 Not Modified") \ + _ (305, USE_PROXY, "305 Use Proxy") \ + _ (307, TEMPORARY_REDIRECT, "307 Temporary Redirect") \ + _ (308, PERMANENT_REDIRECT, "308 Permanent Redirect") \ _ (400, BAD_REQUEST, "400 Bad Request") \ + _ (401, UNAUTHORIZED, "401 Unauthorized") \ + _ (402, PAYMENT_REQUIRED, "402 Payment Required") \ + _ (403, FORBIDDEN, "403 Forbidden") \ _ (404, NOT_FOUND, "404 Not Found") \ _ (405, METHOD_NOT_ALLOWED, "405 Method Not Allowed") \ - _ (500, INTERNAL_ERROR, "500 Internal Server Error") + _ (406, NOT_ACCEPTABLE, "406 Not Acceptable") \ + _ (407, PROXY_AUTHENTICATION_REQUIRED, "407 Proxy Authentication Required") \ + _ (408, REQUEST_TIMEOUT, "408 Request Timeout") \ + _ (409, CONFLICT, "409 Conflict") \ + _ (410, GONE, "410 Gone") \ + _ (411, LENGTH_REQUIRED, "411 Length Required") \ + _ (412, PRECONDITION_FAILED, "412 Precondition Failed") \ + _ (413, CONTENT_TOO_LARGE, "413 Content Too Large") \ + _ (414, URI_TOO_LONG, "414 URI Too Long") \ + _ (415, UNSUPPORTED_MEDIA_TYPE, "415 Unsupported Media Type") \ + _ (416, RANGE_NOT_SATISFIABLE, "416 Range Not Satisfiable") \ + _ (417, EXPECTATION_FAILED, "417 Expectation Failed") \ + _ (421, MISDIRECTED_REQUEST, "421 Misdirected Request") \ + _ (422, UNPROCESSABLE_CONTENT, "422 Unprocessable_Content") \ + _ (426, UPGRADE_REQUIRED, "426 Upgrade Required") \ + _ (500, INTERNAL_ERROR, "500 Internal Server Error") \ + _ (501, NOT_IMPLEMENTED, "501 Not Implemented") \ + _ (502, BAD_GATEWAY, "502 Bad Gateway") \ + _ (503, SERVICE_UNAVAILABLE, "503 Service Unavailable") \ + _ (504, GATEWAY_TIMEOUT, "504 Gateway Timeout") \ + _ (505, HTTP_VERSION_NOT_SUPPORTED, "505 HTTP Version Not Supported") typedef enum http_status_code_ { @@ -187,6 +245,101 @@ typedef enum http_status_code_ HTTP_N_STATUS } http_status_code_t; +#define foreach_http_header_name \ + _ (ACCEPT, "Accept") \ + _ (ACCEPT_CHARSET, "Accept-Charset") \ + _ (ACCEPT_ENCODING, "Accept-Encoding") \ + _ (ACCEPT_LANGUAGE, "Accept-Language") \ + _ (ACCEPT_RANGES, "Accept-Ranges") \ + _ (ACCESS_CONTROL_ALLOW_CREDENTIALS, "Access-Control-Allow-Credentials") \ + _ (ACCESS_CONTROL_ALLOW_HEADERS, "Access-Control-Allow-Headers") \ + _ (ACCESS_CONTROL_ALLOW_METHODS, "Access-Control-Allow-Methods") \ + _ (ACCESS_CONTROL_ALLOW_ORIGIN, "Access-Control-Allow-Origin") \ + _ (ACCESS_CONTROL_EXPOSE_HEADERS, "Access-Control-Expose-Headers") \ + _ (ACCESS_CONTROL_MAX_AGE, "Access-Control-Max-Age") \ + _ (ACCESS_CONTROL_REQUEST_HEADERS, "Access-Control-Request-Headers") \ + _ (ACCESS_CONTROL_REQUEST_METHOD, "Access-Control-Request-Method") \ + _ (AGE, "Age") \ + _ (ALLOW, "Allow") \ + _ (ALPN, "ALPN") \ + _ (ALT_SVC, "Alt-Svc") \ + _ (ALT_USED, "Alt-Used") \ + _ (ALTERNATES, "Alternates") \ + _ (AUTHENTICATION_CONTROL, "Authentication-Control") \ + _ (AUTHENTICATION_INFO, "Authentication-Info") \ + _ (AUTHORIZATION, "Authorization") \ + _ (CACHE_CONTROL, "Cache-Control") \ + _ (CACHE_STATUS, "Cache-Status") \ + _ (CAPSULE_PROTOCOL, "Capsule-Protocol") \ + _ (CDN_CACHE_CONTROL, "CDN-Cache-Control") \ + _ (CDN_LOOP, "CDN-Loop") \ + _ (CLIENT_CERT, "Client-Cert") \ + _ (CLIENT_CERT_CHAIN, "Client-Cert-Chain") \ + _ (CLOSE, "Close") \ + _ (CONNECTION, "Connection") \ + _ (CONTENT_DIGEST, "Content-Digest") \ + _ (CONTENT_DISPOSITION, "Content-Disposition") \ + _ (CONTENT_ENCODING, "Content-Encoding") \ + _ (CONTENT_LANGUAGE, "Content-Language") \ + _ (CONTENT_LENGTH, "Content-Length") \ + _ (CONTENT_LOCATION, "Content-Location") \ + _ (CONTENT_RANGE, "Content-Range") \ + _ (CONTENT_TYPE, "Content-Type") \ + _ (COOKIE, "Cookie") \ + _ (DATE, "Date") \ + _ (DIGEST, "Digest") \ + _ (DPOP, "DPoP") \ + _ (DPOP_NONCE, "DPoP-Nonce") \ + _ (EARLY_DATA, "Early-Data") \ + _ (ETAG, "ETag") \ + _ (EXPECT, "Expect") \ + _ (EXPIRES, "Expires") \ + _ (FORWARDED, "Forwarded") \ + _ (FROM, "From") \ + _ (HOST, "Host") \ + _ (IF_MATCH, "If-Match") \ + _ (IF_MODIFIED_SINCE, "If-Modified-Since") \ + _ (IF_NONE_MATCH, "If-None-Match") \ + _ (IF_RANGE, "If-Range") \ + _ (IF_UNMODIFIED_SINCE, "If-Unmodified-Since") \ + _ (KEEP_ALIVE, "Keep-Alive") \ + _ (LAST_MODIFIED, "Last-Modified") \ + _ (LINK, "Link") \ + _ (LOCATION, "Location") \ + _ (MAX_FORWARDS, "Max-Forwards") \ + _ (ORIGIN, "Origin") \ + _ (PRIORITY, "Priority") \ + _ (PROXY_AUTHENTICATE, "Proxy-Authenticate") \ + _ (PROXY_AUTHENTICATION_INFO, "Proxy-Authentication-Info") \ + _ (PROXY_AUTHORIZATION, "Proxy-Authorization") \ + _ (PROXY_STATUS, "Proxy-Status") \ + _ (RANGE, "Range") \ + _ (REFERER, "Referer") \ + _ (REPR_DIGEST, "Repr-Digest") \ + _ (SET_COOKIE, "Set-Cookie") \ + _ (SIGNATURE, "Signature") \ + _ (SIGNATURE_INPUT, "Signature-Input") \ + _ (STRICT_TRANSPORT_SECURITY, "Strict-Transport-Security") \ + _ (RETRY_AFTER, "Retry-After") \ + _ (SERVER, "Server") \ + _ (TE, "TE") \ + _ (TRAILER, "Trailer") \ + _ (TRANSFER_ENCODING, "Transfer-Encoding") \ + _ (UPGRADE, "Upgrade") \ + _ (USER_AGENT, "User-Agent") \ + _ (VARY, "Vary") \ + _ (VIA, "Via") \ + _ (WANT_CONTENT_DIGEST, "Want-Content-Digest") \ + _ (WANT_REPR_DIGEST, "Want-Repr-Digest") \ + _ (WWW_AUTHENTICATE, "WWW-Authenticate") + +typedef enum http_header_name_ +{ +#define _(sym, str) HTTP_HEADER_##sym, + foreach_http_header_name +#undef _ +} http_header_name_t; + typedef enum http_msg_data_type_ { HTTP_MSG_DATA_INLINE, @@ -197,6 +350,15 @@ typedef struct http_msg_data_ { http_msg_data_type_t type; u64 len; + http_target_form_t target_form; + u32 target_path_offset; + u32 target_path_len; + u32 target_query_offset; + u32 target_query_len; + u32 headers_offset; + u32 headers_len; + u32 body_offset; + u64 body_len; u8 data[0]; } http_msg_data_t; @@ -208,7 +370,6 @@ typedef struct http_msg_ http_req_method_t method_type; http_status_code_t code; }; - http_content_type_t content_type; http_msg_data_t data; } http_msg_t; @@ -228,6 +389,8 @@ typedef struct http_tc_ http_conn_state_t state; u32 timer_handle; u8 *app_name; + u8 *host; + u8 is_server; /* * Current request @@ -237,8 +400,19 @@ typedef struct http_tc_ u8 *rx_buf; u32 rx_buf_offset; http_buffer_t tx_buf; - u32 to_recv; + u64 to_recv; u32 bytes_dequeued; + u32 control_data_len; /* start line + headers + empty line */ + http_target_form_t target_form; + u32 target_path_offset; + u32 target_path_len; + u32 target_query_offset; + u32 target_query_len; + u32 headers_offset; + u32 headers_len; + u32 body_offset; + u64 body_len; + u16 status_code; } http_conn_t; typedef struct http_worker_ @@ -254,10 +428,12 @@ typedef struct http_main_ clib_timebase_t timebase; + u16 *sc_by_u16; /* * Runtime config */ u8 debug_level; + u8 is_init; /* * Config @@ -267,14 +443,535 @@ typedef struct http_main_ u32 fifo_size; } http_main_t; -static inline int -http_state_is_tx_valid (http_conn_t *hc) +always_inline int +_validate_target_syntax (u8 *target, int is_query, int *is_encoded) +{ + int i, encoded = 0; + + static uword valid_chars[4] = { + /* !$&'()*+,-./0123456789:;= */ + 0x2fffffd200000000, + /* @ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz~ */ + 0x47fffffe87ffffff, + 0x0000000000000000, + 0x0000000000000000, + }; + + for (i = 0; i < vec_len (target); i++) + { + if (clib_bitmap_get_no_check (valid_chars, target[i])) + continue; + /* target was already split after first question mark, + * for query it is valid character */ + if (is_query && target[i] == '?') + continue; + /* pct-encoded = "%" HEXDIG HEXDIG */ + if (target[i] == '%') + { + if ((i + 2) > vec_len (target)) + return -1; + if (!isxdigit (target[i + 1]) || !isxdigit (target[i + 2])) + return -1; + i += 2; + encoded = 1; + continue; + } + clib_warning ("invalid character %d", target[i]); + return -1; + } + if (is_encoded) + *is_encoded = encoded; + return 0; +} + +/** + * An "absolute-path" rule validation (RFC9110 section 4.1). + * + * @param path Target path to validate. + * @param is_encoded Return flag that indicates if percent-encoded (optional). + * + * @return @c 0 on success. + */ +always_inline int +http_validate_abs_path_syntax (u8 *path, int *is_encoded) +{ + return _validate_target_syntax (path, 0, is_encoded); +} + +/** + * A "query" rule validation (RFC3986 section 2.1). + * + * @param query Target query to validate. + * @param is_encoded Return flag that indicates if percent-encoded (optional). + * + * @return @c 0 on success. + */ +always_inline int +http_validate_query_syntax (u8 *query, int *is_encoded) +{ + return _validate_target_syntax (query, 1, is_encoded); +} + +#define htoi(x) (isdigit (x) ? (x - '0') : (tolower (x) - 'a' + 10)) + +/** + * Decode percent-encoded data. + * + * @param src Data to decode. + * + * @return New vector with decoded data. + * + * The caller is always responsible to free the returned vector. + */ +always_inline u8 * +http_percent_decode (u8 *src) +{ + int i; + u8 *decoded_uri = 0; + + for (i = 0; i < vec_len (src); i++) + { + if (src[i] == '%') + { + u8 c = (htoi (src[i + 1]) << 4) | htoi (src[i + 2]); + vec_add1 (decoded_uri, c); + i += 2; + } + else + vec_add1 (decoded_uri, src[i]); + } + return decoded_uri; +} + +/** + * Remove dot segments from path (RFC3986 section 5.2.4) + * + * @param path Path to sanitize. + * + * @return New vector with sanitized path. + * + * The caller is always responsible to free the returned vector. + */ +always_inline u8 * +http_path_remove_dot_segments (u8 *path) +{ + u32 *segments = 0, *segments_len = 0, segment_len; + u8 *new_path = 0; + int i, ii; + + if (!path) + return vec_new (u8, 0); + + segments = vec_new (u32, 1); + /* first segment */ + segments[0] = 0; + /* find all segments */ + for (i = 1; i < (vec_len (path) - 1); i++) + { + if (path[i] == '/') + vec_add1 (segments, i + 1); + } + /* dummy tail */ + vec_add1 (segments, vec_len (path)); + + /* scan all segments for "." and ".." */ + segments_len = vec_new (u32, vec_len (segments) - 1); + for (i = 0; i < vec_len (segments_len); i++) + { + segment_len = segments[i + 1] - segments[i]; + if (segment_len == 2 && path[segments[i]] == '.') + segment_len = 0; + else if (segment_len == 3 && path[segments[i]] == '.' && + path[segments[i] + 1] == '.') + { + segment_len = 0; + /* remove parent (if any) */ + for (ii = i - 1; ii >= 0; ii--) + { + if (segments_len[ii]) + { + segments_len[ii] = 0; + break; + } + } + } + segments_len[i] = segment_len; + } + + /* we might end with empty path, so return at least empty vector */ + new_path = vec_new (u8, 0); + /* append all valid segments */ + for (i = 0; i < vec_len (segments_len); i++) + { + if (segments_len[i]) + vec_add (new_path, path + segments[i], segments_len[i]); + } + vec_free (segments); + vec_free (segments_len); + return new_path; +} + +always_inline int +_parse_field_name (u8 **pos, u8 *end, u8 **field_name_start, + u32 *field_name_len) +{ + u32 name_len = 0; + u8 *p; + + static uword tchar[4] = { + /* !#$%'*+-.0123456789 */ + 0x03ff6cba00000000, + /* ABCDEFGHIJKLMNOPQRSTUVWXYZ^_`abcdefghijklmnopqrstuvwxyz|~ */ + 0x57ffffffc7fffffe, + 0x0000000000000000, + 0x0000000000000000, + }; + + p = *pos; + + *field_name_start = p; + while (p != end) + { + if (clib_bitmap_get_no_check (tchar, *p)) + { + name_len++; + p++; + } + else if (*p == ':') + { + if (name_len == 0) + { + clib_warning ("empty field name"); + return -1; + } + *field_name_len = name_len; + p++; + *pos = p; + return 0; + } + else + { + clib_warning ("invalid character %d", *p); + return -1; + } + } + clib_warning ("field name end not found"); + return -1; +} + +always_inline int +_parse_field_value (u8 **pos, u8 *end, u8 **field_value_start, + u32 *field_value_len) +{ + u32 value_len = 0; + u8 *p; + + p = *pos; + + /* skip leading whitespace */ + while (1) + { + if (p == end) + { + clib_warning ("field value not found"); + return -1; + } + else if (*p != ' ' && *p != '\t') + { + break; + } + p++; + } + + *field_value_start = p; + while (p != end) + { + if (*p == '\r') + { + if ((end - p) < 1) + { + clib_warning ("incorrect field line end"); + return -1; + } + p++; + if (*p == '\n') + { + if (value_len == 0) + { + clib_warning ("empty field value"); + return -1; + } + p++; + *pos = p; + /* skip trailing whitespace */ + p = *field_value_start + value_len - 1; + while (*p == ' ' || *p == '\t') + { + p--; + value_len--; + } + *field_value_len = value_len; + return 0; + } + clib_warning ("CR without LF"); + return -1; + } + if (*p < ' ' && *p != '\t') + { + clib_warning ("invalid character %d", *p); + return -1; + } + p++; + value_len++; + } + + clib_warning ("field value end not found"); + return -1; +} + +typedef struct +{ + u8 *name; + u8 *value; +} http_header_ht_t; + +typedef struct +{ + http_token_t name; + http_token_t value; +} http_header_t; + +typedef struct +{ + http_header_ht_t *headers; + uword *value_by_name; +} http_header_table_t; + +/** + * Free header table's memory. + * + * @param ht Header table to free. + */ +always_inline void +http_free_header_table (http_header_table_t *ht) +{ + http_header_ht_t *header; + vec_foreach (header, ht->headers) + { + vec_free (header->name); + vec_free (header->value); + } + vec_free (ht->headers); + hash_free (ht->value_by_name); + clib_mem_free (ht); +} + +/** + * Parse headers in given vector. + * + * @param headers Vector to parse. + * @param [out] header_table Parsed headers in case of success. + * + * @return @c 0 on success. + * + * The caller is responsible to free the returned @c header_table + * using @c http_free_header_table . + */ +always_inline int +http_parse_headers (u8 *headers, http_header_table_t **header_table) +{ + u8 *pos, *end, *name_start, *value_start, *name; + u32 name_len, value_len; + int rv; + http_header_ht_t *header; + http_header_table_t *ht; + uword *p; + + end = headers + vec_len (headers); + pos = headers; + + ht = clib_mem_alloc (sizeof (*ht)); + ht->value_by_name = hash_create_string (0, sizeof (uword)); + ht->headers = 0; + do + { + rv = _parse_field_name (&pos, end, &name_start, &name_len); + if (rv != 0) + { + http_free_header_table (ht); + return rv; + } + rv = _parse_field_value (&pos, end, &value_start, &value_len); + if (rv != 0) + { + http_free_header_table (ht); + return rv; + } + name = vec_new (u8, name_len); + clib_memcpy (name, name_start, name_len); + vec_terminate_c_string (name); + /* check if header is repeated */ + p = hash_get_mem (ht->value_by_name, name); + if (p) + { + /* if yes combine values */ + header = vec_elt_at_index (ht->headers, p[0]); + vec_pop (header->value); /* drop null byte */ + header->value = format (header->value, ", %U%c", format_ascii_bytes, + value_start, value_len, 0); + vec_free (name); + continue; + } + /* or create new record */ + vec_add2 (ht->headers, header, sizeof (*header)); + header->name = name; + header->value = vec_new (u8, value_len); + clib_memcpy (header->value, value_start, value_len); + vec_terminate_c_string (header->value); + hash_set_mem (ht->value_by_name, header->name, header - ht->headers); + } + while (pos != end); + + *header_table = ht; + + return 0; +} + +/** + * Try to find given header name in header table. + * + * @param header_table Header table to search. + * @param name Header name to match. + * + * @return Header's value in case of success, @c 0 otherwise. + */ +always_inline const char * +http_get_header (http_header_table_t *header_table, const char *name) { - http_state_t state = hc->http_state; - return (state == HTTP_STATE_APP_IO_MORE_DATA || - state == HTTP_STATE_CLIENT_IO_MORE_DATA || - state == HTTP_STATE_WAIT_APP_REPLY || - state == HTTP_STATE_WAIT_APP_METHOD); + uword *p; + http_header_ht_t *header; + + p = hash_get_mem (header_table->value_by_name, name); + if (p) + { + header = vec_elt_at_index (header_table->headers, p[0]); + return (const char *) header->value; + } + + return 0; +} + +/** + * Add header to the list. + * + * @param headers Header list. + * @param name Pointer to header's name buffer. + * @param name_len Length of the name. + * @param value Pointer to header's value buffer. + * @param value_len Length of the value. + * + * @note Headers added at protocol layer: Date, Server, Content-Length + */ +always_inline void +http_add_header (http_header_t **headers, const char *name, uword name_len, + const char *value, uword value_len) +{ + http_header_t *header; + vec_add2 (*headers, header, 1); + header->name.base = (char *) name; + header->name.len = name_len; + header->value.base = (char *) value; + header->value.len = value_len; +} + +/** + * Serialize the header list. + * + * @param headers Header list to serialize. + * + * @return New vector with serialized headers. + * + * The caller is always responsible to free the returned vector. + */ +always_inline u8 * +http_serialize_headers (http_header_t *headers) +{ + u8 *headers_buf = 0, *dst; + u32 headers_buf_len = 2; + http_header_t *header; + + vec_foreach (header, headers) + headers_buf_len += header->name.len + header->value.len + 4; + + vec_validate (headers_buf, headers_buf_len - 1); + dst = headers_buf; + + vec_foreach (header, headers) + { + clib_memcpy (dst, header->name.base, header->name.len); + dst += header->name.len; + *dst++ = ':'; + *dst++ = ' '; + clib_memcpy (dst, header->value.base, header->value.len); + dst += header->value.len; + *dst++ = '\r'; + *dst++ = '\n'; + } + *dst++ = '\r'; + *dst = '\n'; + return headers_buf; +} + +typedef struct +{ + ip46_address_t ip; + u16 port; + u8 is_ip4; +} http_uri_t; + +always_inline int +http_parse_authority_form_target (u8 *target, http_uri_t *authority) +{ + unformat_input_t input; + u32 port; + int rv = 0; + + unformat_init_vector (&input, vec_dup (target)); + if (unformat (&input, "[%U]:%d", unformat_ip6_address, &authority->ip.ip6, + &port)) + { + authority->port = clib_host_to_net_u16 (port); + authority->is_ip4 = 0; + } + else if (unformat (&input, "%U:%d", unformat_ip4_address, &authority->ip.ip4, + &port)) + { + authority->port = clib_host_to_net_u16 (port); + authority->is_ip4 = 1; + } + /* TODO reg-name resolution */ + else + { + clib_warning ("unsupported format '%v'", target); + rv = -1; + } + unformat_free (&input); + return rv; +} + +always_inline u8 * +http_serialize_authority_form_target (http_uri_t *authority) +{ + u8 *s; + + if (authority->is_ip4) + s = format (0, "%U:%d", format_ip4_address, &authority->ip.ip4, + clib_net_to_host_u16 (authority->port)); + else + s = format (0, "[%U]:%d", format_ip6_address, &authority->ip.ip6, + clib_net_to_host_u16 (authority->port)); + + return s; } #endif /* SRC_PLUGINS_HTTP_HTTP_H_ */ diff --git a/src/plugins/http/http_buffer.c b/src/plugins/http/http_buffer.c index f3dc308dbf8..bc1b8c08630 100644 --- a/src/plugins/http/http_buffer.c +++ b/src/plugins/http/http_buffer.c @@ -173,7 +173,7 @@ buf_ptr_drain (http_buffer_t *hb, u32 len) bf->segs[1].data += len; bf->segs[0].len -= len; - HTTP_DBG (1, "drained %u left %u", len, bf->segs[1].len); + HTTP_DBG (1, "drained %u left %u", len, bf->segs[0].len); if (!bf->segs[0].len) { diff --git a/src/plugins/http/http_content_types.h b/src/plugins/http/http_content_types.h new file mode 100644 index 00000000000..ddc02566db7 --- /dev/null +++ b/src/plugins/http/http_content_types.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2024 Cisco Systems, Inc. + */ + +#ifndef SRC_PLUGINS_HTTP_HTTP_CONTENT_TYPES_H_ +#define SRC_PLUGINS_HTTP_HTTP_CONTENT_TYPES_H_ + +#include <http/http.h> + +static http_token_t http_content_types[] = { +#define _(s, ext, str) { http_token_lit (str) }, + foreach_http_content_type +#undef _ +}; + +#define http_content_type_token(e) \ + http_content_types[e].base, http_content_types[e].len + +#endif /* SRC_PLUGINS_HTTP_HTTP_CONTENT_TYPES_H_ */ diff --git a/src/plugins/http/http_header_names.h b/src/plugins/http/http_header_names.h new file mode 100644 index 00000000000..99acac786db --- /dev/null +++ b/src/plugins/http/http_header_names.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2024 Cisco Systems, Inc. + */ + +#ifndef SRC_PLUGINS_HTTP_HTTP_HEADER_NAMES_H_ +#define SRC_PLUGINS_HTTP_HTTP_HEADER_NAMES_H_ + +#include <http/http.h> + +static http_token_t http_header_names[] = { +#define _(sym, str) { http_token_lit (str) }, + foreach_http_header_name +#undef _ +}; + +#define http_header_name_token(e) \ + http_header_names[e].base, http_header_names[e].len + +#define http_header_name_str(e) http_header_names[e].base + +#endif /* SRC_PLUGINS_HTTP_HTTP_HEADER_NAMES_H_ */ diff --git a/src/plugins/http/http_plugin.rst b/src/plugins/http/http_plugin.rst new file mode 100644 index 00000000000..56da3a810b9 --- /dev/null +++ b/src/plugins/http/http_plugin.rst @@ -0,0 +1,507 @@ +.. _http_plugin: + +.. toctree:: + +HTTP Plugin +=========== + +Overview +-------- + +This plugin adds the HTTP protocol to VPP's Host Stack. +As a result parsing and serializing of HTTP/1 requests or responses are available for internal VPP applications. + +Usage +----- + +The plugin exposes following inline functions: ``http_validate_abs_path_syntax``, ``http_validate_query_syntax``, +``http_percent_decode``, ``http_path_remove_dot_segments``, ``http_parse_headers``, ``http_get_header``, +``http_free_header_table``, ``http_add_header``, ``http_serialize_headers``. + +It relies on the hoststack constructs and uses ``http_msg_data_t`` data structure for passing metadata to/from applications. + +Server application +^^^^^^^^^^^^^^^^^^ + +Server application sets ``TRANSPORT_PROTO_HTTP`` as ``transport_proto`` in session endpoint configuration when registering to listen. + +Receiving data +"""""""""""""" + +HTTP plugin sends message header with metadata for parsing, in form of offset and length, followed by all data bytes as received from transport. + +Application will get pre-parsed following items: + +* HTTP method +* target form +* target path offset and length +* target query offset and length +* header section offset and length +* body offset and length + +The example below reads HTTP message header in ``builtin_app_rx_callback``, which is first step application should do: + +.. code-block:: C + + #include <http/http.h> + http_msg_t msg; + rv = svm_fifo_dequeue (ts->rx_fifo, sizeof (msg), (u8 *) &msg); + ASSERT (rv == sizeof (msg)); + +As next step application might validate message and method type, for example application only expects to receive GET requests: + +.. code-block:: C + + if (msg.type != HTTP_MSG_REQUEST || msg.method_type != HTTP_REQ_GET) + { + /* your error handling */ + } + +Now application can start reading HTTP data. First let's read the target path: + +.. code-block:: C + + u8 *target_path; + vec_validate (target_path, msg.data.target_path_len - 1); + rv = svm_fifo_peek (ts->rx_fifo, msg.data.target_path_offset, msg.data.target_path_len, target_path); + ASSERT (rv == msg.data.target_path_len); + +Application might also want to know target form which is stored in ``msg.data.target_form``, you can read more about target forms in RFC9112 section 3.2. +In case of origin form HTTP plugin always sets ``target_path_offset`` after leading slash character. + +Example bellow validates "absolute-path" rule, as described in RFC9110 section 4.1, in case of target in origin form, additionally application can get information if percent encoding is used and decode path: + +.. code-block:: C + + int is_encoded = 0; + if (msg.data.target_form == HTTP_TARGET_ORIGIN_FORM) + { + if (http_validate_abs_path_syntax (target_path, &is_encoded)) + { + /* your error handling */ + } + if (is_encoded) + { + u8 *decoded = http_percent_decode (target_path); + vec_free (target_path); + target_path = decoded; + } + } + +More on topic when to decode in RFC3986 section 2.4. + +When application serves static files, it is highly recommended to sanitize target path by removing dot segments (you don't want to risk path traversal attack): + +.. code-block:: C + + u8 *sanitized_path; + sanitized_path = http_path_remove_dot_segments (target_path); + +Let's move to target query which is optional. Percent encoding might be used too, but we skip it for brevity: + +.. code-block:: C + + u8 *target_query = 0; + if (msg.data.target_query_len) + { + vec_validate (target_query, msg.data.target_query_len - 1); + rv = svm_fifo_peek (ts->rx_fifo, msg.data.target_query_offset, + msg.data.target_query_len, target_query); + ASSERT (rv == msg.data.target_query_len); + if (http_validate_query_syntax (target_query, 0)) + { + /* your error handling */ + } + } + +And now for something completely different, headers. +Headers are parsed using a generic algorithm, independent of the individual header names. +When header is repeated, its combined value consists of all values separated by comma, concatenated in order as received. +Following example shows how to parse headers: + +.. code-block:: C + + #include <http/http_header_names.h> + if (msg.data.headers_len) + { + u8 *headers = 0; + http_header_table_t *ht; + vec_validate (headers, msg.data.headers_len - 1); + rv = svm_fifo_peek (ts->rx_fifo, msg.data.headers_offset, + msg.data.headers_len, headers); + ASSERT (rv == msg.data.headers_len); + if (http_parse_headers (headers, &ht)) + { + /* your error handling */ + } + /* get Accept header */ + const char *accept_value = http_get_header (ht, http_header_name_str (HTTP_HEADER_ACCEPT)); + if (accept_value) + { + /* do something interesting */ + } + http_free_header_table (ht); + vec_free (headers); + } + +Finally application reads body (if any), which might be received in multiple pieces (depends on size), so we might need some state machine in ``builtin_app_rx_callback``. +We will add following members to our session context structure: + +.. code-block:: C + + typedef struct + { + /* ... */ + u64 to_recv; + u8 *resp_body; + } session_ctx_t; + +First we prepare vector for response body, do it only once when you are reading metadata: + +.. code-block:: C + + /* drop everything up to body */ + svm_fifo_dequeue_drop (ts->rx_fifo, msg.data.body_offset); + ctx->to_recv = msg.data.body_len; + /* prepare vector for response body */ + vec_validate (ctx->resp_body, msg.data.body_len - 1); + vec_reset_length (ctx->resp_body); + +Now we can start reading body content, following block of code could be executed multiple times: + +.. code-block:: C + + /* dequeue */ + u32 n_deq = svm_fifo_max_dequeue (ts->rx_fifo); + /* current offset */ + u64 curr = vec_len (ctx->resp_body); + rv = svm_fifo_dequeue (ts->rx_fifo, n_deq, ctx->resp_body + curr); + ASSERT (rv == n_deq); + /* update length of the vector */ + vec_set_len (ctx->resp_body, curr + n_deq); + /* update number of remaining bytes to receive */ + ctx->to_recv -= rv; + /* check if all data received */ + if (ctx->to_recv == 0) + { + /* we are done */ + /* send 200 OK response */ + } + +Sending data +"""""""""""""" + +When server application sends response back to HTTP layer it starts with message metadata, followed by optional serialized headers and finally body (if any). + +Application should set following items: + +* Status code +* target form +* header section offset and length +* body offset and length + +Application could pass headers back to HTTP layer. Header list is created dynamically as vector of ``http_header_t``, +where we store only pointers to buffers (zero copy). +Well known header names are predefined. +The list is serialized just before you send buffer to HTTP layer. + +.. note:: + Following headers are added at protocol layer and **MUST NOT** be set by application: Date, Server, Content-Length + +Following example shows how to create headers section: + +.. code-block:: C + + #include <http/http.h> + #include <http/http_header_names.h> + #include <http/http_content_types.h> + http_header_t *resp_headers = 0; + u8 *headers_buf = 0; + http_add_header (resp_headers, + http_header_name_token (HTTP_HEADER_CONTENT_TYPE), + http_content_type_token (HTTP_CONTENT_TEXT_HTML)); + http_add_header (resp_headers, + http_header_name_token (HTTP_HEADER_CACHE_CONTROL), + http_token_lit ("max-age=600")); + http_add_header (resp_headers, + http_header_name_token (HTTP_HEADER_LOCATION), + (const char *) redirect, vec_len (redirect)); + headers_buf = http_serialize_headers (resp_headers); + +The example below show how to create and send response HTTP message metadata: + +.. code-block:: C + + http_msg_t msg; + msg.type = HTTP_MSG_REPLY; + msg.code = HTTP_STATUS_MOVED + msg.data.headers_offset = 0; + msg.data.headers_len = vec_len (headers_buf); + msg.data.type = HTTP_MSG_DATA_INLINE; + msg.data.body_len = vec_len (tx_buf); + msg.data.body_offset = msg.data.headers_len; + msg.data.len = msg.data.body_len + msg.data.headers_len; + ts = session_get (hs->vpp_session_index, hs->thread_index); + rv = svm_fifo_enqueue (ts->tx_fifo, sizeof (msg), (u8 *) &msg); + ASSERT (rv == sizeof (msg)); + +Next you will send your serialized headers: + +.. code-block:: C + + rv = svm_fifo_enqueue (ts->tx_fifo, vec_len (headers_buf), headers_buf); + ASSERT (rv == msg.data.headers_len); + vec_free (headers_buf); + +Finally application sends response body: + +.. code-block:: C + + rv = svm_fifo_enqueue (ts->tx_fifo, vec_len (tx_buf), tx_buf); + if (rv != vec_len (hs->tx_buf)) + { + hs->tx_offset = rv; + svm_fifo_add_want_deq_ntf (ts->tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); + } + else + { + vec_free (tx_buf); + } + if (svm_fifo_set_event (ts->tx_fifo)) + session_program_tx_io_evt (ts->handle, SESSION_IO_EVT_TX); + +Examples above shows how to send body and headers by copy, alternatively you could pass them as pointer: + +.. code-block:: C + + msg.data.type = HTTP_MSG_DATA_PTR; + /* code omitted for brevity */ + if (msg.data.headers_len) + { + uword headers = pointer_to_uword (headers_buf); + rv = svm_fifo_enqueue (ts->tx_fifo, sizeof (headers), (u8 *) &headers); + ASSERT (rv == sizeof (headers)); + } + uword data = pointer_to_uword (tx_buf); + rv = svm_fifo_enqueue (ts->tx_fifo, sizeof (data), (u8 *) &data); + ASSERT (rv == sizeof (data)); + +In this case you need to free data when you receive next request or when session is closed. + + +Client application +^^^^^^^^^^^^^^^^^^ + +Client application opens connection with vnet URI where transport protocol is set to ``http``. + +Sending data +"""""""""""""" + +HTTP request is sent when connection is successfully established in ``session_connected_callback``. + +When client application sends message to HTTP layer it starts with message metadata, followed by request target, optional headers and body (if any) buffers. + +Application should set following items: + +* HTTP method +* target form, offset and length +* header section offset and length +* body offset and length + +Application could pass headers to HTTP layer. Header list is created dynamically as vector of ``http_header_t``, +where we store only pointers to buffers (zero copy). +Well known header names are predefined. +The list is serialized just before you send buffer to HTTP layer. + +.. note:: + Following headers are added at protocol layer and **MUST NOT** be set by application: Host, User-Agent + + +The example below shows how to create headers section: + +.. code-block:: C + + #include <http/http.h> + #include <http/http_header_names.h> + #include <http/http_content_types.h> + http_header_t *req_headers = 0; + u8 *headers_buf = 0; + http_add_header (req_headers, + http_header_name_token (HTTP_HEADER_ACCEPT), + http_content_type_token (HTTP_CONTENT_TEXT_HTML)); + headers_buf = http_serialize_headers (req_headers); + vec_free (hs->req_headers); + +Following example shows how to set message metadata: + +.. code-block:: C + + http_msg_t msg; + msg.type = HTTP_MSG_REQUEST; + msg.method_type = HTTP_REQ_GET; + msg.data.headers_offset = 0; + /* request target */ + msg.data.target_form = HTTP_TARGET_ORIGIN_FORM; + msg.data.target_path_offset = 0; + msg.data.target_path_len = vec_len (target); + /* custom headers */ + msg.data.headers_offset = msg.data.target_path_len; + msg.data.headers_len = vec_len (headers_buf); + /* no request body because we are doing GET request */ + msg.data.body_len = 0; + /* data type and total length */ + msg.data.type = HTTP_MSG_DATA_INLINE; + msg.data.len = msg.data.target_path_len + msg.data.headers_len + msg.data.body_len; + +Finally application sends everything to HTTP layer: + +.. code-block:: C + + svm_fifo_seg_t segs[3] = { { (u8 *) &msg, sizeof (msg) }, /* message metadata */ + { target, vec_len (target) }, /* request target */ + { headers_buf, vec_len (headers_buf) } }; /* serialized headers */ + rv = svm_fifo_enqueue_segments (as->tx_fifo, segs, 3, 0 /* allow partial */); + vec_free (headers_buf); + if (rv < 0 || rv != sizeof (msg) + msg.data.len) + { + clib_warning ("failed app enqueue"); + return -1; + } + if (svm_fifo_set_event (as->tx_fifo)) + session_program_tx_io_evt (as->handle, SESSION_IO_EVT_TX); + +Examples above shows how to send buffers by copy, alternatively you could pass them as pointer: + +.. code-block:: C + + msg.data.type = HTTP_MSG_DATA_PTR; + msg.method_type = HTTP_REQ_POST; + msg.data.body_len = vec_len (data); + /* code omitted for brevity */ + uword target = pointer_to_uword (target); + uword headers = pointer_to_uword (headers_buf); + uword body = pointer_to_uword (data); + svm_fifo_seg_t segs[4] = { + { (u8 *) &msg, sizeof (msg) }, + { (u8 *) &target, sizeof (target) }, + { (u8 *) &headers, sizeof (headers) }, + { (u8 *) &body, sizeof (body) }, + }; + rv = svm_fifo_enqueue_segments (s->tx_fifo, segs, 4, 0 /* allow partial */); + ASSERT (rv == (sizeof (msg) + sizeof (target) + sizeof (headers) + sizeof (body))); + +In this case you need to free data when you receive response or when session is closed. + +Receiving data +"""""""""""""" + +HTTP plugin sends message header with metadata for parsing, in form of offset and length, followed by all data bytes as received from transport. + +Application will get pre-parsed following items: + +* status code +* header section offset and length +* body offset and length + +The example below reads HTTP message header in ``builtin_app_rx_callback``, which is first step application should do: + +.. code-block:: C + + #include <http/http.h> + http_msg_t msg; + rv = svm_fifo_dequeue (ts->rx_fifo, sizeof (msg), (u8 *) &msg); + ASSERT (rv == sizeof (msg)); + +As next step application might validate message type and status code: + +.. code-block:: C + + if (msg.type != HTTP_MSG_REPLY) + { + /* your error handling */ + } + if (msg.code != HTTP_STATUS_OK) + { + /* your error handling */ + /* of course you can continue with steps bellow */ + /* you might be interested in some headers or body content (if any) */ + } + +Headers are parsed using a generic algorithm, independent of the individual header names. +When header is repeated, its combined value consists of all values separated by comma, concatenated in order as received. +Following example shows how to parse headers: + +.. code-block:: C + + #include <http/http_header_names.h> + if (msg.data.headers_len) + { + u8 *headers = 0; + http_header_table_t *ht; + vec_validate (headers, msg.data.headers_len - 1); + rv = svm_fifo_peek (ts->rx_fifo, msg.data.headers_offset, + msg.data.headers_len, headers); + ASSERT (rv == msg.data.headers_len); + if (http_parse_headers (headers, &ht)) + { + /* your error handling */ + } + /* get Content-Type header */ + const char *content_type = http_get_header (ht, http_header_name_str (HTTP_HEADER_CONTENT_TYPE)); + if (content_type) + { + /* do something interesting */ + } + http_free_header_table (ht); + vec_free (headers); + } + +Finally application reads body, which might be received in multiple pieces (depends on size), so we might need some state machine in ``builtin_app_rx_callback``. +We will add following members to our session context structure: + +.. code-block:: C + + typedef struct + { + /* ... */ + u64 to_recv; + u8 *resp_body; + } session_ctx_t; + +First we prepare vector for response body, do it only once when you are reading metadata: + +.. code-block:: C + + /* drop everything up to body */ + svm_fifo_dequeue_drop (ts->rx_fifo, msg.data.body_offset); + ctx->to_recv = msg.data.body_len; + /* prepare vector for response body */ + vec_validate (ctx->resp_body, msg.data.body_len - 1); + vec_reset_length (ctx->resp_body); + +Now we can start reading body content, following block of code could be executed multiple times: + +.. code-block:: C + + /* dequeue */ + u32 max_deq = svm_fifo_max_dequeue (ts->rx_fifo); + u32 n_deq = clib_min (to_recv, max_deq); + /* current offset */ + u64 curr = vec_len (ctx->resp_body); + rv = svm_fifo_dequeue (ts->rx_fifo, n_deq, ctx->resp_body + curr); + if (rv < 0 || rv != n_deq) + { + /* your error handling */ + } + /* update length of the vector */ + vec_set_len (ctx->resp_body, curr + n_deq); + /* update number of remaining bytes to receive */ + ASSERT (to_recv >= rv); + ctx->to_recv -= rv; + /* check if all data received */ + if (ctx->to_recv == 0) + { + /* we are done */ + /* close the session if you don't want to send another request */ + /* and update state machine... */ + } diff --git a/src/plugins/http/http_status_codes.h b/src/plugins/http/http_status_codes.h new file mode 100644 index 00000000000..100095c8f42 --- /dev/null +++ b/src/plugins/http/http_status_codes.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2024 Cisco Systems, Inc. + */ + +#ifndef SRC_PLUGINS_HTTP_HTTP_STATUS_CODES_H_ +#define SRC_PLUGINS_HTTP_HTTP_STATUS_CODES_H_ + +#include <http/http.h> + +static const char *http_status_code_str[] = { +#define _(c, s, str) str, + foreach_http_status_code +#undef _ +}; + +static inline u8 * +format_http_status_code (u8 *s, va_list *va) +{ + http_status_code_t status_code = va_arg (*va, http_status_code_t); + if (status_code < HTTP_N_STATUS) + s = format (s, "%s", http_status_code_str[status_code]); + else + s = format (s, "invalid status code %d", status_code); + return s; +} + +#endif /* SRC_PLUGINS_HTTP_HTTP_STATUS_CODES_H_ */ diff --git a/src/plugins/http/http_test.c b/src/plugins/http/http_test.c new file mode 100644 index 00000000000..1f2f21dd19a --- /dev/null +++ b/src/plugins/http/http_test.c @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2024 Cisco Systems, Inc. + */ + +#include <http/http.h> + +static clib_error_t * +test_http_authority_command_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + u8 *target = 0; + http_uri_t authority; + int rv; + + if (!unformat (input, "%v", &target)) + return clib_error_return (0, "error: no input provided"); + + rv = http_parse_authority_form_target (target, &authority); + vec_free (target); + if (rv) + return clib_error_return (0, "error: parsing failed"); + + target = http_serialize_authority_form_target (&authority); + vlib_cli_output (vm, "%v", target); + vec_free (target); + + return 0; +} + +VLIB_CLI_COMMAND (test_http_authority_command) = { + .path = "test http authority-form", + .short_help = "test dns authority-form", + .function = test_http_authority_command_fn, +}; diff --git a/src/plugins/http/http_timer.c b/src/plugins/http/http_timer.c index 42fe69076fe..5ee8efc8551 100644 --- a/src/plugins/http/http_timer.c +++ b/src/plugins/http/http_timer.c @@ -71,6 +71,8 @@ http_timers_init (vlib_main_t *vm, http_conn_timeout_fn *cb_fn) http_tw_ctx_t *twc = &http_tw_ctx; vlib_node_t *n; + ASSERT (twc->tw.timers == 0); + tw_timer_wheel_init_2t_1w_2048sl (&twc->tw, http_timer_process_expired_cb, 1.0 /* timer interval */, ~0); clib_spinlock_init (&twc->tw_lock); diff --git a/src/plugins/http_static/builtinurl/json_urls.c b/src/plugins/http_static/builtinurl/json_urls.c index 808893aac79..19c5245e4b2 100644 --- a/src/plugins/http_static/builtinurl/json_urls.c +++ b/src/plugins/http_static/builtinurl/json_urls.c @@ -20,77 +20,68 @@ hss_url_handler_rc_t handle_get_version (hss_url_handler_args_t *args) { u8 *s = 0; + unformat_input_t input; + int verbose = 0; + + if (args->query) + { + unformat_init_vector (&input, args->query); + if (unformat (&input, "verbose=")) + { + if (unformat (&input, "true")) + verbose = 1; + } + } s = format (s, "{\"vpp_details\": {"); s = format (s, " \"version\": \"%s\",", VPP_BUILD_VER); + if (verbose) + { + s = format (s, " \"build_by\": \"%s\",", VPP_BUILD_USER); + s = format (s, " \"build_host\": \"%s\",", VPP_BUILD_HOST); + s = format (s, " \"build_dir\": \"%s\",", VPP_BUILD_TOPDIR); + } s = format (s, " \"build_date\": \"%s\"}}\r\n", VPP_BUILD_DATE); args->data = s; args->data_len = vec_len (s); + args->ct = HTTP_CONTENT_APP_JSON; args->free_vec_data = 1; return HSS_URL_HANDLER_OK; } -void -trim_path_from_request (u8 *s, char *path) -{ - u8 *cp; - int trim_length = strlen (path) + 1 /* remove '?' */; - - /* Get rid of the path and question-mark */ - vec_delete (s, trim_length, 0); - - /* Tail trim irrelevant browser info */ - cp = s; - while ((cp - s) < vec_len (s)) - { - if (*cp == ' ') - { - /* - * Makes request a vector which happens to look - * like a c-string. - */ - *cp = 0; - vec_set_len (s, cp - s); - break; - } - cp++; - } -} - hss_url_handler_rc_t handle_get_interface_stats (hss_url_handler_args_t *args) { u8 *s = 0, *stats = 0; - uword *p; - u32 *sw_if_indices = 0; + u32 sw_if_index, *sw_if_indices = 0; vnet_hw_interface_t *hi; vnet_sw_interface_t *si; char *q = "\""; int i; int need_comma = 0; + unformat_input_t input; u8 *format_vnet_sw_interface_cntrs (u8 * s, vnet_interface_main_t * im, vnet_sw_interface_t * si, int json); vnet_main_t *vnm = vnet_get_main (); vnet_interface_main_t *im = &vnm->interface_main; /* Get stats for a single interface via http POST */ - if (args->reqtype == HTTP_REQ_POST) + if (args->req_type == HTTP_REQ_POST) { - trim_path_from_request (args->request, "interface_stats.json"); - + unformat_init_vector (&input, args->req_data); /* Find the sw_if_index */ - p = hash_get (im->hw_interface_by_name, args->request); - if (!p) + if (!unformat (&input, "%U", unformat_vnet_sw_interface, vnm, + &sw_if_index)) { s = format (s, "{\"interface_stats\": {[\n"); - s = format (s, " \"name\": \"%s\",", args->request); + s = format (s, " \"name\": \"%s\",", args->req_data); s = format (s, " \"error\": \"%s\"", "UnknownInterface"); s = format (s, "]}\n"); goto out; } - vec_add1 (sw_if_indices, p[0]); + vec_add1 (sw_if_indices, sw_if_index); } else /* default, HTTP_BUILTIN_METHOD_GET */ { @@ -127,6 +118,7 @@ handle_get_interface_stats (hss_url_handler_args_t *args) out: args->data = s; args->data_len = vec_len (s); + args->ct = HTTP_CONTENT_APP_JSON; args->free_vec_data = 1; vec_free (sw_if_indices); vec_free (stats); @@ -167,6 +159,7 @@ handle_get_interface_list (hss_url_handler_args_t *args) args->data = s; args->data_len = vec_len (s); + args->ct = HTTP_CONTENT_APP_JSON; args->free_vec_data = 1; return HSS_URL_HANDLER_OK; } diff --git a/src/plugins/http_static/http_cache.c b/src/plugins/http_static/http_cache.c index 8b9751b7f78..2e63e335d47 100644 --- a/src/plugins/http_static/http_cache.c +++ b/src/plugins/http_static/http_cache.c @@ -17,6 +17,8 @@ #include <vppinfra/bihash_template.c> #include <vppinfra/unix.h> #include <vlib/vlib.h> +#include <sys/stat.h> +#include <vppinfra/time_range.h> static void hss_cache_lock (hss_cache_t *hc) @@ -153,7 +155,7 @@ lru_update (hss_cache_t *hc, hss_cache_entry_t *ep, f64 now) static void hss_cache_attach_entry (hss_cache_t *hc, u32 ce_index, u8 **data, - u64 *data_len) + u64 *data_len, u8 **last_modified) { hss_cache_entry_t *ce; @@ -162,6 +164,7 @@ hss_cache_attach_entry (hss_cache_t *hc, u32 ce_index, u8 **data, ce->inuse++; *data = ce->data; *data_len = vec_len (ce->data); + *last_modified = ce->last_modified; /* Update the cache entry, mark it in-use */ lru_update (hc, ce, vlib_time_now (vlib_get_main ())); @@ -209,16 +212,15 @@ hss_cache_lookup (hss_cache_t *hc, u8 *path) u32 hss_cache_lookup_and_attach (hss_cache_t *hc, u8 *path, u8 **data, - u64 *data_len) + u64 *data_len, u8 **last_modified) { u32 ce_index; - /* Make sure nobody removes the entry while we look it up */ hss_cache_lock (hc); ce_index = hss_cache_lookup (hc, path); if (ce_index != ~0) - hss_cache_attach_entry (hc, ce_index, data, data_len); + hss_cache_attach_entry (hc, ce_index, data, data_len, last_modified); hss_cache_unlock (hc); @@ -260,6 +262,7 @@ hss_cache_do_evictions (hss_cache_t *hc) hc->cache_evictions++; vec_free (ce->filename); vec_free (ce->data); + vec_free (ce->last_modified); if (hc->debug_level > 1) clib_warning ("pool put index %d", ce - hc->cache_pool); @@ -271,13 +274,15 @@ hss_cache_do_evictions (hss_cache_t *hc) } u32 -hss_cache_add_and_attach (hss_cache_t *hc, u8 *path, u8 **data, u64 *data_len) +hss_cache_add_and_attach (hss_cache_t *hc, u8 *path, u8 **data, u64 *data_len, + u8 **last_modified) { BVT (clib_bihash_kv) kv; hss_cache_entry_t *ce; clib_error_t *error; u8 *file_data; u32 ce_index; + struct stat dm; hss_cache_lock (hc); @@ -298,11 +303,17 @@ hss_cache_add_and_attach (hss_cache_t *hc, u8 *path, u8 **data, u64 *data_len) pool_get_zero (hc->cache_pool, ce); ce->filename = vec_dup (path); ce->data = file_data; + if (stat ((char *) path, &dm) == 0) + { + ce->last_modified = + format (0, "%U GMT", format_clib_timebase_time, (f64) dm.st_mtime); + } /* Attach cache entry without additional lock */ ce->inuse++; *data = file_data; *data_len = vec_len (file_data); + *last_modified = ce->last_modified; lru_add (hc, ce, vlib_time_now (vlib_get_main ())); hc->cache_size += vec_len (ce->data); @@ -364,6 +375,7 @@ hss_cache_clear (hss_cache_t *hc) hc->cache_evictions++; vec_free (ce->filename); vec_free (ce->data); + vec_free (ce->last_modified); if (hc->debug_level > 1) clib_warning ("pool put index %d", ce - hc->cache_pool); pool_put (hc->cache_pool, ce); @@ -421,19 +433,19 @@ format_hss_cache (u8 *s, va_list *args) { s = format (s, "cache size %lld bytes, limit %lld bytes, evictions %lld", hc->cache_size, hc->cache_limit, hc->cache_evictions); - return 0; + return s; } vm = vlib_get_main (); now = vlib_time_now (vm); - s = format (s, "%U", format_hss_cache_entry, 0 /* header */, now); + s = format (s, "%U\n", format_hss_cache_entry, 0 /* header */, now); for (index = hc->first_index; index != ~0;) { ce = pool_elt_at_index (hc->cache_pool, index); index = ce->next_index; - s = format (s, "%U", format_hss_cache_entry, ce, now); + s = format (s, "%U\n", format_hss_cache_entry, ce, now); } s = format (s, "%40s%12lld", "Total Size", hc->cache_size); diff --git a/src/plugins/http_static/http_cache.h b/src/plugins/http_static/http_cache.h index a89ed5e7e94..21f71a924d5 100644 --- a/src/plugins/http_static/http_cache.h +++ b/src/plugins/http_static/http_cache.h @@ -22,6 +22,9 @@ typedef struct hss_cache_entry_ { /** Name of the file */ u8 *filename; + /** Last modified date, format: + * <day-name>, <day> <month> <year> <hour>:<minute>:<second> GMT */ + u8 *last_modified; /** Contents of the file, as a u8 * vector */ u8 *data; /** Last time the cache entry was used */ @@ -58,9 +61,9 @@ typedef struct hss_cache_ } hss_cache_t; u32 hss_cache_lookup_and_attach (hss_cache_t *hc, u8 *path, u8 **data, - u64 *data_len); + u64 *data_len, u8 **last_modified); u32 hss_cache_add_and_attach (hss_cache_t *hc, u8 *path, u8 **data, - u64 *data_len); + u64 *data_len, u8 **last_modified); void hss_cache_detach_entry (hss_cache_t *hc, u32 ce_index); u32 hss_cache_clear (hss_cache_t *hc); void hss_cache_init (hss_cache_t *hc, uword cache_size, u8 debug_level); diff --git a/src/plugins/http_static/http_static.api b/src/plugins/http_static/http_static.api index 4d6d8bfe9b5..dd4f513a420 100644 --- a/src/plugins/http_static/http_static.api +++ b/src/plugins/http_static/http_static.api @@ -2,7 +2,8 @@ /** \file This file defines static http server control-plane API messages */ -option version = "2.1.0"; + +option version = "2.2.0"; /** \brief Configure and enable the static http server @param client_index - opaque cookie to identify the sender @@ -16,6 +17,39 @@ option version = "2.1.0"; */ autoreply define http_static_enable { + option deprecated; + + /* Client identifier, set from api_main.my_client_index */ + u32 client_index; + + /* Arbitrary context, so client can match reply to request */ + u32 context; + /* Typical options */ + u32 fifo_size; + u32 cache_size_limit; + /* Unusual options */ + u32 prealloc_fifos; + u32 private_segment_size; + + /* Root of the html path */ + string www_root[256]; + /* The bind URI */ + string uri[256]; +}; + +/** \brief Configure and enable the static http server + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param fifo_size - size (in bytes) of the session FIFOs + @param cache_size_limit - size (in bytes) of the in-memory file data cache + @param max_age - how long a response is considered fresh (in seconds) + @param prealloc_fifos - number of preallocated fifos (usually 0) + @param private_segment_size - fifo segment size (usually 0) + @param www_root - html root path + @param uri - bind URI, defaults to "tcp://0.0.0.0/80" +*/ + +autoreply define http_static_enable_v2 { /* Client identifier, set from api_main.my_client_index */ u32 client_index; @@ -24,6 +58,7 @@ autoreply define http_static_enable { /* Typical options */ u32 fifo_size; u32 cache_size_limit; + u32 max_age [default=600]; /* Unusual options */ u32 prealloc_fifos; u32 private_segment_size; diff --git a/src/plugins/http_static/http_static.c b/src/plugins/http_static/http_static.c index 8f8fe37b7c1..967b8474af8 100644 --- a/src/plugins/http_static/http_static.c +++ b/src/plugins/http_static/http_static.c @@ -66,7 +66,7 @@ hss_register_url_handler (hss_url_handler_fn fp, const char *url, */ static int hss_enable_api (u32 fifo_size, u32 cache_limit, u32 prealloc_fifos, - u32 private_segment_size, u8 *www_root, u8 *uri) + u32 private_segment_size, u8 *www_root, u8 *uri, u32 max_age) { hss_main_t *hsm = &hss_main; int rv; @@ -77,6 +77,7 @@ hss_enable_api (u32 fifo_size, u32 cache_limit, u32 prealloc_fifos, hsm->private_segment_size = private_segment_size; hsm->www_root = format (0, "%s%c", www_root, 0); hsm->uri = format (0, "%s%c", uri, 0); + hsm->max_age = max_age; if (vec_len (hsm->www_root) < 2) return VNET_API_ERROR_INVALID_VALUE; @@ -84,7 +85,10 @@ hss_enable_api (u32 fifo_size, u32 cache_limit, u32 prealloc_fifos, if (hsm->app_index != ~0) return VNET_API_ERROR_APP_ALREADY_ATTACHED; - vnet_session_enable_disable (hsm->vlib_main, 1 /* turn on TCP, etc. */); + session_enable_disable_args_t args = { .is_en = 1, + .rt_engine_type = + RT_BACKEND_ENGINE_RULE_TABLE }; + vnet_session_enable_disable (hsm->vlib_main, &args); rv = hss_create (hsm->vlib_main); switch (rv) @@ -110,14 +114,33 @@ static void vl_api_http_static_enable_t_handler mp->uri[ARRAY_LEN (mp->uri) - 1] = 0; mp->www_root[ARRAY_LEN (mp->www_root) - 1] = 0; - rv = - hss_enable_api (ntohl (mp->fifo_size), ntohl (mp->cache_size_limit), - ntohl (mp->prealloc_fifos), - ntohl (mp->private_segment_size), mp->www_root, mp->uri); + rv = hss_enable_api (ntohl (mp->fifo_size), ntohl (mp->cache_size_limit), + ntohl (mp->prealloc_fifos), + ntohl (mp->private_segment_size), mp->www_root, mp->uri, + HSS_DEFAULT_MAX_AGE); REPLY_MACRO (VL_API_HTTP_STATIC_ENABLE_REPLY); } +/* API message handler */ +static void +vl_api_http_static_enable_v2_t_handler (vl_api_http_static_enable_v2_t *mp) +{ + vl_api_http_static_enable_v2_reply_t *rmp; + hss_main_t *hsm = &hss_main; + int rv; + + mp->uri[ARRAY_LEN (mp->uri) - 1] = 0; + mp->www_root[ARRAY_LEN (mp->www_root) - 1] = 0; + + rv = hss_enable_api (ntohl (mp->fifo_size), ntohl (mp->cache_size_limit), + ntohl (mp->prealloc_fifos), + ntohl (mp->private_segment_size), mp->www_root, mp->uri, + ntohl (mp->max_age)); + + REPLY_MACRO (VL_API_HTTP_STATIC_ENABLE_V2_REPLY); +} + #include <http_static/http_static.api.c> static clib_error_t * hss_api_init (vlib_main_t *vm) diff --git a/src/plugins/http_static/http_static.h b/src/plugins/http_static/http_static.h index 2850d356b74..bee79090d2b 100644 --- a/src/plugins/http_static/http_static.h +++ b/src/plugins/http_static/http_static.h @@ -23,6 +23,8 @@ #include <vppinfra/error.h> #include <http_static/http_cache.h> +#define HSS_DEFAULT_MAX_AGE 600 + /** @file http_static.h * Static http server definitions */ @@ -45,13 +47,15 @@ typedef struct /** Data length */ u64 data_len; /** Current data send offset */ - u32 data_offset; + u64 data_offset; /** Need to free data in detach_cache_entry */ int free_data; /** File cache pool index */ u32 cache_pool_index; - /** Content type, e.g. text, text/javascript, etc. */ - http_content_type_t content_type; + /** Response header list */ + http_header_t *resp_headers; + /** Serialized headers to send */ + u8 *headers_buf; } hss_session_t; typedef struct hss_session_handle_ @@ -79,8 +83,9 @@ typedef struct hss_url_handler_args_ /* Request args */ struct { - u8 *request; - http_req_method_t reqtype; + u8 *query; + u8 *req_data; + http_req_method_t req_type; }; /* Reply args */ @@ -90,6 +95,7 @@ typedef struct hss_url_handler_args_ uword data_len; u8 free_vec_data; http_status_code_t sc; + http_content_type_t ct; }; }; } hss_url_handler_args_t; @@ -152,6 +158,10 @@ typedef struct u8 enable_url_handlers; /** Max cache size before LRU occurs */ u64 cache_size; + /** How long a response is considered fresh (in seconds) */ + u32 max_age; + /** Formatted max_age: "max-age=xyz" */ + u8 *max_age_formatted; /** hash table of file extensions to mime types string indices */ uword *mime_type_indices_by_file_extensions; diff --git a/src/plugins/http_static/http_static_test.c b/src/plugins/http_static/http_static_test.c index 3503a1b0812..f701c8b9ee7 100644 --- a/src/plugins/http_static/http_static_test.c +++ b/src/plugins/http_static/http_static_test.c @@ -18,6 +18,7 @@ #include <vlibapi/api.h> #include <vlibmemory/api.h> #include <vppinfra/error.h> +#include <http_static/http_static.h> uword unformat_sw_if_index (unformat_input_t * input, va_list * args); @@ -126,6 +127,96 @@ api_http_static_enable (vat_main_t * vam) return ret; } +static int +api_http_static_enable_v2 (vat_main_t *vam) +{ + unformat_input_t *line_input = vam->input; + vl_api_http_static_enable_v2_t *mp; + u64 tmp; + u8 *www_root = 0; + u8 *uri = 0; + u32 prealloc_fifos = 0; + u32 private_segment_size = 0; + u32 fifo_size = 8 << 10; + u32 cache_size_limit = 1 << 20; + u32 max_age = HSS_DEFAULT_MAX_AGE; + int ret; + + /* Parse args required to build the message */ + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "www-root %s", &www_root)) + ; + else if (unformat (line_input, "prealloc-fifos %d", &prealloc_fifos)) + ; + else if (unformat (line_input, "private-segment-size %U", + unformat_memory_size, &tmp)) + { + if (tmp >= 0x100000000ULL) + { + errmsg ("private segment size %llu, too large", tmp); + return -99; + } + private_segment_size = (u32) tmp; + } + else if (unformat (line_input, "fifo-size %U", unformat_memory_size, + &tmp)) + { + if (tmp >= 0x100000000ULL) + { + errmsg ("fifo-size %llu, too large", tmp); + return -99; + } + fifo_size = (u32) tmp; + } + else if (unformat (line_input, "cache-size %U", unformat_memory_size, + &tmp)) + { + if (tmp < (128ULL << 10)) + { + errmsg ("cache-size must be at least 128kb"); + return -99; + } + cache_size_limit = (u32) tmp; + } + else if (unformat (line_input, "max-age %d", &max_age)) + ; + else if (unformat (line_input, "uri %s", &uri)) + ; + else + { + errmsg ("unknown input `%U'", format_unformat_error, line_input); + return -99; + } + } + + if (www_root == 0) + { + errmsg ("Must specify www-root"); + return -99; + } + + if (uri == 0) + uri = format (0, "tcp://0.0.0.0/80%c", 0); + + /* Construct the API message */ + M (HTTP_STATIC_ENABLE_V2, mp); + strncpy_s ((char *) mp->www_root, 256, (const char *) www_root, 256); + strncpy_s ((char *) mp->uri, 256, (const char *) uri, 256); + mp->fifo_size = ntohl (fifo_size); + mp->cache_size_limit = ntohl (cache_size_limit); + mp->prealloc_fifos = ntohl (prealloc_fifos); + mp->private_segment_size = ntohl (private_segment_size); + mp->max_age = ntohl (max_age); + + /* send it... */ + S (mp); + + /* Wait for a reply... */ + W (ret); + return ret; +} + #include <http_static/http_static.api_test.c> /* diff --git a/src/plugins/http_static/static_server.c b/src/plugins/http_static/static_server.c index 040cdca9d7a..48e71f51629 100644 --- a/src/plugins/http_static/static_server.c +++ b/src/plugins/http_static/static_server.c @@ -19,6 +19,9 @@ #include <sys/stat.h> #include <unistd.h> +#include <http/http_header_names.h> +#include <http/http_content_types.h> + /** @file static_server.c * Static http server, sufficient to serve .html / .css / .js content. */ @@ -55,8 +58,6 @@ hss_session_free (hss_session_t *hs) { hss_main_t *hsm = &hss_main; - pool_put (hsm->sessions[hs->thread_index], hs); - if (CLIB_DEBUG) { u32 save_thread_index; @@ -65,6 +66,8 @@ hss_session_free (hss_session_t *hs) memset (hs, 0xfa, sizeof (*hs)); hs->thread_index = save_thread_index; } + + pool_put (hsm->sessions[hs->thread_index], hs); } /** \brief Disconnect a session @@ -83,48 +86,84 @@ start_send_data (hss_session_t *hs, http_status_code_t status) { http_msg_t msg; session_t *ts; + u8 *headers_buf = 0; + u32 n_enq; + u64 to_send; int rv; ts = session_get (hs->vpp_session_index, hs->thread_index); + if (vec_len (hs->resp_headers)) + { + headers_buf = http_serialize_headers (hs->resp_headers); + vec_free (hs->resp_headers); + msg.data.headers_offset = 0; + msg.data.headers_len = vec_len (headers_buf); + } + else + { + msg.data.headers_offset = 0; + msg.data.headers_len = 0; + } + msg.type = HTTP_MSG_REPLY; msg.code = status; - msg.content_type = hs->content_type; - msg.data.len = hs->data_len; + msg.data.body_len = hs->data_len; + msg.data.len = msg.data.body_len + msg.data.headers_len; - if (hs->data_len > hss_main.use_ptr_thresh) + if (msg.data.len > hss_main.use_ptr_thresh) { msg.data.type = HTTP_MSG_DATA_PTR; rv = svm_fifo_enqueue (ts->tx_fifo, sizeof (msg), (u8 *) &msg); ASSERT (rv == sizeof (msg)); + if (msg.data.headers_len) + { + hs->headers_buf = headers_buf; + uword headers = pointer_to_uword (hs->headers_buf); + rv = + svm_fifo_enqueue (ts->tx_fifo, sizeof (headers), (u8 *) &headers); + ASSERT (rv == sizeof (headers)); + } + uword data = pointer_to_uword (hs->data); rv = svm_fifo_enqueue (ts->tx_fifo, sizeof (data), (u8 *) &data); - ASSERT (rv == sizeof (sizeof (data))); + ASSERT (rv == sizeof (data)); goto done; } msg.data.type = HTTP_MSG_DATA_INLINE; + msg.data.body_offset = msg.data.headers_len; rv = svm_fifo_enqueue (ts->tx_fifo, sizeof (msg), (u8 *) &msg); ASSERT (rv == sizeof (msg)); - if (!msg.data.len) + if (msg.data.headers_len) + { + rv = svm_fifo_enqueue (ts->tx_fifo, vec_len (headers_buf), headers_buf); + ASSERT (rv == msg.data.headers_len); + vec_free (headers_buf); + } + + if (!msg.data.body_len) goto done; - rv = svm_fifo_enqueue (ts->tx_fifo, hs->data_len, hs->data); + to_send = hs->data_len; + n_enq = clib_min (svm_fifo_size (ts->tx_fifo), to_send); - if (rv != hs->data_len) + rv = svm_fifo_enqueue (ts->tx_fifo, n_enq, hs->data); + + if (rv < to_send) { - hs->data_offset = rv; + hs->data_offset = (rv > 0) ? rv : 0; svm_fifo_add_want_deq_ntf (ts->tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); } done: if (svm_fifo_set_event (ts->tx_fifo)) - session_send_io_evt_to_thread (ts->tx_fifo, SESSION_IO_EVT_TX); + session_program_tx_io_evt (ts->handle, SESSION_IO_EVT_TX); } __clib_export void @@ -142,6 +181,15 @@ hss_session_send_data (hss_url_handler_args_t *args) hs->data = args->data; hs->data_len = args->data_len; hs->free_data = args->free_vec_data; + + /* Set content type only if we have some response data */ + if (hs->data_len) + { + http_add_header (&hs->resp_headers, + http_header_name_token (HTTP_HEADER_CONTENT_TYPE), + http_content_type_token (args->ct)); + } + start_send_data (hs, args->sc); } @@ -212,30 +260,27 @@ content_type_from_request (u8 *request) static int try_url_handler (hss_main_t *hsm, hss_session_t *hs, http_req_method_t rt, - u8 *request) + u8 *target_path, u8 *target_query, u8 *data) { http_status_code_t sc = HTTP_STATUS_OK; hss_url_handler_args_t args = {}; uword *p, *url_table; - http_content_type_t type; int rv; - if (!hsm->enable_url_handlers || !request) + if (!hsm->enable_url_handlers || !target_path) return -1; /* zero-length? try "index.html" */ - if (vec_len (request) == 0) + if (vec_len (target_path) == 0) { - request = format (request, "index.html"); + target_path = format (target_path, "index.html"); } - type = content_type_from_request (request); - /* Look for built-in GET / POST handlers */ url_table = (rt == HTTP_REQ_GET) ? hsm->get_url_handlers : hsm->post_url_handlers; - p = hash_get_mem (url_table, request); + p = hash_get_mem (url_table, target_path); if (!p) return -1; @@ -244,10 +289,12 @@ try_url_handler (hss_main_t *hsm, hss_session_t *hs, http_req_method_t rt, hs->cache_pool_index = ~0; if (hsm->debug_level > 0) - clib_warning ("%s '%s'", (rt == HTTP_REQ_GET) ? "GET" : "POST", request); + clib_warning ("%s '%s'", (rt == HTTP_REQ_GET) ? "GET" : "POST", + target_path); - args.reqtype = rt; - args.request = request; + args.req_type = rt; + args.query = target_query; + args.req_data = data; args.sh.thread_index = hs->thread_index; args.sh.session_index = hs->session_index; @@ -260,18 +307,25 @@ try_url_handler (hss_main_t *hsm, hss_session_t *hs, http_req_method_t rt, if (rv == HSS_URL_HANDLER_ERROR) { clib_warning ("builtin handler %llx hit on %s '%s' but failed!", p[0], - (rt == HTTP_REQ_GET) ? "GET" : "POST", request); - sc = HTTP_STATUS_NOT_FOUND; + (rt == HTTP_REQ_GET) ? "GET" : "POST", target_path); + sc = HTTP_STATUS_BAD_GATEWAY; } hs->data = args.data; hs->data_len = args.data_len; hs->free_data = args.free_vec_data; - hs->content_type = type; + + /* Set content type only if we have some response data */ + if (hs->data_len) + { + http_add_header (&hs->resp_headers, + http_header_name_token (HTTP_HEADER_CONTENT_TYPE), + http_content_type_token (args.ct)); + } start_send_data (hs, sc); - if (!hs->data) + if (!hs->data_len) hss_session_disconnect_transport (hs); return 0; @@ -335,18 +389,20 @@ try_index_file (hss_main_t *hsm, hss_session_t *hs, u8 *path) } redirect = - format (0, - "Location: http%s://%U%s%s\r\n\r\n", - proto == TRANSPORT_PROTO_TLS ? "s" : "", format_ip46_address, - &endpt.ip, endpt.is_ip4, print_port ? port_str : (u8 *) "", path); + format (0, "http%s://%U%s%s", proto == TRANSPORT_PROTO_TLS ? "s" : "", + format_ip46_address, &endpt.ip, endpt.is_ip4, + print_port ? port_str : (u8 *) "", path); if (hsm->debug_level > 0) clib_warning ("redirect: %s", redirect); vec_free (port_str); - hs->data = redirect; - hs->data_len = vec_len (redirect); + http_add_header (&hs->resp_headers, + http_header_name_token (HTTP_HEADER_LOCATION), + (const char *) redirect, vec_len (redirect)); + hs->data = redirect; /* TODO: find better way */ + hs->data_len = 0; hs->free_data = 1; return HTTP_STATUS_MOVED; @@ -354,29 +410,28 @@ try_index_file (hss_main_t *hsm, hss_session_t *hs, u8 *path) static int try_file_handler (hss_main_t *hsm, hss_session_t *hs, http_req_method_t rt, - u8 *request) + u8 *target) { http_status_code_t sc = HTTP_STATUS_OK; - u8 *path; + u8 *path, *sanitized_path; u32 ce_index; http_content_type_t type; + u8 *last_modified; /* Feature not enabled */ if (!hsm->www_root) return -1; - type = content_type_from_request (request); + /* Remove dot segments to prevent path traversal */ + sanitized_path = http_path_remove_dot_segments (target); /* * Construct the file to open - * Browsers are capable of sporadically including a leading '/' */ - if (!request) + if (!target) path = format (0, "%s%c", hsm->www_root, 0); - else if (request[0] == '/') - path = format (0, "%s%s%c", hsm->www_root, request, 0); else - path = format (0, "%s/%s%c", hsm->www_root, request, 0); + path = format (0, "%s/%s%c", hsm->www_root, sanitized_path, 0); if (hsm->debug_level > 0) clib_warning ("%s '%s'", (rt == HTTP_REQ_GET) ? "GET" : "POST", path); @@ -386,8 +441,8 @@ try_file_handler (hss_main_t *hsm, hss_session_t *hs, http_req_method_t rt, hs->data_offset = 0; - ce_index = - hss_cache_lookup_and_attach (&hsm->cache, path, &hs->data, &hs->data_len); + ce_index = hss_cache_lookup_and_attach (&hsm->cache, path, &hs->data, + &hs->data_len, &last_modified); if (ce_index == ~0) { if (!file_path_is_valid (path)) @@ -406,8 +461,8 @@ try_file_handler (hss_main_t *hsm, hss_session_t *hs, http_req_method_t rt, sc = try_index_file (hsm, hs, path); goto done; } - ce_index = - hss_cache_add_and_attach (&hsm->cache, path, &hs->data, &hs->data_len); + ce_index = hss_cache_add_and_attach (&hsm->cache, path, &hs->data, + &hs->data_len, &last_modified); if (ce_index == ~0) { sc = HTTP_STATUS_INTERNAL_ERROR; @@ -418,43 +473,61 @@ try_file_handler (hss_main_t *hsm, hss_session_t *hs, http_req_method_t rt, hs->path = path; hs->cache_pool_index = ce_index; -done: + /* Set following headers only for happy path: + * Content-Type + * Cache-Control max-age + */ + type = content_type_from_request (target); + http_add_header (&hs->resp_headers, + http_header_name_token (HTTP_HEADER_CONTENT_TYPE), + http_content_type_token (type)); + http_add_header ( + &hs->resp_headers, http_header_name_token (HTTP_HEADER_CACHE_CONTROL), + (const char *) hsm->max_age_formatted, vec_len (hsm->max_age_formatted)); + http_add_header (&hs->resp_headers, + http_header_name_token (HTTP_HEADER_LAST_MODIFIED), + (const char *) last_modified, vec_len (last_modified)); - hs->content_type = type; +done: + vec_free (sanitized_path); start_send_data (hs, sc); - if (!hs->data) + if (!hs->data_len) hss_session_disconnect_transport (hs); return 0; } -static int -handle_request (hss_session_t *hs, http_req_method_t rt, u8 *request) +static void +handle_request (hss_session_t *hs, http_req_method_t rt, u8 *target_path, + u8 *target_query, u8 *data) { hss_main_t *hsm = &hss_main; - if (!try_url_handler (hsm, hs, rt, request)) - return 0; + if (!try_url_handler (hsm, hs, rt, target_path, target_query, data)) + return; - if (!try_file_handler (hsm, hs, rt, request)) - return 0; + if (!try_file_handler (hsm, hs, rt, target_path)) + return; /* Handler did not find anything return 404 */ start_send_data (hs, HTTP_STATUS_NOT_FOUND); hss_session_disconnect_transport (hs); - - return 0; } static int hss_ts_rx_callback (session_t *ts) { hss_session_t *hs; - u8 *request = 0; + u8 *target_path = 0, *target_query = 0, *data = 0; http_msg_t msg; int rv; hs = hss_session_get (ts->thread_index, ts->opaque); + if (hs->free_data) + vec_free (hs->data); + hs->data = 0; + hs->resp_headers = 0; + vec_free (hs->headers_buf); /* Read the http message header */ rv = svm_fifo_dequeue (ts->rx_fifo, sizeof (msg), (u8 *) &msg); @@ -463,26 +536,66 @@ hss_ts_rx_callback (session_t *ts) if (msg.type != HTTP_MSG_REQUEST || (msg.method_type != HTTP_REQ_GET && msg.method_type != HTTP_REQ_POST)) { - hs->data = 0; + http_add_header (&hs->resp_headers, + http_header_name_token (HTTP_HEADER_ALLOW), + http_token_lit ("GET, POST")); start_send_data (hs, HTTP_STATUS_METHOD_NOT_ALLOWED); - return 0; + goto done; } - /* Read request */ - if (msg.data.len) + if (msg.data.target_form != HTTP_TARGET_ORIGIN_FORM) { - vec_validate (request, msg.data.len - 1); - rv = svm_fifo_dequeue (ts->rx_fifo, msg.data.len, request); - ASSERT (rv == msg.data.len); - /* request must be a proper C-string in addition to a vector */ - vec_add1 (request, 0); + start_send_data (hs, HTTP_STATUS_BAD_REQUEST); + goto done; } - /* Find and send data */ - handle_request (hs, msg.method_type, request); + /* Read target path */ + if (msg.data.target_path_len) + { + vec_validate (target_path, msg.data.target_path_len - 1); + rv = svm_fifo_peek (ts->rx_fifo, msg.data.target_path_offset, + msg.data.target_path_len, target_path); + ASSERT (rv == msg.data.target_path_len); + if (http_validate_abs_path_syntax (target_path, 0)) + { + start_send_data (hs, HTTP_STATUS_BAD_REQUEST); + goto done; + } + /* Target path must be a proper C-string in addition to a vector */ + vec_add1 (target_path, 0); + } + + /* Read target query */ + if (msg.data.target_query_len) + { + vec_validate (target_query, msg.data.target_query_len - 1); + rv = svm_fifo_peek (ts->rx_fifo, msg.data.target_query_offset, + msg.data.target_query_len, target_query); + ASSERT (rv == msg.data.target_query_len); + if (http_validate_query_syntax (target_query, 0)) + { + start_send_data (hs, HTTP_STATUS_BAD_REQUEST); + goto done; + } + } - vec_free (request); + /* Read body */ + if (msg.data.body_len) + { + vec_validate (data, msg.data.body_len - 1); + rv = svm_fifo_peek (ts->rx_fifo, msg.data.body_offset, msg.data.body_len, + data); + ASSERT (rv == msg.data.body_len); + } + + /* Find and send data */ + handle_request (hs, msg.method_type, target_path, target_query, data); +done: + vec_free (target_path); + vec_free (target_query); + vec_free (data); + svm_fifo_dequeue_drop (ts->rx_fifo, msg.data.len); return 0; } @@ -490,7 +603,8 @@ static int hss_ts_tx_callback (session_t *ts) { hss_session_t *hs; - u32 to_send; + u32 n_enq; + u64 to_send; int rv; hs = hss_session_get (ts->thread_index, ts->opaque); @@ -498,7 +612,9 @@ hss_ts_tx_callback (session_t *ts) return 0; to_send = hs->data_len - hs->data_offset; - rv = svm_fifo_enqueue (ts->tx_fifo, to_send, hs->data + hs->data_offset); + n_enq = clib_min (svm_fifo_size (ts->tx_fifo), to_send); + + rv = svm_fifo_enqueue (ts->tx_fifo, n_enq, hs->data + hs->data_offset); if (rv <= 0) { @@ -513,7 +629,7 @@ hss_ts_tx_callback (session_t *ts) } if (svm_fifo_set_event (ts->tx_fifo)) - session_send_io_evt_to_thread (ts->tx_fifo, SESSION_IO_EVT_TX); + session_program_tx_io_evt (ts->handle, SESSION_IO_EVT_TX); return 0; } @@ -607,6 +723,7 @@ hss_ts_cleanup (session_t *s, session_cleanup_ntf_t ntf) hs->data = 0; hs->data_offset = 0; hs->free_data = 0; + vec_free (hs->headers_buf); vec_free (hs->path); hss_session_free (hs); @@ -757,6 +874,8 @@ hss_create (vlib_main_t *vm) if (hsm->enable_url_handlers) hss_url_handlers_init (hsm); + hsm->max_age_formatted = format (0, "max-age=%d", hsm->max_age); + return 0; } @@ -777,6 +896,7 @@ hss_create_command_fn (vlib_main_t *vm, unformat_input_t *input, hsm->private_segment_size = 0; hsm->fifo_size = 0; hsm->cache_size = 10 << 20; + hsm->max_age = HSS_DEFAULT_MAX_AGE; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -808,6 +928,8 @@ hss_create_command_fn (vlib_main_t *vm, unformat_input_t *input, ; else if (unformat (line_input, "url-handlers")) hsm->enable_url_handlers = 1; + else if (unformat (line_input, "max-age %d", &hsm->max_age)) + ; else { error = clib_error_return (0, "unknown input `%U'", @@ -836,7 +958,10 @@ no_input: goto done; } - vnet_session_enable_disable (vm, 1 /* turn on TCP, etc. */ ); + session_enable_disable_args_t args = { .is_en = 1, + .rt_engine_type = + RT_BACKEND_ENGINE_RULE_TABLE }; + vnet_session_enable_disable (vm, &args); if ((rv = hss_create (vm))) { @@ -865,8 +990,8 @@ VLIB_CLI_COMMAND (hss_create_command, static) = { .path = "http static server", .short_help = "http static server www-root <path> [prealloc-fifos <nn>]\n" - "[private-segment-size <nnMG>] [fifo-size <nbytes>] [uri <uri>]\n" - "[ptr-thresh <nn>] [url-handlers] [debug [nn]]\n", + "[private-segment-size <nnMG>] [fifo-size <nbytes>] [max-age <nseconds>]\n" + "[uri <uri>] [ptr-thresh <nn>] [url-handlers] [debug [nn]]\n", .function = hss_create_command_fn, }; @@ -876,7 +1001,7 @@ format_hss_session (u8 *s, va_list *args) hss_session_t *hs = va_arg (*args, hss_session_t *); int __clib_unused verbose = va_arg (*args, int); - s = format (s, "\n path %s, data length %u, data_offset %u", + s = format (s, "\n path %s, data length %llu, data_offset %llu", hs->path ? hs->path : (u8 *) "[none]", hs->data_len, hs->data_offset); return s; diff --git a/src/plugins/ikev2/CMakeLists.txt b/src/plugins/ikev2/CMakeLists.txt index 568271ed7d9..dd2b49d6651 100644 --- a/src/plugins/ikev2/CMakeLists.txt +++ b/src/plugins/ikev2/CMakeLists.txt @@ -27,6 +27,7 @@ add_vpp_plugin(ikev2 ikev2_crypto.c ikev2_format.c ikev2_payload.c + ikev2_handoff.c API_FILES ikev2_types.api diff --git a/src/plugins/ikev2/ikev2.api b/src/plugins/ikev2/ikev2.api index de276e7f3ea..e2ff8fb8268 100644 --- a/src/plugins/ikev2/ikev2.api +++ b/src/plugins/ikev2/ikev2.api @@ -658,6 +658,12 @@ counters ikev2 { units "packets"; description "IKE AUTH SA requests received"; }; + handoff { + severity info; + type counter64; + units "packets"; + description "IKE packets handoff"; + }; }; paths { "/err/ikev2-ip4" "ike"; diff --git a/src/plugins/ikev2/ikev2.c b/src/plugins/ikev2/ikev2.c index 9bea2c96d12..f66469a24d1 100644 --- a/src/plugins/ikev2/ikev2.c +++ b/src/plugins/ikev2/ikev2.c @@ -97,6 +97,7 @@ format_ikev2_gen_sa_error (u8 * s, va_list * args) typedef enum { IKEV2_NEXT_IP4_LOOKUP, + IKEV2_NEXT_IP4_HANDOFF, IKEV2_NEXT_IP4_ERROR_DROP, IKEV2_IP4_N_NEXT, } ikev2_ip4_next_t; @@ -104,6 +105,7 @@ typedef enum typedef enum { IKEV2_NEXT_IP6_LOOKUP, + IKEV2_NEXT_IP6_HANDOFF, IKEV2_NEXT_IP6_ERROR_DROP, IKEV2_IP6_N_NEXT, } ikev2_ip6_next_t; @@ -3187,6 +3189,7 @@ ikev2_node_internal (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b; u16 nexts[VLIB_FRAME_SIZE], *next = nexts; ikev2_main_per_thread_data_t *ptd = ikev2_get_per_thread_data (); + u32 thread_index = vm->thread_index; ikev2_stats_t _stats, *stats = &_stats; int res; @@ -3213,6 +3216,14 @@ ikev2_node_internal (vlib_main_t *vm, vlib_node_runtime_t *node, int ip_hdr_sz = 0; int is_req = 0; + if (PREDICT_TRUE (thread_index != km->handoff_thread)) + { + vlib_node_increment_counter (vm, node->node_index, + IKEV2_ERROR_HANDOFF, 1); + + next[0] = is_ip4 ? IKEV2_NEXT_IP4_HANDOFF : IKEV2_NEXT_IP6_HANDOFF; + goto out; + } if (natt) { u8 *ptr = vlib_buffer_get_current (b0); @@ -3723,6 +3734,8 @@ ikev2_node_internal (vlib_main_t *vm, vlib_node_runtime_t *node, ikev2_delete_sa (ptd, sa0); } + + out: if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) && (b0->flags & VLIB_BUFFER_IS_TRACED))) { @@ -3775,6 +3788,7 @@ VLIB_REGISTER_NODE (ikev2_node_ip4,static) = { .n_next_nodes = IKEV2_IP4_N_NEXT, .next_nodes = { [IKEV2_NEXT_IP4_LOOKUP] = "ip4-lookup", + [IKEV2_NEXT_IP4_HANDOFF] = "ikev2-ip4-handoff", [IKEV2_NEXT_IP4_ERROR_DROP] = "error-drop", }, }; @@ -3792,6 +3806,7 @@ VLIB_REGISTER_NODE (ikev2_node_ip4_natt,static) = { .n_next_nodes = IKEV2_IP4_N_NEXT, .next_nodes = { [IKEV2_NEXT_IP4_LOOKUP] = "ip4-lookup", + [IKEV2_NEXT_IP4_HANDOFF] = "ikev2-ip4-natt-handoff", [IKEV2_NEXT_IP4_ERROR_DROP] = "error-drop", }, }; @@ -3809,6 +3824,7 @@ VLIB_REGISTER_NODE (ikev2_node_ip6,static) = { .n_next_nodes = IKEV2_IP6_N_NEXT, .next_nodes = { [IKEV2_NEXT_IP6_LOOKUP] = "ip6-lookup", + [IKEV2_NEXT_IP4_HANDOFF] = "ikev2-ip6-handoff", [IKEV2_NEXT_IP6_ERROR_DROP] = "error-drop", }, }; @@ -5126,6 +5142,8 @@ ikev2_init (vlib_main_t * vm) km->liveness_period = IKEV2_LIVENESS_PERIOD_CHECK; km->liveness_max_retries = IKEV2_LIVENESS_RETRIES; + km->handoff_thread = vlib_num_workers () ? 1 : 0; + return 0; } @@ -5133,6 +5151,31 @@ VLIB_INIT_FUNCTION (ikev2_init) = { .runs_after = VLIB_INITS ("ipsec_init", "ipsec_punt_init"), }; +static clib_error_t * +ikev2_config (vlib_main_t *vm, unformat_input_t *input) +{ + ikev2_main_t *km = &ikev2_main; + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "handoff-thread %d", &km->handoff_thread)) + { + if (km->handoff_thread > vlib_num_workers ()) + { + return clib_error_return (0, "wrong handoff-thread %d", + km->handoff_thread); + } + } + else + return clib_error_return (0, "unknown input `%U'", format_unformat_error, + input); + } + + return 0; +} + +VLIB_CONFIG_FUNCTION (ikev2_config, "ikev2"); + static u8 ikev2_mngr_process_child_sa (ikev2_sa_t * sa, ikev2_child_sa_t * csa, u8 del_old_ids) @@ -5447,6 +5490,7 @@ ikev2_send_informational_request (ikev2_sa_t * sa) } dp = sa->dst_port ? sa->dst_port : ikev2_get_port (sa); + ikev2_send_ike (km->vlib_main, src, dst, bi0, len, ikev2_get_port (sa), dp, sa->sw_if_index); } @@ -5625,6 +5669,15 @@ ikev2_lazy_init (ikev2_main_t *km) if (!km->dns_resolve_name_ptr) ikev2_log_error ("cannot load symbols from dns plugin"); + km->handoff_ip4_fq_index = + vlib_frame_queue_main_init (ikev2_node_ip4.index, 0); + + km->handoff_ip4_natt_fq_index = + vlib_frame_queue_main_init (ikev2_node_ip4_natt.index, 0); + + km->handoff_ip6_fq_index = + vlib_frame_queue_main_init (ikev2_node_ip6.index, 0); + /* wake up ikev2 process */ vlib_process_signal_event (vlib_get_first_main (), ikev2_mngr_process_node.index, 0, 0); diff --git a/src/plugins/ikev2/ikev2_api.c b/src/plugins/ikev2/ikev2_api.c index c9608aa660b..e09bde3cbe2 100644 --- a/src/plugins/ikev2/ikev2_api.c +++ b/src/plugins/ikev2/ikev2_api.c @@ -577,6 +577,7 @@ vl_api_ikev2_child_sa_dump_t_handler (vl_api_ikev2_child_sa_dump_t * mp) vec_foreach (child, sa->childs) { u32 child_sa_index = child - sa->childs; + sai = ikev2_encode_sa_index (sai, tkm - im->per_thread_data); send_child_sa (child, mp, child_sa_index, sai); } } diff --git a/src/plugins/ikev2/ikev2_crypto.c b/src/plugins/ikev2/ikev2_crypto.c index 3d4ad0a28ed..58167e2322e 100644 --- a/src/plugins/ikev2/ikev2_crypto.c +++ b/src/plugins/ikev2/ikev2_crypto.c @@ -481,15 +481,14 @@ ikev2_encrypt_data (ikev2_main_per_thread_data_t * ptd, ikev2_sa_t * sa, int BN_bn2binpad (const BIGNUM * a, unsigned char *to, int tolen) { - int r = BN_bn2bin (a, to); + int r = BN_num_bytes (a); ASSERT (tolen >= r); int pad = tolen - r; if (pad) { - vec_insert (to, pad, 0); clib_memset (to, 0, pad); - vec_dec_len (to, pad); } + BN_bn2bin (a, to + pad); return tolen; } #endif diff --git a/src/plugins/ikev2/ikev2_handoff.c b/src/plugins/ikev2/ikev2_handoff.c new file mode 100644 index 00000000000..8f55985bce8 --- /dev/null +++ b/src/plugins/ikev2/ikev2_handoff.c @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <vlib/vlib.h> +#include <vnet/vnet.h> +#include <ikev2/ikev2_priv.h> + +extern ikev2_main_t ikev2_main; + +#define foreach_ikev2_handoff_error _ (CONGESTION_DROP, "congestion drop") + +typedef enum +{ +#define _(sym, str) IKEV2_HANDOFF_ERROR_##sym, + foreach_ikev2_handoff_error +#undef _ + IKEV2_HANDOFF_N_ERROR, +} ikev2_handoff_error_t; + +static char *ikev2_handoff_error_strings[] = { +#define _(sym, string) string, + foreach_ikev2_handoff_error +#undef _ +}; + +typedef struct ikev2_handoff_trace_t_ +{ + u32 current_worker_index; + u32 next_worker_index; +} ikev2_handoff_trace_t; + +u8 * +format_ikev2_handoff_trace (u8 *s, va_list *args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + + ikev2_handoff_trace_t *t = va_arg (*args, ikev2_handoff_trace_t *); + s = format (s, "ikev2 handoff %d to %d", t->current_worker_index, + t->next_worker_index); + return s; +} + +static_always_inline uword +ikev2_handoff_inline (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_frame_t *frame, u32 fq_index) +{ + ikev2_main_t *km = &ikev2_main; + vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b; + u16 thread_indices[VLIB_FRAME_SIZE], *ti; + u32 n_enq, n_left_from, *from; + u32 this_thread; + + this_thread = vm->thread_index; + + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + vlib_get_buffers (vm, from, bufs, n_left_from); + + b = bufs; + ti = thread_indices; + + while (n_left_from > 0) + { + ti[0] = km->handoff_thread; + + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) && + b[0]->flags & VLIB_BUFFER_IS_TRACED)) + { + ikev2_handoff_trace_t *t = + vlib_add_trace (vm, node, b[0], sizeof (*t)); + t->current_worker_index = this_thread; + t->next_worker_index = ti[0]; + } + n_left_from--; + ti++; + b++; + } + + n_enq = vlib_buffer_enqueue_to_thread (vm, node, fq_index, from, + thread_indices, frame->n_vectors, 1); + + if (n_enq < frame->n_vectors) + vlib_node_increment_counter (vm, node->node_index, + IKEV2_HANDOFF_ERROR_CONGESTION_DROP, + frame->n_vectors - n_enq); + return n_enq; +} + +/* Do worker handoff based on the ikev2's thread_index */ +VLIB_NODE_FN (ikev2_ip4_handoff) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame) +{ + ikev2_main_t *km = &ikev2_main; + + return ikev2_handoff_inline (vm, node, from_frame, km->handoff_ip4_fq_index); +} + +VLIB_NODE_FN (ikev2_ip4_natt_handoff) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame) +{ + ikev2_main_t *km = &ikev2_main; + + return ikev2_handoff_inline (vm, node, from_frame, + km->handoff_ip4_natt_fq_index); +} + +VLIB_NODE_FN (ikev2_ip6_handoff) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame) +{ + ikev2_main_t *km = &ikev2_main; + + return ikev2_handoff_inline (vm, node, from_frame, km->handoff_ip6_fq_index); +} + +VLIB_REGISTER_NODE (ikev2_ip4_handoff) = { + .name = "ikev2-ip4-handoff", + .vector_size = sizeof (u32), + .format_trace = format_ikev2_handoff_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = ARRAY_LEN(ikev2_handoff_error_strings), + .error_strings = ikev2_handoff_error_strings, + .n_next_nodes = 1, + .next_nodes = { + [0] = "error-drop", + }, +}; + +VLIB_REGISTER_NODE (ikev2_ip4_natt_handoff) = { + .name = "ikev2-ip4-natt-handoff", + .vector_size = sizeof (u32), + .format_trace = format_ikev2_handoff_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = ARRAY_LEN(ikev2_handoff_error_strings), + .error_strings = ikev2_handoff_error_strings, + .n_next_nodes = 1, + .next_nodes = { + [0] = "error-drop", + }, +}; + +VLIB_REGISTER_NODE (ikev2_ip6_handoff) = { + .name = "ikev2-ip6-handoff", + .vector_size = sizeof (u32), + .format_trace = format_ikev2_handoff_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = ARRAY_LEN(ikev2_handoff_error_strings), + .error_strings = ikev2_handoff_error_strings, + .n_next_nodes = 1, + .next_nodes = { + [0] = "error-drop", + }, +}; diff --git a/src/plugins/ikev2/ikev2_priv.h b/src/plugins/ikev2/ikev2_priv.h index 0639809e9b1..96313182552 100644 --- a/src/plugins/ikev2/ikev2_priv.h +++ b/src/plugins/ikev2/ikev2_priv.h @@ -571,6 +571,12 @@ typedef struct /* punt handle for IPsec NATT IPSEC_PUNT_IP4_SPI_UDP_0 reason */ vlib_punt_hdl_t punt_hdl; + /** Worker handoff */ + u32 handoff_thread; + u32 handoff_ip4_fq_index; + u32 handoff_ip4_natt_fq_index; + u32 handoff_ip6_fq_index; + } ikev2_main_t; extern ikev2_main_t ikev2_main; diff --git a/src/plugins/linux-cp/lcp_interface.c b/src/plugins/linux-cp/lcp_interface.c index e1f4a6a1d69..61665ad4146 100644 --- a/src/plugins/linux-cp/lcp_interface.c +++ b/src/plugins/linux-cp/lcp_interface.c @@ -258,7 +258,11 @@ lcp_itf_pair_add (u32 host_sw_if_index, u32 phy_sw_if_index, u8 *host_name, vec_validate_init_empty (lip_db_by_host, host_sw_if_index, INDEX_INVALID); lip_db_by_phy[phy_sw_if_index] = lipi; lip_db_by_host[host_sw_if_index] = lipi; - hash_set (lip_db_by_vif, host_index, lipi); + + if (clib_strcmp ((char *) ns, (char *) lcp_get_default_ns ()) == 0) + { + hash_set (lip_db_by_vif, host_index, lipi); + } lip->lip_host_sw_if_index = host_sw_if_index; lip->lip_phy_sw_if_index = phy_sw_if_index; @@ -997,7 +1001,8 @@ lcp_itf_pair_create (u32 phy_sw_if_index, u8 *host_if_name, clib_max (1, lcp_get_default_num_queues (0 /* is_tx */)), .num_tx_queues = clib_max (1, lcp_get_default_num_queues (1 /* is_tx */)), - .id = hw->hw_if_index, + .id = ~0, + .auto_id_offset = 4096, .sw_if_index = ~0, .rx_ring_sz = 256, .tx_ring_sz = 256, @@ -1094,7 +1099,7 @@ lcp_itf_pair_create (u32 phy_sw_if_index, u8 *host_if_name, * This controls whether the host can RX/TX. */ sw = vnet_get_sw_interface (vnm, phy_sw_if_index); - lip = lcp_itf_pair_get (lcp_itf_pair_find_by_vif (vif_index)); + lip = lcp_itf_pair_get (lcp_itf_pair_find_by_phy (phy_sw_if_index)); LCP_ITF_PAIR_INFO ("pair create: %U sw-flags %u hw-flags %u", format_lcp_itf_pair, lip, sw->flags, hw->flags); vnet_sw_interface_admin_up (vnm, host_sw_if_index); diff --git a/src/plugins/mactime/builtins.c b/src/plugins/mactime/builtins.c index c487d0375bf..f726d3c03ed 100644 --- a/src/plugins/mactime/builtins.c +++ b/src/plugins/mactime/builtins.c @@ -147,6 +147,7 @@ handle_get_mactime (hss_url_handler_args_t *args) args->data = s; args->data_len = vec_len (s); + args->ct = HTTP_CONTENT_APP_JSON; args->free_vec_data = 1; return HSS_URL_HANDLER_OK; } diff --git a/src/plugins/marvell/CMakeLists.txt b/src/plugins/marvell/CMakeLists.txt deleted file mode 100644 index b48ac72aa08..00000000000 --- a/src/plugins/marvell/CMakeLists.txt +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) 2018 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*)") - return() -endif() - -find_path(MUSDK_INCLUDE_DIR NAMES mv_std.h) -find_library(MUSDK_LIB NAMES libmusdk.a) - -if(MUSDK_INCLUDE_DIR AND MUSDK_LIB) - get_filename_component(MUSDK_LIB_DIR ${MUSDK_LIB} DIRECTORY) - set(MUSDK_LINK_FLAGS "-Wl,--whole-archive,${MUSDK_LIB_DIR}/libmusdk.a,--no-whole-archive") - add_vpp_plugin(marvell - SOURCES - plugin.c - pp2/cli.c - pp2/format.c - pp2/input.c - pp2/output.c - pp2/pp2.c - pp2/pp2_api.c - - API_FILES - pp2/pp2.api - - API_TEST_SOURCES - pp2/pp2_test.c - - LINK_FLAGS - ${MUSDK_LINK_FLAGS} - ) - include_directories(${MUSDK_INCLUDE_DIR}) - message(STATUS "Found Marvell MUSDK in ${MUSDK_INCLUDE_DIR}") -else() - message(WARNING "Marvell MUSDK not found - marvell_plugin disabled") -endif() diff --git a/src/plugins/marvell/README.rst b/src/plugins/marvell/README.rst deleted file mode 100644 index 19cf1c49d0e..00000000000 --- a/src/plugins/marvell/README.rst +++ /dev/null @@ -1,85 +0,0 @@ -Marvell device plugin -===================== - -Overview --------- - -This plugins provides native device support for Marvell PP2 network -device, by use of Marvell Usermode SDK -(`MUSDK <https://github.com/MarvellEmbeddedProcessors/musdk-marvell>`__). -Code is developed and tested on -`MACCHIATObin <http://macchiatobin.net>`__ board. - -Prerequisites -------------- - -Plugins depends on installed MUSDK and Marvell provided linux -`kernel <https://github.com/MarvellEmbeddedProcessors/linux-marvell>`__ -with MUSDK provided kernel patches (see ``patches/linux`` in musdk repo -and relevant documentation. Kernel version used: **4.14.22 -armada-18.09.3** MUSDK version used: **armada-18.09.3** Following kernel -modules from MUSDK must be loaded for plugin to work: \* -``musdk_cma.ko`` \* ``mv_pp_uio.ko`` - -Musdk 18.09.3 compilation steps -------------------------------- - -:: - - ./bootstrap - ./configure --prefix=/opt/vpp/external/aarch64/ CFLAGS="-Wno-error=unused-result -g -fPIC" --enable-shared=no - sed -i -e 's/marvell,mv-pp-uio/generic-uio/' modules/pp2/mv_pp_uio.c - sed -i -e 's/O_CREAT/O_CREAT, S_IRUSR | S_IWUSR/' src/lib/file_utils.c - make - sudo make install - -Usage ------ - -Interface Creation -~~~~~~~~~~~~~~~~~~ - -Interfaces are dynamically created with following CLI: - -:: - - create interface marvell pp2 name eth0 - set interface state mv-ppio-0/0 up - -Where ``eth0`` is linux interface name and ``mv-ppio-X/Y`` is VPP -interface name where X is PP2 device ID and Y is PPIO ID Interface needs -to be assigned to MUSDK in FDT configuration and linux interface state -must be up. - -Interface Deletion -~~~~~~~~~~~~~~~~~~ - -Interface can be deleted with following CLI: - -:: - - delete interface marvell pp2 <interface name> - -Interface Statistics -~~~~~~~~~~~~~~~~~~~~ - -Interface statistics can be displayed with -``sh hardware-interface mv-ppio0/0`` command. - -Interaction with DPDK plugin -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This plugin doesn’t have any dependency on DPDK or DPDK plugin but it -can work with DPDK plugin enabled or disabled. It is observed that -performance is better around 30% when DPDK plugin is disabled, as DPDK -plugin registers own buffer manager, which needs to deal with additional -metadata in each packet. - -DPKD plugin can be disabled by adding following config to the -startup.conf. - -:: - - plugins { - dpdk_plugin.so { disable } - } diff --git a/src/plugins/marvell/pp2/cli.c b/src/plugins/marvell/pp2/cli.c deleted file mode 100644 index 5072a3c035b..00000000000 --- a/src/plugins/marvell/pp2/cli.c +++ /dev/null @@ -1,135 +0,0 @@ -/* - *------------------------------------------------------------------ - * Copyright (c) 2018 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *------------------------------------------------------------------ - */ -#include <stdint.h> -#include <net/if.h> -#include <sys/ioctl.h> -#include <inttypes.h> - -#include <vlib/vlib.h> -#include <vlib/unix/unix.h> -#include <vnet/ethernet/ethernet.h> - -#include <marvell/pp2/pp2.h> - -static clib_error_t * -mrvl_pp2_create_command_fn (vlib_main_t * vm, unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - unformat_input_t _line_input, *line_input = &_line_input; - mrvl_pp2_create_if_args_t args = { 0 }; - unsigned int val; - - /* Get a line of input. */ - if (!unformat_user (input, unformat_line_input, line_input)) - return 0; - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (line_input, "name %s", &args.name)) - ; - else if (unformat (line_input, "rx-queue-size %u", &val)) - args.rx_q_sz = val; - else if (unformat (line_input, "tx-queue-size %u", &val)) - args.tx_q_sz = val; - else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); - } - unformat_free (line_input); - - - mrvl_pp2_create_if (&args); - - vec_free (args.name); - - return args.error; -} - -VLIB_CLI_COMMAND (mrvl_pp2_create_command, static) = { - .path = "create interface marvell pp2", - .short_help = "create interface marvell pp2 [name <ifname>] [rx-queue-size slots] [tx-queue-size slots]", - .function = mrvl_pp2_create_command_fn, -}; - -static clib_error_t * -mrvl_pp2_delete_command_fn (vlib_main_t * vm, unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - unformat_input_t _line_input, *line_input = &_line_input; - u32 sw_if_index = ~0; - vnet_hw_interface_t *hw; - mrvl_pp2_main_t *mm = &mrvl_pp2_main; - mrvl_pp2_if_t *dif; - vnet_main_t *vnm = vnet_get_main (); - - /* Get a line of input. */ - if (!unformat_user (input, unformat_line_input, line_input)) - return 0; - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (line_input, "sw_if_index %d", &sw_if_index)) - ; - else if (unformat (line_input, "%U", unformat_vnet_sw_interface, - vnm, &sw_if_index)) - ; - else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); - } - unformat_free (line_input); - - if (sw_if_index == ~0) - return clib_error_return (0, - "please specify interface name or sw_if_index"); - - hw = vnet_get_sup_hw_interface_api_visible_or_null (vnm, sw_if_index); - if (hw == NULL || mrvl_pp2_device_class.index != hw->dev_class_index) - return clib_error_return (0, "not a Marvell PP2 interface"); - - dif = pool_elt_at_index (mm->interfaces, hw->dev_instance); - - mrvl_pp2_delete_if (dif); - - return 0; -} - -VLIB_CLI_COMMAND (mrvl_pp2_delete_command, static) = { - .path = "delete interface marvell pp2", - .short_help = "delete interface marvell pp2 " - "{<interface> | sw_if_index <sw_idx>}", - .function = mrvl_pp2_delete_command_fn, -}; - -clib_error_t * -mrvl_pp2_cli_init (vlib_main_t * vm) -{ - /* initialize binary API */ - mrvl_pp2_plugin_api_hookup (vm); - - return 0; -} - -VLIB_INIT_FUNCTION (mrvl_pp2_cli_init); - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/plugins/marvell/pp2/format.c b/src/plugins/marvell/pp2/format.c deleted file mode 100644 index 877010ea561..00000000000 --- a/src/plugins/marvell/pp2/format.c +++ /dev/null @@ -1,200 +0,0 @@ -/* - *------------------------------------------------------------------ - * Copyright (c) 2018 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *------------------------------------------------------------------ - */ - -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> -#include <sys/ioctl.h> - -#include <vlib/vlib.h> -#include <vlib/unix/unix.h> -#include <vnet/plugin/plugin.h> -#include <marvell/pp2/pp2.h> - -static inline u32 -mrvl_get_u32_bits (void *start, int offset, int first, int last) -{ - u32 value = *(u32 *) (((u8 *) start) + offset); - if ((last == 0) && (first == 31)) - return value; - value >>= last; - value &= (1 << (first - last + 1)) - 1; - return value; -} - -u8 * -format_mrvl_pp2_interface_name (u8 * s, va_list * args) -{ - mrvl_pp2_main_t *ppm = &mrvl_pp2_main; - u32 dev_instance = va_arg (*args, u32); - mrvl_pp2_if_t *ppif = pool_elt_at_index (ppm->interfaces, dev_instance); - return format (s, "mv-ppio-%d/%d", ppif->ppio->pp2_id, ppif->ppio->port_id); -} - -#define foreach_ppio_statistics_entry \ - _(rx_packets) \ - _(rx_fullq_dropped) \ - _(rx_bm_dropped) \ - _(rx_early_dropped) \ - _(rx_fifo_dropped) \ - _(rx_cls_dropped) \ - _(tx_packets) - -#define foreach_ppio_inq_statistics_entry \ - _(enq_desc) \ - _(drop_early) \ - _(drop_fullq) \ - _(drop_bm) - -#define foreach_ppio_outq_statistics_entry \ - _(enq_desc) \ - _(enq_dec_to_ddr) \ - _(enq_buf_to_ddr) \ - _(deq_desc) - -u8 * -format_mrvl_pp2_interface (u8 * s, va_list * args) -{ - mrvl_pp2_main_t *ppm = &mrvl_pp2_main; - u32 dev_instance = va_arg (*args, u32); - u32 indent = format_get_indent (s); - mrvl_pp2_if_t *ppif = pool_elt_at_index (ppm->interfaces, dev_instance); - struct pp2_ppio_statistics stat; - int i; - u8 *s2 = 0; - - pp2_ppio_get_statistics (ppif->ppio, &stat, 0); - -#define _(c) if (stat.c) \ - s2 = format (s2, "\n%U%-25U%16Ld", \ - format_white_space, indent + 2, \ - format_c_identifier, #c, stat.c); - foreach_ppio_statistics_entry; - - if (vec_len (s2)) - s = format (s, "Interface statistics:%v", s2); - vec_reset_length (s2); - - vec_foreach_index (i, ppif->inqs) - { - struct pp2_ppio_inq_statistics stat = { 0 }; - pp2_ppio_inq_get_statistics (ppif->ppio, 0, i, &stat, 0); - - foreach_ppio_inq_statistics_entry; - - if (vec_len (s2)) - s = format (s, "\n%UInput queue %u statistics:%v", - format_white_space, indent, i, s2); - vec_reset_length (s2); - } - vec_foreach_index (i, ppif->outqs) - { - struct pp2_ppio_outq_statistics stat = { 0 }; - - pp2_ppio_outq_get_statistics (ppif->ppio, i, &stat, 0); - - foreach_ppio_outq_statistics_entry; - - if (vec_len (s2)) - s = format (s, "\n%UOutput queue %u statistics:%v", - format_white_space, indent, i, s2); - vec_reset_length (s2); - } -#undef _ - vec_free (s2); - return s; -} - -#define foreach_pp2_rx_desc_field \ - _(0x00, 6, 0, l3_offset) \ - _(0x00, 12, 8, ip_hdlen) \ - _(0x00, 14, 13, ec) \ - _(0x00, 15, 15, es) \ - _(0x00, 19, 16, pool_id) \ - _(0x00, 21, 21, hwf_sync) \ - _(0x00, 22, 22, l4_chk_ok) \ - _(0x00, 23, 23, ip_frg) \ - _(0x00, 24, 24, ipv4_hdr_err) \ - _(0x00, 27, 25, l4_info) \ - _(0x00, 30, 28, l3_info) \ - _(0x00, 31, 31, buf_header) \ - _(0x04, 5, 0, lookup_id) \ - _(0x04, 8, 6, cpu_code) \ - _(0x04, 9, 9, pppoe) \ - _(0x04, 11, 10, l3_cast_info) \ - _(0x04, 13, 12, l2_cast_info) \ - _(0x04, 15, 14, vlan_info) \ - _(0x04, 31, 16, byte_count) \ - _(0x08, 11, 0, gem_port_id) \ - _(0x08, 13, 12, color) \ - _(0x08, 14, 14, gop_sop_u) \ - _(0x08, 15, 15, key_hash_enable) \ - _(0x08, 31, 16, l4chk) \ - _(0x0c, 31, 0, timestamp) \ - _(0x10, 31, 0, buf_phys_ptr_lo) \ - _(0x14, 7, 0, buf_phys_ptr_hi) \ - _(0x14, 31, 8, key_hash) \ - _(0x18, 31, 0, buf_virt_ptr_lo) \ - _(0x1c, 7, 0, buf_virt_ptr_hi) \ - _(0x1c, 14, 8, buf_qset_no) \ - _(0x1c, 15, 15, buf_type) \ - _(0x1c, 21, 16, mod_dscp) \ - _(0x1c, 24, 22, mod_pri) \ - _(0x1c, 25, 25, mdscp) \ - _(0x1c, 26, 26, mpri) \ - _(0x1c, 27, 27, mgpid) \ - _(0x1c, 31, 29, port_num) - -u8 * -format_mrvl_pp2_input_trace (u8 * s, va_list * args) -{ - vlib_main_t *vm = va_arg (*args, vlib_main_t *); - vlib_node_t *node = va_arg (*args, vlib_node_t *); - mrvl_pp2_input_trace_t *t = va_arg (*args, mrvl_pp2_input_trace_t *); - vnet_main_t *vnm = vnet_get_main (); - vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, t->hw_if_index); - u32 indent = format_get_indent (s); - struct pp2_ppio_desc *d = &t->desc; - u32 r32; - - s = format (s, "pp2: %v (%d) next-node %U", - hi->name, t->hw_if_index, format_vlib_next_node_name, vm, - node->index, t->next_index); - s = format (s, "\n%U", format_white_space, indent + 2); - -#define _(a, b, c, n) \ - r32 = mrvl_get_u32_bits (d, a, b, c); \ - if (r32 > 9) \ - s = format (s, "%s %u (0x%x)", #n, r32, r32); \ - else \ - s = format (s, "%s %u", #n,r32); \ - if (format_get_indent (s) > 72) \ - s = format (s, "\n%U", format_white_space, indent + 2); \ - else s = format (s, " "); - - foreach_pp2_rx_desc_field; -#undef _ - return s; -} - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/plugins/marvell/pp2/input.c b/src/plugins/marvell/pp2/input.c deleted file mode 100644 index 2545f91becb..00000000000 --- a/src/plugins/marvell/pp2/input.c +++ /dev/null @@ -1,392 +0,0 @@ -/* - *------------------------------------------------------------------ - * Copyright (c) 2018 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *------------------------------------------------------------------ - */ - -#define _GNU_SOURCE -#include <stdint.h> -#include <net/if.h> -#include <sys/ioctl.h> -#include <sys/uio.h> - -#include <vlib/vlib.h> -#include <vlib/unix/unix.h> -#include <vnet/ethernet/ethernet.h> -#include <vnet/devices/devices.h> -#include <vnet/interface/rx_queue_funcs.h> - -#include <marvell/pp2/pp2.h> - -#define foreach_mrvl_pp2_input_error \ - _(PPIO_RECV, "pp2_ppio_recv error") \ - _(BPOOL_GET_NUM_BUFFS, "pp2_bpool_get_num_buffs error") \ - _(BPOOL_PUT_BUFFS, "pp2_bpool_put_buffs error") \ - _(BUFFER_ALLOC, "buffer alloc error") \ - _(MAC_CE, "MAC error (CRC error)") \ - _(MAC_OR, "overrun error") \ - _(MAC_RSVD, "unknown MAC error") \ - _(MAC_RE, "resource error") \ - _(IP_HDR, "ip4 header error") - -typedef enum -{ -#define _(f,s) MRVL_PP2_INPUT_ERROR_##f, - foreach_mrvl_pp2_input_error -#undef _ - MRVL_PP2_INPUT_N_ERROR, -} mrvl_pp2_input_error_t; - -static __clib_unused char *mrvl_pp2_input_error_strings[] = { -#define _(n,s) s, - foreach_mrvl_pp2_input_error -#undef _ -}; - -static_always_inline void -mrvl_pp2_input_trace (vlib_main_t * vm, vlib_node_runtime_t * node, u32 next0, - vlib_buffer_t * b0, uword * n_trace, - mrvl_pp2_if_t * ppif, struct pp2_ppio_desc *d) -{ - if (PREDICT_TRUE ( - vlib_trace_buffer (vm, node, next0, b0, /* follow_chain */ 0))) - { - mrvl_pp2_input_trace_t *tr; - vlib_set_trace_count (vm, node, --(*n_trace)); - tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - tr->next_index = next0; - tr->hw_if_index = ppif->hw_if_index; - clib_memcpy_fast (&tr->desc, d, sizeof (struct pp2_ppio_desc)); - } -} - -static_always_inline u16 -mrvl_pp2_set_buf_data_len_flags (vlib_buffer_t * b, struct pp2_ppio_desc *d, - u32 add_flags) -{ - u16 len; - len = pp2_ppio_inq_desc_get_pkt_len (d); - b->total_length_not_including_first_buffer = 0; - b->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID | add_flags; - - if (add_flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID) - vnet_buffer (b)->l2_hdr_offset = 2; - - if (add_flags & VNET_BUFFER_F_L3_HDR_OFFSET_VALID) - { - u16 offset = DM_RXD_GET_L3_OFF (d); - vnet_buffer (b)->l3_hdr_offset = offset; - b->current_data = offset; - b->current_length = len - offset + 2; - } - else - { - b->current_data = 2; - b->current_length = len; - } - - if (add_flags & VNET_BUFFER_F_L3_HDR_OFFSET_VALID) - vnet_buffer (b)->l4_hdr_offset = vnet_buffer (b)->l3_hdr_offset + - DM_RXD_GET_IPHDR_LEN (d) * 4; - - return len; -} - -static_always_inline u16 -mrvl_pp2_next_from_desc (vlib_node_runtime_t * node, struct pp2_ppio_desc * d, - vlib_buffer_t * b, u32 * next) -{ - u8 l3_info; - /* ES bit set means MAC error - drop and count */ - if (PREDICT_FALSE (DM_RXD_GET_ES (d))) - { - *next = VNET_DEVICE_INPUT_NEXT_DROP; - u8 ec = DM_RXD_GET_EC (d); - if (ec == 0) - b->error = node->errors[MRVL_PP2_INPUT_ERROR_MAC_CE]; - else if (ec == 1) - b->error = node->errors[MRVL_PP2_INPUT_ERROR_MAC_OR]; - else if (ec == 2) - b->error = node->errors[MRVL_PP2_INPUT_ERROR_MAC_RSVD]; - else if (ec == 3) - b->error = node->errors[MRVL_PP2_INPUT_ERROR_MAC_RE]; - return mrvl_pp2_set_buf_data_len_flags (b, d, 0); - } - l3_info = DM_RXD_GET_L3_PRS_INFO (d); - - /* ipv4 packet can be value 1, 2 or 3 */ - if (PREDICT_TRUE ((l3_info - 1) < 3)) - { - if (PREDICT_FALSE (DM_RXD_GET_L3_IP4_HDR_ERR (d) != 0)) - { - *next = VNET_DEVICE_INPUT_NEXT_DROP; - b->error = node->errors[MRVL_PP2_INPUT_ERROR_IP_HDR]; - return mrvl_pp2_set_buf_data_len_flags (b, d, 0); - } - *next = VNET_DEVICE_INPUT_NEXT_IP4_NCS_INPUT; - return mrvl_pp2_set_buf_data_len_flags - (b, d, - VNET_BUFFER_F_L2_HDR_OFFSET_VALID | - VNET_BUFFER_F_L3_HDR_OFFSET_VALID | - VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP4); - } - - /* ipv4 packet can be value 4 or 5 */ - if (PREDICT_TRUE ((l3_info - 4) < 2)) - { - *next = VNET_DEVICE_INPUT_NEXT_IP6_INPUT; - return mrvl_pp2_set_buf_data_len_flags - (b, d, - VNET_BUFFER_F_L2_HDR_OFFSET_VALID | - VNET_BUFFER_F_L3_HDR_OFFSET_VALID | - VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP6); - } - - *next = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; - return mrvl_pp2_set_buf_data_len_flags (b, d, - VNET_BUFFER_F_L2_HDR_OFFSET_VALID); -} - -static_always_inline uword -mrvl_pp2_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_frame_t * frame, mrvl_pp2_if_t * ppif, - u16 qid) -{ - vnet_main_t *vnm = vnet_get_main (); - mrvl_pp2_main_t *ppm = &mrvl_pp2_main; - u32 thread_index = vm->thread_index; - mrvl_pp2_inq_t *inq = vec_elt_at_index (ppif->inqs, qid); - uword n_trace = vlib_get_trace_count (vm, node); - mrvl_pp2_per_thread_data_t *ptd = - vec_elt_at_index (ppm->per_thread_data, thread_index); - u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; - u32 sw_if_index[VLIB_N_RX_TX]; - u32 n_rx_packets = 0; - u32 n_rx_bytes = 0; - u32 *to_next = 0; - struct pp2_ppio_desc *d; - u16 n_desc = VLIB_FRAME_SIZE; - u32 n_bufs; - u32 *buffers; - int i; - - vec_validate_aligned (ptd->descs, n_desc, CLIB_CACHE_LINE_BYTES); - if (PREDICT_FALSE (pp2_ppio_recv (ppif->ppio, 0, qid, ptd->descs, &n_desc))) - { - vlib_error_count (vm, node->node_index, MRVL_PP2_INPUT_ERROR_PPIO_RECV, - 1); - n_desc = 0; - } - n_rx_packets = n_desc; - - for (i = 0; i < n_desc; i++) - ptd->buffers[i] = pp2_ppio_inq_desc_get_cookie (&ptd->descs[i]); - - d = ptd->descs; - buffers = ptd->buffers; - sw_if_index[VLIB_RX] = ppif->sw_if_index; - sw_if_index[VLIB_TX] = (u32) ~ 0; - while (n_desc) - { - u32 n_left_to_next; - vlib_buffer_t *b0, *b1; - u32 bi0, bi1; - u32 next0, next1; - vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); - while (n_desc >= 4 && n_left_to_next >= 2) - { - /* prefetch */ - bi0 = buffers[0]; - bi1 = buffers[1]; - to_next[0] = bi0; - to_next[1] = bi1; - b0 = vlib_get_buffer (vm, bi0); - b1 = vlib_get_buffer (vm, bi1); - - if (PREDICT_TRUE (ppif->per_interface_next_index == ~0)) - { - n_rx_bytes += mrvl_pp2_next_from_desc (node, d, b0, &next0); - n_rx_bytes += mrvl_pp2_next_from_desc (node, d + 1, b1, &next1); - vnet_feature_start_device_input (ppif->sw_if_index, &next0, b0); - vnet_feature_start_device_input (ppif->sw_if_index, &next1, b1); - } - else - { - n_rx_bytes += mrvl_pp2_set_buf_data_len_flags (b0, d, 0); - n_rx_bytes += mrvl_pp2_set_buf_data_len_flags (b1, d + 1, 0); - next0 = next1 = ppif->per_interface_next_index; - } - - clib_memcpy_fast (vnet_buffer (b0)->sw_if_index, sw_if_index, - sizeof (sw_if_index)); - clib_memcpy_fast (vnet_buffer (b1)->sw_if_index, sw_if_index, - sizeof (sw_if_index)); - - if (PREDICT_FALSE (n_trace > 0)) - { - mrvl_pp2_input_trace (vm, node, next0, b0, &n_trace, ppif, d); - if (n_trace > 0) - mrvl_pp2_input_trace (vm, node, next1, b1, &n_trace, ppif, - d + 1); - } - - to_next += 2; - n_left_to_next -= 2; - d += 2; - buffers += 2; - n_desc -= 2; - - /* enqueue */ - vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next, - n_left_to_next, bi0, bi1, next0, - next1); - - } - while (n_desc && n_left_to_next) - { - u32 bi0 = buffers[0]; - to_next[0] = bi0; - b0 = vlib_get_buffer (vm, bi0); - - if (PREDICT_TRUE (ppif->per_interface_next_index == ~0)) - { - n_rx_bytes += mrvl_pp2_next_from_desc (node, d, b0, &next0); - vnet_feature_start_device_input (ppif->sw_if_index, &next0, b0); - } - else - { - n_rx_bytes += mrvl_pp2_set_buf_data_len_flags (b0, d, 0); - next0 = ppif->per_interface_next_index; - } - - clib_memcpy_fast (vnet_buffer (b0)->sw_if_index, sw_if_index, - sizeof (sw_if_index)); - - if (PREDICT_FALSE (n_trace > 0)) - mrvl_pp2_input_trace (vm, node, next0, b0, &n_trace, ppif, d); - - to_next += 1; - n_left_to_next--; - d++; - buffers++; - n_desc--; - - /* enqueue */ - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, - n_left_to_next, bi0, next0); - } - vlib_put_next_frame (vm, node, next_index, n_left_to_next); - } - vlib_increment_combined_counter (vnm-> - interface_main.combined_sw_if_counters + - VNET_INTERFACE_COUNTER_RX, thread_index, - ppif->hw_if_index, n_rx_packets, - n_rx_bytes); - - if (PREDICT_FALSE (pp2_bpool_get_num_buffs (inq->bpool, &n_bufs))) - { - vlib_error_count (vm, node->node_index, - MRVL_PP2_INPUT_ERROR_BPOOL_GET_NUM_BUFFS, 1); - goto done; - } - - n_bufs = inq->size - n_bufs; - while (n_bufs >= MRVL_PP2_BUFF_BATCH_SZ) - { - u16 n_alloc, i; - struct buff_release_entry *e = ptd->bre; - u32 *buffers = ptd->buffers; - - n_alloc = vlib_buffer_alloc (vm, ptd->buffers, MRVL_PP2_BUFF_BATCH_SZ); - i = n_alloc; - - if (PREDICT_FALSE (n_alloc == 0)) - { - vlib_error_count (vm, node->node_index, - MRVL_PP2_INPUT_ERROR_BUFFER_ALLOC, 1); - goto done; - } - - while (i--) - { - u32 bi = buffers[0]; - vlib_buffer_t *b = vlib_get_buffer (vm, bi); - e->buff.addr = vlib_buffer_get_pa (vm, b) - 64; - e->buff.cookie = bi; - e->bpool = inq->bpool; - e++; - buffers++; - } - - i = n_alloc; - if (PREDICT_FALSE (pp2_bpool_put_buffs (ptd->hif, ptd->bre, &i))) - { - vlib_error_count (vm, node->node_index, - MRVL_PP2_INPUT_ERROR_BPOOL_PUT_BUFFS, 1); - vlib_buffer_free (vm, ptd->buffers, n_alloc); - goto done; - } - - if (PREDICT_FALSE (i != n_alloc)) - vlib_buffer_free (vm, ptd->buffers + i, n_alloc - i); - - n_bufs -= i; - } - -done: - return n_rx_packets; -} - -uword -mrvl_pp2_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_frame_t * frame) -{ - u32 n_rx = 0; - mrvl_pp2_main_t *ppm = &mrvl_pp2_main; - vnet_hw_if_rxq_poll_vector_t *pv; - - pv = vnet_hw_if_get_rxq_poll_vector (vm, node); - - for (int i = 0; i < vec_len (pv); i++) - { - mrvl_pp2_if_t *ppif; - ppif = vec_elt_at_index (ppm->interfaces, pv[i].dev_instance); - if (ppif->flags & MRVL_PP2_IF_F_ADMIN_UP) - n_rx += - mrvl_pp2_device_input_inline (vm, node, frame, ppif, pv[i].queue_id); - } - return n_rx; -} - -VLIB_REGISTER_NODE (mrvl_pp2_input_node) = { - .function = mrvl_pp2_input_fn, - .flags = VLIB_NODE_FLAG_TRACE_SUPPORTED, - .name = "mrvl-pp2-input", - .sibling_of = "device-input", - .format_trace = format_mrvl_pp2_input_trace, - .type = VLIB_NODE_TYPE_INPUT, - .state = VLIB_NODE_STATE_POLLING, - .n_errors = MRVL_PP2_INPUT_N_ERROR, - .error_strings = mrvl_pp2_input_error_strings, -}; - - - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/plugins/marvell/pp2/output.c b/src/plugins/marvell/pp2/output.c deleted file mode 100644 index 911b2f55a17..00000000000 --- a/src/plugins/marvell/pp2/output.c +++ /dev/null @@ -1,121 +0,0 @@ -/* - *------------------------------------------------------------------ - * Copyright (c) 2018 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *------------------------------------------------------------------ - */ - -#include <stdint.h> -#include <net/if.h> -#include <sys/ioctl.h> -#include <sys/uio.h> - -#include <vlib/vlib.h> -#include <vlib/unix/unix.h> -#include <vnet/ethernet/ethernet.h> -#include <vnet/devices/devices.h> - -#include <marvell/pp2/pp2.h> - -uword -mrvl_pp2_interface_tx (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * frame) -{ - mrvl_pp2_main_t *ppm = &mrvl_pp2_main; - vnet_interface_output_runtime_t *rd = (void *) node->runtime_data; - mrvl_pp2_if_t *ppif = pool_elt_at_index (ppm->interfaces, rd->dev_instance); - u32 thread_index = vm->thread_index; - mrvl_pp2_per_thread_data_t *ptd = - vec_elt_at_index (ppm->per_thread_data, thread_index); - u8 qid = thread_index; - mrvl_pp2_outq_t *outq = vec_elt_at_index (ppif->outqs, qid); - u32 *buffers = vlib_frame_vector_args (frame); - u16 n_desc = frame->n_vectors, n_left = n_desc, n_sent = n_desc, n_done; - struct pp2_ppio_desc *d; - u16 mask = outq->size - 1; - - if (PREDICT_FALSE (pp2_ppio_get_num_outq_done (ppif->ppio, ptd->hif, qid, - &n_done))) - { - n_done = 0; - vlib_error_count (vm, node->node_index, - MRVL_PP2_TX_ERROR_PPIO_GET_NUM_OUTQ_DONE, 1); - } - - if (n_done) - { - u16 n_free = clib_min (n_done, outq->size - (outq->tail & mask)); - vlib_buffer_free (vm, outq->buffers + (outq->tail & mask), n_free); - if (PREDICT_FALSE (n_free < n_done)) - vlib_buffer_free (vm, outq->buffers, n_done - n_free); - outq->tail += n_done; - } - - vec_validate_aligned (ptd->descs, n_left, CLIB_CACHE_LINE_BYTES); - d = ptd->descs; - while (n_left) - { - u32 bi0 = buffers[0]; - vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0); - u64 paddr = vlib_buffer_get_pa (vm, b0); - - pp2_ppio_outq_desc_reset (d); - pp2_ppio_outq_desc_set_phys_addr (d, paddr + b0->current_data); - pp2_ppio_outq_desc_set_pkt_offset (d, 0); - pp2_ppio_outq_desc_set_pkt_len (d, b0->current_length); - d++; - buffers++; - n_left--; - } - - if (pp2_ppio_send (ppif->ppio, ptd->hif, qid, ptd->descs, &n_sent)) - { - n_sent = 0; - vlib_error_count (vm, node->node_index, MRVL_PP2_TX_ERROR_PPIO_SEND, 1); - } - - /* free unsent buffers */ - if (PREDICT_FALSE (n_sent != n_desc)) - { - vlib_buffer_free (vm, vlib_frame_vector_args (frame) + n_sent, - frame->n_vectors - n_sent); - vlib_error_count (vm, node->node_index, MRVL_PP2_TX_ERROR_NO_FREE_SLOTS, - frame->n_vectors - n_sent); - } - - /* store buffer index for each enqueued packet into the ring - so we can know what to free after packet is sent */ - if (n_sent) - { - u16 slot = outq->head & mask; - buffers = vlib_frame_vector_args (frame); - u16 n_copy = clib_min (outq->size - slot, n_sent); - - vlib_buffer_copy_indices (outq->buffers + slot, buffers, n_copy); - if (PREDICT_FALSE (n_copy < n_sent)) - clib_memcpy_fast (outq->buffers, buffers + n_copy, - (n_sent - n_copy) * sizeof (u32)); - - outq->head += n_sent; - } - - return n_sent; -} - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/plugins/marvell/pp2/pp2.c b/src/plugins/marvell/pp2/pp2.c deleted file mode 100644 index 030ab9b4496..00000000000 --- a/src/plugins/marvell/pp2/pp2.c +++ /dev/null @@ -1,403 +0,0 @@ -/* - *------------------------------------------------------------------ - * Copyright (c) 2018 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *------------------------------------------------------------------ - */ - -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> -#include <sys/ioctl.h> - -#include <vlib/vlib.h> -#include <vlib/unix/unix.h> -#include <vnet/plugin/plugin.h> -#include <marvell/pp2/pp2.h> -#include <vnet/interface/rx_queue_funcs.h> - -/* size of DMA memory used by musdk (not used for buffers) */ -#define MV_SYS_DMA_MEM_SZ (2 << 20) -/* number of HIFs reserved (first X) */ -#define NUM_HIFS_RSVD 4 -/* number of buffer pools reserved (first X) */ -#define NUM_BPOOLS_RSVD 7 - -mrvl_pp2_main_t mrvl_pp2_main; -extern vnet_device_class_t ppa2_device_class; - -static void -mrvl_pp2_main_deinit () -{ - mrvl_pp2_main_t *ppm = &mrvl_pp2_main; - int i; - vec_foreach_index (i, ppm->per_thread_data) - { - mrvl_pp2_per_thread_data_t *ptd = vec_elt_at_index (ppm->per_thread_data, - i); - if (ptd->hif) - pp2_hif_deinit (ptd->hif); - vec_free (ptd->descs); - } - vec_free (ppm->per_thread_data); - pp2_deinit (); - mv_sys_dma_mem_destroy (); -} - -static clib_error_t * -mrvl_pp2_main_init () -{ - mrvl_pp2_main_t *ppm = &mrvl_pp2_main; - vlib_thread_main_t *tm = vlib_get_thread_main (); - clib_error_t *err = 0; - struct pp2_init_params init_params = { 0 }; - int i, rv; - u8 *s = 0; - - rv = mv_sys_dma_mem_init (MV_SYS_DMA_MEM_SZ); - if (rv) - return clib_error_return (0, "mv_sys_dma_mem_init failed, rv = %u", rv); - - init_params.hif_reserved_map = ((1 << NUM_HIFS_RSVD) - 1); - init_params.bm_pool_reserved_map = ((1 << NUM_BPOOLS_RSVD) - 1); - rv = pp2_init (&init_params); - if (rv) - { - err = clib_error_return (0, "mrvl_pp2_init failed, rv = %u", rv); - goto done; - } - - vec_validate_aligned (ppm->per_thread_data, tm->n_vlib_mains - 1, - CLIB_CACHE_LINE_BYTES); - - vec_foreach_index (i, ppm->per_thread_data) - { - mrvl_pp2_per_thread_data_t *ptd = vec_elt_at_index (ppm->per_thread_data, - i); - struct pp2_hif_params hif_params = { 0 }; - vec_reset_length (s); - s = format (s, "hif-%d%c", NUM_HIFS_RSVD + i, 0); - hif_params.match = (char *) s; - hif_params.out_size = 2048; /* FIXME */ - if (pp2_hif_init (&hif_params, &ptd->hif)) - { - err = clib_error_return (0, "hif '%s' init failed", s); - goto done; - } - } - -done: - if (err) - mrvl_pp2_main_deinit (); - vec_free (s); - return err; -} - -static u32 -mrvl_pp2_eth_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi, - u32 flags) -{ - /* nothing for now */ - return 0; -} - -void -mrvl_pp2_delete_if (mrvl_pp2_if_t * ppif) -{ - vlib_main_t *vm = vlib_get_main (); - vnet_main_t *vnm = vnet_get_main (); - mrvl_pp2_main_t *ppm = &mrvl_pp2_main; - mrvl_pp2_outq_t *outq; - mrvl_pp2_inq_t *inq; - - if (ppif->hw_if_index != ~0) - ethernet_delete_interface (vnm, ppif->hw_if_index); - - if (ppif->ppio) - { - pp2_ppio_disable (ppif->ppio); - pp2_ppio_deinit (ppif->ppio); - } - - /* free buffers hanging in the tx ring */ - vec_foreach (outq, ppif->outqs) - { - while (outq->tail < outq->head) - { - u16 slot = outq->tail & (outq->size - 1); - vlib_buffer_free (vm, outq->buffers + slot, 1); - outq->tail++; - } - vec_free (outq->buffers); - } - vec_free (ppif->outqs); - - /* free buffers hangin in the rx buffer pool */ - vec_foreach (inq, ppif->inqs) - if (inq->bpool) - { - u32 n_bufs = 0; - pp2_bpool_get_num_buffs (inq->bpool, &n_bufs); - while (n_bufs--) - { - struct pp2_buff_inf binf; - if (pp2_bpool_get_buff (ppm->per_thread_data[0].hif, inq->bpool, - &binf) == 0) - { - u32 bi = binf.cookie; - vlib_buffer_free (vm, &bi, 1); - } - } - pp2_bpool_deinit (inq->bpool); - } - vec_free (ppif->inqs); - - - pool_put (ppm->interfaces, ppif); - - if (pool_elts (ppm->interfaces) == 0) - mrvl_pp2_main_deinit (); -} - -void -mrvl_pp2_create_if (mrvl_pp2_create_if_args_t * args) -{ - vlib_main_t *vm = vlib_get_main (); - vnet_main_t *vnm = vnet_get_main (); - vlib_thread_main_t *tm = vlib_get_thread_main (); - vnet_eth_interface_registration_t eir = {}; - mrvl_pp2_main_t *ppm = &mrvl_pp2_main; - struct pp2_bpool_params bpool_params = { 0 }; - struct pp2_ppio_params ppio_params = { 0 }; - struct pp2_ppio_inq_params inq_params = { 0 }; - vnet_sw_interface_t *sw; - mrvl_pp2_if_t *ppif = 0; - u8 pp2_id, port_id, *s = 0; - eth_addr_t mac_addr; - u8 n_outqs, n_inqs = 1; - int i; - - if (tm->n_vlib_mains > PP2_PPIO_MAX_NUM_OUTQS) - { - args->rv = VNET_API_ERROR_INIT_FAILED; - args->error = clib_error_return (0, "number of threads (main + workers)" - " is bigger than number of output " - "queues (%u)", PP2_PPIO_MAX_NUM_OUTQS); - return; - } - n_outqs = tm->n_vlib_mains; - - /* defaults */ - args->tx_q_sz = args->tx_q_sz ? args->tx_q_sz : 2 * VLIB_FRAME_SIZE; - args->rx_q_sz = args->rx_q_sz ? args->rx_q_sz : 2 * VLIB_FRAME_SIZE; - - if (vec_len (ppm->per_thread_data) == 0) - { - if ((args->error = mrvl_pp2_main_init ()) != 0) - { - args->rv = VNET_API_ERROR_INIT_FAILED; - return; - } - } - - pool_get_zero (ppm->interfaces, ppif); - ppif->dev_instance = ppif - ppm->interfaces; - ppif->hw_if_index = ~0; - vec_validate_aligned (ppif->inqs, n_inqs - 1, CLIB_CACHE_LINE_BYTES); - vec_validate_aligned (ppif->outqs, n_outqs - 1, CLIB_CACHE_LINE_BYTES); - - for (i = 0; i < n_inqs; i++) - { - mrvl_pp2_inq_t *inq = vec_elt_at_index (ppif->inqs, i); - inq->size = args->rx_q_sz; - } - for (i = 0; i < n_outqs; i++) - { - mrvl_pp2_outq_t *outq = vec_elt_at_index (ppif->outqs, i); - outq->size = args->tx_q_sz; - vec_validate_aligned (outq->buffers, outq->size, CLIB_CACHE_LINE_BYTES); - } - - if (pp2_netdev_get_ppio_info ((char *) args->name, &pp2_id, &port_id)) - { - args->rv = VNET_API_ERROR_INVALID_INTERFACE; - args->error = clib_error_return (0, "Invalid interface '%s'", - args->name); - goto error; - } - - /* FIXME bpool bit select per pp */ - s = format (s, "pool-%d:%d%c", pp2_id, pp2_id + 8, 0); - bpool_params.match = (char *) s; - bpool_params.buff_len = vlib_buffer_get_default_data_size (vm); - /* FIXME +64 ? */ - if (pp2_bpool_init (&bpool_params, &ppif->inqs[0].bpool)) - { - args->rv = VNET_API_ERROR_INIT_FAILED; - args->error = clib_error_return (0, "bpool '%s' init failed", s); - goto error; - } - vec_reset_length (s); - - s = format (s, "ppio-%d:%d%c", pp2_id, port_id, 0); - ppio_params.match = (char *) s; - ppio_params.type = PP2_PPIO_T_NIC; - inq_params.size = args->rx_q_sz; - ppio_params.inqs_params.num_tcs = 1; - ppio_params.inqs_params.tcs_params[0].pkt_offset = 0; - ppio_params.inqs_params.tcs_params[0].num_in_qs = n_inqs; - ppio_params.inqs_params.tcs_params[0].inqs_params = &inq_params; - ppio_params.inqs_params.tcs_params[0].pools[0][0] = ppif->inqs[0].bpool; - ppio_params.outqs_params.num_outqs = n_outqs; - for (i = 0; i < n_outqs; i++) - { - ppio_params.outqs_params.outqs_params[i].weight = 1; - ppio_params.outqs_params.outqs_params[i].size = args->tx_q_sz; - } - if (pp2_ppio_init (&ppio_params, &ppif->ppio)) - { - args->rv = VNET_API_ERROR_INIT_FAILED; - args->error = clib_error_return (0, "ppio '%s' init failed", s); - goto error; - } - vec_reset_length (s); - - if (pp2_ppio_get_mac_addr (ppif->ppio, mac_addr)) - { - args->rv = VNET_API_ERROR_INIT_FAILED; - args->error = - clib_error_return (0, "%s: pp2_ppio_get_mac_addr failed", s); - goto error; - } - - eir.dev_class_index = mrvl_pp2_device_class.index; - eir.dev_instance = ppif->dev_instance; - eir.address = mac_addr; - eir.cb.flag_change = mrvl_pp2_eth_flag_change; - ppif->hw_if_index = vnet_eth_register_interface (vnm, &eir); - - sw = vnet_get_hw_sw_interface (vnm, ppif->hw_if_index); - ppif->sw_if_index = sw->sw_if_index; - ppif->per_interface_next_index = ~0; - args->sw_if_index = sw->sw_if_index; - vnet_hw_if_set_input_node (vnm, ppif->hw_if_index, - mrvl_pp2_input_node.index); - /* FIXME: only one RX queue ? */ - ppif->inqs[0].queue_index = vnet_hw_if_register_rx_queue ( - vnm, ppif->hw_if_index, 0, VNET_HW_IF_RXQ_THREAD_ANY); - - vnet_hw_if_set_rx_queue_mode (vnm, ppif->inqs[0].queue_index, - VNET_HW_IF_RX_MODE_POLLING); - vnet_hw_if_update_runtime_data (vnm, ppif->hw_if_index); - vnet_hw_interface_set_flags (vnm, ppif->hw_if_index, - VNET_HW_INTERFACE_FLAG_LINK_UP); - goto done; - -error: - mrvl_pp2_delete_if (ppif); -done: - vec_free (s); -} - -static clib_error_t * -mrvl_pp2_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, - u32 flags) -{ - mrvl_pp2_main_t *ppm = &mrvl_pp2_main; - vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); - mrvl_pp2_if_t *ppif = pool_elt_at_index (ppm->interfaces, hw->dev_instance); - static clib_error_t *error = 0; - int is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0; - int rv; - - if (is_up) - rv = pp2_ppio_enable (ppif->ppio); - else - rv = pp2_ppio_disable (ppif->ppio); - - if (rv) - return clib_error_return (0, "failed to %s interface", - is_up ? "enable" : "disable"); - - if (is_up) - ppif->flags |= MRVL_PP2_IF_F_ADMIN_UP; - else - ppif->flags &= ~MRVL_PP2_IF_F_ADMIN_UP; - - return error; -} - -static void -mrvl_pp2_clear_interface_counters (u32 instance) -{ - mrvl_pp2_main_t *ppm = &mrvl_pp2_main; - mrvl_pp2_if_t *ppif = pool_elt_at_index (ppm->interfaces, instance); - struct pp2_ppio_statistics stats; - - pp2_ppio_get_statistics (ppif->ppio, &stats, 1); -} - -static void -mrvl_pp2_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index, - u32 node_index) -{ - mrvl_pp2_main_t *ppm = &mrvl_pp2_main; - vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); - mrvl_pp2_if_t *ppif = pool_elt_at_index (ppm->interfaces, hw->dev_instance); - - /* Shut off redirection */ - if (node_index == ~0) - { - ppif->per_interface_next_index = node_index; - return; - } - - ppif->per_interface_next_index = - vlib_node_add_next (vlib_get_main (), mrvl_pp2_input_node.index, - node_index); -} - -static char *mrvl_pp2_tx_func_error_strings[] = { -#define _(n,s) s, - foreach_mrvl_pp2_tx_func_error -#undef _ -}; - -VNET_DEVICE_CLASS (mrvl_pp2_device_class,) = -{ - .name = "Marvell PPv2 interface", - .format_device_name = format_mrvl_pp2_interface_name, - .format_device = format_mrvl_pp2_interface, - .tx_function = mrvl_pp2_interface_tx, - .tx_function_n_errors = MRVL_PP2_TX_N_ERROR, - .tx_function_error_strings = mrvl_pp2_tx_func_error_strings, - .admin_up_down_function = mrvl_pp2_interface_admin_up_down, - .clear_counters = mrvl_pp2_clear_interface_counters, - .rx_redirect_to_node = mrvl_pp2_set_interface_next_node, -}; - -static clib_error_t * -mrvl_pp2_init (vlib_main_t * vm) -{ - return 0; -} - -VLIB_INIT_FUNCTION (mrvl_pp2_init); - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/plugins/marvell/pp2/pp2.h b/src/plugins/marvell/pp2/pp2.h deleted file mode 100644 index abb8e573a37..00000000000 --- a/src/plugins/marvell/pp2/pp2.h +++ /dev/null @@ -1,147 +0,0 @@ -/* - *------------------------------------------------------------------ - * Copyright (c) 2018 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *------------------------------------------------------------------ - */ - -#define MVCONF_DBG_LEVEL 0 -#define MVCONF_PP2_BPOOL_COOKIE_SIZE 32 -#define MVCONF_PP2_BPOOL_DMA_ADDR_SIZE 64 -#define MVCONF_DMA_PHYS_ADDR_T_SIZE 64 -#define MVCONF_SYS_DMA_UIO -#define MVCONF_TYPES_PUBLIC -#define MVCONF_DMA_PHYS_ADDR_T_PUBLIC - -#include <vlib/vlib.h> - -#include "mv_std.h" -#include "env/mv_sys_dma.h" -#include "drivers/mv_pp2.h" -#include <drivers/mv_pp2_bpool.h> -#include <drivers/mv_pp2_ppio.h> - -typedef struct -{ - CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); - u16 size; - u32 queue_index; - struct pp2_bpool *bpool; -} mrvl_pp2_inq_t; - -typedef struct -{ - CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); - u16 size; - u32 *buffers; - u16 head; - u16 tail; -} mrvl_pp2_outq_t; - -typedef struct -{ - CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); - u32 flags; -#define MRVL_PP2_IF_F_ADMIN_UP (1 << 0) - struct pp2_ppio *ppio; - u32 per_interface_next_index; - - mrvl_pp2_inq_t *inqs; - mrvl_pp2_outq_t *outqs; - - u32 dev_instance; - u32 sw_if_index; - u32 hw_if_index; -} mrvl_pp2_if_t; - -#define MRVL_PP2_BUFF_BATCH_SZ VLIB_FRAME_SIZE - -typedef struct -{ - CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); - struct pp2_hif *hif; - struct pp2_ppio_desc *descs; - struct buff_release_entry bre[MRVL_PP2_BUFF_BATCH_SZ]; - u32 buffers[VLIB_FRAME_SIZE]; -} mrvl_pp2_per_thread_data_t; - -typedef struct -{ - mrvl_pp2_if_t *interfaces; - mrvl_pp2_per_thread_data_t *per_thread_data; - - /* API message ID base */ - u16 msg_id_base; -} mrvl_pp2_main_t; - -extern vnet_device_class_t mrvl_pp2_device_class; -extern mrvl_pp2_main_t mrvl_pp2_main; - -typedef struct -{ - u8 *name; - u16 rx_q_sz; - u16 tx_q_sz; - - /* return */ - i32 rv; - u32 sw_if_index; - clib_error_t *error; -} mrvl_pp2_create_if_args_t; - -void mrvl_pp2_create_if (mrvl_pp2_create_if_args_t * args); -void mrvl_pp2_delete_if (mrvl_pp2_if_t * dfif); -clib_error_t *mrvl_pp2_plugin_api_hookup (vlib_main_t * vm); - -/* output.c */ - -#define foreach_mrvl_pp2_tx_func_error \ - _(NO_FREE_SLOTS, "no free tx slots") \ - _(PPIO_SEND, "pp2_ppio_send errors") \ - _(PPIO_GET_NUM_OUTQ_DONE, "pp2_ppio_get_num_outq_done errors") - -typedef enum -{ -#define _(f,s) MRVL_PP2_TX_ERROR_##f, - foreach_mrvl_pp2_tx_func_error -#undef _ - MRVL_PP2_TX_N_ERROR, -} mrvl_pp2_tx_func_error_t; - -uword mrvl_pp2_interface_tx (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_frame_t * frame); - -/* input.c */ - -typedef struct -{ - u32 next_index; - u32 hw_if_index; - struct pp2_ppio_desc desc; -} mrvl_pp2_input_trace_t; - -extern vlib_node_registration_t mrvl_pp2_input_node; - -/* format.c */ -format_function_t format_mrvl_pp2_input_trace; -format_function_t format_mrvl_pp2_interface; -format_function_t format_mrvl_pp2_interface_name; - - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/plugins/marvell/pp2/pp2_api.c b/src/plugins/marvell/pp2/pp2_api.c deleted file mode 100644 index c1f3a9e1d1d..00000000000 --- a/src/plugins/marvell/pp2/pp2_api.c +++ /dev/null @@ -1,100 +0,0 @@ -/* - *------------------------------------------------------------------ - * Copyright (c) 2019 Arm Limited. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *------------------------------------------------------------------ - */ - -#include <vlib/vlib.h> -#include <vlib/unix/unix.h> -#include <vnet/ethernet/ethernet.h> - -#include <marvell/pp2/pp2.h> - -#include <vlibapi/api.h> -#include <vlibmemory/api.h> - -/* define message IDs */ -#include <marvell/pp2/pp2.api_enum.h> -#include <marvell/pp2/pp2.api_types.h> - -#define REPLY_MSG_ID_BASE (pp2->msg_id_base) -#include <vlibapi/api_helper_macros.h> - -static void -vl_api_mrvl_pp2_create_t_handler (vl_api_mrvl_pp2_create_t * mp) -{ - mrvl_pp2_main_t *pp2 = &mrvl_pp2_main; - mrvl_pp2_create_if_args_t args = { 0 }; - vl_api_mrvl_pp2_create_reply_t *rmp; - int rv; - - args.name = format (0, "%s", mp->if_name); - args.rx_q_sz = ntohs (mp->rx_q_sz); - args.tx_q_sz = ntohs (mp->tx_q_sz); - mrvl_pp2_create_if (&args); - rv = args.rv; - vec_free (args.name); - if (args.error) - { - clib_error_free (args.error); - } - REPLY_MACRO2 (VL_API_MRVL_PP2_CREATE_REPLY, - ({ rmp->sw_if_index = ntohl (args.sw_if_index); })); -} - -static void -vl_api_mrvl_pp2_delete_t_handler (vl_api_mrvl_pp2_delete_t * mp) -{ - vnet_main_t *vnm = vnet_get_main (); - vnet_hw_interface_t *hw; - mrvl_pp2_main_t *pp2 = &mrvl_pp2_main; - vl_api_mrvl_pp2_delete_reply_t *rmp; - mrvl_pp2_if_t *dif; - int rv = 0; - mp->sw_if_index = ntohl (mp->sw_if_index); - hw = vnet_get_sup_hw_interface (vnm, mp->sw_if_index); - if (hw == NULL || mrvl_pp2_device_class.index != hw->dev_class_index) - { - rv = VNET_API_ERROR_INVALID_SW_IF_INDEX; - goto reply; - } - - dif = pool_elt_at_index (pp2->interfaces, hw->dev_instance); - - mrvl_pp2_delete_if (dif); - -reply: - REPLY_MACRO (VL_API_MRVL_PP2_DELETE_REPLY); -} - -#include <marvell/pp2/pp2.api.c> -/* set up the API message handling tables */ -clib_error_t * -mrvl_pp2_plugin_api_hookup (vlib_main_t * vm) -{ - mrvl_pp2_main_t *pp2 = &mrvl_pp2_main; - - /* ask for a correctly-sized block of API message decode slots */ - pp2->msg_id_base = setup_message_id_table (); - - return 0; -} - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/plugins/marvell/pp2/pp2_test.c b/src/plugins/marvell/pp2/pp2_test.c deleted file mode 100644 index 26a9e9a6e34..00000000000 --- a/src/plugins/marvell/pp2/pp2_test.c +++ /dev/null @@ -1,144 +0,0 @@ -/* - *------------------------------------------------------------------ - * Copyright (c) 2019 Arm Limited. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *------------------------------------------------------------------ - */ - -#include <vlib/vlib.h> -#include <vlib/unix/unix.h> -#include <vnet/ethernet/ethernet.h> - -#include <vat/vat.h> -#include <vlibapi/api.h> -#include <vlibmemory/api.h> - -#include <vppinfra/error.h> -#include <marvell/pp2/pp2.h> - -#define __plugin_msg_base pp2_test_main.msg_id_base -#include <vlibapi/vat_helper_macros.h> - -/* declare message IDs */ -#include <marvell/pp2/pp2.api_enum.h> -#include <marvell/pp2/pp2.api_types.h> - -typedef struct -{ - /* API message ID base */ - u16 msg_id_base; - vat_main_t *vat_main; -} pp2_test_main_t; - -pp2_test_main_t pp2_test_main; - -/* mrvl_pp2 create API */ -static int -api_mrvl_pp2_create (vat_main_t * vam) -{ - unformat_input_t *i = vam->input; - vl_api_mrvl_pp2_create_t *mp; - mrvl_pp2_create_if_args_t args; - int ret; - u16 size; - - clib_memset (&args, 0, sizeof (mrvl_pp2_create_if_args_t)); - while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) - { - if (unformat (i, "name %s", &args.name)) - ; - else if (unformat (i, "rx-queue-size %u", &size)) - args.rx_q_sz = size; - else if (unformat (i, "tx-queue-size %u", &size)) - args.tx_q_sz = size; - else - { - clib_warning ("unknown input '%U'", format_unformat_error, i); - return -99; - } - } - - M (MRVL_PP2_CREATE, mp); - - strncpy_s ((char *) mp->if_name, ARRAY_LEN (mp->if_name), - (char *) (args.name), strlen ((char *) args.name)); - mp->rx_q_sz = clib_host_to_net_u16 (args.rx_q_sz); - mp->tx_q_sz = clib_host_to_net_u16 (args.tx_q_sz); - - S (mp); - W (ret); - - vec_free (args.name); - - return ret; -} - -/* mrvl_pp2 create reply handler */ -static void -vl_api_mrvl_pp2_create_reply_t_handler (vl_api_mrvl_pp2_create_reply_t * mp) -{ - vat_main_t *vam = pp2_test_main.vat_main; - i32 retval = ntohl (mp->retval); - - if (retval == 0) - { - fformat (vam->ofp, "created mrvl_pp2 with sw_if_index %d\n", - ntohl (mp->sw_if_index)); - } - - vam->retval = retval; - vam->result_ready = 1; - vam->regenerate_interface_table = 1; -} - - -/* mrvl_pp2 delete API */ -static int -api_mrvl_pp2_delete (vat_main_t * vam) -{ - unformat_input_t *i = vam->input; - //vnet_main_t *vnm = vnet_get_main (); - vl_api_mrvl_pp2_delete_t *mp; - u32 sw_if_index = 0; - int ret; - - while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) - { - if (unformat (i, "sw_if_index %d", &sw_if_index)) - ; - else - { - clib_warning ("unknown input '%U'", format_unformat_error, i); - return -99; - } - } - - M (MRVL_PP2_DELETE, mp); - - mp->sw_if_index = clib_host_to_net_u32 (sw_if_index); - - S (mp); - W (ret); - - return ret; -} - -#include <marvell/pp2/pp2.api_test.c> - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/plugins/npt66/npt66.api b/src/plugins/npt66/npt66.api index 63640ac2097..dab09cda31f 100644 --- a/src/plugins/npt66/npt66.api +++ b/src/plugins/npt66/npt66.api @@ -36,5 +36,16 @@ counters npt66 { units "packets"; description "packet translation failed"; }; - + icmp6_checksum { + severity error; + type counter64; + units "packets"; + description "ICMP6 checksum validation failed"; + }; + icmp6_truncated { + severity error; + type counter64; + units "packets"; + description "ICMP6 packet truncated"; + }; };
\ No newline at end of file diff --git a/src/plugins/npt66/npt66_node.c b/src/plugins/npt66/npt66_node.c index f74f9143998..0d0c475f2c3 100644 --- a/src/plugins/npt66/npt66_node.c +++ b/src/plugins/npt66/npt66_node.c @@ -127,10 +127,7 @@ npt66_translate (ip6_header_t *ip, npt66_binding_t *binding, int dir) if (!ip6_prefix_cmp (ip->src_address, binding->internal, binding->internal_plen)) { - clib_warning ( - "npt66_translate: src address is not internal (%U -> %U)", - format_ip6_address, &ip->src_address, format_ip6_address, - &ip->dst_address); + /* Packet is not for us */ goto done; } ip->src_address = ip6_prefix_copy (ip->src_address, binding->external, @@ -144,10 +141,7 @@ npt66_translate (ip6_header_t *ip, npt66_binding_t *binding, int dir) if (!ip6_prefix_cmp (ip->dst_address, binding->external, binding->external_plen)) { - clib_warning ( - "npt66_translate: dst address is not external (%U -> %U)", - format_ip6_address, &ip->src_address, format_ip6_address, - &ip->dst_address); + /* Packet is not for us */ goto done; } ip->dst_address = ip6_prefix_copy (ip->dst_address, binding->internal, @@ -162,7 +156,7 @@ done: static int npt66_icmp6_translate (vlib_buffer_t *b, ip6_header_t *outer_ip, icmp46_header_t *icmp, npt66_binding_t *binding, - int dir) + int dir, u32 *error) { ip6_header_t *ip = (ip6_header_t *) (icmp + 2); int rv = 0; @@ -171,7 +165,7 @@ npt66_icmp6_translate (vlib_buffer_t *b, ip6_header_t *outer_ip, if (clib_net_to_host_u16 (outer_ip->payload_length) < sizeof (icmp46_header_t) + 4 + sizeof (ip6_header_t)) { - clib_warning ("ICMP6 payload too short"); + *error = NPT66_ERROR_ICMP6_TRUNCATED; return -1; } @@ -181,7 +175,7 @@ npt66_icmp6_translate (vlib_buffer_t *b, ip6_header_t *outer_ip, sum16 = ip6_tcp_udp_icmp_compute_checksum (vm, b, outer_ip, &bogus_length); if (sum16 != 0 && sum16 != 0xffff) { - clib_warning ("ICMP6 checksum failed"); + *error = NPT66_ERROR_ICMP6_CHECKSUM; return -1; } if (dir == VLIB_RX) @@ -189,10 +183,7 @@ npt66_icmp6_translate (vlib_buffer_t *b, ip6_header_t *outer_ip, if (!ip6_prefix_cmp (ip->src_address, binding->external, binding->external_plen)) { - clib_warning ( - "npt66_icmp6_translate: src address is not internal (%U -> %U)", - format_ip6_address, &ip->src_address, format_ip6_address, - &ip->dst_address); + /* Not for us */ goto done; } ip->src_address = ip6_prefix_copy (ip->src_address, binding->internal, @@ -206,10 +197,7 @@ npt66_icmp6_translate (vlib_buffer_t *b, ip6_header_t *outer_ip, if (!ip6_prefix_cmp (ip->dst_address, binding->external, binding->external_plen)) { - clib_warning ( - "npt66_icmp6_translate: dst address is not external (%U -> %U)", - format_ip6_address, &ip->src_address, format_ip6_address, - &ip->dst_address); + /* Not for us */ goto done; } ip->dst_address = ip6_prefix_copy (ip->dst_address, binding->internal, @@ -217,8 +205,8 @@ npt66_icmp6_translate (vlib_buffer_t *b, ip6_header_t *outer_ip, rv = npt66_adjust_checksum (binding->internal_plen, false, binding->delta, &ip->dst_address); } -done: +done: return rv; } @@ -243,10 +231,12 @@ npt66_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, n_left_from = frame->n_vectors; vlib_get_buffers (vm, from, b, n_left_from); npt66_binding_t *binding; + u32 translated = 0; /* Stage 1: build vector of flow hash (based on lookup mask) */ while (n_left_from > 0) { + u32 error = NPT66_ERROR_TRANSLATION; u32 sw_if_index = vnet_buffer (b[0])->sw_if_index[dir]; u32 iph_offset = dir == VLIB_TX ? vnet_buffer (b[0])->ip.save_rewrite_length : 0; @@ -261,28 +251,26 @@ npt66_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, icmp46_header_t *icmp = (icmp46_header_t *) (ip + 1); if (ip->protocol == IP_PROTOCOL_ICMP6 && icmp->type < 128) { - rv = npt66_icmp6_translate (b[0], ip, icmp, binding, dir); + rv = npt66_icmp6_translate (b[0], ip, icmp, binding, dir, &error); if (rv < 0) { - clib_warning ("ICMP6 npt66_translate failed"); *next = NPT66_NEXT_DROP; + b[0]->error = node->errors[error]; goto next; } } - rv = npt66_translate (ip, binding, dir); + rv = npt66_translate (ip, binding, dir); if (rv < 0) { - vlib_node_increment_counter (vm, node->node_index, - NPT66_ERROR_TRANSLATION, 1); + b[0]->error = node->errors[error]; *next = NPT66_NEXT_DROP; goto next; } - else if (dir == VLIB_TX) - vlib_node_increment_counter (vm, node->node_index, NPT66_ERROR_TX, 1); else - vlib_node_increment_counter (vm, node->node_index, NPT66_ERROR_RX, 1); - + { + translated++; + } next: next += 1; n_left_from -= 1; @@ -321,6 +309,9 @@ npt66_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, break; } } + vlib_node_increment_counter ( + vm, node->node_index, dir == VLIB_TX ? NPT66_ERROR_TX : NPT66_ERROR_RX, + translated); vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors); return frame->n_vectors; @@ -338,17 +329,17 @@ VLIB_NODE_FN (npt66_output_node) } VLIB_REGISTER_NODE(npt66_input_node) = { - .name = "npt66-input", - .vector_size = sizeof(u32), - .format_trace = format_npt66_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - .n_errors = NPT66_N_ERROR, - .error_counters = npt66_error_counters, - .n_next_nodes = NPT66_N_NEXT, - .next_nodes = - { - [NPT66_NEXT_DROP] = "error-drop", - }, + .name = "npt66-input", + .vector_size = sizeof(u32), + .format_trace = format_npt66_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = NPT66_N_ERROR, + .error_counters = npt66_error_counters, + .n_next_nodes = NPT66_N_NEXT, + .next_nodes = + { + [NPT66_NEXT_DROP] = "error-drop", + }, }; VLIB_REGISTER_NODE (npt66_output_node) = { diff --git a/src/plugins/builtinurl/CMakeLists.txt b/src/plugins/osi/CMakeLists.txt index ddbca5e50f1..8ab014770ea 100644 --- a/src/plugins/builtinurl/CMakeLists.txt +++ b/src/plugins/osi/CMakeLists.txt @@ -1,5 +1,4 @@ - -# Copyright (c) <current-year> <your-organization> +# Copyright (c) 2023 Cisco and/or its affiliates # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -12,15 +11,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -add_vpp_plugin(builtinurl - SOURCES - builtins.c - builtinurl.c - builtinurl.h +add_vpp_plugin(osi - API_FILES - builtinurl.api + SOURCES + osi.c + node.c + pg.c + plugin.c - API_TEST_SOURCES - builtinurl_test.c + INSTALL_HEADERS + osi.h ) diff --git a/src/plugins/osi/FEATURE.yaml b/src/plugins/osi/FEATURE.yaml new file mode 100644 index 00000000000..337be1c7146 --- /dev/null +++ b/src/plugins/osi/FEATURE.yaml @@ -0,0 +1,11 @@ +--- +name: OSI plugin +maintainer: + - community <vpp-dev@lists.fd.io> +features: + - Adds support for OSI protocols (SAP types) + - Registered as input protocol for PPP, HDLC, and LLC +missing: + - No tests for this feature currently exist +description: "" +state: experimental diff --git a/src/vnet/osi/node.c b/src/plugins/osi/node.c index 9edc354cda7..a36b1525e0e 100644 --- a/src/vnet/osi/node.c +++ b/src/plugins/osi/node.c @@ -39,7 +39,7 @@ #include <vlib/vlib.h> #include <vnet/pg/pg.h> -#include <vnet/osi/osi.h> +#include <osi/osi.h> #include <vnet/ppp/ppp.h> #include <vnet/hdlc/hdlc.h> #include <vnet/llc/llc.h> diff --git a/src/vnet/osi/osi.c b/src/plugins/osi/osi.c index 9556481448a..67c7053f388 100644 --- a/src/vnet/osi/osi.c +++ b/src/plugins/osi/osi.c @@ -38,7 +38,7 @@ */ #include <vnet/vnet.h> -#include <vnet/osi/osi.h> +#include <osi/osi.h> /* Global main structure. */ osi_main_t osi_main; @@ -169,13 +169,8 @@ add_protocol (osi_main_t * pm, osi_protocol_t protocol, char *protocol_name) static clib_error_t * osi_init (vlib_main_t * vm) { - clib_error_t *error = 0; osi_main_t *pm = &osi_main; - /* init order dependency: llc_init -> osi_init */ - if ((error = vlib_call_init_function (vm, llc_init))) - return error; - clib_memset (pm, 0, sizeof (pm[0])); pm->vlib_main = vm; @@ -189,8 +184,11 @@ osi_init (vlib_main_t * vm) return vlib_call_init_function (vm, osi_input_init); } -VLIB_INIT_FUNCTION (osi_init); - +/* init order dependency: llc_init -> osi_init -> snap_init*/ +/* Otherwise, osi_input_init will wipe out e.g. the snap init */ +VLIB_INIT_FUNCTION (osi_init) = { + .init_order = VLIB_INITS ("llc_init", "osi_init", "snap_init"), +}; /* * fd.io coding-style-patch-verification: ON diff --git a/src/vnet/osi/osi.h b/src/plugins/osi/osi.h index fb248ed9cc5..fb248ed9cc5 100644 --- a/src/vnet/osi/osi.h +++ b/src/plugins/osi/osi.h diff --git a/src/vnet/osi/pg.c b/src/plugins/osi/pg.c index c87a869b28d..3bac693c127 100644 --- a/src/vnet/osi/pg.c +++ b/src/plugins/osi/pg.c @@ -39,7 +39,7 @@ #include <vlib/vlib.h> #include <vnet/pg/pg.h> -#include <vnet/osi/osi.h> +#include <osi/osi.h> typedef struct { diff --git a/src/plugins/marvell/plugin.c b/src/plugins/osi/plugin.c index ed90776ba95..5fc412e093e 100644 --- a/src/plugins/marvell/plugin.c +++ b/src/plugins/osi/plugin.c @@ -1,6 +1,7 @@ /* - *------------------------------------------------------------------ - * Copyright (c) 2018 Cisco and/or its affiliates. + * plugin.c: osi + * + * Copyright (c) 2023 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -12,22 +13,11 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - *------------------------------------------------------------------ */ - #include <vlib/vlib.h> #include <vnet/plugin/plugin.h> #include <vpp/app/version.h> - VLIB_PLUGIN_REGISTER () = { .version = VPP_BUILD_VER, - .description = "Marvell PP2 Device Driver", -}; - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ + .description = "OSI plugin", +};
\ No newline at end of file diff --git a/src/plugins/prom/prom.c b/src/plugins/prom/prom.c index 934e8480d3c..475e98b1038 100644 --- a/src/plugins/prom/prom.c +++ b/src/plugins/prom/prom.c @@ -191,6 +191,7 @@ send_data_to_hss (hss_session_handle_t sh) args.sh = sh; args.data = vec_dup (pm->stats); args.data_len = vec_len (pm->stats); + args.ct = HTTP_CONTENT_TEXT_PLAIN; args.sc = HTTP_STATUS_OK; args.free_vec_data = 1; @@ -207,7 +208,7 @@ static uword prom_scraper_process (vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f) { - uword *event_data = 0, event_type; + uword *event_data = 0, event_type, *sh_as_uword; prom_main_t *pm = &prom_main; hss_session_handle_t sh; f64 timeout = 10000.0; @@ -222,12 +223,15 @@ prom_scraper_process (vlib_main_t *vm, vlib_node_runtime_t *rt, /* timeout, do nothing */ break; case PROM_SCRAPER_EVT_RUN: - sh.as_u64 = event_data[0]; vec_reset_length (pm->stats); pm->stats = scrape_stats_segment (pm->stats, pm->stats_patterns, pm->used_only); - session_send_rpc_evt_to_thread_force (sh.thread_index, - send_data_to_hss_rpc, &sh); + vec_foreach (sh_as_uword, event_data) + { + sh.as_u64 = (u64) *sh_as_uword; + session_send_rpc_evt_to_thread_force ( + sh.thread_index, send_data_to_hss_rpc, sh_as_uword); + } pm->last_scrape = vlib_time_now (vm); break; default: diff --git a/src/plugins/pvti/CMakeLists.txt b/src/plugins/pvti/CMakeLists.txt new file mode 100644 index 00000000000..900b662d54a --- /dev/null +++ b/src/plugins/pvti/CMakeLists.txt @@ -0,0 +1,40 @@ +# Copyright (c) 2024 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_vpp_plugin(pvti + SOURCES + pvti_if.c + pvti.c + input.h + input.c + input-main.c + output.h + output.c + output-main.c + bypass.h + bypass.c + bypass-main.c + api.c + pvti.h + + MULTIARCH_SOURCES + input.c + output.c + bypass.c + + API_FILES + pvti.api + + # API_TEST_SOURCES + # pvti_test.c +) diff --git a/src/plugins/pvti/FEATURE.yaml b/src/plugins/pvti/FEATURE.yaml new file mode 100644 index 00000000000..52dbe5b7c1b --- /dev/null +++ b/src/plugins/pvti/FEATURE.yaml @@ -0,0 +1,8 @@ +--- +name: Packet Vector Tunnel +maintainer: Andrew Yourtchenko <ayourtch@gmail.com> +features: + - support inner MTU up to ~8K over standard 1280..1500 MTU substrate +description: "Large MTU Tunnels" +state: development +properties: [API, CLI] diff --git a/src/plugins/pvti/api.c b/src/plugins/pvti/api.c new file mode 100644 index 00000000000..cda39ad44e8 --- /dev/null +++ b/src/plugins/pvti/api.c @@ -0,0 +1,137 @@ + +#include <vnet/vnet.h> +#include <vlibmemory/api.h> + +#include <vnet/format_fns.h> +#include <vnet/ip/ip_types_api.h> +#include <vlibapi/api.h> + +#include <pvti/pvti.api_enum.h> +#include <pvti/pvti.api_types.h> + +#include <pvti/pvti.h> +#include <pvti/pvti_if.h> + +#define REPLY_MSG_ID_BASE pvm->msg_id_base +#include <vlibapi/api_helper_macros.h> + +typedef struct +{ + vl_api_registration_t *reg; + u32 context; +} pvti_if_details_ctx_t; + +typedef struct +{ + +} pvti_interface_dump_ctx_t; + +static walk_rc_t +pvti_if_send_details (index_t pvtii, void *data) +{ + vl_api_pvti_interface_details_t *rmp; + pvti_if_details_ctx_t *ctx = data; + const pvti_if_t *pvi; + + pvi = pvti_if_get (pvtii); + + rmp = vl_msg_api_alloc_zero (sizeof (*rmp)); + rmp->_vl_msg_id = + htons (VL_API_PVTI_INTERFACE_DETAILS + pvti_main.msg_id_base); + + rmp->interface.sw_if_index = htonl (pvi->sw_if_index); + rmp->interface.local_port = htons (pvi->local_port); + rmp->interface.remote_port = htons (pvi->remote_port); + rmp->interface.underlay_mtu = htons (pvi->underlay_mtu); + + ip_address_encode2 (&pvi->local_ip, &rmp->interface.local_ip); + ip_address_encode2 (&pvi->remote_ip, &rmp->interface.remote_ip); + + rmp->context = ctx->context; + + vl_api_send_msg (ctx->reg, (u8 *) rmp); + + return (WALK_CONTINUE); +} + +static void +vl_api_pvti_interface_dump_t_handler (vl_api_pvti_interface_dump_t *mp) +{ + vl_api_registration_t *reg; + // pvti_main_t *pvm = &pvti_main; + + reg = vl_api_client_index_to_registration (mp->client_index); + if (reg == 0) + return; + + pvti_if_details_ctx_t ctx = { + .reg = reg, + .context = mp->context, + }; + + u32 sw_if_index = ntohl (mp->sw_if_index); + if (sw_if_index == ~0) + pvti_if_walk (pvti_if_send_details, &ctx); + else + { + index_t pvtii = pvti_if_find_by_sw_if_index (sw_if_index); + if (pvtii != INDEX_INVALID) + pvti_if_send_details (pvtii, &ctx); + } +} + +static void +vl_api_pvti_interface_create_t_handler (vl_api_pvti_interface_create_t *mp) +{ + vl_api_pvti_interface_create_reply_t *rmp; + pvti_main_t *pvm = &pvti_main; + int rv = ~0; + u32 sw_if_index = ~0; + ip_address_t local_ip; + ip_address_t remote_ip; + + ip_address_decode2 (&mp->interface.local_ip, &local_ip); + ip_address_decode2 (&mp->interface.remote_ip, &remote_ip); + u16 lport = clib_host_to_net_u16 (mp->interface.local_port); + u16 rport = clib_host_to_net_u16 (mp->interface.remote_port); + u16 underlay_mtu = clib_host_to_net_u16 (mp->interface.underlay_mtu); + u32 underlay_fib_index = + clib_host_to_net_u32 (mp->interface.underlay_fib_index); + pvti_peer_address_method_t peer_address_method = + mp->interface.peer_address_from_payload ? PVTI_PEER_ADDRESS_FROM_PAYLOAD : + PVTI_PEER_ADDRESS_FIXED; + + if (underlay_mtu == 0) + { + underlay_mtu = 1500; + } + + rv = + pvti_if_create (&local_ip, lport, &remote_ip, rport, peer_address_method, + underlay_mtu, underlay_fib_index, &sw_if_index); + + REPLY_MACRO2 (VL_API_PVTI_INTERFACE_CREATE_REPLY, + { rmp->sw_if_index = htonl (sw_if_index); }); +} + +static void +vl_api_pvti_interface_delete_t_handler (vl_api_pvti_interface_delete_t *mp) +{ + vl_api_pvti_interface_delete_reply_t *rmp; + pvti_main_t *pvm = &pvti_main; + int rv = 0; + + rv = pvti_if_delete (ntohl (mp->sw_if_index)); + REPLY_MACRO (VL_API_PVTI_INTERFACE_DELETE_REPLY); +} + +/* API definitions */ +#include <pvti/pvti.api.c> + +void +pvti_api_init () +{ + pvti_main_t *pvm = &pvti_main; + /* Add our API messages to the global name_crc hash table */ + pvm->msg_id_base = setup_message_id_table (); +} diff --git a/src/plugins/pvti/bypass-main.c b/src/plugins/pvti/bypass-main.c new file mode 100644 index 00000000000..db79ccd2113 --- /dev/null +++ b/src/plugins/pvti/bypass-main.c @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2024 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <pvti/bypass.h> + +/* packet trace format function */ +static u8 * +format_pvti_bypass_trace (u8 *s, va_list *args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + pvti_bypass_trace_t *t = va_arg (*args, pvti_bypass_trace_t *); + + s = format (s, "PVTI-BYPASS: sw_if_index %d, next index %d\n", + t->sw_if_index, t->next_index); + s = format (s, " src %U sport %d dport %d\n", format_ip_address, + &t->remote_ip, t->remote_port, t->local_port); + s = format (s, " seq: %d", t->seq); + return s; +} + +vlib_node_registration_t pvti4_bypass_node; +vlib_node_registration_t pvti6_bypass_node; + +static char *pvti_bypass_error_strings[] = { +#define _(sym, string) string, + foreach_pvti_bypass_error +#undef _ +}; + +VLIB_REGISTER_NODE (pvti4_bypass_node) = +{ + .name = "ip4-pvti-bypass", + .vector_size = sizeof (u32), + .format_trace = format_pvti_bypass_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_errors = ARRAY_LEN(pvti_bypass_error_strings), + .error_strings = pvti_bypass_error_strings, + + .n_next_nodes = PVTI_BYPASS_N_NEXT, + + .next_nodes = { + [PVTI_BYPASS_NEXT_DROP] = "error-drop", + [PVTI_BYPASS_NEXT_PVTI_INPUT] = "pvti4-input", + }, + +}; + +VLIB_REGISTER_NODE (pvti6_bypass_node) = +{ + .name = "ip6-pvti-bypass", + .vector_size = sizeof (u32), + .format_trace = format_pvti_bypass_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_errors = ARRAY_LEN(pvti_bypass_error_strings), + .error_strings = pvti_bypass_error_strings, + + .n_next_nodes = PVTI_BYPASS_N_NEXT, + + .next_nodes = { + [PVTI_BYPASS_NEXT_DROP] = "error-drop", + [PVTI_BYPASS_NEXT_PVTI_INPUT] = "pvti6-input", + }, + +}; diff --git a/src/plugins/pvti/bypass.c b/src/plugins/pvti/bypass.c new file mode 100644 index 00000000000..14c976439eb --- /dev/null +++ b/src/plugins/pvti/bypass.c @@ -0,0 +1,202 @@ +/* + * Copyright (c) 2024 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <vlib/vlib.h> +#include <vnet/vnet.h> +#include <vnet/pg/pg.h> +#include <vppinfra/error.h> +#include <pvti/pvti.h> +#include <pvti/pvti_if.h> +#include <pvti/bypass.h> + +always_inline u16 +pvti_bypass_node_common (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_frame_t *frame, bool is_ip6) +{ + u32 n_left_from, *from, *to_next; + pvti_bypass_next_t next_index; + vlib_node_runtime_t *error_node = + vlib_node_get_runtime (vm, ip4_input_node.index); + + u32 pkts_processed = 0; + + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + next_index = node->cached_next_index; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + + while (n_left_from > 0 && n_left_to_next > 0) + { + vlib_buffer_t *b0; + u32 sw_if_index0 = 0; + ip4_header_t *ip40; + ip6_header_t *ip60; + udp_header_t *udp0; + u32 bi0, ip_len0, udp_len0, flags0, next0; + u8 error0, good_udp0, proto0; + i32 len_diff0; + + bi0 = to_next[0] = from[0]; + from += 1; + n_left_from -= 1; + to_next += 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + + /* setup the packet for the next feature */ + vnet_feature_next (&next0, b0); + + if (is_ip6) + { + ip60 = vlib_buffer_get_current (b0); + } + else + { + ip40 = vlib_buffer_get_current (b0); + } + + if (is_ip6) + { + proto0 = ip60->protocol; + } + else + { + /* Treat IP frag packets as "experimental" protocol for now */ + proto0 = ip4_is_fragment (ip40) ? 0xfe : ip40->protocol; + } + + /* Process packet 0 */ + if (proto0 != IP_PROTOCOL_UDP) + goto exit; /* not UDP packet */ + + if (is_ip6) + udp0 = ip6_next_header (ip60); + else + udp0 = ip4_next_header (ip40); + + /* look up the destination ip and port */ + u32 pvti_index0 = INDEX_INVALID; + if (is_ip6) + { + pvti_index0 = pvti_if_find_by_remote_ip6_and_port ( + &ip60->src_address, clib_net_to_host_u16 (udp0->src_port)); + } + else + { + pvti_index0 = pvti_if_find_by_remote_ip4_and_port ( + &ip40->src_address, clib_net_to_host_u16 (udp0->src_port)); + } + if (pvti_index0 == INDEX_INVALID) + goto exit; + + flags0 = b0->flags; + good_udp0 = (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0; + + /* Don't verify UDP checksum for packets with explicit zero checksum. + */ + good_udp0 |= udp0->checksum == 0; + + /* Verify UDP length */ + if (is_ip6) + ip_len0 = clib_net_to_host_u16 (ip60->payload_length); + else + ip_len0 = clib_net_to_host_u16 (ip40->length); + udp_len0 = clib_net_to_host_u16 (udp0->length); + len_diff0 = ip_len0 - udp_len0; + + /* Verify UDP checksum */ + if (PREDICT_FALSE (!good_udp0)) + { + if (is_ip6) + flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, b0); + else + flags0 = ip4_tcp_udp_validate_checksum (vm, b0); + good_udp0 = (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0; + } + + if (is_ip6) + { + error0 = good_udp0 ? 0 : IP6_ERROR_UDP_CHECKSUM; + error0 = (len_diff0 >= 0) ? error0 : IP6_ERROR_UDP_LENGTH; + } + else + { + error0 = good_udp0 ? 0 : IP4_ERROR_UDP_CHECKSUM; + error0 = (len_diff0 >= 0) ? error0 : IP4_ERROR_UDP_LENGTH; + } + + next0 = error0 ? PVTI_BYPASS_NEXT_DROP : PVTI_BYPASS_NEXT_PVTI_INPUT; + b0->error = error0 ? error_node->errors[error0] : 0; + + /* pvtiX-input node expect current at PVTI header */ + if (is_ip6) + vlib_buffer_advance (b0, sizeof (ip6_header_t) + + sizeof (udp_header_t)); + else + vlib_buffer_advance (b0, sizeof (ip4_header_t) + + sizeof (udp_header_t)); + exit: + + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) && + (b0->flags & VLIB_BUFFER_IS_TRACED))) + { + pvti_bypass_trace_t *t = + vlib_add_trace (vm, node, b0, sizeof (*t)); + t->sw_if_index = sw_if_index0; + t->next_index = next0; + t->seq = 0; // clib_net_to_host_u32 (pvti0->seq); + if (is_ip6) + { + } + else + { + t->remote_ip.ip.ip4 = ip40->src_address; + t->remote_ip.version = AF_IP4; + } + // t->local_port = h0->udp.dst_port; + // t->remote_port = h0->udp.src_port; + } + + pkts_processed += 1; + + /* verify speculative enqueue, maybe switch current next frame */ + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, + n_left_to_next, bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + vlib_node_increment_counter (vm, node->node_index, + PVTI_BYPASS_ERROR_PROCESSED, pkts_processed); + return frame->n_vectors; +} + +VLIB_NODE_FN (pvti4_bypass_node) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) +{ + return pvti_bypass_node_common (vm, node, frame, 0); +} + +VLIB_NODE_FN (pvti6_bypass_node) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) +{ + return pvti_bypass_node_common (vm, node, frame, 1); +} diff --git a/src/plugins/pvti/bypass.h b/src/plugins/pvti/bypass.h new file mode 100644 index 00000000000..611d5770ad3 --- /dev/null +++ b/src/plugins/pvti/bypass.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2024 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __included_pvti_bypass_h__ +#define __included_pvti_bypass_h__ + +#include <vlib/vlib.h> +#include <vnet/vnet.h> +#include <vnet/pg/pg.h> +#include <vppinfra/error.h> +#include <pvti/pvti.h> +#include <pvti/pvti_if.h> + +typedef struct +{ + u32 next_index; + u32 sw_if_index; + ip_address_t remote_ip; + u16 remote_port; + u16 local_port; + u32 seq; +} pvti_bypass_trace_t; + +#define foreach_pvti_bypass_error \ + _ (PROCESSED, "PVTI bypass tunnel packets processed") + +typedef enum +{ +#define _(sym, str) PVTI_BYPASS_ERROR_##sym, + foreach_pvti_bypass_error +#undef _ + PVTI_BYPASS_N_ERROR, +} pvti_bypass_error_t; + +typedef enum +{ + PVTI_BYPASS_NEXT_DROP, + PVTI_BYPASS_NEXT_PVTI_INPUT, + PVTI_BYPASS_N_NEXT, +} pvti_bypass_next_t; + +#endif // pvti_bypass_h diff --git a/src/plugins/pvti/input-main.c b/src/plugins/pvti/input-main.c new file mode 100644 index 00000000000..8ab8b18dd7c --- /dev/null +++ b/src/plugins/pvti/input-main.c @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2024 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <pvti/input.h> + +static char *pvti_input_error_strings[] = { +#define _(sym, string) string, + foreach_pvti_input_error +#undef _ +}; + +#define _(f, s) s, +static char *pvti_input_trace_type_names[] = { foreach_pvti_input_trace_type }; +#undef _ + +static char * +get_pvti_trace_type_name (u8 ptype) +{ + if (ptype < PVTI_INPUT_TRACE_N_TYPES) + { + return pvti_input_trace_type_names[ptype]; + } + else + { + return "unknown"; + } +} + +/* packet trace format function */ +static u8 * +format_pvti_input_trace (u8 *s, va_list *args) +{ + int i; + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + pvti_input_trace_t *t = va_arg (*args, pvti_input_trace_t *); + + u32 indent = format_get_indent (s); + + s = format (s, + "PVTI-IN: sw_if_index %d, next index %d, trace_type: %s(%d), " + "chunkcnt: %d\n", + t->sw_if_index, t->next_index, + get_pvti_trace_type_name (t->trace_type), t->trace_type, + t->chunk_count); + s = format (s, " src %U sport %d dport %d\n", format_ip_address, + &t->remote_ip, t->remote_port, t->local_port); + s = format (s, " seq: %d, chunk_count: %d\n", t->seq, t->chunk_count); + u16 max = t->chunk_count > MAX_CHUNKS ? MAX_CHUNKS : t->chunk_count; + for (i = 0; i < max; i++) + { + s = format (s, " %02d: sz %d\n", i, t->chunks[i].total_chunk_length); + } + s = format (s, "\n%U%U", format_white_space, indent, + format_ip_adjacency_packet_data, t->packet_data, + sizeof (t->packet_data)); + + return s; +} + +vlib_node_registration_t pvti4_input_node; +vlib_node_registration_t pvti6_input_node; + +VLIB_REGISTER_NODE (pvti4_input_node) = +{ + .name = "pvti4-input", + .vector_size = sizeof (u32), + .format_trace = format_pvti_input_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_errors = ARRAY_LEN(pvti_input_error_strings), + .error_strings = pvti_input_error_strings, + + .n_next_nodes = PVTI_INPUT_N_NEXT, + + .next_nodes = { + [PVTI_INPUT_NEXT_DROP] = "error-drop", + [PVTI_INPUT_NEXT_IP4_INPUT] = "ip4-input-no-checksum", + [PVTI_INPUT_NEXT_IP6_INPUT] = "ip6-input", + [PVTI_INPUT_NEXT_PUNT] = "error-punt", + }, + +}; +VLIB_REGISTER_NODE (pvti6_input_node) = +{ + .name = "pvti6-input", + .vector_size = sizeof (u32), + .format_trace = format_pvti_input_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_errors = ARRAY_LEN(pvti_input_error_strings), + .error_strings = pvti_input_error_strings, + + .n_next_nodes = PVTI_INPUT_N_NEXT, + + .next_nodes = { + [PVTI_INPUT_NEXT_DROP] = "error-drop", + [PVTI_INPUT_NEXT_IP4_INPUT] = "ip4-input-no-checksum", + [PVTI_INPUT_NEXT_IP6_INPUT] = "ip6-input", + [PVTI_INPUT_NEXT_PUNT] = "error-punt", + }, + +}; diff --git a/src/plugins/pvti/input.c b/src/plugins/pvti/input.c new file mode 100644 index 00000000000..6a8806e2795 --- /dev/null +++ b/src/plugins/pvti/input.c @@ -0,0 +1,496 @@ +/* + * Copyright (c) 2024 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <vlib/vlib.h> +#include <vnet/vnet.h> +#include <vnet/pg/pg.h> +#include <vppinfra/error.h> +#include <pvti/pvti.h> +#include <pvti/pvti_if.h> +#include <pvti/input.h> + +always_inline void +pvti_enqueue_rx_bi_to_next_and_trace (vlib_main_t *vm, + vlib_node_runtime_t *node, + pvti_per_thread_data_t *ptd, u32 bi0, + u16 next0) +{ + vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0); + + if (PREDICT_TRUE (vlib_trace_buffer (vm, node, next0, b0, + /* follow_chain */ 0))) + { + pvti_input_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t)); + t->next_index = next0; + t->trace_type = PVTI_INPUT_TRACE_decap; + clib_memcpy (t->packet_data, vlib_buffer_get_current (b0), + sizeof (t->packet_data)); + } + vec_add1 (ptd->pending_rx_buffers, bi0); + vec_add1 (ptd->pending_rx_nexts, next0); +} + +always_inline pvti_rx_peer_t * +pvti_try_find_or_create_rx_peer (pvti_per_thread_data_t *ptd, + vlib_buffer_t *b0, bool is_ip6) +{ + pvti_rx_peer_t *peer; + + ip_address_t remote_ip = { 0 }; + u16 remote_port; + if (is_ip6) + { + pvti_ip6_encap_header_t *h0 = + ((pvti_ip6_encap_header_t *) vlib_buffer_get_current (b0)) - 1; + ip_address_set (&remote_ip, &h0->ip6.src_address, AF_IP6); + remote_port = clib_net_to_host_u16 (h0->udp.src_port); + } + else + { + pvti_ip4_encap_header_t *h0 = + ((pvti_ip4_encap_header_t *) vlib_buffer_get_current (b0)) - 1; + ip_address_set (&remote_ip, &h0->ip4.src_address, AF_IP4); + remote_port = clib_net_to_host_u16 (h0->udp.src_port); + } + + pool_foreach (peer, ptd->rx_peers) + { + if (peer->remote_port == remote_port && + 0 == ip_address_cmp (&remote_ip, &peer->remote_ip)) + { + if (peer->deleted) + { + // The peer has been marked as deleted - wipe it. + clib_memset (peer, 0xca, sizeof (*peer)); + pool_put (ptd->rx_peers, peer); + continue; + } + return peer; + } + } + + index_t pvti_if_index0 = + pvti_if_find_by_remote_ip_and_port (&remote_ip, remote_port); + if (INDEX_INVALID == pvti_if_index0) + { + // no suitable interface found, bail + return 0; + } + pvti_if_t *pvti_if0 = pvti_if_get (pvti_if_index0); + + pvti_rx_peer_t new_peer = { + .local_ip = pvti_if0->local_ip, + .local_port = pvti_if0->local_port, + .remote_ip = remote_ip, + .remote_port = remote_port, + .pvti_if_index = pvti_if_index0, + .rx_streams = { { 0 } }, + }; + pvti_rx_peer_t *rx_new_peer; + pool_get (ptd->rx_peers, rx_new_peer); + *rx_new_peer = new_peer; + + int i; + for (i = 0; i < MAX_RX_STREAMS; i++) + { + rx_new_peer->rx_streams[i].rx_bi0 = INDEX_INVALID; + rx_new_peer->rx_streams[i].rx_bi0_first = INDEX_INVALID; + rx_new_peer->rx_streams[i].rx_next0 = 0; + } + + return rx_new_peer; +} + +always_inline u16 +pvti_input_node_common (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_frame_t *frame, bool is_ip6) +{ + u32 n_left_from, *from; + pvti_chunk_header_t *chunks[MAX_CHUNKS]; + u32 pkts_processed = 0; + u32 pkts_decapsulated = 0; + u32 decap_failed_no_buffers = 0; + + pvti_main_t *pvm = &pvti_main; + + u32 thread_index = vlib_get_thread_index (); + pvti_per_thread_data_t *ptd = + vec_elt_at_index (pvm->per_thread_data[is_ip6], thread_index); + + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + + while (n_left_from > 0) + { + u32 bi0; + vlib_buffer_t *b0; + u32 next0 = PVTI_INPUT_NEXT_DROP; + u32 sw_if_index0; + u8 true_chunk_count = 0; + u8 max_chunk_count; + + bi0 = from[0]; + from += 1; + n_left_from -= 1; + + b0 = vlib_get_buffer (vm, bi0); + sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; + pvti_ip4_encap_header_t *h0 = + ((pvti_ip4_encap_header_t *) vlib_buffer_get_current (b0)) - 1; + pvti_rx_peer_t *pvti_rx_peer0 = + pvti_try_find_or_create_rx_peer (ptd, b0, is_ip6); + if (!pvti_rx_peer0) + { + b0->error = node->errors[PVTI_INPUT_ERROR_PEER]; + goto drop_and_maybe_trace; + } + + b0 = vlib_get_buffer (vm, bi0); + pvti_packet_header_t *pvti0 = vlib_buffer_get_current (b0); + u8 stream_index = pvti0->stream_index; + max_chunk_count = + pvti0->chunk_count < MAX_CHUNKS ? pvti0->chunk_count : MAX_CHUNKS; + u16 pvti_packet_header_sz0 = + pvti0->pad_bytes + offsetof (pvti_packet_header_t, pad); + if (b0->current_length < pvti_packet_header_sz0) + { + b0->error = node->errors[PVTI_INPUT_ERROR_PACKET_TOO_SHORT]; + goto drop_and_maybe_trace; + } + vlib_buffer_advance (b0, pvti_packet_header_sz0); + + if (max_chunk_count == 0) + { + b0->error = node->errors[PVTI_INPUT_ERROR_NOCHUNKS]; + goto drop_and_maybe_trace; + } + if (pvti0->reass_chunk_count > max_chunk_count) + { + b0->error = node->errors[PVTI_INPUT_ERROR_TOOMANYREASS]; + goto drop_and_maybe_trace; + } + pvti_per_rx_stream_data_t *rx_stream0 = + &pvti_rx_peer0->rx_streams[stream_index]; + + u32 new_seq0 = clib_net_to_host_u32 (pvti0->seq); + if (new_seq0 == rx_stream0->last_rx_seq + 1) + { + /* Sequence# matches, we can attempt adding the leading chunks to + * reassembly */ + rx_stream0->last_rx_seq = new_seq0; + + while ((b0->current_length > 0) && + true_chunk_count < pvti0->reass_chunk_count) + { + /* attempt to either incorporate the first chunk into + * reassembly or skip it. */ + pvti_chunk_header_t *pvc0 = vlib_buffer_get_current (b0); + const u16 chunk_payload_length = + clib_net_to_host_u16 (pvc0->total_chunk_length) - + sizeof (*pvc0); + vlib_buffer_advance (b0, sizeof (*pvc0)); + + if (rx_stream0->rx_bi0 == INDEX_INVALID) + { + clib_warning ( + "RX internal error: not-first chunk but no wip block"); + } + else + { + + vlib_buffer_t *rb0 = + vlib_get_buffer (vm, rx_stream0->rx_bi0); + u16 allowed_length = + PVTI_RX_MAX_LENGTH - rb0->current_length; + if (allowed_length > chunk_payload_length) + { + // simple case - there is space in the buffer to fit + // the whole chunk + void *tail = + vlib_buffer_put_uninit (rb0, chunk_payload_length); + clib_memcpy (tail, vlib_buffer_get_current (b0), + chunk_payload_length); + } + else + { + // The current chunk can not fit - need to make two + // copies, one into the current buffer, and one into + // a newly allocated chained buffer. + void *tail = + vlib_buffer_put_uninit (rb0, allowed_length); + clib_memcpy (tail, vlib_buffer_get_current (b0), + allowed_length); + u16 remaining_payload_length = + chunk_payload_length - allowed_length; + u32 nrbi0 = pvti_get_new_buffer (vm); + if (INDEX_INVALID == nrbi0) + { + ASSERT (0); // FIXME what the recovery is + // supposed to look like ? + } + else + { + // link up the new buffer and copy the remainder + // there + vlib_buffer_t *nrb0 = vlib_get_buffer (vm, nrbi0); + rb0->flags |= VLIB_BUFFER_NEXT_PRESENT; + rb0->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID; + rb0->next_buffer = nrbi0; + rx_stream0->rx_bi0 = nrbi0; + void *tail = vlib_buffer_put_uninit ( + nrb0, remaining_payload_length); + clib_memcpy (tail, + vlib_buffer_get_current (b0) + + allowed_length, + remaining_payload_length); + } + } + pvti_rx_peer0->rx_streams[stream_index] + .rx_received_inner_length += chunk_payload_length; + if (pvti_rx_peer0->rx_streams[stream_index] + .rx_received_inner_length == + pvti_rx_peer0->rx_streams[stream_index] + .rx_expected_inner_length) + { + next0 = rx_stream0->rx_next0; + pvti_enqueue_rx_bi_to_next_and_trace ( + vm, node, ptd, rx_stream0->rx_bi0_first, next0); + pkts_decapsulated += 1; + + // clean out the current reassemly state + rx_stream0->rx_bi0 = INDEX_INVALID; + rx_stream0->rx_bi0_first = INDEX_INVALID; + pvti_rx_peer0->rx_streams[stream_index] + .rx_received_inner_length = 0; + pvti_rx_peer0->rx_streams[stream_index] + .rx_expected_inner_length = 0; + rx_stream0->rx_next0 = 0; + } + } + chunks[true_chunk_count] = pvc0; + true_chunk_count += 1; + vlib_buffer_advance (b0, chunk_payload_length); + } + } + else + { + /* Sequence does not match, skip the reassembly chunks and reset + * the reassembly state */ + + while ((b0->current_length > 0) && + true_chunk_count < pvti0->reass_chunk_count) + { + /* skip the reassembly chunks */ + pvti_chunk_header_t *pvc0 = vlib_buffer_get_current (b0); + chunks[true_chunk_count] = pvc0; + true_chunk_count += 1; + vlib_buffer_advance ( + b0, clib_net_to_host_u16 (pvc0->total_chunk_length)); + } + // FIXME: discard the current reassembly state, reset the seq# + if (rx_stream0->rx_bi0_first != INDEX_INVALID) + { + clib_warning ("RX PVTI: discard chunk being reassembled"); + vlib_buffer_free_one (vm, rx_stream0->rx_bi0_first); + rx_stream0->rx_bi0 = INDEX_INVALID; + rx_stream0->rx_bi0_first = INDEX_INVALID; + rx_stream0->rx_received_inner_length = 0; + rx_stream0->rx_expected_inner_length = 0; + rx_stream0->rx_next0 = 0; + } + } + + while ((b0->current_length > 0) && true_chunk_count < max_chunk_count) + { + if (b0->current_length < sizeof (pvti_chunk_header_t)) + { + clib_warning ("RX ERR: length too short for a chunk"); + break; + } + pvti_chunk_header_t *pvc0 = vlib_buffer_get_current (b0); + chunks[true_chunk_count] = pvc0; + true_chunk_count += 1; + u16 total_chunk_length = + clib_net_to_host_u16 (pvc0->total_chunk_length); + if (b0->current_length < total_chunk_length) + { + clib_warning ("RX ERR: length 0x%x too big for a chunk", + true_chunk_count); + break; + } + u8 *pkt = (u8 *) (pvc0 + 1); + u16 inner_length; + if (rx_stream0->rx_bi0_first != INDEX_INVALID) + { + vlib_buffer_free_one (vm, rx_stream0->rx_bi0_first); + rx_stream0->rx_bi0 = INDEX_INVALID; + rx_stream0->rx_bi0_first = INDEX_INVALID; + rx_stream0->rx_received_inner_length = 0; + rx_stream0->rx_expected_inner_length = 0; + rx_stream0->rx_next0 = 0; + } + + switch (*pkt & 0xf0) + { + case 0x40: + next0 = PVTI_INPUT_NEXT_IP4_INPUT; + inner_length = clib_net_to_host_u16 (*((u16 *) (pkt + 2))); + break; + case 0x60: + next0 = PVTI_INPUT_NEXT_IP6_INPUT; + inner_length = clib_net_to_host_u16 (*((u16 *) (pkt + 4))) + + sizeof (ip6_header_t); + break; + default: + next0 = PVTI_INPUT_NEXT_DROP; + vlib_buffer_advance (b0, total_chunk_length); + continue; + } + vlib_buffer_advance (b0, sizeof (pvti_chunk_header_t)); + + if (inner_length + sizeof (pvti_chunk_header_t) > total_chunk_length) + { + /* FIXME: the packet size is larger than the chunk -> it's a + * first fragment */ + // enqueue the chunk and finish packet processing. + // There must be no active reassembly. + ASSERT (rx_stream0->rx_bi0_first == INDEX_INVALID); + rx_stream0->rx_next0 = next0; + rx_stream0->rx_bi0 = bi0; + rx_stream0->rx_bi0_first = bi0; + rx_stream0->rx_expected_inner_length = inner_length; + rx_stream0->rx_received_inner_length = + total_chunk_length - sizeof (pvti_chunk_header_t); + rx_stream0->last_rx_seq = new_seq0; + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) && + (b0->flags & VLIB_BUFFER_IS_TRACED))) + { + pvti_input_trace_t *t = + vlib_add_trace (vm, node, b0, sizeof (*t)); + t->next_index = ~0; + t->trace_type = PVTI_INPUT_TRACE_enqueue; + clib_memcpy (t->packet_data, vlib_buffer_get_current (b0), + sizeof (t->packet_data)); + } + goto continue_outer; + } + + u32 nbi0 = pvti_get_new_buffer (vm); + if (INDEX_INVALID == nbi0) + { + decap_failed_no_buffers += 1; + continue; + }; + vlib_buffer_t *nb0 = vlib_get_buffer (vm, nbi0); + pvti_if_t *pvti_if0 = pvti_if_get (pvti_rx_peer0->pvti_if_index); + vnet_buffer (nb0)->sw_if_index[VLIB_RX] = pvti_if0->sw_if_index; + void *new_packet = vlib_buffer_put_uninit (nb0, inner_length); + clib_memcpy (new_packet, pvc0 + 1, inner_length); + vlib_buffer_advance (b0, inner_length); + + pvti_enqueue_rx_bi_to_next_and_trace (vm, node, ptd, nbi0, next0); + pkts_decapsulated += 1; + } + /* we have processed all the chunks from the buffer, but the buffer + * remains. Free it. */ + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) && + (b0->flags & VLIB_BUFFER_IS_TRACED))) + { + pvti_input_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t)); + t->next_index = ~0; + t->trace_type = PVTI_INPUT_TRACE_free; + t->seq = clib_net_to_host_u32 (pvti0->seq); + t->chunk_count = pvti0->chunk_count; + u8 chunk_count = + pvti0->chunk_count < MAX_CHUNKS ? pvti0->chunk_count : MAX_CHUNKS; + for (int i = 0; i < chunk_count; i++) + { + t->chunks[i].total_chunk_length = + clib_net_to_host_u16 (chunks[i]->total_chunk_length); + } + clib_memcpy (t->packet_data, vlib_buffer_get_current (b0), + sizeof (t->packet_data)); + } + vlib_buffer_free_one (vm, bi0); + + continue_outer: + pkts_processed += 1; + continue; + + drop_and_maybe_trace: + next0 = PVTI_INPUT_NEXT_DROP; + + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) && + (b0->flags & VLIB_BUFFER_IS_TRACED))) + { + int i; + pvti_input_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t)); + t->sw_if_index = sw_if_index0; + t->trace_type = PVTI_INPUT_TRACE_drop; + t->next_index = next0; + t->remote_ip.ip.ip4 = h0->ip4.src_address; + t->remote_ip.version = AF_IP4; + t->local_port = h0->udp.dst_port; + t->remote_port = h0->udp.src_port; + if (!pvti_rx_peer0) + { + t->seq = 0xdeaddead; + } + else + { + t->seq = clib_net_to_host_u32 (pvti0->seq); + t->chunk_count = pvti0->chunk_count; + u8 chunk_count = pvti0->chunk_count < MAX_CHUNKS ? + pvti0->chunk_count : + MAX_CHUNKS; + for (i = 0; i < chunk_count; i++) + { + t->chunks[i].total_chunk_length = + clib_net_to_host_u16 (chunks[i]->total_chunk_length); + } + } + } + + pkts_processed += 1; + vec_add1 (ptd->pending_rx_buffers, bi0); + vec_add1 (ptd->pending_rx_nexts, next0); + } + + vlib_buffer_enqueue_to_next_vec (vm, node, &ptd->pending_rx_buffers, + &ptd->pending_rx_nexts, + vec_len (ptd->pending_rx_nexts)); + vec_reset_length (ptd->pending_rx_buffers); + vec_reset_length (ptd->pending_rx_nexts); + + vlib_node_increment_counter (vm, node->node_index, + PVTI_INPUT_ERROR_PROCESSED, pkts_processed); + vlib_node_increment_counter ( + vm, node->node_index, PVTI_INPUT_ERROR_DECAPSULATED, pkts_decapsulated); + vlib_node_increment_counter (vm, node->node_index, + PVTI_INPUT_ERROR_NO_BUFFERS, + decap_failed_no_buffers); + return frame->n_vectors; +} + +VLIB_NODE_FN (pvti4_input_node) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) +{ + return pvti_input_node_common (vm, node, frame, 0); +} + +VLIB_NODE_FN (pvti6_input_node) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) +{ + return pvti_input_node_common (vm, node, frame, 1); +} diff --git a/src/plugins/pvti/input.h b/src/plugins/pvti/input.h new file mode 100644 index 00000000000..02a186cde05 --- /dev/null +++ b/src/plugins/pvti/input.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2024 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __included_pvti_input_h__ +#define __included_pvti_input_h__ + +#include <vlib/vlib.h> +#include <vnet/vnet.h> +#include <vnet/pg/pg.h> +#include <vppinfra/error.h> +#include <pvti/pvti.h> +#include <pvti/pvti_if.h> + +typedef struct +{ + u16 total_chunk_length; +} pvti_input_chunk_t; + +#define MAX_CHUNKS 32 +#define PVTI_RX_MAX_LENGTH 2048 + +typedef struct +{ + u32 next_index; + u32 sw_if_index; + ip_address_t remote_ip; + u16 remote_port; + u16 local_port; + u32 seq; + pvti_input_chunk_t chunks[MAX_CHUNKS]; + u8 chunk_count; + u8 trace_type; + u8 packet_data[64]; +} pvti_input_trace_t; + +#define foreach_pvti_input_trace_type \ + _ (drop, "drop") \ + _ (decap, "decapsulate") \ + _ (free, "free") \ + _ (enqueue, "enqueue") + +typedef enum +{ +#define _(f, s) PVTI_INPUT_TRACE_##f, + foreach_pvti_input_trace_type +#undef _ + PVTI_INPUT_TRACE_N_TYPES, +} pvti_input_trace_type_t; + +#define foreach_pvti_input_error \ + _ (PROCESSED, "PVTI tunneled packets processed") \ + _ (DECAPSULATED, "PVTI inner packets decapsulated") \ + _ (PEER, "Could not find a peer") \ + _ (NOCHUNKS, "Packet has no chunks") \ + _ (NO_BUFFERS, "No buffers available to decapsulate") \ + _ (TOOMANYREASS, "Packet has more reassembly chunks than total") \ + _ (PACKET_TOO_SHORT, "Packet too short") + +typedef enum +{ +#define _(sym, str) PVTI_INPUT_ERROR_##sym, + foreach_pvti_input_error +#undef _ + PVTI_INPUT_N_ERROR, +} pvti_input_error_t; + +typedef enum +{ + PVTI_INPUT_NEXT_DROP, + PVTI_INPUT_NEXT_IP4_INPUT, + PVTI_INPUT_NEXT_IP6_INPUT, + PVTI_INPUT_NEXT_PUNT, + PVTI_INPUT_N_NEXT, +} pvti_input_next_t; + +#endif // pvti_input_h diff --git a/src/plugins/pvti/output-main.c b/src/plugins/pvti/output-main.c new file mode 100644 index 00000000000..ae4ae5f8e98 --- /dev/null +++ b/src/plugins/pvti/output-main.c @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2024 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <pvti/output.h> + +/* packet trace format function */ +static u8 * +format_pvti_output_trace (u8 *s, va_list *args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + pvti_output_trace_t *t = va_arg (*args, pvti_output_trace_t *); + + u32 indent = format_get_indent (s); + s = + format (s, "PVTI-OUT(%d): sw_if_index %d, next index %d, underlay_mtu %d,", + t->trace_type, t->sw_if_index, t->next_index, t->underlay_mtu); + s = format (s, "\n%U stream_index %d, bi0_max_current_length %d, tx_seq %d", + format_white_space, indent, t->stream_index, + t->bi0_max_current_length, t->tx_seq); + s = format (s, "\n%U%U", format_white_space, indent, + format_ip_adjacency_packet_data, t->packet_data, + sizeof (t->packet_data)); + + return s; +} + +vlib_node_registration_t pvti_output_node; + +static char *pvti_output_error_strings[] = { +#define _(sym, string) string, + foreach_pvti_output_error +#undef _ +}; + +VLIB_REGISTER_NODE (pvti4_output_node) = +{ + .name = "pvti4-output", + .vector_size = sizeof (u32), + .format_trace = format_pvti_output_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = ARRAY_LEN(pvti_output_error_strings), + .error_strings = pvti_output_error_strings, + + .n_next_nodes = PVTI_OUTPUT_N_NEXT, + + .next_nodes = { + [PVTI_OUTPUT_NEXT_DROP] = "error-drop", + [PVTI_OUTPUT_NEXT_INTERFACE_OUTPUT] = "adj-midchain-tx", + [PVTI_OUTPUT_NEXT_IP4_LOOKUP] = "ip4-lookup", + [PVTI_OUTPUT_NEXT_IP6_LOOKUP] = "ip6-lookup", + }, + +}; +VLIB_REGISTER_NODE (pvti6_output_node) = +{ + .name = "pvti6-output", + .vector_size = sizeof (u32), + .format_trace = format_pvti_output_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = ARRAY_LEN(pvti_output_error_strings), + .error_strings = pvti_output_error_strings, + + .n_next_nodes = PVTI_OUTPUT_N_NEXT, + + .next_nodes = { + [PVTI_OUTPUT_NEXT_DROP] = "error-drop", + [PVTI_OUTPUT_NEXT_INTERFACE_OUTPUT] = "adj-midchain-tx", + [PVTI_OUTPUT_NEXT_IP4_LOOKUP] = "ip4-lookup", + [PVTI_OUTPUT_NEXT_IP6_LOOKUP] = "ip6-lookup", + }, + +}; diff --git a/src/plugins/pvti/output.c b/src/plugins/pvti/output.c new file mode 100644 index 00000000000..1939c6f585a --- /dev/null +++ b/src/plugins/pvti/output.c @@ -0,0 +1,543 @@ +/* + * Copyright (c) 2024 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <vlib/vlib.h> +#include <vnet/vnet.h> +#include <vnet/pg/pg.h> +#include <vppinfra/error.h> +#include <pvti/pvti.h> +#include <pvti/pvti_if.h> +#include <pvti/output.h> + +static_always_inline u32 +ip6_vtcfl (u8 stream_index) +{ + u32 vtcfl = 0x6 << 28; + vtcfl |= stream_index; + + return (clib_host_to_net_u32 (vtcfl)); +} + +always_inline vlib_buffer_t * +pvti_alloc_new_tx_buffer (vlib_main_t *vm) +{ + u32 bi0 = INDEX_INVALID; + if (vlib_buffer_alloc (vm, &bi0, 1) != 1) + { + return 0; + } + vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0); + b0->current_data = 0; + b0->current_length = 0; + return b0; +} + +always_inline bool +pvti_find_or_try_create_tx_peer (vlib_main_t *vm, pvti_per_thread_data_t *ptd, + pvti_if_t *pvti_if0, ip_address_t *remote_ip, + u16 remote_port, u32 *out_index) +{ + + pvti_tx_peer_t *peer; + pool_foreach (peer, ptd->tx_peers) + { + if (peer->remote_port == remote_port && + 0 == ip_address_cmp (remote_ip, &peer->remote_ip)) + { + if (peer->deleted) + { + // Bad luck, the peer has been deleted. + u32 boi0 = vlib_get_buffer_index (vm, peer->bo0); + if (peer->bo0) + { + vlib_buffer_free (vm, &boi0, 1); + } + clib_memset (peer, 0xca, sizeof (*peer)); + pool_put (ptd->tx_peers, peer); + continue; + } + *out_index = peer - ptd->tx_peers; + return 1; + } + } + + ip_address_family_t dst_ver = ip_addr_version (&pvti_if0->remote_ip); + + u16 pvti_encap_overhead = (dst_ver == AF_IP6) ? + sizeof (pvti_ip6_encap_header_t) : + sizeof (pvti_ip4_encap_header_t); + + u16 pvti_packet_overhead = + pvti_encap_overhead + sizeof (pvti_packet_header_t) + PVTI_ALIGN_BYTES; + + ASSERT (pvti_if0->underlay_mtu > pvti_packet_overhead); + + u32 bo0_max_current_length = pvti_if0->underlay_mtu - pvti_packet_overhead; + + vlib_buffer_t *bo0 = pvti_alloc_new_tx_buffer (vm); + + if (!bo0) + { + return 0; + } + + pvti_tx_peer_t new_peer = { + .local_ip = pvti_if0->local_ip, + .remote_ip = *remote_ip, + .local_port = pvti_if0->local_port, + .remote_port = remote_port, + .underlay_mtu = pvti_if0->underlay_mtu, + .underlay_fib_index = pvti_if0->underlay_fib_index, + .bo0_max_current_length = bo0_max_current_length, + .pvti_if_index = pvti_if_get_index (pvti_if0), + .deleted = 0, + .bo0 = bo0, + .chunk_count = 0, + .reass_chunk_count = 0, + .current_tx_seq = 42, + }; + + pvti_tx_peer_t *tx_new_peer; + pool_get (ptd->tx_peers, tx_new_peer); + + *tx_new_peer = new_peer; + *out_index = tx_new_peer - ptd->tx_peers; + return 1; +} + +always_inline bool +pvti_try_get_tx_peer_index (vlib_main_t *vm, pvti_per_thread_data_t *ptd, + pvti_if_t *pvti_if0, vlib_buffer_t *b0, + bool is_ip6, u32 *out_index) +{ + if (pvti_if0->peer_address_from_payload) + { + ip_address_t remote_ip = { 0 }; + if (is_ip6) + { + ip6_header_t *ip6 = vlib_buffer_get_current (b0); + ip_address_set (&remote_ip, &ip6->dst_address, AF_IP6); + } + else + { + ip4_header_t *ip4 = vlib_buffer_get_current (b0); + ip_address_set (&remote_ip, &ip4->dst_address, AF_IP4); + } + return pvti_find_or_try_create_tx_peer ( + vm, ptd, pvti_if0, &remote_ip, pvti_if0->remote_port, out_index); + } + else + { + return pvti_find_or_try_create_tx_peer ( + vm, ptd, pvti_if0, &pvti_if0->remote_ip, pvti_if0->remote_port, + out_index); + } + /* not reached */ +} + +always_inline void +pvti_finalize_chunk (pvti_tx_peer_t *tx_peer, + pvti_chunk_header_t *chunk_header, u8 *tail, + bool is_reassembly_chunk) +{ + clib_memset (chunk_header, 0xab, sizeof (pvti_chunk_header_t)); + chunk_header->total_chunk_length = + clib_host_to_net_u16 (tail - (u8 *) chunk_header); + tx_peer->chunk_count++; + if (is_reassembly_chunk) + { + tx_peer->reass_chunk_count++; + } +} + +always_inline pvti_output_next_t +encap_pvti_buffer_ip46 (vlib_main_t *vm, vlib_node_runtime_t *node, + pvti_tx_peer_t *tx_peer, int is_ip6) +{ + ip_address_family_t src_ver = ip_addr_version (&tx_peer->local_ip); + ip_address_family_t dst_ver = ip_addr_version (&tx_peer->remote_ip); + u8 stream_index = 0; + + ASSERT (src_ver == dst_ver); + bool is_ip6_encap = (AF_IP6 == src_ver); + + vlib_buffer_t *b0 = tx_peer->bo0; + vlib_buffer_advance (b0, + -(sizeof (pvti_packet_header_t) + PVTI_ALIGN_BYTES)); + + pvti_packet_header_t *pvti0 = vlib_buffer_get_current (b0); + clib_memset (pvti0, 0xca, sizeof (*pvti0) + PVTI_ALIGN_BYTES); + pvti0->pad_bytes = PVTI_ALIGN_BYTES; + + pvti0->seq = clib_host_to_net_u32 (tx_peer->current_tx_seq); + pvti0->stream_index = stream_index; + pvti0->reass_chunk_count = tx_peer->reass_chunk_count; + pvti0->chunk_count = tx_peer->chunk_count; + pvti0->mandatory_flags_mask = 0; + pvti0->flags_value = 0; + + if (is_ip6_encap) + { + vlib_buffer_advance (b0, -(sizeof (pvti_ip6_encap_header_t))); + if (b0->current_data < -VLIB_BUFFER_PRE_DATA_SIZE) + { + // undo the change + vlib_buffer_advance (b0, (sizeof (pvti_ip6_encap_header_t))); + b0->error = node->errors[PVTI_OUTPUT_ERROR_NO_PRE_SPACE]; + return PVTI_OUTPUT_NEXT_DROP; + } + pvti_ip6_encap_header_t *ve = vlib_buffer_get_current (b0); + + ve->udp.src_port = clib_host_to_net_u16 (tx_peer->local_port); + ve->udp.dst_port = clib_host_to_net_u16 (tx_peer->remote_port); + ve->udp.length = clib_host_to_net_u16 ( + b0->current_length - offsetof (pvti_ip6_encap_header_t, udp)); + ve->udp.checksum = 0; + + ve->ip6.ip_version_traffic_class_and_flow_label = + ip6_vtcfl (stream_index); + ve->ip6.payload_length = ve->udp.length; + ve->ip6.protocol = 17; + ve->ip6.hop_limit = 128; + ip_address_copy_addr (&ve->ip6.src_address, &tx_peer->local_ip); + ip_address_copy_addr (&ve->ip6.dst_address, &tx_peer->remote_ip); + } + else + { + vlib_buffer_advance (b0, -(sizeof (pvti_ip4_encap_header_t))); + if (b0->current_data < -VLIB_BUFFER_PRE_DATA_SIZE) + { + // undo the change + vlib_buffer_advance (b0, (sizeof (pvti_ip4_encap_header_t))); + b0->error = node->errors[PVTI_OUTPUT_ERROR_NO_PRE_SPACE]; + return PVTI_OUTPUT_NEXT_DROP; + } + pvti_ip4_encap_header_t *ve = vlib_buffer_get_current (b0); + + ve->udp.src_port = clib_host_to_net_u16 (tx_peer->local_port); + ve->udp.dst_port = clib_host_to_net_u16 (tx_peer->remote_port); + ve->udp.length = clib_host_to_net_u16 ( + b0->current_length - offsetof (pvti_ip4_encap_header_t, udp)); + ve->udp.checksum = 0; + + ve->ip4.ip_version_and_header_length = 0x45; + ve->ip4.tos = 0; + ve->ip4.length = clib_host_to_net_u16 (b0->current_length); + ve->ip4.fragment_id = + clib_host_to_net_u16 (tx_peer->current_tx_seq & 0xffff); + ve->ip4.flags_and_fragment_offset = 0; + ve->ip4.ttl = 128; + ve->ip4.protocol = 17; + + ve->ip4.dst_address.as_u32 = ip_addr_v4 (&tx_peer->remote_ip).data_u32; + ve->ip4.src_address.as_u32 = ip_addr_v4 (&tx_peer->local_ip).data_u32; + ve->ip4.checksum = ip4_header_checksum (&ve->ip4); + } + + // This is important, if not reset, causes a crash + vnet_buffer (b0)->sw_if_index[VLIB_TX] = tx_peer->underlay_fib_index; + + // vnet_buffer (b0)->oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM; + return is_ip6_encap ? PVTI_OUTPUT_NEXT_IP6_LOOKUP : + PVTI_OUTPUT_NEXT_IP4_LOOKUP; +} + +always_inline void +pvti_enqueue_tx_and_trace (vlib_main_t *vm, vlib_node_runtime_t *node, + pvti_per_thread_data_t *ptd, vlib_buffer_t *b0, + u16 next0, u8 stream_index, pvti_tx_peer_t *tx_peer) +{ + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) && + tx_peer->is_bo0_traced)) + { + if (PREDICT_TRUE ( + vlib_trace_buffer (vm, node, next0, b0, /* follow_chain */ 0))) + { + + pvti_output_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t)); + t->next_index = next0; + t->underlay_mtu = tx_peer->underlay_mtu; + t->stream_index = stream_index; + t->trace_type = 1; + t->bi0_max_current_length = tx_peer->bo0_max_current_length; + t->tx_seq = tx_peer->current_tx_seq; + clib_memcpy (t->packet_data, vlib_buffer_get_current (b0), + sizeof (t->packet_data)); + } + } + u32 bi0 = vlib_get_buffer_index (vm, b0); + vec_add1 (ptd->pending_tx_buffers, bi0); + vec_add1 (ptd->pending_tx_nexts, next0); +} + +always_inline void +pvti_enqueue_tx_drop_and_trace (vlib_main_t *vm, vlib_node_runtime_t *node, + pvti_per_thread_data_t *ptd, vlib_buffer_t *b0, + u8 stream_index) +{ + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) && + (b0->flags & VLIB_BUFFER_IS_TRACED))) + { + pvti_output_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t)); + t->next_index = PVTI_OUTPUT_NEXT_DROP; + t->stream_index = stream_index; + t->trace_type = 0; + clib_memcpy (t->packet_data, vlib_buffer_get_current (b0), + sizeof (t->packet_data)); + } + u32 bi0 = vlib_get_buffer_index (vm, b0); + vec_add1 (ptd->pending_tx_buffers, bi0); + vec_add1 (ptd->pending_tx_nexts, PVTI_OUTPUT_NEXT_DROP); +} + +always_inline bool +pvti_flush_peer_and_recharge (vlib_main_t *vm, vlib_node_runtime_t *node, + pvti_per_thread_data_t *ptd, u32 tx_peer_index, + u8 stream_index, const bool is_ip6) +{ + pvti_tx_peer_t *tx_peer = pool_elt_at_index (ptd->tx_peers, tx_peer_index); + u16 next0 = encap_pvti_buffer_ip46 (vm, node, tx_peer, is_ip6); + + pvti_enqueue_tx_and_trace (vm, node, ptd, tx_peer->bo0, next0, stream_index, + tx_peer); + + tx_peer->bo0 = pvti_alloc_new_tx_buffer (vm); + tx_peer->reass_chunk_count = 0; + tx_peer->chunk_count = 0; + tx_peer->current_tx_seq++; + + return 1; +} + +always_inline u16 +pvti_output_node_common (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_frame_t *frame, const bool is_ip6) +{ + pvti_main_t *pvm = &pvti_main; + + u32 n_left_from, *from; + u32 pkts_encapsulated = 0; + u32 pkts_processed = 0; + u32 pkts_chopped = 0; + u32 pkts_overflow = 0; + u32 pkts_overflow_cantfit = 0; + + bool is_node_traced = (node->flags & VLIB_NODE_FLAG_TRACE) ? 1 : 0; + + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + + u8 stream_index = pvti_get_stream_index (is_ip6); + + u32 thread_index = vlib_get_thread_index (); + pvti_per_thread_data_t *ptd = + vec_elt_at_index (pvm->per_thread_data[is_ip6], thread_index); + + vlib_buffer_t *ibufs[VLIB_FRAME_SIZE], **ib = ibufs; + + vlib_get_buffers (vm, from, ibufs, n_left_from); + + n_left_from = frame->n_vectors; + while (1 && n_left_from > 0) + { + n_left_from -= 1; + vlib_buffer_t *b0 = ib[0]; + ib++; + u32 bi0 = vlib_get_buffer_index (vm, b0); + bool is_b0_traced = + is_node_traced && ((b0->flags & VLIB_BUFFER_IS_TRACED) ? 1 : 0); + pkts_processed += 1; + + u32 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX]; + u32 pvti_index0 = pvti_if_find_by_sw_if_index (sw_if_index0); + if (pvti_index0 == INDEX_INVALID) + { + b0->error = node->errors[PVTI_OUTPUT_ERROR_PEER]; + pvti_enqueue_tx_drop_and_trace (vm, node, ptd, b0, stream_index); + continue; + } + pvti_if_t *pvti_if0 = pvti_if_get (pvti_index0); + u32 tx_peer_index; + if (!pvti_try_get_tx_peer_index (vm, ptd, pvti_if0, b0, is_ip6, + &tx_peer_index)) + { + b0->error = node->errors[PVTI_OUTPUT_ERROR_MAKE_PEER]; + pvti_enqueue_tx_drop_and_trace (vm, node, ptd, b0, stream_index); + continue; + } + pvti_tx_peer_t *tx_peer = &ptd->tx_peers[tx_peer_index]; + + u32 b0_len = vlib_buffer_length_in_chain (vm, b0); + u32 total_chunk_len = sizeof (pvti_chunk_header_t) + b0_len; + + if (tx_peer->bo0_max_current_length >= + tx_peer->bo0->current_length + total_chunk_len) + { + /* Happy case, we can fit the entire new chunk */ + pvti_chunk_header_t *chunk_header = vlib_buffer_put_uninit ( + tx_peer->bo0, sizeof (pvti_chunk_header_t)); + u8 *tail = vlib_buffer_put_uninit (tx_peer->bo0, b0_len); + vlib_buffer_t *b0_curr; + b0_curr = b0; + while (b0_len > 0) + { + clib_memcpy (tail, vlib_buffer_get_current (b0_curr), + b0_curr->current_length); + tail += b0_curr->current_length; + b0_len -= b0_curr->current_length; + ASSERT ((b0_len == 0) || + (b0_curr->flags & VLIB_BUFFER_NEXT_PRESENT)); + if (b0_curr->flags & VLIB_BUFFER_NEXT_PRESENT) + { + b0_curr = vlib_get_buffer (vm, b0_curr->next_buffer); + } + } + tx_peer->is_bo0_traced |= is_b0_traced; + pvti_finalize_chunk (tx_peer, chunk_header, tail, false); + } + else + { + bool is_reassembly = false; + /* FIXME: here, flush a packet if we want to avoid fragmenting it */ +#define PVTI_TINY_PACKET_SZ 20 + int threshold_len = + sizeof (pvti_chunk_header_t) + PVTI_TINY_PACKET_SZ; + + /* Can we fit anything meaningful into bo0 ? if not - flush */ + if (tx_peer->bo0_max_current_length <= + tx_peer->bo0->current_length + threshold_len) + { + if (!pvti_flush_peer_and_recharge (vm, node, ptd, tx_peer_index, + stream_index, is_ip6)) + { + b0->error = node->errors[PVTI_OUTPUT_ERROR_RECHARGE0]; + pvti_enqueue_tx_drop_and_trace (vm, node, ptd, b0, + stream_index); + continue; + } + pkts_encapsulated += 1; + } + + pvti_chunk_header_t *chunk_header = vlib_buffer_put_uninit ( + tx_peer->bo0, sizeof (pvti_chunk_header_t)); + + u8 *tail; + vlib_buffer_t *b0_curr; + /* append the chained buffers and flush as necessary */ + b0_curr = b0; + + int curr_b0_start_offset = 0; + + while (b0_len > 0) + { + ASSERT (tx_peer->bo0_max_current_length > + tx_peer->bo0->current_length); + int copy_len = + clib_min (b0_curr->current_length - curr_b0_start_offset, + tx_peer->bo0_max_current_length - + tx_peer->bo0->current_length); + tail = vlib_buffer_put_uninit (tx_peer->bo0, copy_len); + clib_memcpy (tail, + (u8 *) vlib_buffer_get_current (b0_curr) + + curr_b0_start_offset, + copy_len); + tail += copy_len; + b0_len -= copy_len; + // Advance the start offset or reset it if we copied the entire + // block + curr_b0_start_offset = + curr_b0_start_offset + copy_len == b0_curr->current_length ? + 0 : + curr_b0_start_offset + copy_len; + ASSERT ((b0_len == 0) || (curr_b0_start_offset > 0) || + (b0_curr->flags & VLIB_BUFFER_NEXT_PRESENT)); + if (curr_b0_start_offset > 0) + { + pvti_finalize_chunk (tx_peer, chunk_header, tail, + is_reassembly); + tx_peer->is_bo0_traced |= is_b0_traced; + if (!pvti_flush_peer_and_recharge ( + vm, node, ptd, tx_peer_index, stream_index, is_ip6)) + { + b0->error = node->errors[PVTI_OUTPUT_ERROR_RECHARGE1]; + pvti_enqueue_tx_drop_and_trace (vm, node, ptd, b0, + stream_index); + continue; + } + pkts_encapsulated += 1; + /* next chunk(s) will be reassembly until the next block */ + is_reassembly = true; + chunk_header = vlib_buffer_put_uninit ( + tx_peer->bo0, sizeof (pvti_chunk_header_t)); + } + else + { + if ((b0_curr->flags & VLIB_BUFFER_NEXT_PRESENT)) + { + b0_curr = vlib_get_buffer (vm, b0_curr->next_buffer); + } + else + { + pvti_finalize_chunk (tx_peer, chunk_header, tail, + is_reassembly); + tx_peer->is_bo0_traced |= is_b0_traced; + } + } + } + } + vlib_buffer_free_one (vm, bi0); + } + + int i; + for (i = 0; i < vec_len (ptd->tx_peers); i++) + { + if (ptd->tx_peers[i].chunk_count) + { + pvti_flush_peer_and_recharge (vm, node, ptd, i, stream_index, + is_ip6); + pkts_encapsulated += 1; + } + } + + vlib_buffer_enqueue_to_next_vec (vm, node, &ptd->pending_tx_buffers, + &ptd->pending_tx_nexts, + vec_len (ptd->pending_tx_nexts)); + vec_reset_length (ptd->pending_tx_buffers); + vec_reset_length (ptd->pending_tx_nexts); + + vlib_node_increment_counter ( + vm, node->node_index, PVTI_OUTPUT_ERROR_ENCAPSULATED, pkts_encapsulated); + vlib_node_increment_counter (vm, node->node_index, + PVTI_OUTPUT_ERROR_PROCESSED, pkts_processed); + vlib_node_increment_counter (vm, node->node_index, PVTI_OUTPUT_ERROR_CHOPPED, + pkts_chopped); + vlib_node_increment_counter (vm, node->node_index, + PVTI_OUTPUT_ERROR_OVERFLOW, pkts_overflow); + vlib_node_increment_counter (vm, node->node_index, + PVTI_OUTPUT_ERROR_OVERFLOW_CANTFIT, + pkts_overflow_cantfit); + return frame->n_vectors; +} + +VLIB_NODE_FN (pvti4_output_node) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) +{ + return pvti_output_node_common (vm, node, frame, 0); +} + +VLIB_NODE_FN (pvti6_output_node) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) +{ + return pvti_output_node_common (vm, node, frame, 1); +} diff --git a/src/plugins/pvti/output.h b/src/plugins/pvti/output.h new file mode 100644 index 00000000000..95e78ba9720 --- /dev/null +++ b/src/plugins/pvti/output.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2024 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __included_pvti_output_h__ +#define __included_pvti_output_h__ + +#include <vlib/vlib.h> +#include <vnet/vnet.h> +#include <vnet/pg/pg.h> +#include <vppinfra/error.h> +#include <pvti/pvti.h> +#include <pvti/pvti_if.h> + +typedef struct +{ + u32 next_index; + u32 sw_if_index; + u32 tx_seq; + u16 underlay_mtu; + u16 bi0_max_current_length; + u8 stream_index; + u8 trace_type; + u8 packet_data[96]; +} pvti_output_trace_t; + +#define foreach_pvti_output_error \ + _ (NONE, "No error") \ + _ (PROCESSED, "Packets processed") \ + _ (ENCAPSULATED, "Packets encapsulated") \ + _ (PEER, "No peer found") \ + _ (MAKE_PEER, "Could not make peer") \ + _ (RECHARGE0, "Could not recharge 0") \ + _ (RECHARGE1, "Could not recharge 1") \ + _ (NO_PRE_SPACE, "Not enought pre-data space") \ + _ (CHOPPED, "Packets chopped") \ + _ (OVERFLOW, "Packets overflowed") \ + _ (OVERFLOW_CANTFIT, "Packets overflowed and cant fit excess") + +typedef enum +{ +#define _(sym, str) PVTI_OUTPUT_ERROR_##sym, + foreach_pvti_output_error +#undef _ + PVTI_OUTPUT_N_ERROR, +} pvti_output_error_t; + +typedef enum +{ + PVTI_INDEPENDENT_CHUNK = 0, + PVTI_REASS_CHUNK, +} pvti_chunk_type_t; + +#define MAX_CURR_LEN_UNKNOWN 0xffff + +typedef enum +{ + PVTI_OUTPUT_NEXT_DROP, + PVTI_OUTPUT_NEXT_INTERFACE_OUTPUT, + PVTI_OUTPUT_NEXT_IP4_LOOKUP, + PVTI_OUTPUT_NEXT_IP6_LOOKUP, + PVTI_OUTPUT_N_NEXT, +} pvti_output_next_t; + +#endif // pvti_output_h diff --git a/src/plugins/pvti/pvti.api b/src/plugins/pvti/pvti.api new file mode 100644 index 00000000000..859ed1ab6b0 --- /dev/null +++ b/src/plugins/pvti/pvti.api @@ -0,0 +1,111 @@ +/* + * pvti.api - binary API skeleton + * + * Copyright (c) 2024 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @file pvti.api + * @brief VPP control-plane API messages. + * + * This file defines VPP control-plane binary API messages which are generally + * called through a shared memory interface. + */ + +/* Version and type recitations */ + +option version = "0.0.1"; +import "vnet/interface_types.api"; +import "vnet/ip/ip_types.api"; + +/** \brief A composite type uniquely defining a PVTI tunnel. + @param sw_if_index - ignored on create/delete, present in details. + @param src_ip - Source IP address + @param src_port - Source UDP port + @param dst_ip - Destination IP address + @param dst_port - Destination UDP port + @param underlay_mtu - Underlay MTU for packet splitting/coalescing + @param underlay_fib_index - Underlay FIB index to be used after encap +*/ +typedef pvti_tunnel +{ + vl_api_interface_index_t sw_if_index; + vl_api_address_t local_ip; + u16 local_port; + vl_api_address_t remote_ip; + bool peer_address_from_payload; + u16 remote_port; + u16 underlay_mtu; + u32 underlay_fib_index; +}; + + +/** @brief API to enable / disable pvti on an interface + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param enable_disable - 1 to enable, 0 to disable the feature + @param sw_if_index - interface handle +*/ + +define pvti_interface_create +{ + option status="in_progress"; + + /* Client identifier, set from api_main.my_client_index */ + u32 client_index; + + /* Arbitrary context, so client can match reply to request */ + u32 context; + vl_api_pvti_tunnel_t interface; +}; + +define pvti_interface_create_reply +{ + option status="in_progress"; + u32 context; + i32 retval; + + /* Index for the newly created interface */ + vl_api_interface_index_t sw_if_index; +}; + +autoreply define pvti_interface_delete { + option status="in_progress"; + + /* Client identifier, set from api_main.my_client_index */ + u32 client_index; + + /* Arbitrary context, so client can match reply to request */ + u32 context; + + vl_api_interface_index_t sw_if_index; +}; + + +define pvti_interface_dump +{ + option status="in_progress"; + u32 client_index; + u32 context; + vl_api_interface_index_t sw_if_index; +}; + +define pvti_interface_details +{ + option status="in_progress"; + u32 context; + vl_api_pvti_tunnel_t interface; +}; + + diff --git a/src/plugins/pvti/pvti.c b/src/plugins/pvti/pvti.c new file mode 100644 index 00000000000..646276dec09 --- /dev/null +++ b/src/plugins/pvti/pvti.c @@ -0,0 +1,479 @@ +/* + * Copyright (c) 2024 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <vnet/vnet.h> +#include <vnet/plugin/plugin.h> +#include <vnet/fib/fib_table.h> +#include <pvti/pvti.h> + +#include <vlibapi/api.h> +#include <vlibmemory/api.h> +#include <vpp/app/version.h> +#include <stdbool.h> + +#include <pvti/pvti.api_enum.h> +#include <pvti/pvti.api_types.h> + +#include <pvti/pvti_if.h> + +#define REPLY_MSG_ID_BASE pmp->msg_id_base +#include <vlibapi/api_helper_macros.h> +#include <vnet/ip/ip_format_fns.h> + +pvti_main_t pvti_main; + +u8 * +format_pvti_tx_peer_ptr (u8 *s, va_list *args) +{ + pvti_tx_peer_t *peer = va_arg (*args, pvti_tx_peer_t *); + + s = format ( + s, + "[%p]%s local:%U:%d remote:%U:%d underlay_mtu:%d underlay_fib_idx:%d " + "pvti_idx:%d b0_max_clen:%d cseq:%d chunk_count:%d reass_chunk_count:%d", + peer, peer->deleted ? " DELETED" : "", format_ip46_address, + &peer->local_ip, IP46_TYPE_ANY, peer->local_port, format_ip46_address, + &peer->remote_ip, IP46_TYPE_ANY, peer->remote_port, peer->underlay_mtu, + peer->underlay_fib_index, peer->pvti_if_index, + peer->bo0_max_current_length, peer->current_tx_seq, peer->chunk_count, + peer->reass_chunk_count); + + return (s); +} + +u8 * +format_pvti_rx_peer_ptr (u8 *s, va_list *args) +{ + pvti_rx_peer_t *peer = va_arg (*args, pvti_rx_peer_t *); + + s = format (s, "[%p]%s local:%U:%d remote:%U:%d pvti_idx:%d", peer, + peer->deleted ? " DELETED" : "", format_ip46_address, + &peer->local_ip, IP46_TYPE_ANY, peer->local_port, + format_ip46_address, &peer->remote_ip, IP46_TYPE_ANY, + peer->remote_port, peer->pvti_if_index); + + return (s); +} + +void +pvti_verify_initialized (pvti_main_t *pvm) +{ + if (!pvm->is_initialized) + { + const int n_threads = vlib_get_n_threads (); + vec_validate (pvm->per_thread_data[0], n_threads - 1); + vec_validate (pvm->per_thread_data[1], n_threads - 1); + pvm->is_initialized = 1; + } +} + +void +vnet_int_pvti_bypass_mode (u32 sw_if_index, u8 is_ip6, u8 is_enable) +{ + pvti_main_t *pvm = &pvti_main; + + if (pool_is_free_index (pvm->vnet_main->interface_main.sw_interfaces, + sw_if_index)) + return; + + pvti_verify_initialized (pvm); + + is_enable = !!is_enable; + + if (is_ip6) + { + if (clib_bitmap_get (pvm->bm_ip6_bypass_enabled_by_sw_if, sw_if_index) != + is_enable) + { + vnet_feature_enable_disable ("ip6-unicast", "ip6-pvti-bypass", + sw_if_index, is_enable, 0, 0); + pvm->bm_ip6_bypass_enabled_by_sw_if = clib_bitmap_set ( + pvm->bm_ip6_bypass_enabled_by_sw_if, sw_if_index, is_enable); + } + } + else + { + if (clib_bitmap_get (pvm->bm_ip4_bypass_enabled_by_sw_if, sw_if_index) != + is_enable) + { + vnet_feature_enable_disable ("ip4-unicast", "ip4-pvti-bypass", + sw_if_index, is_enable, 0, 0); + pvm->bm_ip4_bypass_enabled_by_sw_if = clib_bitmap_set ( + pvm->bm_ip4_bypass_enabled_by_sw_if, sw_if_index, is_enable); + } + } +} + +static clib_error_t * +set_ip_pvti_bypass (u32 is_ip6, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + unformat_input_t _line_input, *line_input = &_line_input; + vnet_main_t *vnm = vnet_get_main (); + clib_error_t *error = 0; + u32 sw_if_index, is_enable; + + sw_if_index = ~0; + is_enable = 1; + + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat_user (line_input, unformat_vnet_sw_interface, vnm, + &sw_if_index)) + ; + else if (unformat (line_input, "del")) + is_enable = 0; + else + { + error = unformat_parse_error (line_input); + goto done; + } + } + + if (~0 == sw_if_index) + { + error = clib_error_return (0, "unknown interface `%U'", + format_unformat_error, line_input); + goto done; + } + + vnet_int_pvti_bypass_mode (sw_if_index, is_ip6, is_enable); + +done: + unformat_free (line_input); + + return error; +} + +static clib_error_t * +set_ip4_pvti_bypass (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + return set_ip_pvti_bypass (0, input, cmd); +} + +VLIB_CLI_COMMAND (set_interface_ip_pvti_bypass_command, static) = { + .path = "set interface ip pvti-bypass", + .function = set_ip4_pvti_bypass, + .short_help = "set interface ip pvti-bypass <interface> [del]", +}; + +static clib_error_t * +set_ip6_pvti_bypass (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + return set_ip_pvti_bypass (1, input, cmd); +} + +VLIB_CLI_COMMAND (set_interface_ip6_pvti_bypass_command, static) = { + .path = "set interface ip6 pvti-bypass", + .function = set_ip6_pvti_bypass, + .short_help = "set interface ip6 pvti-bypass <interface> [del]", +}; + +static clib_error_t * +pvti_interface_create_command_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + unformat_input_t _line_input, *line_input = &_line_input; + clib_error_t *error = 0; + + // pvti_main_t * pmp = &pvti_main; + u32 sw_if_index = ~0; + int rv = 0; + ip_address_t peer_ip = { 0 }; + ip_address_t local_ip = { 0 }; + u32 peer_port = 0; + u32 local_port = 12345; + u32 underlay_mtu = 1500; + u32 underlay_fib_index = ~0; + u32 underlay_table_id = ~0; + pvti_peer_address_method_t peer_address_method = PVTI_PEER_ADDRESS_FIXED; + bool peer_set = 0; + + /* Get a line of input. */ + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "peer %U %d %d", unformat_ip_address, &peer_ip, + &peer_port, &local_port)) + { + peer_set = 1; + } + else if (unformat (line_input, "underlay-mtu %d", &underlay_mtu)) + { + // MTU set + } + else if (unformat (line_input, "local-ip %U", unformat_ip_address, + &local_ip)) + { + // local IP set + } + else if (unformat (line_input, "underlay-fib %d", &underlay_fib_index)) + { + // underlay fib set + } + else if (unformat (line_input, "peer-address-from-payload")) + { + peer_address_method = PVTI_PEER_ADDRESS_FROM_PAYLOAD; + } + else if (unformat (line_input, "underlay-table %d", &underlay_table_id)) + { + fib_protocol_t fib_proto = FIB_PROTOCOL_IP4; + if (peer_ip.version == AF_IP6) + { + fib_proto = FIB_PROTOCOL_IP6; + } + u32 fib_index = fib_table_find (fib_proto, underlay_table_id); + + if (~0 == fib_index) + { + error = clib_error_return (0, "Nonexistent table id %d", + underlay_table_id); + goto done; + } + underlay_fib_index = fib_index; + } + else + break; + } + if (!peer_set) + { + error = clib_error_return (0, "Please specify a peer..."); + goto done; + } + + rv = pvti_if_create (&local_ip, local_port, &peer_ip, peer_port, + peer_address_method, underlay_mtu, underlay_fib_index, + &sw_if_index); + + switch (rv) + { + case 0: + break; + + case VNET_API_ERROR_INVALID_SW_IF_INDEX: + error = clib_error_return (0, "Invalid interface"); + break; + + default: + error = clib_error_return (0, "pvti_if_create returned %d", rv); + } +done: + unformat_free (line_input); + return error; +} + +static clib_error_t * +pvti_interface_delete_command_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + // pvti_main_t * pmp = &pvti_main; + u32 sw_if_index = ~0; + int rv = 0; + bool if_index_set = 0; + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "if-index %d", &sw_if_index)) + { + if_index_set = 1; + } + else + break; + } + if (!if_index_set) + return clib_error_return (0, "Please specify a sw_if_index..."); + + rv = pvti_if_delete (sw_if_index); + + switch (rv) + { + case 0: + break; + + case VNET_API_ERROR_INVALID_SW_IF_INDEX: + return clib_error_return (0, "Invalid interface"); + break; + + default: + return clib_error_return (0, "pvti_if_delete returned %d", rv); + } + return 0; +} + +VLIB_CLI_COMMAND (pvti_interface_create_command, static) = { + .path = "pvti interface create", + .short_help = + "pvti interface create peer <remote-ip> <remote-port> <local-port> [ " + "local-ip <ip-addr> ][ underlay-mtu <MTU>][underlay-table " + "<table-index>][inderlay-fib <fib-index>]", + .function = pvti_interface_create_command_fn, +}; + +VLIB_CLI_COMMAND (pvti_interface_delete_command, static) = { + .path = "pvti interface delete", + .short_help = "pvti interface delete if-index <sw-ifindex>", + .function = pvti_interface_delete_command_fn, +}; + +static clib_error_t * +pvti_show_interface_command_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + pvti_if_t *pvti_if; + vec_foreach (pvti_if, pvti_main.if_pool) + { + int index = pvti_if - pvti_main.if_pool; + vlib_cli_output (vm, "%U", format_pvti_if, index); + }; + return 0; +} + +static clib_error_t * +pvti_show_tx_peers_command_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + pvti_per_thread_data_t *ptd; + int is_ip6; + for (is_ip6 = 0; is_ip6 <= 1; is_ip6++) + { + vec_foreach (ptd, pvti_main.per_thread_data[is_ip6]) + { + vlib_cli_output (vm, "thread %d (%s)", + ptd - pvti_main.per_thread_data[is_ip6], + is_ip6 ? "IPv6" : "IPv4"); + pvti_tx_peer_t *peer; + vec_foreach (peer, ptd->tx_peers) + { + vlib_cli_output (vm, " %U", format_pvti_tx_peer_ptr, peer); + } + } + } + return 0; +} + +static clib_error_t * +pvti_show_rx_peers_command_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + pvti_per_thread_data_t *ptd; + int is_ip6; + for (is_ip6 = 0; is_ip6 <= 1; is_ip6++) + { + vec_foreach (ptd, pvti_main.per_thread_data[is_ip6]) + { + vlib_cli_output (vm, "thread %d (%s)", + ptd - pvti_main.per_thread_data[is_ip6], + is_ip6 ? "IPv6" : "IPv4"); + pvti_rx_peer_t *peer; + vec_foreach (peer, ptd->rx_peers) + { + vlib_cli_output (vm, " %U", format_pvti_rx_peer_ptr, peer); + } + } + } + return 0; +} + +VLIB_CLI_COMMAND (pvti_show_interface_command, static) = { + .path = "show pvti interface", + .short_help = "show pvti interface", + .function = pvti_show_interface_command_fn, +}; + +VLIB_CLI_COMMAND (pvti_show_tx_peers_command, static) = { + .path = "show pvti tx peers", + .short_help = "show pvti tx peers", + .function = pvti_show_tx_peers_command_fn, +}; + +VLIB_CLI_COMMAND (pvti_show_rx_peers_command, static) = { + .path = "show pvti rx peers", + .short_help = "show pvti rx peers", + .function = pvti_show_rx_peers_command_fn, +}; + +void pvti_api_init (); + +VNET_FEATURE_INIT (pvti4_bypass, static) = { + .arc_name = "ip4-unicast", + .node_name = "ip4-pvti-bypass", + .runs_before = 0, +}; + +VNET_FEATURE_INIT (pvti6_bypass, static) = { + .arc_name = "ip6-unicast", + .node_name = "ip6-pvti-bypass", + .runs_before = 0, +}; + +static clib_error_t * +pvti_early_config (vlib_main_t *vm, unformat_input_t *input) +{ + u8 *runs_before = 0; + int rbi = 0; + if (vec_len (vnet_feat_pvti4_bypass.runs_before) == 0) + { + rbi = 0; + } + else + { + rbi = vec_len (vnet_feat_pvti4_bypass.runs_before) - 1; + } + vec_validate (vnet_feat_pvti4_bypass.runs_before, rbi); + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "runs-before %v", &runs_before)) + { + vec_add1 (runs_before, 0); + vnet_feat_pvti4_bypass.runs_before[rbi] = (char *) runs_before; + vec_add1 (vnet_feat_pvti4_bypass.runs_before, 0); + } + else + return clib_error_return (0, "unknown input"); + } + + return NULL; +} + +VLIB_EARLY_CONFIG_FUNCTION (pvti_early_config, "pvti"); + +static clib_error_t * +pvti_init (vlib_main_t *vm) +{ + pvti_main_t *pmp = &pvti_main; + clib_error_t *error = 0; + + pmp->vlib_main = vm; + pmp->vnet_main = vnet_get_main (); + pmp->is_initialized = 0; + + pvti_api_init (); + return error; +} + +VLIB_INIT_FUNCTION (pvti_init); + +VLIB_PLUGIN_REGISTER () = { + .version = VPP_BUILD_VER, + .description = "Packet Vector Tunnel Interface plugin", +}; diff --git a/src/plugins/pvti/pvti.h b/src/plugins/pvti/pvti.h new file mode 100644 index 00000000000..ac097c5ecca --- /dev/null +++ b/src/plugins/pvti/pvti.h @@ -0,0 +1,257 @@ +/* + * pvti.h - skeleton vpp engine plug-in header file + * + * Copyright (c) 2024 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __included_pvti_h__ +#define __included_pvti_h__ + +#include <vnet/vnet.h> +#include <vnet/ip/ip.h> +#include <vnet/ethernet/ethernet.h> + +#include <vppinfra/hash.h> +#include <vppinfra/error.h> + +#define VPP_MAX_THREADS (1 << 8) + +#define MAX_RX_STREAMS 256 + +#define PVTI_ALIGN_BYTES 9 + +typedef CLIB_PACKED (struct { + u32 seq; + u8 stream_index; // set to the cpu# on the sending side + u8 chunk_count; + u8 reass_chunk_count; // number of chunks in the front that are related to + // previously started buffer + // mandatory_flags_mask highlights which of the flags cause packet drop if + // not understood, and which of them can be just ignored. + u8 mandatory_flags_mask; + u8 flags_value; + u8 pad_bytes; + u8 pad[0]; +}) pvti_packet_header_t; + +typedef CLIB_PACKED (struct { + ip4_header_t ip4; + udp_header_t udp; + // not part of encap header pvti_packet_header_t pv; +}) pvti_ip4_encap_header_t; + +typedef CLIB_PACKED (struct { + ip6_header_t ip6; + udp_header_t udp; + // not part of encap header pvti_packet_header_t pv; +}) pvti_ip6_encap_header_t; + +typedef CLIB_PACKED (struct { + u16 total_chunk_length; + // More fragments: this chunk is not the last block fragment +#define CHUNK_FLAGS_MF (1 << 0) + // More blocks: this block has chained blocks that follow +#define CHUNK_FLAGS_MB (1 << 1) + u16 _pad0; + u32 _pad1; + u8 chunk_data[0]; +}) pvti_chunk_header_t; + +typedef struct +{ + // a buffer being built from the smaller packets + u32 bi0; + + // how big can this buffer grow + u32 bi0_max_current_length; + + // how many chunks are already in the buffer + u8 chunk_count; + // leading reassembly chunk count + u8 reass_chunk_count; + + u32 current_tx_seq; +} pvti_per_tx_stream_data_t; + +typedef struct +{ + /* The seq# that we last processed */ + u32 last_rx_seq; + + // a current buffer that is being reassembled + u32 rx_bi0; + // The root buffer, most of the times == rx_bi0 except in the case of chained + // buffers. + u32 rx_bi0_first; + + // Next index for dispatch when the reassembly is done + u16 rx_next0; + // expected totall inner length for the packet + u16 rx_expected_inner_length; + u16 rx_received_inner_length; + +} pvti_per_rx_stream_data_t; + +typedef struct +{ + ip_address_t local_ip; + ip_address_t remote_ip; + u16 remote_port; + u16 local_port; + u16 underlay_mtu; + u32 underlay_fib_index; + + u32 pvti_if_index; + bool deleted; + bool is_bo0_traced; + + u32 bo0_max_current_length; + + u8 chunk_count; + u8 reass_chunk_count; + u32 current_tx_seq; + vlib_buffer_t *bo0; + +} pvti_tx_peer_t; + +typedef struct +{ + ip_address_t local_ip; + ip_address_t remote_ip; + u16 remote_port; + u16 local_port; + + pvti_per_rx_stream_data_t rx_streams[MAX_RX_STREAMS]; + + u32 pvti_if_index; + bool deleted; +} pvti_rx_peer_t; + +typedef struct +{ + /* pool of destination-based structures which are used to build the packets + */ + pvti_tx_peer_t *tx_peers; + + /* vector of buffers to send */ + u32 *pending_tx_buffers; + u16 *pending_tx_nexts; + /* pool of source-based structures for the remote peers' data tracking + */ + pvti_rx_peer_t *rx_peers; + + /* vector of buffers being decapsulated */ + u32 *pending_rx_buffers; + u16 *pending_rx_nexts; + +} pvti_per_thread_data_t; + +typedef struct +{ + ip_address_t local_ip; + ip_address_t remote_ip; + u16 remote_port; + u16 local_port; + u16 underlay_mtu; + u32 underlay_fib_index; + bool peer_address_from_payload; + u64 created_at; + + u32 sw_if_index; + u32 hw_if_index; + + // per-stream data for TX + pvti_per_tx_stream_data_t tx_streams[256]; + pvti_per_rx_stream_data_t rx_streams[256]; + +} pvti_if_t; + +typedef struct +{ + /* API message ID base */ + u16 msg_id_base; + + /* have we initialized the data structures ? */ + bool is_initialized; + + /* interface pool */ + pvti_if_t *if_pool; + + /* if_index in the pool above by sw_if_index */ + index_t *if_index_by_sw_if_index; + + /* indices by port */ + index_t **if_indices_by_port; + + /* per-thread data, ip4[0] and ip6[1] */ + pvti_per_thread_data_t *per_thread_data[2]; + + /* on/off switch for the periodic function */ + u8 periodic_timer_enabled; + /* Node index, non-zero if the periodic process has been created */ + u32 periodic_node_index; + + /* graph node state */ + uword *bm_ip4_bypass_enabled_by_sw_if; + uword *bm_ip6_bypass_enabled_by_sw_if; + + /* convenience */ + vlib_main_t *vlib_main; + vnet_main_t *vnet_main; + ethernet_main_t *ethernet_main; +} pvti_main_t; + +extern pvti_main_t pvti_main; + +extern vlib_node_registration_t pvti_node; +extern vlib_node_registration_t pvti4_input_node; +extern vlib_node_registration_t pvti4_output_node; +extern vlib_node_registration_t pvti6_input_node; +extern vlib_node_registration_t pvti6_output_node; +extern vlib_node_registration_t pvti_periodic_node; + +always_inline u8 +pvti_get_stream_index (int is_ip6) +{ + u32 thread_index = vlib_get_thread_index (); + + ASSERT ((thread_index & 0xffffff80) == 0); + + u8 stream_index = (thread_index & 0x7f) | (is_ip6 ? 0x80 : 0); + return stream_index; +} + +/* attempt to get a new buffer */ +always_inline u32 +pvti_get_new_buffer (vlib_main_t *vm) +{ + u32 bi0 = INDEX_INVALID; + if (vlib_buffer_alloc (vm, &bi0, 1) != 1) + { + return INDEX_INVALID; + } + vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0); + b0->current_data = 0; + b0->current_length = 0; + return bi0; +} + +/* Periodic function events */ +#define PVTI_EVENT1 1 +#define PVTI_EVENT2 2 +#define PVTI_EVENT_PERIODIC_ENABLE_DISABLE 3 + +void pvti_create_periodic_process (pvti_main_t *); +void pvti_verify_initialized (pvti_main_t *pvm); + +#endif /* __included_pvti_h__ */ diff --git a/src/plugins/pvti/pvti_if.c b/src/plugins/pvti/pvti_if.c new file mode 100644 index 00000000000..4f83994a1a4 --- /dev/null +++ b/src/plugins/pvti/pvti_if.c @@ -0,0 +1,376 @@ +/* + * Copyright (c) 2020 Cisco and/or its affiliates. + * Copyright (c) 2020 Doc.ai and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <vnet/adj/adj_midchain.h> +#include <vnet/udp/udp.h> + +#include <pvti/pvti.h> +#include <pvti/pvti_if.h> + +static u8 * +format_pvti_if_name (u8 *s, va_list *args) +{ + u32 dev_instance = va_arg (*args, u32); + // wg_if_t *wgi = wg_if_get (dev_instance); + return format (s, "pvti%d", dev_instance); +} + +u8 * +format_pvti_if (u8 *s, va_list *args) +{ + index_t pvtii = va_arg (*args, u32); + pvti_if_t *pvti_if = pvti_if_get (pvtii); + + s = format ( + s, "[%d] %U local:%U:%d remote:%U:%d underlay_mtu:%d underlay_fib_idx:%d", + pvtii, format_vnet_sw_if_index_name, vnet_get_main (), + pvti_if->sw_if_index, format_ip46_address, &pvti_if->local_ip, + IP46_TYPE_ANY, pvti_if->local_port, format_ip46_address, + &pvti_if->remote_ip, IP46_TYPE_ANY, pvti_if->remote_port, + pvti_if->underlay_mtu, pvti_if->underlay_fib_index); + + return (s); +} + +index_t +pvti_if_find_by_sw_if_index (u32 sw_if_index) +{ + if (vec_len (pvti_main.if_index_by_sw_if_index) <= sw_if_index) + return INDEX_INVALID; + u32 ti = pvti_main.if_index_by_sw_if_index[sw_if_index]; + if (ti == ~0) + return INDEX_INVALID; + + return (ti); +} + +index_t +pvti_if_find_by_remote_ip4_and_port (ip4_address_t *remote_ip4, + u16 remote_port) +{ + pvti_if_t *ifc; + pool_foreach (ifc, pvti_main.if_pool) + { + if ((ifc->remote_port == remote_port) && + (ifc->remote_ip.version == AF_IP4) && + ((ifc->remote_ip.ip.ip4.as_u32 == remote_ip4->as_u32) || + ifc->peer_address_from_payload)) + { + return (ifc - pvti_main.if_pool); + } + } + return INDEX_INVALID; +} + +index_t +pvti_if_find_by_remote_ip6_and_port (ip6_address_t *remote_ip6, + u16 remote_port) +{ + pvti_if_t *ifc; + pool_foreach (ifc, pvti_main.if_pool) + { + if ((ifc->remote_port == remote_port) && + (ifc->remote_ip.version == AF_IP6) && + ((0 == memcmp (&ifc->remote_ip.ip.ip6, remote_ip6, + sizeof (*remote_ip6))) || + ifc->peer_address_from_payload)) + { + return (ifc - pvti_main.if_pool); + } + } + return INDEX_INVALID; +} + +index_t +pvti_if_find_by_remote_ip_and_port (ip_address_t *remote_ip, u16 remote_port) +{ + pvti_if_t *ifc; + pool_foreach (ifc, pvti_main.if_pool) + { + if ((ifc->remote_port == remote_port) && + (ifc->peer_address_from_payload || + (0 == ip_address_cmp (remote_ip, &ifc->remote_ip)))) + { + return (ifc - pvti_main.if_pool); + } + } + return INDEX_INVALID; +} + +static void +pvti_add_tidx_by_port (index_t t_index, u16 port) +{ + pvti_main_t *pvm = &pvti_main; + vec_validate_init_empty (pvm->if_indices_by_port, port, NULL); + vec_add1 (pvm->if_indices_by_port[port], t_index); +} + +static void +pvti_del_tidx_by_port (index_t t_index, u16 port) +{ + pvti_main_t *pvm = &pvti_main; + index_t *ii; + if (!pvm->if_indices_by_port) + { + return; + } + if (port >= vec_len (pvm->if_indices_by_port)) + { + return; + } + if (vec_len (pvm->if_indices_by_port[port]) == 0) + { + ALWAYS_ASSERT (pvm->if_indices_by_port[port] > 0); + /* not reached */ + return; + } + + vec_foreach (ii, pvm->if_indices_by_port[port]) + { + if (*ii == t_index) + { + vec_del1 (pvm->if_indices_by_port[port], + pvm->if_indices_by_port[port] - ii); + break; + } + } +} + +static u32 +pvti_get_tunnel_count_by_port (u16 port) +{ + pvti_main_t *pvm = &pvti_main; + if (!pvm->if_indices_by_port) + { + return 0; + } + return vec_len (vec_elt (pvm->if_indices_by_port, port)); +} + +static clib_error_t * +pvti_if_admin_up_down (vnet_main_t *vnm, u32 hw_if_index, u32 flags) +{ + // vnet_hw_interface_t *hi; + u32 hw_flags; + + // hi = vnet_get_hw_interface (vnm, hw_if_index); + hw_flags = + (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP ? VNET_HW_INTERFACE_FLAG_LINK_UP : + 0); + vnet_hw_interface_set_flags (vnm, hw_if_index, hw_flags); + + return (NULL); +} + +void +pvti_if_update_adj (vnet_main_t *vnm, u32 sw_if_index, adj_index_t ai) +{ + + /* Convert any neighbour adjacency that has a next-hop reachable through + * the wg interface into a midchain. This is to avoid sending ARP/ND to + * resolve the next-hop address via the wg interface. Then, if one of the + * peers has matching prefix among allowed prefixes, the midchain will be + * updated to the corresponding one. + */ + adj_nbr_midchain_update_rewrite (ai, NULL, NULL, ADJ_FLAG_NONE, NULL); + + // wgii = wg_if_find_by_sw_if_index (sw_if_index); + // wg_if_peer_walk (wg_if_get (wgii), wg_peer_if_adj_change, &ai); +} + +VNET_DEVICE_CLASS (pvti_if_device_class) = { + .name = "Packet Vectorizer Tunnel", + .format_device_name = format_pvti_if_name, + .admin_up_down_function = pvti_if_admin_up_down, +}; + +VNET_HW_INTERFACE_CLASS (pvti_hw_interface_class) = { + .name = "PVTunnel", + .update_adjacency = pvti_if_update_adj, + .flags = VNET_HW_INTERFACE_CLASS_FLAG_P2P, + // .flags = VNET_HW_INTERFACE_CLASS_FLAG_NBMA, +}; + +int +pvti_if_create (ip_address_t *local_ip, u16 local_port, + ip_address_t *remote_ip, u16 remote_port, + pvti_peer_address_method_t peer_address_method, + u16 underlay_mtu, u32 underlay_fib_index, u32 *sw_if_indexp) +{ + vnet_main_t *vnm = vnet_get_main (); + pvti_main_t *pvm = &pvti_main; + u32 hw_if_index; + vnet_hw_interface_t *hi; + pvti_verify_initialized (pvm); + + pvti_if_t *pvti_if; + + ASSERT (sw_if_indexp); + + *sw_if_indexp = (u32) ~0; + + pool_get_zero (pvti_main.if_pool, pvti_if); + pvti_if->local_ip = *local_ip; + pvti_if->local_port = local_port; + pvti_if->remote_ip = *remote_ip; + if (peer_address_method == PVTI_PEER_ADDRESS_FROM_PAYLOAD) + { + pvti_if->peer_address_from_payload = 1; + } + pvti_if->remote_port = remote_port; + pvti_if->underlay_mtu = underlay_mtu; + pvti_if->underlay_fib_index = underlay_fib_index; + pvti_if->created_at = clib_cpu_time_now (); + + /* tunnel index (or instance) */ + u32 t_idx = pvti_if - pvti_main.if_pool; + + hw_if_index = + vnet_register_interface (vnm, pvti_if_device_class.index, t_idx, + pvti_hw_interface_class.index, t_idx); + + pvti_if->hw_if_index = hw_if_index; + + hi = vnet_get_hw_interface (vnm, hw_if_index); + pvti_if->sw_if_index = *sw_if_indexp = hi->sw_if_index; + + vec_validate_init_empty (pvm->if_index_by_sw_if_index, hi->sw_if_index, + INDEX_INVALID); + + vec_elt (pvm->if_index_by_sw_if_index, hi->sw_if_index) = t_idx; + pvti_if_t *pvti_if0 = pool_elt_at_index (pvti_main.if_pool, t_idx); + int i; + for (i = 0; i < 256; i++) + { + pvti_if0->tx_streams[i].bi0 = INDEX_INVALID; + pvti_if0->tx_streams[i].current_tx_seq = 42; + + pvti_if0->rx_streams[i].rx_bi0 = INDEX_INVALID; + pvti_if0->rx_streams[i].rx_bi0_first = INDEX_INVALID; + } + + /* + int is_ip6 = 0; + u32 encap_index = !is_ip6 ? + pvti4_output_node.index : pvti6_output_node.index; + vnet_set_interface_output_node (vnm, pvti_if->hw_if_index, encap_index); + */ + vnet_set_interface_l3_output_node (vnm->vlib_main, hi->sw_if_index, + (u8 *) "pvti4-output"); + + pvti_add_tidx_by_port (t_idx, local_port); + if (1 == pvti_get_tunnel_count_by_port (local_port)) + { + clib_warning ("Registering local port %d", local_port); + udp_register_dst_port (vlib_get_main (), local_port, + pvti4_input_node.index, UDP_IP4); + udp_register_dst_port (vlib_get_main (), local_port, + pvti6_input_node.index, UDP_IP6); + } + else + { + clib_warning ("Not registering the port"); + } + + vnet_hw_interface_set_flags (vnm, pvti_if->hw_if_index, + VNET_HW_INTERFACE_FLAG_LINK_UP); + + return 0; +} + +void +pvti_if_walk (pvti_if_walk_cb_t fn, void *data) +{ + index_t pvtii; + + pool_foreach_index (pvtii, pvti_main.if_pool) + { + if (WALK_STOP == fn (pvtii, data)) + break; + } +} + +int +pvti_if_delete (u32 sw_if_index) +{ + vnet_main_t *vnm = vnet_get_main (); + pvti_main_t *pvm = &pvti_main; + + if (pool_is_free_index (vnm->interface_main.sw_interfaces, sw_if_index)) + return VNET_API_ERROR_INVALID_SW_IF_INDEX; + + vnet_hw_interface_t *hw = vnet_get_sup_hw_interface (vnm, sw_if_index); + if (hw == 0 || hw->dev_class_index != pvti_if_device_class.index) + return VNET_API_ERROR_INVALID_VALUE; + + pvti_if_t *ifc; + bool found = 0; + pool_foreach (ifc, pvm->if_pool) + { + if (ifc->sw_if_index == sw_if_index) + { + found = 1; + break; + } + } + if (!found) + { + return VNET_API_ERROR_INVALID_VALUE_2; + } + index_t tidx = ifc - pvm->if_pool; + + u16 local_port = ifc->local_port; + pvti_del_tidx_by_port (tidx, local_port); + pvm->if_index_by_sw_if_index[sw_if_index] = INDEX_INVALID; + + if (0 == pvti_get_tunnel_count_by_port (local_port)) + { + udp_unregister_dst_port (vlib_get_main (), local_port, 1); + udp_unregister_dst_port (vlib_get_main (), local_port, 0); + } + + vnet_reset_interface_l3_output_node (vnm->vlib_main, sw_if_index); + vnet_delete_hw_interface (vnm, hw->hw_if_index); + pool_put (pvti_main.if_pool, ifc); + + /* mark per-thread peers as deleted */ + pvti_per_thread_data_t *ptd; + + vec_foreach (ptd, pvm->per_thread_data[0]) + { + pvti_tx_peer_t *peer; + vec_foreach (peer, ptd->tx_peers) + { + if (tidx == peer->pvti_if_index) + { + peer->deleted = 1; + } + } + } + vec_foreach (ptd, pvm->per_thread_data[1]) + { + pvti_tx_peer_t *peer; + vec_foreach (peer, ptd->tx_peers) + { + if (tidx == peer->pvti_if_index) + { + peer->deleted = 1; + } + } + } + + return 0; +} diff --git a/src/plugins/pvti/pvti_if.h b/src/plugins/pvti/pvti_if.h new file mode 100644 index 00000000000..44bf22ce825 --- /dev/null +++ b/src/plugins/pvti/pvti_if.h @@ -0,0 +1,47 @@ +#ifndef PVTI_IF_H +#define PVTI_IF_H + +#include <vnet/interface_funcs.h> + +typedef enum +{ + PVTI_PEER_ADDRESS_FIXED = 0, + PVTI_PEER_ADDRESS_FROM_PAYLOAD +} pvti_peer_address_method_t; + +typedef walk_rc_t (*pvti_if_walk_cb_t) (index_t wgi, void *data); +void pvti_if_walk (pvti_if_walk_cb_t fn, void *data); + +int pvti_if_create (ip_address_t *local_ip, u16 local_port, + ip_address_t *remote_ip, u16 remote_port, + pvti_peer_address_method_t peer_address_method, + u16 underlay_mtu, u32 underlay_fib_index, + u32 *sw_if_indexp); +index_t pvti_if_find_by_sw_if_index (u32 sw_if_index); +index_t pvti_if_find_by_remote_ip4_and_port (ip4_address_t *remote_ip4, + u16 remote_port); +index_t pvti_if_find_by_remote_ip6_and_port (ip6_address_t *remote_ip4, + u16 remote_port); + +index_t pvti_if_find_by_remote_ip_and_port (ip_address_t *remote_ip, + u16 remote_port); + +int pvti_if_delete (u32 sw_if_index); + +u8 *format_pvti_if (u8 *s, va_list *args); + +static_always_inline pvti_if_t * +pvti_if_get (index_t pvtii) +{ + if (INDEX_INVALID == pvtii) + return (NULL); + return (pool_elt_at_index (pvti_main.if_pool, pvtii)); +} + +static_always_inline index_t +pvti_if_get_index (pvti_if_t *pvti_if) +{ + return pvti_if - pvti_main.if_pool; +} + +#endif diff --git a/src/plugins/quic/quic.c b/src/plugins/quic/quic.c index 60d4ac21c19..3f7a3426069 100644 --- a/src/plugins/quic/quic.c +++ b/src/plugins/quic/quic.c @@ -1058,6 +1058,8 @@ quic_on_stream_open (quicly_stream_open_t * self, quicly_stream_t * stream) svm_fifo_add_want_deq_ntf (stream_session->rx_fifo, SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL | SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY); + svm_fifo_init_ooo_lookup (stream_session->rx_fifo, 0 /* ooo enq */); + svm_fifo_init_ooo_lookup (stream_session->tx_fifo, 1 /* ooo deq */); stream_session->session_state = SESSION_STATE_ACCEPTING; if ((rv = app_worker_accept_notify (app_wrk, stream_session))) @@ -1302,6 +1304,8 @@ quic_connect_stream (session_t * quic_session, session_endpoint_cfg_t * sep) return app_worker_connect_notify (app_wrk, NULL, rv, sep->opaque); } + svm_fifo_init_ooo_lookup (stream_session->rx_fifo, 0 /* ooo enq */); + svm_fifo_init_ooo_lookup (stream_session->tx_fifo, 1 /* ooo deq */); svm_fifo_add_want_deq_ntf (stream_session->rx_fifo, SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL | SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY); @@ -1679,6 +1683,9 @@ quic_on_quic_session_connected (quic_ctx_t * ctx) return; } + svm_fifo_init_ooo_lookup (quic_session->rx_fifo, 0 /* ooo enq */); + svm_fifo_init_ooo_lookup (quic_session->tx_fifo, 1 /* ooo deq */); + quic_session->session_state = SESSION_STATE_CONNECTING; if ((rv = app_worker_connect_notify (app_wrk, quic_session, SESSION_E_NONE, ctx->client_opaque))) @@ -2137,6 +2144,9 @@ quic_accept_connection (quic_rx_packet_ctx_t * pctx) return; } + svm_fifo_init_ooo_lookup (quic_session->rx_fifo, 0 /* ooo enq */); + svm_fifo_init_ooo_lookup (quic_session->tx_fifo, 1 /* ooo deq */); + app_wrk = app_worker_get (quic_session->app_wrk_index); quic_session->session_state = SESSION_STATE_ACCEPTING; if ((rv = app_worker_accept_notify (app_wrk, quic_session))) diff --git a/src/plugins/snort/CMakeLists.txt b/src/plugins/snort/CMakeLists.txt index bd9dcdc4fdd..3fc2bd625a4 100644 --- a/src/plugins/snort/CMakeLists.txt +++ b/src/plugins/snort/CMakeLists.txt @@ -7,6 +7,10 @@ add_vpp_plugin(snort dequeue.c main.c cli.c + snort_api.c + + API_FILES + snort.api MULTIARCH_SOURCES enqueue.c diff --git a/src/plugins/snort/cli.c b/src/plugins/snort/cli.c index 08740f41b37..4b6dbc742a7 100644 --- a/src/plugins/snort/cli.c +++ b/src/plugins/snort/cli.c @@ -25,6 +25,7 @@ snort_create_instance_command_fn (vlib_main_t *vm, unformat_input_t *input, u8 *name = 0; u32 queue_size = 1024; u8 drop_on_diconnect = 1; + int rv = 0; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -60,8 +61,30 @@ snort_create_instance_command_fn (vlib_main_t *vm, unformat_input_t *input, goto done; } - err = snort_instance_create (vm, (char *) name, min_log2 (queue_size), - drop_on_diconnect); + rv = snort_instance_create (vm, (char *) name, min_log2 (queue_size), + drop_on_diconnect); + + switch (rv) + { + case 0: + break; + case VNET_API_ERROR_ENTRY_ALREADY_EXISTS: + err = clib_error_return (0, "instance '%s' already exists", name); + break; + case VNET_API_ERROR_SYSCALL_ERROR_1: + err = clib_error_return (0, "memory fd failure: %U", format_clib_error, + clib_mem_get_last_error ()); + break; + case VNET_API_ERROR_SYSCALL_ERROR_2: + err = clib_error_return (0, "ftruncate failure"); + break; + case VNET_API_ERROR_SYSCALL_ERROR_3: + err = clib_error_return (0, "mmap failure"); + break; + default: + err = clib_error_return (0, "snort_instance_create returned %d", rv); + break; + } done: vec_free (name); @@ -77,6 +100,118 @@ VLIB_CLI_COMMAND (snort_create_instance_command, static) = { }; static clib_error_t * +snort_disconnect_instance_command_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + unformat_input_t _line_input, *line_input = &_line_input; + clib_error_t *err = 0; + u8 *name = 0; + snort_instance_t *si; + int rv = 0; + + if (!unformat_user (input, unformat_line_input, line_input)) + return clib_error_return (0, "please specify instance name"); + + if (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + unformat (line_input, "%s", &name); + + if (!name) + { + err = clib_error_return (0, "please specify instance name"); + goto done; + } + + si = snort_get_instance_by_name ((char *) name); + if (!si) + rv = VNET_API_ERROR_NO_SUCH_ENTRY; + else + rv = snort_instance_disconnect (vm, si->index); + + switch (rv) + { + case 0: + break; + case VNET_API_ERROR_NO_SUCH_ENTRY: + err = clib_error_return (0, "unknown instance '%s'", name); + break; + case VNET_API_ERROR_FEATURE_DISABLED: + err = clib_error_return (0, "instance '%s' is not connected", name); + break; + case VNET_API_ERROR_INVALID_VALUE: + err = clib_error_return (0, "failed to disconnect a broken client"); + break; + default: + err = clib_error_return (0, "snort_instance_disconnect returned %d", rv); + break; + } + +done: + vec_free (name); + unformat_free (line_input); + return err; +} + +VLIB_CLI_COMMAND (snort_disconnect_instance_command, static) = { + .path = "snort disconnect instance", + .short_help = "snort disconnect instance <name>", + .function = snort_disconnect_instance_command_fn, +}; + +static clib_error_t * +snort_delete_instance_command_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + unformat_input_t _line_input, *line_input = &_line_input; + clib_error_t *err = 0; + u8 *name = 0; + int rv = 0; + + if (!unformat_user (input, unformat_line_input, line_input)) + return clib_error_return (0, "please specify instance name"); + + if (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + unformat (line_input, "%s", &name); + + if (!name) + { + err = clib_error_return (0, "please specify instance name"); + goto done; + } + + snort_instance_t *si = snort_get_instance_by_name ((char *) name); + if (!si) + err = clib_error_return (0, "unknown instance '%s' requested", name); + else + rv = snort_instance_delete (vm, si->index); + + switch (rv) + { + case 0: + break; + case VNET_API_ERROR_NO_SUCH_ENTRY: + err = clib_error_return (0, "instance '%s' deletion failure", name); + break; + case VNET_API_ERROR_INSTANCE_IN_USE: + err = clib_error_return (0, "instance '%s' has connected client", name); + break; + default: + err = clib_error_return (0, "snort_instance_delete returned %d", rv); + break; + } + +done: + vec_free (name); + unformat_free (line_input); + return err; +} + +VLIB_CLI_COMMAND (snort_delete_instance_command, static) = { + .path = "snort delete instance", + .short_help = "snort delete instance <name>", + .function = snort_delete_instance_command_fn, +}; + +static clib_error_t * snort_attach_command_fn (vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd) { @@ -86,6 +221,7 @@ snort_attach_command_fn (vlib_main_t *vm, unformat_input_t *input, u8 *name = 0; u32 sw_if_index = ~0; snort_attach_dir_t dir = SNORT_INOUT; + int rv = 0; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -124,8 +260,29 @@ snort_attach_command_fn (vlib_main_t *vm, unformat_input_t *input, goto done; } - err = - snort_interface_enable_disable (vm, (char *) name, sw_if_index, 1, dir); + rv = snort_interface_enable_disable (vm, (char *) name, sw_if_index, 1, dir); + + switch (rv) + { + case 0: + break; + case VNET_API_ERROR_FEATURE_ALREADY_ENABLED: + /* already attached to same instance */ + break; + case VNET_API_ERROR_INSTANCE_IN_USE: + err = clib_error_return (0, + "interface %U already assigned to " + "an instance", + format_vnet_sw_if_index_name, vnm, sw_if_index); + break; + case VNET_API_ERROR_NO_SUCH_ENTRY: + err = clib_error_return (0, "unknown instance '%s'", name); + break; + default: + err = clib_error_return (0, "snort_interface_enable_disable returned %d", + rv); + break; + } done: vec_free (name); @@ -148,6 +305,7 @@ snort_detach_command_fn (vlib_main_t *vm, unformat_input_t *input, vnet_main_t *vnm = vnet_get_main (); clib_error_t *err = 0; u32 sw_if_index = ~0; + int rv = 0; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -172,7 +330,23 @@ snort_detach_command_fn (vlib_main_t *vm, unformat_input_t *input, goto done; } - err = snort_interface_enable_disable (vm, 0, sw_if_index, 0, SNORT_INOUT); + rv = snort_interface_enable_disable (vm, 0, sw_if_index, 0, SNORT_INOUT); + + switch (rv) + { + case 0: + break; + case VNET_API_ERROR_INVALID_INTERFACE: + err = clib_error_return (0, + "interface %U is not assigned to snort " + "instance!", + format_vnet_sw_if_index_name, vnm, sw_if_index); + break; + default: + err = clib_error_return (0, "snort_interface_enable_disable returned %d", + rv); + break; + } done: unformat_free (line_input); @@ -213,7 +387,7 @@ snort_show_interfaces_command_fn (vlib_main_t *vm, unformat_input_t *input, snort_instance_t *si; u32 *index; - vlib_cli_output (vm, "interface\tsnort instance"); + vlib_cli_output (vm, "interface\t\tsnort instance"); vec_foreach (index, sm->instance_by_sw_if_index) { if (index[0] != ~0) @@ -237,7 +411,18 @@ snort_show_clients_command_fn (vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd) { snort_main_t *sm = &snort_main; - vlib_cli_output (vm, "number of clients: %d", pool_elts (sm->clients)); + u32 n_clients = pool_elts (sm->clients); + snort_client_t *c; + snort_instance_t *si; + + vlib_cli_output (vm, "number of clients: %d", n_clients); + if (n_clients) + vlib_cli_output (vm, "client snort instance"); + pool_foreach (c, sm->clients) + { + si = vec_elt_at_index (sm->instances, c->instance_index); + vlib_cli_output (vm, "%6d %s", c - sm->clients, si->name); + } return 0; } @@ -251,14 +436,16 @@ static clib_error_t * snort_mode_polling_command_fn (vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd) { - return snort_set_node_mode (vm, VLIB_NODE_STATE_POLLING); + snort_set_node_mode (vm, VLIB_NODE_STATE_POLLING); + return 0; } static clib_error_t * snort_mode_interrupt_command_fn (vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd) { - return snort_set_node_mode (vm, VLIB_NODE_STATE_INTERRUPT); + snort_set_node_mode (vm, VLIB_NODE_STATE_INTERRUPT); + return 0; } VLIB_CLI_COMMAND (snort_mode_polling_command, static) = { diff --git a/src/plugins/snort/dequeue.c b/src/plugins/snort/dequeue.c index 31745de404c..bc301f6888b 100644 --- a/src/plugins/snort/dequeue.c +++ b/src/plugins/snort/dequeue.c @@ -307,7 +307,7 @@ snort_deq_node_polling (vlib_main_t *vm, vlib_node_runtime_t *node, snort_qpair_t *qp; snort_instance_t *si; - vec_foreach (si, sm->instances) + pool_foreach (si, sm->instances) { qp = vec_elt_at_index (si->qpairs, vm->thread_index); u32 ready = __atomic_load_n (&qp->ready, __ATOMIC_ACQUIRE); diff --git a/src/plugins/snort/enqueue.c b/src/plugins/snort/enqueue.c index 409c0e49078..ce4f34491ec 100644 --- a/src/plugins/snort/enqueue.c +++ b/src/plugins/snort/enqueue.c @@ -133,7 +133,7 @@ snort_enq_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, nexts, n_processed); } - vec_foreach (si, sm->instances) + pool_foreach (si, sm->instances) { u32 head, freelist_len, n_pending, n_enq, mask; u64 ctr = 1; diff --git a/src/plugins/snort/main.c b/src/plugins/snort/main.c index 2430fcdc5c2..50bff027a13 100644 --- a/src/plugins/snort/main.c +++ b/src/plugins/snort/main.c @@ -3,10 +3,24 @@ */ #include <vlib/vlib.h> +#include <vlibapi/api_types.h> #include <vnet/plugin/plugin.h> #include <vpp/app/version.h> #include <snort/snort.h> +#include <snort/snort.api_enum.h> +#include <snort/snort.api_types.h> + +#include <vnet/ip/ip_types_api.h> +#include <vnet/format_fns.h> + +#include <vlibapi/api_helper_macros.h> + +#include <vnet/vnet.h> + +#include <vlibapi/api.h> +#include <vlibmemory/api.h> + #include <sys/eventfd.h> snort_main_t snort_main; @@ -18,6 +32,12 @@ VLIB_REGISTER_LOG_CLASS (snort_log, static) = { #define log_debug(fmt, ...) vlib_log_debug (snort_log.class, fmt, __VA_ARGS__) #define log_err(fmt, ...) vlib_log_err (snort_log.class, fmt, __VA_ARGS__) +snort_main_t * +snort_get_main () +{ + return &snort_main; +} + static void snort_client_disconnect (clib_file_t *uf) { @@ -45,7 +65,38 @@ snort_client_disconnect (clib_file_t *uf) pool_put (sm->clients, c); } -static snort_instance_t * +int +snort_instance_disconnect (vlib_main_t *vm, u32 instance_index) +{ + snort_main_t *sm = &snort_main; + snort_instance_t *si; + snort_client_t *client; + clib_file_main_t *fm = &file_main; + clib_file_t *uf = 0; + int rv = 0; + + si = snort_get_instance_by_index (instance_index); + if (!si) + return VNET_API_ERROR_NO_SUCH_ENTRY; + if (si->client_index == ~0) + return VNET_API_ERROR_FEATURE_DISABLED; + + client = pool_elt_at_index (sm->clients, si->client_index); + uf = clib_file_get (fm, client->file_index); + if (uf) + snort_client_disconnect (uf); + else + { + log_err ("failed to disconnect a broken client from" + "instance '%s'", + si->name); + rv = VNET_API_ERROR_INVALID_VALUE; + } + + return rv; +} + +snort_instance_t * snort_get_instance_by_name (char *name) { snort_main_t *sm = &snort_main; @@ -54,7 +105,16 @@ snort_get_instance_by_name (char *name) return 0; return vec_elt_at_index (sm->instances, p[0]); - ; +} + +snort_instance_t * +snort_get_instance_by_index (u32 instance_index) +{ + snort_main_t *sm = &snort_main; + + if (pool_is_free_index (sm->instances, instance_index)) + return 0; + return pool_elt_at_index (sm->instances, instance_index); } static clib_error_t * @@ -110,6 +170,8 @@ snort_conn_fd_read_ready (clib_file_t *uf) snort_client_disconnect (uf); return 0; } + snort_freelist_init (qp->freelist); + *qp->enq_head = *qp->deq_head = qp->next_desc = 0; } base = (u8 *) si->shm_base; @@ -281,14 +343,13 @@ snort_listener_init (vlib_main_t *vm) return 0; } -clib_error_t * +int snort_instance_create (vlib_main_t *vm, char *name, u8 log2_queue_sz, u8 drop_on_disconnect) { vlib_thread_main_t *tm = vlib_get_thread_main (); snort_main_t *sm = &snort_main; snort_instance_t *si; - clib_error_t *err = 0; u32 index, i; u8 *base = CLIB_MEM_VM_MAP_FAILED; u32 size; @@ -296,9 +357,10 @@ snort_instance_create (vlib_main_t *vm, char *name, u8 log2_queue_sz, u32 qpair_mem_sz = 0; u32 qsz = 1 << log2_queue_sz; u8 align = CLIB_CACHE_LINE_BYTES; + int rv = 0; if (snort_get_instance_by_name (name)) - return clib_error_return (0, "instance already exists"); + return VNET_API_ERROR_ENTRY_ALREADY_EXISTS; /* descriptor table */ qpair_mem_sz += round_pow2 (qsz * sizeof (daq_vpp_desc_t), align); @@ -316,14 +378,13 @@ snort_instance_create (vlib_main_t *vm, char *name, u8 log2_queue_sz, if (fd == -1) { - err = clib_error_return (0, "memory fd failure: %U", format_clib_error, - clib_mem_get_last_error ()); + rv = VNET_API_ERROR_SYSCALL_ERROR_1; goto done; } if ((ftruncate (fd, size)) == -1) { - err = clib_error_return (0, "ftruncate failure"); + rv = VNET_API_ERROR_SYSCALL_ERROR_2; goto done; } @@ -331,7 +392,7 @@ snort_instance_create (vlib_main_t *vm, char *name, u8 log2_queue_sz, if (base == CLIB_MEM_VM_MAP_FAILED) { - err = clib_error_return (0, "mmap failure"); + rv = VNET_API_ERROR_SYSCALL_ERROR_3; goto done; } @@ -399,17 +460,17 @@ snort_instance_create (vlib_main_t *vm, char *name, u8 log2_queue_sz, sm->input_mode); done: - if (err) + if (rv) { if (base != CLIB_MEM_VM_MAP_FAILED) clib_mem_vm_unmap (base); if (fd != -1) close (fd); } - return err; + return rv; } -clib_error_t * +int snort_interface_enable_disable (vlib_main_t *vm, char *instance_name, u32 sw_if_index, int is_enable, snort_attach_dir_t snort_dir) @@ -417,16 +478,16 @@ snort_interface_enable_disable (vlib_main_t *vm, char *instance_name, snort_main_t *sm = &snort_main; vnet_main_t *vnm = vnet_get_main (); snort_instance_t *si; - clib_error_t *err = 0; u64 fa_data; u32 index; + int rv = 0; if (is_enable) { if ((si = snort_get_instance_by_name (instance_name)) == 0) { - err = clib_error_return (0, "unknown instance '%s'", instance_name); - goto done; + log_err ("unknown instance '%s'", instance_name); + return VNET_API_ERROR_NO_SUCH_ENTRY; } vec_validate_init_empty (sm->instance_by_sw_if_index, sw_if_index, ~0); @@ -434,12 +495,13 @@ snort_interface_enable_disable (vlib_main_t *vm, char *instance_name, index = sm->instance_by_sw_if_index[sw_if_index]; if (index != ~0) { + if (index == si->index) + rv = VNET_API_ERROR_FEATURE_ALREADY_ENABLED; + else + rv = VNET_API_ERROR_INSTANCE_IN_USE; si = vec_elt_at_index (sm->instances, index); - err = clib_error_return (0, - "interface %U already assgined to " - "instance '%s'", - format_vnet_sw_if_index_name, vnm, - sw_if_index, si->name); + log_err ("interface %U already assgined to instance '%s'", + format_vnet_sw_if_index_name, vnm, sw_if_index, si->name); goto done; } @@ -462,11 +524,9 @@ snort_interface_enable_disable (vlib_main_t *vm, char *instance_name, if (sw_if_index >= vec_len (sm->instance_by_sw_if_index) || sm->instance_by_sw_if_index[sw_if_index] == ~0) { - err = - clib_error_return (0, - "interface %U is not assigned to snort " - "instance!", - format_vnet_sw_if_index_name, vnm, sw_if_index); + rv = VNET_API_ERROR_INVALID_INTERFACE; + log_err ("interface %U is not assigned to snort instance!", + format_vnet_sw_if_index_name, vnm, sw_if_index); goto done; } index = sm->instance_by_sw_if_index[sw_if_index]; @@ -488,12 +548,66 @@ snort_interface_enable_disable (vlib_main_t *vm, char *instance_name, } done: - if (err) - log_err ("%U", format_clib_error, err); - return 0; + return rv; } -clib_error_t * +static int +snort_strip_instance_interfaces (vlib_main_t *vm, u32 instance_index) +{ + snort_main_t *sm = &snort_main; + u32 *index; + int rv = 0; + + vec_foreach (index, sm->instance_by_sw_if_index) + { + if (*index == instance_index) + rv = snort_interface_enable_disable ( + vm, NULL, index - sm->instance_by_sw_if_index, 0, 0); + if (rv) + break; + } + + return rv; +} + +int +snort_instance_delete (vlib_main_t *vm, u32 instance_index) +{ + snort_main_t *sm = &snort_main; + snort_instance_t *si; + snort_qpair_t *qp; + int rv = 0; + + si = snort_get_instance_by_index (instance_index); + if (!si) + return VNET_API_ERROR_NO_SUCH_ENTRY; + + if (si->client_index != ~0) + return VNET_API_ERROR_INSTANCE_IN_USE; + + if ((rv = snort_strip_instance_interfaces (vm, si->index))) + return rv; + + hash_unset_mem (sm->instance_by_name, si->name); + + clib_mem_vm_unmap (si->shm_base); + close (si->shm_fd); + + vec_foreach (qp, si->qpairs) + { + clib_file_del_by_index (&file_main, qp->deq_fd_file_index); + } + + log_debug ("deleting instance '%s'", si->name); + + vec_free (si->qpairs); + vec_free (si->name); + pool_put (sm->instances, si); + + return rv; +} + +int snort_set_node_mode (vlib_main_t *vm, u32 mode) { int i; diff --git a/src/plugins/snort/snort.api b/src/plugins/snort/snort.api new file mode 100644 index 00000000000..5c65f79e68a --- /dev/null +++ b/src/plugins/snort/snort.api @@ -0,0 +1,226 @@ +option version = "1.0.0"; + +import "vnet/interface_types.api"; +import "vnet/ip/ip_types.api"; + +define snort_instance_create { + u32 client_index; + u32 context; + u32 queue_size; + u8 drop_on_disconnect; + string name[]; +}; + +define snort_instance_create_reply { + u32 context; + i32 retval; + u32 instance_index; +}; + +define snort_instance_delete { + u32 client_index; + u32 context; + u32 instance_index; +}; + +define snort_instance_delete_reply { + u32 context; + i32 retval; +}; + +define snort_client_disconnect { + u32 client_index; + u32 context; + u32 snort_client_index; +}; + +define snort_client_disconnect_reply { + u32 context; + i32 retval; +}; + +define snort_instance_disconnect { + u32 client_index; + u32 context; + u32 instance_index; +}; + +define snort_instance_disconnect_reply { + u32 context; + i32 retval; +}; + +define snort_interface_attach { + u32 client_index; + u32 context; + u32 instance_index; + u32 sw_if_index; + u8 snort_dir; +}; + +define snort_interface_attach_reply { + u32 context; + i32 retval; +}; + +define snort_interface_detach { + u32 client_index; + u32 context; + u32 sw_if_index; +}; + +define snort_interface_detach_reply { + u32 context; + i32 retval; +}; + +define snort_input_mode_get { + u32 client_index; + u32 context; +}; + +define snort_input_mode_get_reply { + u32 context; + i32 retval; + u32 snort_mode; +}; + +define snort_input_mode_set { + u32 client_index; + u32 context; + u8 input_mode; +}; + +define snort_input_mode_set_reply { + u32 context; + i32 retval; +}; + +service { + rpc snort_instance_get returns snort_instance_get_reply + stream snort_instance_details; +}; + +/** \brief Get snort instance(s). + @param client_index - opaque cookie to identify the sender. + @param context - sender context + @param cursor - current iterator value (all requested). + @param instance_index - instance index (~0 for all). +*/ +define snort_instance_get +{ + u32 client_index; + u32 context; + u32 cursor; + u32 instance_index; +}; + +/** \brief Reply for snort instance(s). + @param context - sender context + @param retval - return code for the request. + @param cursor - iterator value to continue with (if there is more). +*/ +define snort_instance_get_reply +{ + u32 context; + i32 retval; + u32 cursor; +}; + +/** \brief Details of a snort instance. + @param context - sender context + @param instance - snort instance info. +*/ +define snort_instance_details { + u32 context; + u32 instance_index; + u32 shm_size; + u32 shm_fd; + u8 drop_on_disconnect; + u32 snort_client_index; + string name[]; +}; + +service { + rpc snort_interface_get returns snort_interface_get_reply + stream snort_interface_details; +}; + +/** \brief Get snort interface(s). + @param client_index - opaque cookie to identify the sender. + @param context - sender context + @param cursor - current iterator value (all requested). + @param sw_if_index - sw if index (~0 for all). +*/ +define snort_interface_get +{ + u32 client_index; + u32 context; + u32 cursor; + u32 sw_if_index; +}; + +/** \brief Reply for snort interface(s). + @param context - sender context + @param retval - return code for the request. + @param cursor - iterator value to continue with (if there is more). +*/ +define snort_interface_get_reply +{ + u32 context; + i32 retval; + u32 cursor; +}; + +/** \brief Details of a snort interface. + @param context - sender context + @param sw_if_index - interface index + @param instance_index - snort instance the interface is attached to. +*/ +define snort_interface_details { + u32 context; + u32 sw_if_index; + u32 instance_index; +}; + +service { + rpc snort_client_get returns snort_client_get_reply + stream snort_client_details; +}; + +/** \brief Get snort clients. + @param client_index - opaque cookie to identify the sender. + @param context - sender context + @param cursor - current iterator value (all requested). + @param client_index (~0 for all). +*/ +define snort_client_get +{ + u32 client_index; + u32 context; + u32 cursor; + u32 snort_client_index; +}; + +/** \brief Reply for snort clients. + @param context - sender context + @param retval - return code for the request. + @param cursor - iterator value to continue with (if there is more). +*/ +define snort_client_get_reply +{ + u32 context; + i32 retval; + u32 cursor; +}; + +/** \brief Details of a snort client. + @param context - sender context + @param client index + @param instance_index - snort instance of the client. +*/ +define snort_client_details { + u32 context; + u32 client_index; + u32 instance_index; +}; diff --git a/src/plugins/snort/snort.h b/src/plugins/snort/snort.h index 79299aa6d91..c7e856c0127 100644 --- a/src/plugins/snort/snort.h +++ b/src/plugins/snort/snort.h @@ -7,6 +7,7 @@ #include <vppinfra/error.h> #include <vppinfra/socket.h> +#include <vppinfra/file.h> #include <vlib/vlib.h> #include <snort/daq_vpp.h> @@ -78,8 +79,11 @@ typedef struct snort_per_thread_data_t *per_thread_data; u32 input_mode; u8 *socket_name; + /* API message ID base */ + u16 msg_id_base; } snort_main_t; +extern clib_file_main_t file_main; extern snort_main_t snort_main; extern vlib_node_registration_t snort_enq_node; extern vlib_node_registration_t snort_deq_node; @@ -103,13 +107,17 @@ typedef enum } /* functions */ -clib_error_t *snort_instance_create (vlib_main_t *vm, char *name, - u8 log2_queue_sz, u8 drop_on_disconnect); -clib_error_t *snort_interface_enable_disable (vlib_main_t *vm, - char *instance_name, - u32 sw_if_index, int is_enable, - snort_attach_dir_t dir); -clib_error_t *snort_set_node_mode (vlib_main_t *vm, u32 mode); +snort_main_t *snort_get_main (); +snort_instance_t *snort_get_instance_by_index (u32 instance_index); +snort_instance_t *snort_get_instance_by_name (char *name); +int snort_instance_create (vlib_main_t *vm, char *name, u8 log2_queue_sz, + u8 drop_on_disconnect); +int snort_interface_enable_disable (vlib_main_t *vm, char *instance_name, + u32 sw_if_index, int is_enable, + snort_attach_dir_t dir); +int snort_set_node_mode (vlib_main_t *vm, u32 mode); +int snort_instance_delete (vlib_main_t *vm, u32 instance_index); +int snort_instance_disconnect (vlib_main_t *vm, u32 instance_index); always_inline void snort_freelist_init (u32 *fl) diff --git a/src/plugins/snort/snort_api.c b/src/plugins/snort/snort_api.c new file mode 100644 index 00000000000..334a84b4341 --- /dev/null +++ b/src/plugins/snort/snort_api.c @@ -0,0 +1,398 @@ +#include <vlib/vlib.h> +#include <vnet/plugin/plugin.h> +#include <snort/snort.h> +#include <vlibapi/api_types.h> + +#include <snort/snort.api_enum.h> +#include <snort/snort.api_types.h> + +#include <vlibmemory/api.h> +#include <vnet/ip/ip_types_api.h> +#include <vnet/format_fns.h> +#include <vnet/api_errno.h> + +/** + * Base message ID fot the plugin + */ +static u32 snort_base_msg_id; +#define REPLY_MSG_ID_BASE snort_base_msg_id + +#include <vlibapi/api_helper_macros.h> + +#include <vnet/vnet.h> + +#include <vlibapi/api.h> +#include <sys/eventfd.h> + +VLIB_REGISTER_LOG_CLASS (snort_log, static) = { + .class_name = "snort", +}; + +#define log_debug(fmt, ...) vlib_log_debug (snort_log.class, fmt, __VA_ARGS__) +#define log_err(fmt, ...) vlib_log_err (snort_log.class, fmt, __VA_ARGS__) + +static void +vl_api_snort_instance_create_t_handler (vl_api_snort_instance_create_t *mp) +{ + vlib_main_t *vm = vlib_get_main (); + vl_api_snort_instance_create_reply_t *rmp; + char *name = vl_api_from_api_to_new_c_string (&mp->name); + u32 queue_sz = clib_net_to_host_u32 (mp->queue_size); + u8 drop_on_disconnect = mp->drop_on_disconnect; + int rv = 0; + u32 instance_index = ~0; + snort_instance_t *si; + + rv = + snort_instance_create (vm, name, min_log2 (queue_sz), drop_on_disconnect); + + if ((si = snort_get_instance_by_name (name))) + { + instance_index = si->index; + } + + REPLY_MACRO2 (VL_API_SNORT_INSTANCE_CREATE_REPLY, ({ + rmp->instance_index = clib_host_to_net_u32 (instance_index); + })); +} + +static void +vl_api_snort_instance_delete_t_handler (vl_api_snort_instance_delete_t *mp) +{ + vlib_main_t *vm = vlib_get_main (); + vl_api_snort_instance_delete_reply_t *rmp; + u32 instance_index = clib_net_to_host_u32 (mp->instance_index); + int rv; + + rv = snort_instance_delete (vm, instance_index); + + REPLY_MACRO (VL_API_SNORT_INSTANCE_DELETE_REPLY); +} + +static void +vl_api_snort_interface_attach_t_handler (vl_api_snort_interface_attach_t *mp) +{ + vlib_main_t *vm = vlib_get_main (); + vl_api_snort_interface_attach_reply_t *rmp; + u32 instance_index = clib_net_to_host_u32 (mp->instance_index); + snort_instance_t *instance = 0; + u32 sw_if_index = clib_net_to_host_u32 (mp->sw_if_index); + u8 snort_dir = mp->snort_dir; + int rv = VNET_API_ERROR_NO_SUCH_ENTRY; + + instance = snort_get_instance_by_index (instance_index); + if (instance) + rv = snort_interface_enable_disable ( + vm, (char *) instance->name, sw_if_index, 1 /* is_enable */, snort_dir); + + REPLY_MACRO (VL_API_SNORT_INTERFACE_ATTACH_REPLY); +} + +static void +send_snort_instance_details (const snort_instance_t *instance, + vl_api_registration_t *rp, u32 context) +{ + vl_api_snort_instance_details_t *rmp; + u32 name_len = vec_len (instance->name); + + REPLY_MACRO_DETAILS5 ( + VL_API_SNORT_INSTANCE_DETAILS, name_len, rp, context, ({ + rmp->instance_index = clib_host_to_net_u32 (instance->index); + vl_api_vec_to_api_string (instance->name, &rmp->name); + rmp->snort_client_index = clib_host_to_net_u32 (instance->client_index); + rmp->shm_size = clib_host_to_net_u32 (instance->shm_size); + rmp->shm_fd = clib_host_to_net_u32 (instance->shm_fd); + rmp->drop_on_disconnect = instance->drop_on_disconnect; + })); +} + +static void +vl_api_snort_instance_get_t_handler (vl_api_snort_instance_get_t *mp) +{ + snort_main_t *sm = snort_get_main (); + snort_instance_t *instance = 0; + vl_api_snort_instance_get_reply_t *rmp; + u32 instance_index; + int rv = 0; + + instance_index = clib_net_to_host_u32 (mp->instance_index); + + if (instance_index == INDEX_INVALID) + { + /* clang-format off */ + REPLY_AND_DETAILS_MACRO ( + VL_API_SNORT_INSTANCE_GET_REPLY, sm->instances, ({ + instance = pool_elt_at_index (sm->instances, cursor); + send_snort_instance_details (instance, rp, mp->context); + })); + /* clang-format on */ + } + else + { + instance = snort_get_instance_by_index (instance_index); + + if (instance) + { + vl_api_registration_t *rp = + vl_api_client_index_to_registration (mp->client_index); + + if (rp == NULL) + { + return; + } + + send_snort_instance_details (instance, rp, mp->context); + } + else + { + rv = VNET_API_ERROR_NO_SUCH_ENTRY; + } + + /* clang-format off */ + REPLY_MACRO2 (VL_API_SNORT_INSTANCE_GET_REPLY, ({ + rmp->cursor = INDEX_INVALID; + })); + /* clang-format on */ + } +} + +static void +send_snort_interface_details (u32 sw_if_index, u32 instance_index, + vl_api_registration_t *rp, u32 context) +{ + vl_api_snort_interface_details_t *rmp; + + if (instance_index != ~0) + { + REPLY_MACRO_DETAILS4 (VL_API_SNORT_INTERFACE_DETAILS, rp, context, ({ + rmp->instance_index = + clib_host_to_net_u32 (instance_index); + rmp->sw_if_index = + clib_host_to_net_u32 (sw_if_index); + })); + } +} + +static void +vl_api_snort_interface_get_t_handler (vl_api_snort_interface_get_t *mp) +{ + snort_main_t *sm = snort_get_main (); + vl_api_snort_interface_get_reply_t *rmp; + u32 sw_if_index; + u32 *index; + int rv = 0; + + sw_if_index = clib_net_to_host_u32 (mp->sw_if_index); + + if (sw_if_index == INDEX_INVALID) + { + /* clang-format off */ + if (vec_len (sm->instance_by_sw_if_index) == 0) + { + REPLY_MACRO2 (VL_API_SNORT_INTERFACE_GET_REPLY, ({ rmp->cursor = ~0; })); + return; + } + + REPLY_AND_DETAILS_VEC_MACRO( + VL_API_SNORT_INTERFACE_GET_REPLY, + sm->instance_by_sw_if_index, + mp, rmp, rv, ({ + index = vec_elt_at_index (sm->instance_by_sw_if_index, cursor); + send_snort_interface_details (cursor, *index, rp, mp->context); + })) + /* clang-format on */ + } + else + { + index = vec_elt_at_index (sm->instance_by_sw_if_index, sw_if_index); + if (snort_get_instance_by_index (index[0])) + { + vl_api_registration_t *rp = + vl_api_client_index_to_registration (mp->client_index); + + if (rp == NULL) + { + return; + } + + send_snort_interface_details (sw_if_index, *index, rp, mp->context); + } + else + { + rv = VNET_API_ERROR_NO_SUCH_ENTRY; + } + + /* clang-format off */ + REPLY_MACRO2 (VL_API_SNORT_INTERFACE_GET_REPLY, ({ + rmp->cursor = INDEX_INVALID; + })); + /* clang-format on */ + } +} + +static void +send_snort_client_details (const snort_client_t *client, + vl_api_registration_t *rp, u32 context) +{ + snort_main_t *sm = snort_get_main (); + vl_api_snort_client_details_t *rmp; + snort_instance_t *instance; + + if (client->instance_index == ~0) + { + return; + } + + instance = pool_elt_at_index (sm->instances, client->instance_index); + if (instance) + { + REPLY_MACRO_DETAILS4 (VL_API_SNORT_CLIENT_DETAILS, rp, context, ({ + rmp->instance_index = + clib_host_to_net_u32 (client->instance_index); + rmp->client_index = + clib_host_to_net_u32 (client - sm->clients); + })); + } +} + +static void +vl_api_snort_client_get_t_handler (vl_api_snort_client_get_t *mp) +{ + snort_main_t *sm = snort_get_main (); + snort_client_t *client; + vl_api_snort_client_get_reply_t *rmp; + u32 client_index; + int rv = 0; + + client_index = clib_net_to_host_u32 (mp->snort_client_index); + + if (client_index == INDEX_INVALID) + { + /* clang-format off */ + REPLY_AND_DETAILS_MACRO ( + VL_API_SNORT_CLIENT_GET_REPLY, sm->clients, ({ + client = pool_elt_at_index (sm->clients, cursor); + send_snort_client_details (client, rp, mp->context); + })); + /* clang-format on */ + } + else + { + client = pool_elt_at_index (sm->clients, client_index); + + if (client) + { + vl_api_registration_t *rp = + vl_api_client_index_to_registration (mp->client_index); + + if (rp == NULL) + { + return; + } + + send_snort_client_details (client, rp, mp->context); + } + else + { + rv = VNET_API_ERROR_NO_SUCH_ENTRY; + } + + /* clang-format off */ + REPLY_MACRO2 (VL_API_SNORT_CLIENT_GET_REPLY, ({ + rmp->cursor = INDEX_INVALID; + })); + /* clang-format on */ + } +} + +static void +vl_api_snort_client_disconnect_t_handler (vl_api_snort_client_disconnect_t *mp) +{ + vlib_main_t *vm = vlib_get_main (); + snort_main_t *sm = snort_get_main (); + snort_client_t *client; + vl_api_snort_client_disconnect_reply_t *rmp; + u32 client_index = clib_net_to_host_u32 (mp->snort_client_index); + int rv = 0; + + if (pool_is_free_index (sm->clients, client_index)) + { + rv = VNET_API_ERROR_NO_SUCH_ENTRY; + } + else + { + client = pool_elt_at_index (sm->clients, client_index); + rv = snort_instance_disconnect (vm, client->instance_index); + } + + REPLY_MACRO (VL_API_SNORT_CLIENT_DISCONNECT_REPLY); +} + +static void +vl_api_snort_instance_disconnect_t_handler ( + vl_api_snort_instance_disconnect_t *mp) +{ + vlib_main_t *vm = vlib_get_main (); + vl_api_snort_instance_disconnect_reply_t *rmp; + u32 instance_index = clib_net_to_host_u32 (mp->instance_index); + int rv = snort_instance_disconnect (vm, instance_index); + + REPLY_MACRO (VL_API_SNORT_INSTANCE_DISCONNECT_REPLY); +} + +static void +vl_api_snort_interface_detach_t_handler (vl_api_snort_interface_detach_t *mp) +{ + vlib_main_t *vm = vlib_get_main (); + vl_api_snort_interface_detach_reply_t *rmp; + u32 sw_if_index = clib_net_to_host_u32 (mp->sw_if_index); + int rv; + + rv = snort_interface_enable_disable (vm, NULL, sw_if_index, + 0 /* is_enable */, 0); + + REPLY_MACRO (VL_API_SNORT_INTERFACE_DETACH_REPLY); +} + +static void +vl_api_snort_input_mode_get_t_handler (vl_api_snort_input_mode_get_t *mp) +{ + snort_main_t *sm = &snort_main; + vl_api_snort_input_mode_get_reply_t *rmp; + int rv = 0; + + REPLY_MACRO2 (VL_API_SNORT_INPUT_MODE_GET_REPLY, ({ + rmp->snort_mode = clib_host_to_net_u32 (sm->input_mode); + })); +} + +static void +vl_api_snort_input_mode_set_t_handler (vl_api_snort_input_mode_set_t *mp) +{ + vlib_main_t *vm = vlib_get_main (); + vl_api_snort_input_mode_set_reply_t *rmp; + u8 mode = mp->input_mode; + int rv = 0; + + if (mode != VLIB_NODE_STATE_INTERRUPT && mode != VLIB_NODE_STATE_POLLING) + { + clib_error_return (0, "invalid input mode %u", mode); + } + snort_set_node_mode (vm, mode); + + REPLY_MACRO (VL_API_SNORT_INPUT_MODE_SET_REPLY); +} + +/* API definitions */ +#include <snort/snort.api.c> + +clib_error_t * +snort_init_api (vlib_main_t *vm) +{ + /* Add our API messages to the global name_crc hash table */ + snort_base_msg_id = setup_message_id_table (); + + return NULL; +} + +VLIB_INIT_FUNCTION (snort_init_api); diff --git a/src/plugins/tlsmbedtls/tls_mbedtls.c b/src/plugins/tlsmbedtls/tls_mbedtls.c index af04f1adeb0..2f4757e28a1 100644 --- a/src/plugins/tlsmbedtls/tls_mbedtls.c +++ b/src/plugins/tlsmbedtls/tls_mbedtls.c @@ -396,6 +396,8 @@ mbedtls_ctx_handshake_rx (tls_ctx_t * ctx) if (mc->ssl.state != MBEDTLS_SSL_HANDSHAKE_OVER) return 0; + ctx->flags |= TLS_CONN_F_HS_DONE; + /* * Handshake complete */ @@ -532,17 +534,10 @@ mbedtls_ctx_read (tls_ctx_t * ctx, session_t * tls_session) return enq; } -static u8 -mbedtls_handshake_is_over (tls_ctx_t * ctx) -{ - mbedtls_ctx_t *mc = (mbedtls_ctx_t *) ctx; - return (mc->ssl.state == MBEDTLS_SSL_HANDSHAKE_OVER); -} - static int mbedtls_transport_close (tls_ctx_t * ctx) { - if (!mbedtls_handshake_is_over (ctx)) + if (!(ctx->flags & TLS_CONN_F_HS_DONE)) { session_close (session_get_from_handle (ctx->tls_session_handle)); return 0; @@ -554,7 +549,7 @@ mbedtls_transport_close (tls_ctx_t * ctx) static int mbedtls_transport_reset (tls_ctx_t *ctx) { - if (!mbedtls_handshake_is_over (ctx)) + if (!(ctx->flags & TLS_CONN_F_HS_DONE)) { session_close (session_get_from_handle (ctx->tls_session_handle)); return 0; @@ -590,7 +585,6 @@ const static tls_engine_vft_t mbedtls_engine = { .ctx_init_client = mbedtls_ctx_init_client, .ctx_write = mbedtls_ctx_write, .ctx_read = mbedtls_ctx_read, - .ctx_handshake_is_over = mbedtls_handshake_is_over, .ctx_start_listen = mbedtls_start_listen, .ctx_stop_listen = mbedtls_stop_listen, .ctx_transport_close = mbedtls_transport_close, diff --git a/src/plugins/tlsopenssl/tls_openssl.c b/src/plugins/tlsopenssl/tls_openssl.c index 5d172a0adcf..c8e685f20c5 100644 --- a/src/plugins/tlsopenssl/tls_openssl.c +++ b/src/plugins/tlsopenssl/tls_openssl.c @@ -1037,15 +1037,6 @@ openssl_ctx_init_server (tls_ctx_t * ctx) return 0; } -static u8 -openssl_handshake_is_over (tls_ctx_t * ctx) -{ - openssl_ctx_t *mc = (openssl_ctx_t *) ctx; - if (!mc->ssl) - return 0; - return SSL_is_init_finished (mc->ssl); -} - static int openssl_transport_close (tls_ctx_t * ctx) { @@ -1054,7 +1045,7 @@ openssl_transport_close (tls_ctx_t * ctx) return 0; #endif - if (!openssl_handshake_is_over (ctx)) + if (!(ctx->flags & TLS_CONN_F_HS_DONE)) { openssl_handle_handshake_failure (ctx); return 0; @@ -1066,7 +1057,7 @@ openssl_transport_close (tls_ctx_t * ctx) static int openssl_transport_reset (tls_ctx_t *ctx) { - if (!openssl_handshake_is_over (ctx)) + if (!(ctx->flags & TLS_CONN_F_HS_DONE)) { openssl_handle_handshake_failure (ctx); return 0; @@ -1166,7 +1157,6 @@ const static tls_engine_vft_t openssl_engine = { .ctx_init_client = openssl_ctx_init_client, .ctx_write = openssl_ctx_write, .ctx_read = openssl_ctx_read, - .ctx_handshake_is_over = openssl_handshake_is_over, .ctx_start_listen = openssl_start_listen, .ctx_stop_listen = openssl_stop_listen, .ctx_transport_close = openssl_transport_close, @@ -1286,7 +1276,10 @@ tls_openssl_set_command_fn (vlib_main_t * vm, unformat_input_t * input, } else { - vnet_session_enable_disable (vm, 1); + session_enable_disable_args_t args = { .is_en = 1, + .rt_engine_type = + RT_BACKEND_ENGINE_RULE_TABLE }; + vnet_session_enable_disable (vm, &args); if (openssl_engine_register (engine_name, engine_alg, async) < 0) { return clib_error_return (0, "Failed to register %s polling", diff --git a/src/plugins/tlspicotls/tls_picotls.c b/src/plugins/tlspicotls/tls_picotls.c index 7375b928206..9459cb776b5 100644 --- a/src/plugins/tlspicotls/tls_picotls.c +++ b/src/plugins/tlspicotls/tls_picotls.c @@ -88,14 +88,6 @@ picotls_lctx_get (u32 lctx_index) return pool_elt_at_index (picotls_main.lctx_pool, lctx_index); } -static u8 -picotls_handshake_is_over (tls_ctx_t * ctx) -{ - picotls_ctx_t *ptls_ctx = (picotls_ctx_t *) ctx; - assert (ptls_ctx->tls); - return ptls_handshake_is_complete (ptls_ctx->tls); -} - static int picotls_try_handshake_write (picotls_ctx_t * ptls_ctx, session_t * tls_session, ptls_buffer_t * buf) @@ -194,7 +186,7 @@ picotls_confirm_app_close (tls_ctx_t * ctx) static int picotls_transport_close (tls_ctx_t * ctx) { - if (!picotls_handshake_is_over (ctx)) + if (!(ctx->flags & TLS_CONN_F_HS_DONE)) { picotls_handle_handshake_failure (ctx); return 0; @@ -206,7 +198,7 @@ picotls_transport_close (tls_ctx_t * ctx) static int picotls_transport_reset (tls_ctx_t *ctx) { - if (!picotls_handshake_is_over (ctx)) + if (!(ctx->flags & TLS_CONN_F_HS_DONE)) { picotls_handle_handshake_failure (ctx); return 0; @@ -435,7 +427,7 @@ picotls_ctx_read (tls_ctx_t *ctx, session_t *tcp_session) if (PREDICT_FALSE (!ptls_handshake_is_complete (ptls_ctx->tls))) { picotls_do_handshake (ptls_ctx, tcp_session); - if (picotls_handshake_is_over (ctx)) + if (ctx->flags & TLS_CONN_F_HS_DONE) { if (ptls_is_server (ptls_ctx->tls)) { @@ -750,7 +742,6 @@ const static tls_engine_vft_t picotls_engine = { .ctx_free = picotls_ctx_free, .ctx_get = picotls_ctx_get, .ctx_get_w_thread = picotls_ctx_get_w_thread, - .ctx_handshake_is_over = picotls_handshake_is_over, .ctx_start_listen = picotls_start_listen, .ctx_stop_listen = picotls_stop_listen, .ctx_init_server = picotls_ctx_init_server, diff --git a/src/plugins/unittest/fib_test.c b/src/plugins/unittest/fib_test.c index fbac809d726..491d135322c 100644 --- a/src/plugins/unittest/fib_test.c +++ b/src/plugins/unittest/fib_test.c @@ -10264,7 +10264,57 @@ fib_test_inherit (void) &l99_o_10_10_10_3), "%U via interposer label", format_fib_prefix,&pfx_10_10_10_21_s_32); + fib_table_entry_special_remove(0, + &pfx_10_10_10_0_s_24, + FIB_SOURCE_SPECIAL); + + const ip46_address_t nh_0_0_0_0 = { + .ip4.as_u32 = clib_host_to_net_u32(0x00000000), + }; + const fib_prefix_t pfx_0_0_0_0_s_0 = { + .fp_len = 0, + .fp_proto = FIB_PROTOCOL_IP4, + .fp_addr = nh_0_0_0_0, + }; + /* we have prio(API) < prio(hi_src) < prio(SPECIAL) */ + /* Add/remove an interposer source from the top of the subtrie. The + * interposer source is inherited. + */ + fib_table_entry_special_dpo_add(0, + &pfx_0_0_0_0_s_0, + hi_src, + (FIB_ENTRY_FLAG_COVERED_INHERIT | + FIB_ENTRY_FLAG_INTERPOSE), + &interposer); + /* + * Add/remove an interposer source from the top of the subtrie. The + * interposer source is inherited, the previous inheritance is discarded. + */ + fib_table_entry_special_dpo_add(0, + &pfx_10_10_10_0_s_24, + FIB_SOURCE_SPECIAL, + (FIB_ENTRY_FLAG_COVERED_INHERIT | + FIB_ENTRY_FLAG_INTERPOSE), + &interposer); + /* force a tree walk */ + fib_table_entry_update_one_path(0, + &pfx_0_0_0_0_s_0, + FIB_SOURCE_API, + FIB_ENTRY_FLAG_NONE, + DPO_PROTO_IP4, + &nh_10_10_10_3, + tm->hw[0]->sw_if_index, + ~0, + 1, + NULL, + FIB_ROUTE_PATH_FLAG_NONE); + fib_table_entry_special_remove(0, + &pfx_10_10_10_0_s_24, + FIB_SOURCE_SPECIAL); + fib_table_entry_special_remove(0, + &pfx_0_0_0_0_s_0, + hi_src); /* * cleanup */ @@ -10275,6 +10325,7 @@ fib_test_inherit (void) fib_table_entry_delete(0, &pfx_10_10_10_0_s_24, FIB_SOURCE_API); fib_table_entry_delete(0, &pfx_10_10_0_0_s_16, FIB_SOURCE_API); fib_table_entry_delete(0, &pfx_10_10_10_0_s_24, FIB_SOURCE_SPECIAL); + fib_table_entry_delete(0, &pfx_0_0_0_0_s_0, FIB_SOURCE_API); adj_unlock(ai_10_10_10_1); adj_unlock(ai_10_10_10_2); adj_unlock(ai_10_10_10_3); diff --git a/src/plugins/unittest/segment_manager_test.c b/src/plugins/unittest/segment_manager_test.c index a106470ee48..29da662e007 100644 --- a/src/plugins/unittest/segment_manager_test.c +++ b/src/plugins/unittest/segment_manager_test.c @@ -739,8 +739,11 @@ segment_manager_test (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd_arg) { int res = 0; + session_enable_disable_args_t args = { .is_en = 1, + .rt_engine_type = + RT_BACKEND_ENGINE_RULE_TABLE }; - vnet_session_enable_disable (vm, 1); + vnet_session_enable_disable (vm, &args); while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { diff --git a/src/plugins/unittest/session_test.c b/src/plugins/unittest/session_test.c index b7627acc129..a70a6ea88bf 100644 --- a/src/plugins/unittest/session_test.c +++ b/src/plugins/unittest/session_test.c @@ -13,13 +13,11 @@ * limitations under the License. */ -#include <vnet/session/application_namespace.h> -#include <vnet/session/application_interface.h> +#include <arpa/inet.h> #include <vnet/session/application.h> #include <vnet/session/session.h> -#include <vnet/session/session_rules_table.h> -#include <vnet/tcp/tcp.h> #include <sys/epoll.h> +#include <vnet/session/session_rules_table.h> #define SESSION_TEST_I(_cond, _comment, _args...) \ ({ \ @@ -133,7 +131,8 @@ session_create_lookpback (u32 table_id, u32 * sw_if_index, if (table_id != 0) { - ip_table_create (FIB_PROTOCOL_IP4, table_id, 0, 0); + ip_table_create (FIB_PROTOCOL_IP4, table_id, 0 /* is_api */, + 1 /* create_mfib */, 0); ip_table_bind (FIB_PROTOCOL_IP4, *sw_if_index, table_id); } @@ -774,10 +773,37 @@ session_test_namespace (vlib_main_t * vm, unformat_input_t * input) return 0; } +static void +session_test_disable_rt_backend_engine (vlib_main_t *vm) +{ + session_enable_disable_args_t args = { .is_en = 0, + .rt_engine_type = + RT_BACKEND_ENGINE_DISABLE }; + vnet_session_enable_disable (vm, &args); +} + +static void +session_test_enable_rule_table_engine (vlib_main_t *vm) +{ + session_enable_disable_args_t args = { .is_en = 1, + .rt_engine_type = + RT_BACKEND_ENGINE_RULE_TABLE }; + vnet_session_enable_disable (vm, &args); +} + +static void +session_test_enable_sdl_engine (vlib_main_t *vm) +{ + session_enable_disable_args_t args = { .is_en = 1, + .rt_engine_type = + RT_BACKEND_ENGINE_SDL }; + vnet_session_enable_disable (vm, &args); +} + static int session_test_rule_table (vlib_main_t * vm, unformat_input_t * input) { - session_rules_table_t _srt, *srt = &_srt; + session_table_t *st = session_table_alloc (); u16 lcl_port = 1234, rmt_port = 4321; u32 action_index = 1, res; ip4_address_t lcl_lkup, rmt_lkup; @@ -795,8 +821,11 @@ session_test_rule_table (vlib_main_t * vm, unformat_input_t * input) } } - clib_memset (srt, 0, sizeof (*srt)); - session_rules_table_init (srt); + session_test_disable_rt_backend_engine (vm); + session_test_enable_rule_table_engine (vm); + + session_table_init (st, FIB_PROTOCOL_MAX); + session_rules_table_init (st, FIB_PROTOCOL_MAX); ip4_address_t lcl_ip = { .as_u32 = clib_host_to_net_u32 (0x01020304), @@ -835,12 +864,13 @@ session_test_rule_table (vlib_main_t * vm, unformat_input_t * input) .action_index = action_index++, .is_add = 1, }; - error = session_rules_table_add_del (srt, &args); + error = + session_rules_table_add_del (st->srtg_handle, TRANSPORT_PROTO_TCP, &args); SESSION_TEST ((error == 0), "Add 1.2.3.4/16 1234 5.6.7.8/16 4321 action %d", action_index - 1); - res = - session_rules_table_lookup4 (srt, &lcl_ip, &rmt_ip, lcl_port, rmt_port); + res = session_rules_table_lookup4 (st->srtg_handle, TRANSPORT_PROTO_TCP, + &lcl_ip, &rmt_ip, lcl_port, rmt_port); SESSION_TEST ((res == 1), "Lookup 1.2.3.4 1234 5.6.7.8 4321, action should " "be 1: %d", res); @@ -851,13 +881,15 @@ session_test_rule_table (vlib_main_t * vm, unformat_input_t * input) args.lcl.fp_addr.ip4 = lcl_ip; args.lcl.fp_len = 24; args.action_index = action_index++; - error = session_rules_table_add_del (srt, &args); + error = + session_rules_table_add_del (st->srtg_handle, TRANSPORT_PROTO_TCP, &args); SESSION_TEST ((error == 0), "Add 1.2.3.4/24 1234 5.6.7.8/16 4321 action %d", action_index - 1); args.rmt.fp_addr.ip4 = rmt_ip; args.rmt.fp_len = 24; args.action_index = action_index++; - error = session_rules_table_add_del (srt, &args); + error = + session_rules_table_add_del (st->srtg_handle, TRANSPORT_PROTO_TCP, &args); SESSION_TEST ((error == 0), "Add 1.2.3.4/24 1234 5.6.7.8/24 4321 action %d", action_index - 1); @@ -869,13 +901,15 @@ session_test_rule_table (vlib_main_t * vm, unformat_input_t * input) args.rmt.fp_addr.ip4 = rmt_ip2; args.rmt.fp_len = 16; args.action_index = action_index++; - error = session_rules_table_add_del (srt, &args); + error = + session_rules_table_add_del (st->srtg_handle, TRANSPORT_PROTO_TCP, &args); SESSION_TEST ((error == 0), "Add 2.2.2.2/24 1234 6.6.6.6/16 4321 action %d", action_index - 1); args.lcl.fp_addr.ip4 = lcl_ip3; args.rmt.fp_addr.ip4 = rmt_ip3; args.action_index = action_index++; - error = session_rules_table_add_del (srt, &args); + error = + session_rules_table_add_del (st->srtg_handle, TRANSPORT_PROTO_TCP, &args); SESSION_TEST ((error == 0), "Add 3.3.3.3/24 1234 7.7.7.7/16 4321 action %d", action_index - 1); @@ -885,7 +919,8 @@ session_test_rule_table (vlib_main_t * vm, unformat_input_t * input) args.lcl.fp_addr.ip4 = lcl_ip3; args.rmt.fp_addr.ip4 = rmt_ip3; args.action_index = action_index++; - error = session_rules_table_add_del (srt, &args); + error = + session_rules_table_add_del (st->srtg_handle, TRANSPORT_PROTO_TCP, &args); SESSION_TEST ((error == 0), "overwrite 3.3.3.3/24 1234 7.7.7.7/16 4321 " "action %d", action_index - 1); @@ -893,23 +928,22 @@ session_test_rule_table (vlib_main_t * vm, unformat_input_t * input) * Lookup 1.2.3.4/32 1234 5.6.7.8/32 4321, 1.2.2.4/32 1234 5.6.7.9/32 4321 * and 3.3.3.3 1234 7.7.7.7 4321 */ - res = - session_rules_table_lookup4 (srt, &lcl_ip, &rmt_ip, lcl_port, rmt_port); + res = session_rules_table_lookup4 (st->srtg_handle, TRANSPORT_PROTO_TCP, + &lcl_ip, &rmt_ip, lcl_port, rmt_port); SESSION_TEST ((res == 3), "Lookup 1.2.3.4 1234 5.6.7.8 4321 action " "should be 3: %d", res); lcl_lkup.as_u32 = clib_host_to_net_u32 (0x01020204); rmt_lkup.as_u32 = clib_host_to_net_u32 (0x05060709); - res = - session_rules_table_lookup4 (srt, &lcl_lkup, - &rmt_lkup, lcl_port, rmt_port); + res = session_rules_table_lookup4 (st->srtg_handle, TRANSPORT_PROTO_TCP, + &lcl_lkup, &rmt_lkup, lcl_port, rmt_port); SESSION_TEST ((res == 1), "Lookup 1.2.2.4 1234 5.6.7.9 4321, action " "should be 1: %d", res); - res = - session_rules_table_lookup4 (srt, &lcl_ip3, &rmt_ip3, lcl_port, rmt_port); + res = session_rules_table_lookup4 (st->srtg_handle, TRANSPORT_PROTO_TCP, + &lcl_ip3, &rmt_ip3, lcl_port, rmt_port); SESSION_TEST ((res == 6), "Lookup 3.3.3.3 1234 7.7.7.7 4321, action " "should be 6 (updated): %d", res); @@ -925,17 +959,17 @@ session_test_rule_table (vlib_main_t * vm, unformat_input_t * input) args.lcl_port = 0; args.rmt_port = 0; args.action_index = action_index++; - error = session_rules_table_add_del (srt, &args); + error = + session_rules_table_add_del (st->srtg_handle, TRANSPORT_PROTO_TCP, &args); SESSION_TEST ((error == 0), "Add 1.2.3.4/24 * 5.6.7.8/24 * action %d", action_index - 1); - res = - session_rules_table_lookup4 (srt, &lcl_ip, &rmt_ip, lcl_port, rmt_port); + res = session_rules_table_lookup4 (st->srtg_handle, TRANSPORT_PROTO_TCP, + &lcl_ip, &rmt_ip, lcl_port, rmt_port); SESSION_TEST ((res == 7), "Lookup 1.2.3.4 1234 5.6.7.8 4321, action should" " be 7 (lpm dst): %d", res); - res = - session_rules_table_lookup4 (srt, &lcl_ip, &rmt_ip, - lcl_port + 1, rmt_port); + res = session_rules_table_lookup4 (st->srtg_handle, TRANSPORT_PROTO_TCP, + &lcl_ip, &rmt_ip, lcl_port + 1, rmt_port); SESSION_TEST ((res == 7), "Lookup 1.2.3.4 1235 5.6.7.8 4321, action should " "be 7: %d", res); @@ -947,7 +981,8 @@ session_test_rule_table (vlib_main_t * vm, unformat_input_t * input) * 1.2.3.4 1235 5.6.7.8 4322 */ args.is_add = 0; - error = session_rules_table_add_del (srt, &args); + error = + session_rules_table_add_del (st->srtg_handle, TRANSPORT_PROTO_TCP, &args); SESSION_TEST ((error == 0), "Del 1.2.3.4/24 * 5.6.7.8/24 *"); args.lcl.fp_addr.ip4 = lcl_ip; @@ -958,7 +993,8 @@ session_test_rule_table (vlib_main_t * vm, unformat_input_t * input) args.rmt_port = 0; args.action_index = action_index++; args.is_add = 1; - error = session_rules_table_add_del (srt, &args); + error = + session_rules_table_add_del (st->srtg_handle, TRANSPORT_PROTO_TCP, &args); SESSION_TEST ((error == 0), "Add 1.2.3.4/16 * 5.6.7.8/16 * action %d", action_index - 1); @@ -970,27 +1006,28 @@ session_test_rule_table (vlib_main_t * vm, unformat_input_t * input) args.rmt_port = rmt_port; args.action_index = action_index++; args.is_add = 1; - error = session_rules_table_add_del (srt, &args); + error = + session_rules_table_add_del (st->srtg_handle, TRANSPORT_PROTO_TCP, &args); SESSION_TEST ((error == 0), "Add 1.2.3.4/24 1235 5.6.7.8/24 4321 action %d", action_index - 1); if (verbose) - session_rules_table_cli_dump (vm, srt, FIB_PROTOCOL_IP4); + session_rules_table_cli_dump (vm, st->srtg_handle, TRANSPORT_PROTO_TCP, + FIB_PROTOCOL_IP4); - res = - session_rules_table_lookup4 (srt, &lcl_ip, &rmt_ip, lcl_port, rmt_port); + res = session_rules_table_lookup4 (st->srtg_handle, TRANSPORT_PROTO_TCP, + &lcl_ip, &rmt_ip, lcl_port, rmt_port); SESSION_TEST ((res == 3), "Lookup 1.2.3.4 1234 5.6.7.8 4321, action should " "be 3: %d", res); - res = - session_rules_table_lookup4 (srt, &lcl_ip, &rmt_ip, - lcl_port + 1, rmt_port); + res = session_rules_table_lookup4 (st->srtg_handle, TRANSPORT_PROTO_TCP, + &lcl_ip, &rmt_ip, lcl_port + 1, rmt_port); SESSION_TEST ((res == 9), "Lookup 1.2.3.4 1235 5.6.7.8 4321, action should " "be 9: %d", res); res = - session_rules_table_lookup4 (srt, &lcl_ip, &rmt_ip, - lcl_port + 1, rmt_port + 1); + session_rules_table_lookup4 (st->srtg_handle, TRANSPORT_PROTO_TCP, &lcl_ip, + &rmt_ip, lcl_port + 1, rmt_port + 1); SESSION_TEST ((res == 8), "Lookup 1.2.3.4 1235 5.6.7.8 4322, action should " "be 8: %d", res); @@ -1004,10 +1041,11 @@ session_test_rule_table (vlib_main_t * vm, unformat_input_t * input) args.lcl.fp_len = 16; args.rmt.fp_len = 16; args.is_add = 0; - error = session_rules_table_add_del (srt, &args); + error = + session_rules_table_add_del (st->srtg_handle, TRANSPORT_PROTO_TCP, &args); SESSION_TEST ((error == 0), "Del 1.2.0.0/16 1234 5.6.0.0/16 4321"); - res = - session_rules_table_lookup4 (srt, &lcl_ip, &rmt_ip, lcl_port, rmt_port); + res = session_rules_table_lookup4 (st->srtg_handle, TRANSPORT_PROTO_TCP, + &lcl_ip, &rmt_ip, lcl_port, rmt_port); SESSION_TEST ((res == 3), "Lookup 1.2.3.4 1234 5.6.7.8 4321, action should " "be 3: %d", res); @@ -1015,10 +1053,11 @@ session_test_rule_table (vlib_main_t * vm, unformat_input_t * input) args.lcl_port = 0; args.rmt_port = 0; args.is_add = 0; - error = session_rules_table_add_del (srt, &args); + error = + session_rules_table_add_del (st->srtg_handle, TRANSPORT_PROTO_TCP, &args); SESSION_TEST ((error == 0), "Del 1.2.0.0/16 * 5.6.0.0/16 *"); - res = - session_rules_table_lookup4 (srt, &lcl_ip, &rmt_ip, lcl_port, rmt_port); + res = session_rules_table_lookup4 (st->srtg_handle, TRANSPORT_PROTO_TCP, + &lcl_ip, &rmt_ip, lcl_port, rmt_port); SESSION_TEST ((res == 3), "Lookup 1.2.3.4 1234 5.6.7.8 4321, action should " "be 3: %d", res); @@ -1033,12 +1072,15 @@ session_test_rule_table (vlib_main_t * vm, unformat_input_t * input) args.lcl_port = 1234; args.rmt_port = 4321; args.is_add = 0; - error = session_rules_table_add_del (srt, &args); + error = + session_rules_table_add_del (st->srtg_handle, TRANSPORT_PROTO_TCP, &args); SESSION_TEST ((error == 0), "Del 1.2.3.4/24 1234 5.6.7.5/24"); - res = - session_rules_table_lookup4 (srt, &lcl_ip, &rmt_ip, lcl_port, rmt_port); + res = session_rules_table_lookup4 (st->srtg_handle, TRANSPORT_PROTO_TCP, + &lcl_ip, &rmt_ip, lcl_port, rmt_port); SESSION_TEST ((res == 2), "Action should be 2: %d", res); + session_table_free (st, FIB_PROTOCOL_MAX); + return 0; } @@ -1074,6 +1116,9 @@ session_test_rules (vlib_main_t * vm, unformat_input_t * input) } } + session_test_disable_rt_backend_engine (vm); + session_test_enable_rule_table_engine (vm); + server_sep.is_ip4 = 1; server_sep.port = placeholder_port; clib_memset (options, 0, sizeof (options)); @@ -2073,13 +2118,284 @@ session_test_mq_basic (vlib_main_t * vm, unformat_input_t * input) return 0; } +static f32 +session_get_memory_usage (void) +{ + clib_mem_heap_t *heap = clib_mem_get_per_cpu_heap (); + u8 *s = 0; + char *ss; + f32 used = 0.0; + + s = format (s, "%U\n", format_clib_mem_heap, heap, 0); + ss = strstr ((char *) s, "used:"); + if (ss) + sscanf (ss, "used: %f", &used); + else + clib_warning ("substring 'used:' not found from show memory"); + vec_free (s); + return (used); +} + +static int +session_test_enable_disable (vlib_main_t *vm, unformat_input_t *input) +{ + u32 iteration = 100, i; + uword was_enabled; + f32 was_using, now_using; + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "repeat %d", &iteration)) + ; + else + { + vlib_cli_output (vm, "parse error: '%U'", format_unformat_error, + input); + return -1; + } + } + + was_enabled = clib_mem_trace_enable_disable (0); + /* warm up */ + for (i = 0; i < 10; i++) + { + session_test_disable_rt_backend_engine (vm); + session_test_enable_sdl_engine (vm); + session_test_disable_rt_backend_engine (vm); + session_test_enable_rule_table_engine (vm); + } + was_using = session_get_memory_usage (); + + for (i = 0; i < iteration; i++) + { + session_test_disable_rt_backend_engine (vm); + session_test_enable_sdl_engine (vm); + session_test_disable_rt_backend_engine (vm); + session_test_enable_rule_table_engine (vm); + } + now_using = session_get_memory_usage (); + + clib_mem_trace_enable_disable (was_enabled); + SESSION_TEST ((was_using == now_using), "was using %.2fM, now using %.2fM", + was_using, now_using); + + return 0; +} + +static int +session_test_sdl (vlib_main_t *vm, unformat_input_t *input) +{ + session_table_t *st = session_table_alloc (); + u16 lcl_port = 0, rmt_port = 0; + u32 action_index = 1, res; + int verbose = 0, error; + ip4_address_t lcl_ip; + const char ip_str_1234[] = "1.2.3.4"; + inet_pton (AF_INET, ip_str_1234, &lcl_ip); + ip4_address_t rmt_ip = { + .as_u32 = clib_host_to_net_u32 (0x0), + }; + ip6_address_t rmt_ip6 = { + .as_u64 = { 0, 0 }, + }; + fib_prefix_t lcl_pref = { + .fp_addr.ip4.as_u32 = lcl_ip.as_u32, + .fp_len = 16, + .fp_proto = FIB_PROTOCOL_IP4, + }; + fib_prefix_t rmt_pref = { + .fp_addr.ip4.as_u32 = rmt_ip.as_u32, + .fp_len = 0, + .fp_proto = 0, + }; + session_rule_table_add_del_args_t args = { + .lcl = lcl_pref, + .rmt = rmt_pref, + .lcl_port = lcl_port, + .rmt_port = rmt_port, + .action_index = action_index++, + .is_add = 1, + }; + const char ip_str_1200[] = "1.2.0.0"; + const char ip_str_1230[] = "1.2.3.0"; + const char ip_str_1111[] = "1.1.1.1"; + const char ip6_str[] = "2501:0db8:85a3:0000:0000:8a2e:0371:1"; + const char ip6_str2[] = "2501:0db8:85a3:0000:0000:8a2e:0372:1"; + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "verbose")) + verbose = 1; + else + { + vlib_cli_output (vm, "parse error: '%U'", format_unformat_error, + input); + return -1; + } + } + + session_test_disable_rt_backend_engine (vm); + session_test_enable_sdl_engine (vm); + + session_table_init (st, FIB_PROTOCOL_MAX); + session_rules_table_init (st, FIB_PROTOCOL_MAX); + + /* Add 1.2.0.0/16 */ + args.lcl.fp_len = 16; + inet_pton (AF_INET, ip_str_1200, &args.lcl.fp_addr.ip4.as_u32); + error = + session_rules_table_add_del (st->srtg_handle, TRANSPORT_PROTO_TCP, &args); + SESSION_TEST ((error == 0), "Add %s/%d action %d", ip_str_1200, + args.lcl.fp_len, action_index - 1); + + /* Lookup 1.2.3.4 */ + res = session_rules_table_lookup4 (st->srtg_handle, TRANSPORT_PROTO_TCP, + &lcl_ip, &rmt_ip, lcl_port, rmt_port); + SESSION_TEST ((res == action_index - 1), + "Lookup %s, action should " + "be 1: %d", + ip_str_1234, action_index - 1); + + /* + * Add 1.2.3.0/24 + */ + args.lcl.fp_len = 24; + inet_pton (AF_INET, ip_str_1230, &args.lcl.fp_addr.ip4.as_u32); + args.action_index = action_index++; + error = + session_rules_table_add_del (st->srtg_handle, TRANSPORT_PROTO_TCP, &args); + SESSION_TEST ((error == 0), "Add %s/%d action %d", ip_str_1230, + args.lcl.fp_len, action_index - 1); + + /* Lookup 1.2.3.4 */ + res = session_rules_table_lookup4 (st->srtg_handle, TRANSPORT_PROTO_TCP, + &lcl_ip, &rmt_ip, lcl_port, rmt_port); + SESSION_TEST ((res == action_index - 1), + "Lookup %s, action should " + "be 2: %d", + ip_str_1234, action_index - 1); + + /* look up 1.1.1.1, should be -1 (invalid index) */ + inet_pton (AF_INET, ip_str_1111, &lcl_ip); + res = session_rules_table_lookup4 (st->srtg_handle, TRANSPORT_PROTO_TCP, + &lcl_ip, &rmt_ip, lcl_port, rmt_port); + SESSION_TEST ((res == SESSION_TABLE_INVALID_INDEX), + "Lookup %s, action should " + "be -1: %d", + ip_str_1111, res); + + /* Add again 1.2.0.0/16, should be rejected */ + args.lcl.fp_len = 16; + inet_pton (AF_INET, ip_str_1200, &args.lcl.fp_addr.ip4.as_u32); + error = + session_rules_table_add_del (st->srtg_handle, TRANSPORT_PROTO_TCP, &args); + SESSION_TEST ((error == SESSION_E_IPINUSE), "Add %s/%d action %d", + ip_str_1200, args.lcl.fp_len, error); + /* + * Add 0.0.0.0/0, should get an error + */ + args.lcl.fp_len = 0; + args.lcl.fp_addr.ip4.as_u32 = 0; + error = + session_rules_table_add_del (st->srtg_handle, TRANSPORT_PROTO_TCP, &args); + SESSION_TEST ((error == SESSION_E_IPINUSE), "Add 0.0.0.0/%d action %d", + args.lcl.fp_len, error); + + /* delete 0.0.0.0 should be rejected */ + args.is_add = 0; + error = + session_rules_table_add_del (st->srtg_handle, TRANSPORT_PROTO_TCP, &args); + SESSION_TEST ((error == SESSION_E_NOROUTE), "Del 0.0.0.0/%d action %d", + args.lcl.fp_len, error); + if (verbose) + session_rules_table_cli_dump (vm, st->srtg_handle, TRANSPORT_PROTO_TCP, + FIB_PROTOCOL_IP4); + + /* + * Clean up + * Delete 1.2.0.0/16 + * Delete 1.2.3.0/24 + */ + inet_pton (AF_INET, ip_str_1200, &args.lcl.fp_addr.ip4.as_u32); + args.lcl.fp_len = 16; + args.is_add = 0; + error = + session_rules_table_add_del (st->srtg_handle, TRANSPORT_PROTO_TCP, &args); + SESSION_TEST ((error == 0), "Del %s/%d should 0: %d", ip_str_1200, + args.lcl.fp_len, error); + + inet_pton (AF_INET, ip_str_1230, &args.lcl.fp_addr.ip4.as_u32); + args.lcl.fp_len = 24; + error = + session_rules_table_add_del (st->srtg_handle, TRANSPORT_PROTO_TCP, &args); + SESSION_TEST ((error == 0), "Del %s/%d, should be 0: %d", ip_str_1230, + args.lcl.fp_len, error); + if (verbose) + session_rules_table_cli_dump (vm, st->srtg_handle, TRANSPORT_PROTO_TCP, + FIB_PROTOCOL_IP4); + + /* ip6 tests */ + + /* + * Add ip6 2001:0db8:85a3:0000:0000:8a2e:0371:1/124 + */ + ip6_address_t lcl_lkup; + inet_pton (AF_INET6, ip6_str, &args.lcl.fp_addr.ip6); + args.lcl.fp_len = 124; + args.lcl.fp_proto = FIB_PROTOCOL_IP6; + args.action_index = action_index++; + args.is_add = 1; + error = + session_rules_table_add_del (st->srtg_handle, TRANSPORT_PROTO_TCP, &args); + SESSION_TEST ((error == 0), "Add %s/%d action %d", ip6_str, args.lcl.fp_len, + action_index - 1); + if (verbose) + session_rules_table_cli_dump (vm, st->srtg_handle, TRANSPORT_PROTO_TCP, + FIB_PROTOCOL_IP6); + + /* Lookup 2001:0db8:85a3:0000:0000:8a2e:0371:1 */ + res = session_rules_table_lookup6 (st->srtg_handle, TRANSPORT_PROTO_TCP, + &args.lcl.fp_addr.ip6, &rmt_ip6, lcl_port, + rmt_port); + SESSION_TEST ((res == action_index - 1), + "Lookup %s action should " + "be 3: %d", + ip6_str, action_index - 1); + + /* Lookup 2001:0db8:85a3:0000:0000:8a2e:0372:1 */ + inet_pton (AF_INET6, ip6_str2, &lcl_lkup); + res = session_rules_table_lookup6 (st->srtg_handle, TRANSPORT_PROTO_TCP, + &lcl_lkup, &rmt_ip6, lcl_port, rmt_port); + SESSION_TEST ((res == SESSION_TABLE_INVALID_INDEX), + "Lookup %s action should " + "be -1: %d", + ip6_str2, res); + + /* + * del ip6 2001:0db8:85a3:0000:0000:8a2e:0371:1/124 + */ + args.is_add = 0; + args.lcl.fp_len = 124; + error = + session_rules_table_add_del (st->srtg_handle, TRANSPORT_PROTO_TCP, &args); + SESSION_TEST ((error == 0), "del %s/%d, should be 0: %d", ip6_str, + args.lcl.fp_len, error); + if (verbose) + session_rules_table_cli_dump (vm, st->srtg_handle, TRANSPORT_PROTO_TCP, + FIB_PROTOCOL_IP6); + + session_table_free (st, FIB_PROTOCOL_MAX); + + return 0; +} + static clib_error_t * session_test (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd_arg) { int res = 0; - vnet_session_enable_disable (vm, 1); + session_test_enable_rule_table_engine (vm); while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { @@ -2099,6 +2415,10 @@ session_test (vlib_main_t * vm, res = session_test_mq_speed (vm, input); else if (unformat (input, "mq-basic")) res = session_test_mq_basic (vm, input); + else if (unformat (input, "enable-disable")) + res = session_test_enable_disable (vm, input); + else if (unformat (input, "sdl")) + res = session_test_sdl (vm, input); else if (unformat (input, "all")) { if ((res = session_test_basic (vm, input))) @@ -2117,6 +2437,10 @@ session_test (vlib_main_t * vm, goto done; if ((res = session_test_mq_basic (vm, input))) goto done; + if ((res = session_test_sdl (vm, input))) + goto done; + if ((res = session_test_enable_disable (vm, input))) + goto done; } else break; diff --git a/src/plugins/unittest/tcp_test.c b/src/plugins/unittest/tcp_test.c index 34033a0b622..bd39474ce93 100644 --- a/src/plugins/unittest/tcp_test.c +++ b/src/plugins/unittest/tcp_test.c @@ -1550,8 +1550,11 @@ tcp_test (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd_arg) { int res = 0; + session_enable_disable_args_t args = { .is_en = 1, + .rt_engine_type = + RT_BACKEND_ENGINE_RULE_TABLE }; - vnet_session_enable_disable (vm, 1); + vnet_session_enable_disable (vm, &args); while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { diff --git a/src/plugins/unittest/util_test.c b/src/plugins/unittest/util_test.c index 53384e55494..5b7e30bc21f 100644 --- a/src/plugins/unittest/util_test.c +++ b/src/plugins/unittest/util_test.c @@ -101,6 +101,36 @@ VLIB_CLI_COMMAND (test_hash_command, static) = .function = test_hash_command_fn, }; +static void * +leak_memory_fn (void *args) +{ + u8 *p = 0; + vec_validate (p, 100); + p = 0; + return 0; +} + +static clib_error_t * +test_mem_leak_command_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + /* do memory leak from thread, so no 'unix_cli' in traceback */ + pthread_t thread; + int rv = pthread_create (&thread, NULL, leak_memory_fn, 0); + if (rv) + { + return clib_error_return (0, "pthread_create failed"); + } + + return 0; +} + +VLIB_CLI_COMMAND (test_mem_leak_command, static) = { + .path = "test mem-leak", + .short_help = "leak some memory", + .function = test_mem_leak_command_fn, +}; + /* * fd.io coding-style-patch-verification: ON * diff --git a/src/plugins/urpf/CMakeLists.txt b/src/plugins/urpf/CMakeLists.txt index 2f44e3b2344..f665d30b0bb 100644 --- a/src/plugins/urpf/CMakeLists.txt +++ b/src/plugins/urpf/CMakeLists.txt @@ -22,6 +22,10 @@ add_vpp_plugin(urpf ip4_urpf.c ip6_urpf.c + INSTALL_HEADERS + urpf_dp.h + urpf.h + API_FILES urpf.api ) diff --git a/src/plugins/urpf/urpf.c b/src/plugins/urpf/urpf.c index e5209caafb4..1e7d6c0fb91 100644 --- a/src/plugins/urpf/urpf.c +++ b/src/plugins/urpf/urpf.c @@ -60,7 +60,17 @@ static const char *urpf_feats[N_AF][VLIB_N_DIR][URPF_N_MODES] = urpf_data_t *urpf_cfgs[N_AF][VLIB_N_DIR]; u8 * -format_urpf_mode (u8 * s, va_list * a) +format_urpf_trace (u8 *s, va_list *va) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *); + urpf_trace_t *t = va_arg (*va, urpf_trace_t *); + + return format (s, "uRPF:%d fib:%d", t->urpf, t->fib_index); +} + +__clib_export u8 * +format_urpf_mode (u8 *s, va_list *a) { urpf_mode_t mode = va_arg (*a, int); @@ -76,8 +86,8 @@ format_urpf_mode (u8 * s, va_list * a) return (format (s, "unknown")); } -static uword -unformat_urpf_mode (unformat_input_t * input, va_list * args) +__clib_export uword +unformat_urpf_mode (unformat_input_t *input, va_list *args) { urpf_mode_t *mode = va_arg (*args, urpf_mode_t *); @@ -94,7 +104,16 @@ unformat_urpf_mode (unformat_input_t * input, va_list * args) return 0; } -int +__clib_export int +urpf_feature_enable_disable (ip_address_family_t af, vlib_dir_t dir, + urpf_mode_t mode, u32 sw_if_index, int enable) +{ + return vnet_feature_enable_disable (urpf_feat_arcs[af][dir], + urpf_feats[af][dir][mode], sw_if_index, + enable, 0, 0); +} + +__clib_export int urpf_update (urpf_mode_t mode, u32 sw_if_index, ip_address_family_t af, vlib_dir_t dir, u32 table_id) { diff --git a/src/plugins/urpf/urpf.h b/src/plugins/urpf/urpf.h index 6983a2b440c..a40a25df16b 100644 --- a/src/plugins/urpf/urpf.h +++ b/src/plugins/urpf/urpf.h @@ -32,7 +32,15 @@ typedef enum urpf_mode_t_ #define URPF_N_MODES (URPF_MODE_STRICT+1) -extern u8 *format_urpf_mode (u8 * s, va_list * a); +typedef struct +{ + index_t urpf; + u32 fib_index; +} urpf_trace_t; + +u8 *format_urpf_trace (u8 *s, va_list *va); +u8 *format_urpf_mode (u8 *s, va_list *a); +uword unformat_urpf_mode (unformat_input_t *input, va_list *args); typedef struct { @@ -43,8 +51,8 @@ typedef struct extern urpf_data_t *urpf_cfgs[N_AF][VLIB_N_DIR]; -extern int urpf_update (urpf_mode_t mode, u32 sw_if_index, - ip_address_family_t af, vlib_dir_t dir, u32 table_id); +int urpf_update (urpf_mode_t mode, u32 sw_if_index, ip_address_family_t af, + vlib_dir_t dir, u32 table_id); #endif diff --git a/src/plugins/urpf/urpf_dp.h b/src/plugins/urpf/urpf_dp.h index 816d8b70b90..b17fed7e04b 100644 --- a/src/plugins/urpf/urpf_dp.h +++ b/src/plugins/urpf/urpf_dp.h @@ -53,22 +53,6 @@ * * This file contains the interface unicast source check. */ -typedef struct -{ - index_t urpf; -} urpf_trace_t; - -static u8 * -format_urpf_trace (u8 * s, va_list * va) -{ - CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *); - CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *); - urpf_trace_t *t = va_arg (*va, urpf_trace_t *); - - s = format (s, "uRPF:%d", t->urpf); - - return s; -} #define foreach_urpf_error \ _(DROP, "uRPF Drop") \ @@ -87,10 +71,157 @@ typedef enum URPF_N_NEXT, } urpf_next_t; +static_always_inline u32 +urpf_get_fib_index (vlib_buffer_t *b, ip_address_family_t af, vlib_dir_t dir) +{ + u32 sw_if_index = vnet_buffer (b)->sw_if_index[dir]; + return vec_elt (urpf_cfgs[af][dir], sw_if_index).fib_index; +} + +static_always_inline void +urpf_perform_check_x1 (ip_address_family_t af, vlib_dir_t dir, + urpf_mode_t mode, vlib_buffer_t *b, const u8 *h, + u32 fib_index, load_balance_t **lb, u32 *pass) +{ + load_balance_t *llb; + u32 lpass; + u32 lb_index; + + ASSERT (fib_index != ~0); + + if (AF_IP4 == af) + { + const ip4_header_t *ip; + + ip = (ip4_header_t *) h; + + lb_index = ip4_fib_forwarding_lookup (fib_index, &ip->src_address); + + /* Pass multicast. */ + lpass = (ip4_address_is_multicast (&ip->src_address) || + ip4_address_is_global_broadcast (&ip->src_address)); + } + else + { + const ip6_header_t *ip; + + ip = (ip6_header_t *) h; + + lb_index = ip6_fib_table_fwding_lookup (fib_index, &ip->src_address); + lpass = ip6_address_is_multicast (&ip->src_address); + } + + llb = load_balance_get (lb_index); + + if (URPF_MODE_STRICT == mode) + { + int res; + + res = fib_urpf_check (llb->lb_urpf, vnet_buffer (b)->sw_if_index[dir]); + if (VLIB_RX == dir) + lpass |= res; + else + { + lpass |= !res && fib_urpf_check_size (llb->lb_urpf); + lpass |= b->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED; + } + } + else + lpass |= fib_urpf_check_size (llb->lb_urpf); + + *lb = llb; + *pass = lpass; +} + +static_always_inline void +urpf_perform_check_x2 (ip_address_family_t af, vlib_dir_t dir, + urpf_mode_t mode, vlib_buffer_t *b0, vlib_buffer_t *b1, + const u8 *h0, const u8 *h1, u32 fib_index0, + u32 fib_index1, load_balance_t **lb0, + load_balance_t **lb1, u32 *pass0, u32 *pass1) +{ + load_balance_t *llb0, *llb1; + u32 lpass0, lpass1; + u32 lb_index0, lb_index1; + + ASSERT (fib_index0 != ~0); + ASSERT (fib_index1 != ~0); + + if (AF_IP4 == af) + { + const ip4_header_t *ip0, *ip1; + + ip0 = (ip4_header_t *) h0; + ip1 = (ip4_header_t *) h1; + + ip4_fib_forwarding_lookup_x2 (fib_index0, fib_index1, &ip0->src_address, + &ip1->src_address, &lb_index0, &lb_index1); + /* Pass multicast. */ + lpass0 = (ip4_address_is_multicast (&ip0->src_address) || + ip4_address_is_global_broadcast (&ip0->src_address)); + lpass1 = (ip4_address_is_multicast (&ip1->src_address) || + ip4_address_is_global_broadcast (&ip1->src_address)); + } + else + { + const ip6_header_t *ip0, *ip1; + + ip0 = (ip6_header_t *) h0; + ip1 = (ip6_header_t *) h1; + + lb_index0 = ip6_fib_table_fwding_lookup (fib_index0, &ip0->src_address); + lb_index1 = ip6_fib_table_fwding_lookup (fib_index1, &ip1->src_address); + lpass0 = ip6_address_is_multicast (&ip0->src_address); + lpass1 = ip6_address_is_multicast (&ip1->src_address); + } + + llb0 = load_balance_get (lb_index0); + llb1 = load_balance_get (lb_index1); + + if (URPF_MODE_STRICT == mode) + { + /* for RX the check is: would this source adddress be + * forwarded out of the interface on which it was recieved, + * if yes allow. For TX it's; would this source address be + * forwarded out of the interface through which it is being + * sent, if yes drop. + */ + int res0, res1; + + res0 = + fib_urpf_check (llb0->lb_urpf, vnet_buffer (b0)->sw_if_index[dir]); + res1 = + fib_urpf_check (llb1->lb_urpf, vnet_buffer (b1)->sw_if_index[dir]); + + if (VLIB_RX == dir) + { + lpass0 |= res0; + lpass1 |= res1; + } + else + { + lpass0 |= !res0 && fib_urpf_check_size (llb0->lb_urpf); + lpass1 |= !res1 && fib_urpf_check_size (llb1->lb_urpf); + + /* allow locally generated */ + lpass0 |= b0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED; + lpass1 |= b1->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED; + } + } + else + { + lpass0 |= fib_urpf_check_size (llb0->lb_urpf); + lpass1 |= fib_urpf_check_size (llb1->lb_urpf); + } + + *lb0 = llb0; + *lb1 = llb1; + *pass0 = lpass0; + *pass1 = lpass1; +} + static_always_inline uword -urpf_inline (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame, +urpf_inline (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, ip_address_family_t af, vlib_dir_t dir, urpf_mode_t mode) { vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b; @@ -106,8 +237,8 @@ urpf_inline (vlib_main_t * vm, while (n_left >= 4) { - u32 pass0, lb_index0, pass1, lb_index1; - const load_balance_t *lb0, *lb1; + u32 pass0, pass1; + load_balance_t *lb0 = 0, *lb1 = 0; u32 fib_index0, fib_index1; const u8 *h0, *h1; @@ -121,87 +252,32 @@ urpf_inline (vlib_main_t * vm, h0 = (u8 *) vlib_buffer_get_current (b[0]); h1 = (u8 *) vlib_buffer_get_current (b[1]); - if (VLIB_TX == dir) { h0 += vnet_buffer (b[0])->ip.save_rewrite_length; h1 += vnet_buffer (b[1])->ip.save_rewrite_length; } - fib_index0 = - urpf_cfgs[af][dir][vnet_buffer (b[0])->sw_if_index[dir]].fib_index; - fib_index1 = - urpf_cfgs[af][dir][vnet_buffer (b[1])->sw_if_index[dir]].fib_index; + fib_index0 = urpf_get_fib_index (b[0], af, dir); + fib_index1 = urpf_get_fib_index (b[1], af, dir); + urpf_perform_check_x2 (af, dir, mode, b[0], b[1], h0, h1, fib_index0, + fib_index1, &lb0, &lb1, &pass0, &pass1); - if (AF_IP4 == af) - { - const ip4_header_t *ip0, *ip1; - - ip0 = (ip4_header_t *) h0; - ip1 = (ip4_header_t *) h1; - - ip4_fib_forwarding_lookup_x2 (fib_index0, - fib_index1, - &ip0->src_address, - &ip1->src_address, - &lb_index0, &lb_index1); - /* Pass multicast. */ - pass0 = (ip4_address_is_multicast (&ip0->src_address) || - ip4_address_is_global_broadcast (&ip0->src_address)); - pass1 = (ip4_address_is_multicast (&ip1->src_address) || - ip4_address_is_global_broadcast (&ip1->src_address)); - } - else + if (b[0]->flags & VLIB_BUFFER_IS_TRACED) { - const ip6_header_t *ip0, *ip1; - - ip0 = (ip6_header_t *) h0; - ip1 = (ip6_header_t *) h1; - - lb_index0 = ip6_fib_table_fwding_lookup (fib_index0, - &ip0->src_address); - lb_index1 = ip6_fib_table_fwding_lookup (fib_index1, - &ip1->src_address); - pass0 = ip6_address_is_multicast (&ip0->src_address); - pass1 = ip6_address_is_multicast (&ip1->src_address); - } - - lb0 = load_balance_get (lb_index0); - lb1 = load_balance_get (lb_index1); + urpf_trace_t *t; - if (URPF_MODE_STRICT == mode) - { - /* for RX the check is: would this source adddress be forwarded - * out of the interface on which it was recieved, if yes allow. - * For TX it's; would this source address be forwarded out of the - * interface through which it is being sent, if yes drop. - */ - int res0, res1; - - res0 = fib_urpf_check (lb0->lb_urpf, - vnet_buffer (b[0])->sw_if_index[dir]); - res1 = fib_urpf_check (lb1->lb_urpf, - vnet_buffer (b[1])->sw_if_index[dir]); - - if (VLIB_RX == dir) - { - pass0 |= res0; - pass1 |= res1; - } - else - { - pass0 |= !res0 && fib_urpf_check_size (lb0->lb_urpf); - pass1 |= !res1 && fib_urpf_check_size (lb1->lb_urpf); - - /* allow locally generated */ - pass0 |= b[0]->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED; - pass1 |= b[1]->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED; - } + t = vlib_add_trace (vm, node, b[0], sizeof (*t)); + t->urpf = lb0 ? lb0->lb_urpf : ~0; + t->fib_index = fib_index0; } - else + if (b[1]->flags & VLIB_BUFFER_IS_TRACED) { - pass0 |= fib_urpf_check_size (lb0->lb_urpf); - pass1 |= fib_urpf_check_size (lb1->lb_urpf); + urpf_trace_t *t; + + t = vlib_add_trace (vm, node, b[1], sizeof (*t)); + t->urpf = lb1 ? lb1->lb_urpf : ~0; + t->fib_index = fib_index1; } if (PREDICT_TRUE (pass0)) @@ -218,22 +294,6 @@ urpf_inline (vlib_main_t * vm, next[1] = URPF_NEXT_DROP; b[1]->error = node->errors[URPF_ERROR_DROP]; } - - if (b[0]->flags & VLIB_BUFFER_IS_TRACED) - { - urpf_trace_t *t; - - t = vlib_add_trace (vm, node, b[0], sizeof (*t)); - t->urpf = lb0->lb_urpf; - } - if (b[1]->flags & VLIB_BUFFER_IS_TRACED) - { - urpf_trace_t *t; - - t = vlib_add_trace (vm, node, b[1], sizeof (*t)); - t->urpf = lb1->lb_urpf; - } - b += 2; next += 2; n_left -= 2; @@ -241,8 +301,8 @@ urpf_inline (vlib_main_t * vm, while (n_left) { - u32 pass0, lb_index0, fib_index0; - const load_balance_t *lb0; + u32 pass0, fib_index0; + load_balance_t *lb0 = 0; const u8 *h0; h0 = (u8 *) vlib_buffer_get_current (b[0]); @@ -250,51 +310,18 @@ urpf_inline (vlib_main_t * vm, if (VLIB_TX == dir) h0 += vnet_buffer (b[0])->ip.save_rewrite_length; - fib_index0 = - urpf_cfgs[af][dir][vnet_buffer (b[0])->sw_if_index[dir]].fib_index; - - if (AF_IP4 == af) - { - const ip4_header_t *ip0; - - ip0 = (ip4_header_t *) h0; - - lb_index0 = ip4_fib_forwarding_lookup (fib_index0, - &ip0->src_address); + fib_index0 = urpf_get_fib_index (b[0], af, dir); + urpf_perform_check_x1 (af, dir, mode, b[0], h0, fib_index0, &lb0, + &pass0); - /* Pass multicast. */ - pass0 = (ip4_address_is_multicast (&ip0->src_address) || - ip4_address_is_global_broadcast (&ip0->src_address)); - } - else + if (b[0]->flags & VLIB_BUFFER_IS_TRACED) { - const ip6_header_t *ip0; - - ip0 = (ip6_header_t *) h0; - - lb_index0 = ip6_fib_table_fwding_lookup (fib_index0, - &ip0->src_address); - pass0 = ip6_address_is_multicast (&ip0->src_address); - } - - lb0 = load_balance_get (lb_index0); + urpf_trace_t *t; - if (URPF_MODE_STRICT == mode) - { - int res0; - - res0 = fib_urpf_check (lb0->lb_urpf, - vnet_buffer (b[0])->sw_if_index[dir]); - if (VLIB_RX == dir) - pass0 |= res0; - else - { - pass0 |= !res0 && fib_urpf_check_size (lb0->lb_urpf); - pass0 |= b[0]->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED; - } + t = vlib_add_trace (vm, node, b[0], sizeof (*t)); + t->urpf = lb0 ? lb0->lb_urpf : ~0; + t->fib_index = fib_index0; } - else - pass0 |= fib_urpf_check_size (lb0->lb_urpf); if (PREDICT_TRUE (pass0)) vnet_feature_next_u16 (&next[0], b[0]); @@ -303,14 +330,6 @@ urpf_inline (vlib_main_t * vm, next[0] = URPF_NEXT_DROP; b[0]->error = node->errors[URPF_ERROR_DROP]; } - - if (b[0]->flags & VLIB_BUFFER_IS_TRACED) - { - urpf_trace_t *t; - - t = vlib_add_trace (vm, node, b[0], sizeof (*t)); - t->urpf = lb0->lb_urpf; - } b++; next++; n_left--; diff --git a/src/svm/svmdb.c b/src/svm/svmdb.c index 3c69dbf45ba..98bb96f3dd4 100644 --- a/src/svm/svmdb.c +++ b/src/svm/svmdb.c @@ -436,7 +436,7 @@ svmdb_local_serialize_strings (svmdb_client_t * client, char *filename) u8 *sanitized_name = 0; int fd = 0; - if (strstr (filename, "..") || index (filename, '/')) + if (strstr (filename, "..") || strchr (filename, '/')) { error = clib_error_return (0, "Illegal characters in filename '%s'", filename); diff --git a/src/tools/vppapigen/vppapigen.py b/src/tools/vppapigen/vppapigen.py index 2b0ce9999d7..0bed578eb93 100755 --- a/src/tools/vppapigen/vppapigen.py +++ b/src/tools/vppapigen/vppapigen.py @@ -9,6 +9,7 @@ import os from subprocess import Popen, PIPE import ply.lex as lex import ply.yacc as yacc +from io import TextIOWrapper assert sys.version_info >= (3, 5), "Not supported Python version: {}".format( sys.version @@ -1166,6 +1167,21 @@ def foldup_crcs(s): f.crc = foldup_blocks(f.block, binascii.crc32(f.crc) & 0xFFFFFFFF) +def write_dependencies(output_file, dependency_file, imports): + r = [] + for i in imports: + for d in dirlist: + f = os.path.abspath(os.path.join(d, i.filename)) + if os.path.exists(f): + r.append(f) + with open(dependency_file, "w", encoding="utf8") as f: + print(f"{output_file}: \\", file=f) + for i in r[:-1]: + print(f" {i} \\", file=f) + if imports: + print(f" {r[-1]}", file=f) + + def run_vppapigen( input_file=None, output=sys.stdout, @@ -1176,6 +1192,7 @@ def run_vppapigen( outputdir=None, pluginpath="", git_revision=None, + dependency_file=None, ): # reset globals dirlist.clear() @@ -1256,6 +1273,9 @@ def run_vppapigen( imports = parser.process_imports(parsed_objects, False, result) s["imported"] = parser.process(imports) + if dependency_file and isinstance(output, TextIOWrapper): + write_dependencies(output.name, dependency_file[0], s["Import"]) + # Add msg_id field s["Define"] = add_msg_id(s["Define"]) @@ -1324,6 +1344,7 @@ def main(): cliparser.add_argument( "--git-revision", help="Git revision to use for opening files" ) + cliparser.add_argument("-MF", nargs=1, help="Dependency file") args = cliparser.parse_args() return run_vppapigen( @@ -1336,6 +1357,7 @@ def main(): pluginpath=args.pluginpath, git_revision=args.git_revision, output=args.output, + dependency_file=args.MF, ) diff --git a/src/tools/vppapigen/vppapigen_c.py b/src/tools/vppapigen/vppapigen_c.py index c2e1e7da7b7..3cd52df1ad0 100755 --- a/src/tools/vppapigen/vppapigen_c.py +++ b/src/tools/vppapigen/vppapigen_c.py @@ -171,14 +171,10 @@ class ToJSON: write(" {\n") # What is length field doing here? write( - ' u8 *s = format(0, "0x%U", format_hex_bytes, ' + ' char *s = format_c_string(0, "0x%U", format_hex_bytes_no_wrap, ' "&a->{n}, {lfield});\n".format(n=o.fieldname, lfield=lfield) ) - write( - ' cJSON_AddStringToObject(o, "{n}", (char *)s);\n'.format( - n=o.fieldname - ) - ) + write(' cJSON_AddStringToObject(o, "{n}", s);\n'.format(n=o.fieldname)) write(" vec_free(s);\n") write(" }\n") return @@ -275,8 +271,12 @@ class ToJSON: "(vl_api_{name}_t *a) {{\n".format(name=o.name) ) - write(' u8 *s = format(0, "%U", format_vl_api_{}_t, a);\n'.format(o.name)) - write(" cJSON *o = cJSON_CreateString((char *)s);\n") + write( + ' char *s = format_c_string(0, "%U", format_vl_api_{}_t, a);\n'.format( + o.name + ) + ) + write(" cJSON *o = cJSON_CreateString(s);\n") write(" vec_free(s);\n") write(" return o;\n") write("}\n") @@ -1143,12 +1143,20 @@ ENDIAN_STRINGS = { } -def get_endian_string(o, fieldtype): +def get_endian_string(fieldtype): """Return proper endian string conversion function""" return ENDIAN_STRINGS[fieldtype] -def endianfun_array(o): +def get_lengthfield_type(fieldname, block): + """Return the type of the length field""" + for o in block: + if o.fieldname == fieldname: + return o.fieldtype + return None + + +def endianfun_array(o, block): """Generate endian functions for arrays""" forloop = """\ ASSERT((u32){length} <= (u32)VL_API_MAX_ARRAY_SIZE); @@ -1167,12 +1175,16 @@ def endianfun_array(o): if o.fieldtype == "u8" or o.fieldtype == "string" or o.fieldtype == "bool": output += " /* a->{n} = a->{n} (no-op) */\n".format(n=o.fieldname) else: - lfield = "a->" + o.lengthfield if o.lengthfield else o.length + # lfield = "a->" + o.lengthfield if o.lengthfield else o.length if o.lengthfield: - output += ( - f" u32 count = to_net ? clib_host_to_net_u32(a->{o.lengthfield}) : " - f"a->{o.lengthfield};\n" - ) + fieldtype = get_lengthfield_type(o.lengthfield, block) + if fieldtype == "u8": + output += f" u32 count = a->{o.lengthfield};\n" + else: + output += ( + f" u32 count = to_net ? {get_endian_string(fieldtype)}(a->{o.lengthfield}) : " + f"a->{o.lengthfield};\n" + ) lfield = "count" else: lfield = o.length @@ -1180,7 +1192,7 @@ def endianfun_array(o): if o.fieldtype in ENDIAN_STRINGS: output += forloop.format( length=lfield, - format=get_endian_string(o, o.fieldtype), + format=get_endian_string(o.fieldtype), name=o.fieldname, ) else: @@ -1193,11 +1205,11 @@ def endianfun_array(o): NO_ENDIAN_CONVERSION = {"client_index": None} -def endianfun_obj(o): +def endianfun_obj(o, block): """Generate endian conversion function for type""" output = "" if o.type == "Array": - return endianfun_array(o) + return endianfun_array(o, block) if o.type != "Field": output += ' s = format(s, "\\n{} {} {} (print not implemented");\n'.format( o.type, o.fieldtype, o.fieldname @@ -1208,7 +1220,7 @@ def endianfun_obj(o): return output if o.fieldtype in ENDIAN_STRINGS: output += " a->{name} = {format}(a->{name});\n".format( - name=o.fieldname, format=get_endian_string(o, o.fieldtype) + name=o.fieldname, format=get_endian_string(o.fieldtype) ) elif o.fieldtype.startswith("vl_api_"): output += " {type}_endian(&a->{name}, to_net);\n".format( @@ -1252,7 +1264,7 @@ static inline void vl_api_{name}_t_endian (vl_api_{name}_t *a, bool to_net) if t.__class__.__name__ == "Enum" or t.__class__.__name__ == "EnumFlag": output += signature.format(name=t.name) if t.enumtype in ENDIAN_STRINGS: - output += " *a = {}(*a);\n".format(get_endian_string(t, t.enumtype)) + output += " *a = {}(*a);\n".format(get_endian_string(t.enumtype)) else: output += " /* a->{name} = a->{name} (no-op) */\n".format( name=t.name @@ -1273,7 +1285,7 @@ static inline void vl_api_{name}_t_endian (vl_api_{name}_t *a, bool to_net) ) elif t.alias["type"] in FORMAT_STRINGS: output += " *a = {}(*a);\n".format( - get_endian_string(t, t.alias["type"]) + get_endian_string(t.alias["type"]) ) else: output += " /* Not Implemented yet {} */".format(t.name) @@ -1283,7 +1295,7 @@ static inline void vl_api_{name}_t_endian (vl_api_{name}_t *a, bool to_net) output += signature.format(name=t.name) for o in t.block: - output += endianfun_obj(o) + output += endianfun_obj(o, t.block) output += "}\n\n" output += "\n#endif" @@ -1349,7 +1361,7 @@ static inline uword vl_api_{name}_t_calc_size (vl_api_{name}_t *a) ) lf = m[0] if lf.fieldtype in ENDIAN_STRINGS: - output += f" + {get_endian_string(b, lf.fieldtype)}(a->{b.lengthfield}) * sizeof(a->{b.fieldname}[0])" + output += f" + {get_endian_string(lf.fieldtype)}(a->{b.lengthfield}) * sizeof(a->{b.fieldname}[0])" elif lf.fieldtype == "u8": output += ( f" + a->{b.lengthfield} * sizeof(a->{b.fieldname}[0])" diff --git a/src/vat/api_format.c b/src/vat/api_format.c index 45ba025f191..48c1ccaa820 100644 --- a/src/vat/api_format.c +++ b/src/vat/api_format.c @@ -2151,7 +2151,7 @@ elog_save (vat_main_t * vam) } /* It's fairly hard to get "../oopsie" through unformat; just in case */ - if (strstr (file, "..") || index (file, '/')) + if (strstr (file, "..") || strchr (file, '/')) { errmsg ("illegal characters in filename '%s'", file); return 0; diff --git a/src/vcl/vcl_locked.c b/src/vcl/vcl_locked.c index 69dd15b0ef4..93ece0027ff 100644 --- a/src/vcl/vcl_locked.c +++ b/src/vcl/vcl_locked.c @@ -374,6 +374,12 @@ vls_worker_get_current (void) return pool_elt_at_index (vlsm->workers, vls_get_worker_index ()); } +static inline u8 +vls_n_workers (void) +{ + return pool_elts (vlsm->workers); +} + static void vls_worker_alloc (void) { @@ -1298,7 +1304,7 @@ vls_mp_checks (vcl_locked_session_t * vls, int is_add) vcl_session_t *s; u32 owner_wrk; - if (vls_mt_wrk_supported ()) + if (vls_mt_wrk_supported () && vls_n_workers () <= 1) return; ASSERT (wrk->wrk_index == vls->vcl_wrk_index); diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c index a557093e897..2ebb1d8e5b6 100644 --- a/src/vcl/vppcom.c +++ b/src/vcl/vppcom.c @@ -1754,6 +1754,10 @@ vppcom_unformat_proto (uint8_t * proto, char *proto_str) *proto = VPPCOM_PROTO_SRTP; else if (!strcmp (proto_str, "srtp")) *proto = VPPCOM_PROTO_SRTP; + else if (!strcmp (proto_str, "HTTP")) + *proto = VPPCOM_PROTO_HTTP; + else if (!strcmp (proto_str, "http")) + *proto = VPPCOM_PROTO_HTTP; else return 1; return 0; @@ -2345,6 +2349,75 @@ vppcom_session_write_inline (vcl_worker_t *wrk, vcl_session_t *s, void *buf, } int +vppcom_session_write_segments (uint32_t session_handle, + vppcom_data_segment_t *ds, uint32_t n_segments) +{ + vcl_worker_t *wrk = vcl_worker_get_current (); + int n_write = 0, n_bytes = 0, is_nonblocking; + vcl_session_t *s = 0; + svm_fifo_t *tx_fifo; + svm_msg_q_t *mq; + u8 is_ct; + u32 i; + + if (PREDICT_FALSE (!ds)) + return VPPCOM_EFAULT; + + /* Accept zero length writes but just return */ + if (PREDICT_FALSE (ds[0].len == 0)) + return VPPCOM_OK; + + s = vcl_session_get_w_handle (wrk, session_handle); + if (PREDICT_FALSE (!s || (s->flags & VCL_SESSION_F_IS_VEP))) + return VPPCOM_EBADFD; + + if (PREDICT_FALSE (!vcl_session_is_open (s))) + return vcl_session_closed_error (s); + + if (PREDICT_FALSE (s->flags & VCL_SESSION_F_WR_SHUTDOWN)) + return VPPCOM_EPIPE; + + is_nonblocking = vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK); + is_ct = vcl_session_is_ct (s); + mq = wrk->app_event_queue; + tx_fifo = is_ct ? s->ct_tx_fifo : s->tx_fifo; + + for (i = 0; i < n_segments; i++) + n_bytes += ds[i].len; + + if (svm_fifo_max_enqueue_prod (tx_fifo) < n_bytes) + { + if (is_nonblocking) + { + return VPPCOM_EWOULDBLOCK; + } + while (svm_fifo_max_enqueue_prod (tx_fifo) < n_bytes) + { + svm_fifo_add_want_deq_ntf (tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); + if (vcl_session_is_closing (s)) + return vcl_session_closing_error (s); + + svm_msg_q_wait (mq, SVM_MQ_WAIT_EMPTY); + vcl_worker_flush_mq_events (wrk); + } + } + + n_write = svm_fifo_enqueue_segments (tx_fifo, (svm_fifo_seg_t *) ds, + n_segments, 0 /* allow_partial */); + + /* The underlying fifo segment can run out of memory */ + if (PREDICT_FALSE (n_write < 0)) + return VPPCOM_EAGAIN; + + if (svm_fifo_set_event (s->tx_fifo)) + app_send_io_evt_to_vpp (s->vpp_evt_q, + s->tx_fifo->shr->master_session_index, + SESSION_IO_EVT_TX, SVM_Q_WAIT); + + return n_write; +} + +int vppcom_session_write (uint32_t session_handle, void *buf, size_t n) { vcl_worker_t *wrk = vcl_worker_get_current (); @@ -2619,6 +2692,7 @@ vppcom_select_eventfd (vcl_worker_t * wrk, int n_bits, vcl_mq_evt_conn_t *mqc; int __clib_unused n_read; int n_mq_evts, i; + double end = -1; u64 buf; if (PREDICT_FALSE (wrk->api_client_handle == ~0)) @@ -2628,23 +2702,45 @@ vppcom_select_eventfd (vcl_worker_t * wrk, int n_bits, } vec_validate (wrk->mq_events, pool_elts (wrk->mq_evt_conns)); - n_mq_evts = epoll_wait (wrk->mqs_epfd, wrk->mq_events, - vec_len (wrk->mq_events), time_to_wait); - for (i = 0; i < n_mq_evts; i++) + if (time_to_wait > 0) + end = clib_time_now (&wrk->clib_time) + (time_to_wait / 1e3); + + do { - if (PREDICT_FALSE (wrk->mq_events[i].data.u32 == ~0)) + n_mq_evts = epoll_wait (wrk->mqs_epfd, wrk->mq_events, + vec_len (wrk->mq_events), time_to_wait); + if (n_mq_evts < 0) { - vcl_api_handle_disconnect (wrk); - continue; + if (errno == EINTR) + continue; + + VDBG (0, "epoll_wait error %u", errno); + return 0; } - mqc = vcl_mq_evt_conn_get (wrk, wrk->mq_events[i].data.u32); - n_read = read (mqc->mq_fd, &buf, sizeof (buf)); - vcl_select_handle_mq (wrk, mqc->mq, n_bits, read_map, write_map, - except_map, 0, bits_set); + if (n_mq_evts == 0) + return 0; + + for (i = 0; i < n_mq_evts; i++) + { + if (PREDICT_FALSE (wrk->mq_events[i].data.u32 == ~0)) + { + vcl_api_handle_disconnect (wrk); + continue; + } + + mqc = vcl_mq_evt_conn_get (wrk, wrk->mq_events[i].data.u32); + n_read = read (mqc->mq_fd, &buf, sizeof (buf)); + vcl_select_handle_mq (wrk, mqc->mq, n_bits, read_map, write_map, + except_map, 0, bits_set); + } + + if (*bits_set || !time_to_wait) + return (int) *bits_set; } + while (end == -1 || clib_time_now (&wrk->clib_time) < end); - return (n_mq_evts > 0 ? (int) *bits_set : 0); + return 0; } int @@ -3407,6 +3503,9 @@ vppcom_epoll_wait_eventfd (vcl_worker_t *wrk, struct epoll_event *events, vec_len (wrk->mq_events), timeout_ms); if (n_mq_evts < 0) { + if (errno == EINTR) + continue; + VDBG (0, "epoll_wait error %u", errno); return n_evts; } @@ -4613,6 +4712,9 @@ vppcom_proto_str (vppcom_proto_t proto) case VPPCOM_PROTO_SRTP: proto_str = "SRTP"; break; + case VPPCOM_PROTO_HTTP: + proto_str = "HTTP"; + break; default: proto_str = "UNKNOWN"; break; diff --git a/src/vcl/vppcom.h b/src/vcl/vppcom.h index 386d7d0c3f7..164dc376ad8 100644 --- a/src/vcl/vppcom.h +++ b/src/vcl/vppcom.h @@ -58,6 +58,7 @@ typedef enum vppcom_proto_ VPPCOM_PROTO_QUIC, VPPCOM_PROTO_DTLS, VPPCOM_PROTO_SRTP, + VPPCOM_PROTO_HTTP, } vppcom_proto_t; typedef enum @@ -259,6 +260,9 @@ extern int vppcom_session_read_segments (uint32_t session_handle, vppcom_data_segment_t * ds, uint32_t n_segments, uint32_t max_bytes); +extern int vppcom_session_write_segments (uint32_t session_handle, + vppcom_data_segment_t * ds, + uint32_t n_segments); extern void vppcom_session_free_segments (uint32_t session_handle, uint32_t n_bytes); extern int vppcom_add_cert_key_pair (vppcom_cert_key_pair_t *ckpair); diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c index 674f15d5dc6..a7952d93c21 100644 --- a/src/vlib/buffer.c +++ b/src/vlib/buffer.c @@ -663,7 +663,8 @@ vlib_buffer_main_init_numa_alloc (struct vlib_main_t *vm, u32 numa_node, u8 unpriv) { vlib_buffer_main_t *bm = vm->buffer_main; - u32 buffers_per_numa = bm->buffers_per_numa; + u32 default_buffers_per_numa = bm->default_buffers_per_numa; + u32 buffers_per_numa = bm->buffers_per_numa[numa_node]; clib_error_t *error; u32 buffer_size; uword n_pages, pagesize; @@ -679,9 +680,13 @@ vlib_buffer_main_init_numa_alloc (struct vlib_main_t *vm, u32 numa_node, return clib_error_return (0, "buffer size (%llu) is greater than page " "size (%llu)", buffer_size, pagesize); - if (buffers_per_numa == 0) - buffers_per_numa = unpriv ? VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV : - VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA; + if (default_buffers_per_numa == 0) + default_buffers_per_numa = unpriv ? + VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV : + VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA; + + if (buffers_per_numa == ~0) + buffers_per_numa = default_buffers_per_numa; name = format (0, "buffers-numa-%d%c", numa_node, 0); n_pages = (buffers_per_numa - 1) / (pagesize / buffer_size) + 1; @@ -821,7 +826,7 @@ clib_error_t * vlib_buffer_main_init (struct vlib_main_t * vm) { vlib_buffer_main_t *bm; - clib_error_t *err; + clib_error_t *err = 0; clib_bitmap_t *bmp = 0, *bmp_has_memory = 0; u32 numa_node; vlib_buffer_pool_t *bp; @@ -853,6 +858,10 @@ vlib_buffer_main_init (struct vlib_main_t * vm) { u8 *index = bm->default_buffer_pool_index_for_numa + numa_node; index[0] = ~0; + + if (bm->buffers_per_numa[numa_node] == 0) + continue; + if ((err = vlib_buffer_main_init_numa_node (vm, numa_node, index))) { clib_error_report (err); @@ -906,18 +915,48 @@ done: } static clib_error_t * +vlib_buffers_numa_configure (vlib_buffer_main_t *bm, u32 numa_node, + unformat_input_t *input) +{ + u32 buffers = ~0; + + if (numa_node >= VLIB_BUFFER_MAX_NUMA_NODES) + return clib_error_return (0, "invalid numa node"); + + if (!input) + return 0; + + unformat_skip_white_space (input); + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "buffers %u", &buffers)) + ; + else + return unformat_parse_error (input); + } + + bm->buffers_per_numa[numa_node] = buffers; + return 0; +} + +static clib_error_t * vlib_buffers_configure (vlib_main_t * vm, unformat_input_t * input) { vlib_buffer_main_t *bm; + u32 numa_node; + unformat_input_t sub_input; + clib_error_t *error = 0; vlib_buffer_main_alloc (vm); bm = vm->buffer_main; bm->log2_page_size = CLIB_MEM_PAGE_SZ_UNKNOWN; + memset (bm->buffers_per_numa, ~0, sizeof (bm->buffers_per_numa)); while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { - if (unformat (input, "buffers-per-numa %u", &bm->buffers_per_numa)) + if (unformat (input, "buffers-per-numa %u", + &bm->default_buffers_per_numa)) ; else if (unformat (input, "page-size %U", unformat_log2_page_size, &bm->log2_page_size)) @@ -925,6 +964,15 @@ vlib_buffers_configure (vlib_main_t * vm, unformat_input_t * input) else if (unformat (input, "default data-size %u", &bm->default_data_size)) ; + else if (unformat (input, "numa %u %U", &numa_node, + unformat_vlib_cli_sub_input, &sub_input)) + { + error = vlib_buffers_numa_configure (bm, numa_node, &sub_input); + unformat_free (&sub_input); + + if (error) + return error; + } else return unformat_parse_error (input); } diff --git a/src/vlib/buffer.h b/src/vlib/buffer.h index 7d45689ed19..aad9701080e 100644 --- a/src/vlib/buffer.h +++ b/src/vlib/buffer.h @@ -508,7 +508,8 @@ typedef struct u8 default_buffer_pool_index_for_numa[VLIB_BUFFER_MAX_NUMA_NODES]; /* config */ - u32 buffers_per_numa; + u32 default_buffers_per_numa; + u32 buffers_per_numa[VLIB_BUFFER_MAX_NUMA_NODES]; u16 ext_hdr_size; u32 default_data_size; clib_mem_page_sz_t log2_page_size; diff --git a/src/vlib/cli.c b/src/vlib/cli.c index 98d57c6ccb0..4198b4b0976 100644 --- a/src/vlib/cli.c +++ b/src/vlib/cli.c @@ -43,6 +43,7 @@ #include <vppinfra/callback.h> #include <vppinfra/cpu.h> #include <vppinfra/elog.h> +#include <vppinfra/cJSON.h> #include <unistd.h> #include <ctype.h> @@ -1115,6 +1116,111 @@ VLIB_CLI_COMMAND (enable_disable_memory_trace_command, static) = { }; static clib_error_t * +save_memory_trace (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + char *file, *chroot_file; + uword was_enabled; + mheap_trace_t *t, *mem_traces = 0; + u8 *tmp; + cJSON *traces, *trace, *traceback, *symbol; + int i; + FILE *fp; + char *json_str = 0; + + cJSON_Hooks cjson_hooks = { + .malloc_fn = clib_mem_alloc, + .free_fn = clib_mem_free, + .realloc_fn = clib_mem_realloc, + }; + cJSON_InitHooks (&cjson_hooks); + + if (!unformat (input, "%s", &file)) + { + vlib_cli_output (vm, "expected file name, got `%U'", + format_unformat_error, input); + return 0; + } + + /* It's fairly hard to get "../oopsie" through unformat; just in case */ + if (strstr (file, "..") || strchr (file, '/')) + { + vlib_cli_output (vm, "illegal characters in filename '%s'", file); + return 0; + } + chroot_file = (char *) format (0, "/tmp/%s%c", file, 0); + vec_free (file); + fp = fopen ((char *) chroot_file, "w"); + if (fp == NULL) + { + vlib_cli_output (vm, "couldn't open output file %s '%s'", chroot_file); + vec_free (chroot_file); + return 0; + } + + was_enabled = clib_mem_trace_enable_disable (0); + vlib_cli_output (vm, "Saving trace to '%s'", chroot_file); + mem_traces = clib_mem_trace_dup (current_traced_heap); + traces = cJSON_CreateArray (); + vec_foreach (t, mem_traces) + { + /* Skip over free elements. */ + if (t->n_allocations == 0) + continue; + + trace = cJSON_CreateObject (); + cJSON_AddNumberToObject (trace, "count", t->n_allocations); + cJSON_AddNumberToObject (trace, "bytes", t->n_bytes); + tmp = format (0, "%p%c", t->offset, 0); + cJSON_AddStringToObject (trace, "sample", (char *) tmp); + vec_free (tmp); + traceback = cJSON_AddArrayToObject (trace, "traceback"); + for (i = 0; i < ARRAY_LEN (t->callers) && t->callers[i]; i++) + { +#if defined(CLIB_UNIX) && !defined(__APPLE__) + tmp = format (0, "%U%c\n", format_clib_elf_symbol_with_address, + t->callers[i], 0); + symbol = cJSON_CreateString ((char *) tmp); + cJSON_AddItemToArray (traceback, symbol); + vec_free (tmp); +#else + tmp = format (0, "%p%c\n", t->callers[i], 0); + symbol = cJSON_CreateString ((char *) tmp); + cJSON_AddItemToArray (traceback, symbol); + vec_free (tmp); +#endif + } + + cJSON_AddItemToArray (traces, trace); + } + json_str = cJSON_PrintUnformatted (traces); + cJSON_Delete (traces); + fputs (json_str, fp); + fclose (fp); + clib_mem_free (json_str); + + vec_free (mem_traces); + clib_mem_trace_enable_disable (was_enabled); + + vec_free (chroot_file); + + return 0; +} + +/*? + * Save memory traces of the currently traced heap in JSON format to file. + * Only filename can be specified, path is fixed (/tmp/<filename>). + * + * @cliexpar + * @cliexcmd{save memory-trace mem_trace.json} +?*/ +VLIB_CLI_COMMAND (save_memory_trace_command, static) = { + .path = "save memory-trace", + .short_help = "save memory-trace <filename>", + .function = save_memory_trace, +}; + +static clib_error_t * restart_cmd_fn (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { diff --git a/src/vlib/format.c b/src/vlib/format.c index 7de6417be69..98010620a5d 100644 --- a/src/vlib/format.c +++ b/src/vlib/format.c @@ -198,7 +198,7 @@ unformat_vlib_tmpfile (unformat_input_t * input, va_list * args) return 0; /* Brain-police user path input */ - if (strstr ((char *) filename, "..") || index ((char *) filename, '/')) + if (strstr ((char *) filename, "..") || strchr ((char *) filename, '/')) { vec_free (filename); return 0; diff --git a/src/vlib/main.c b/src/vlib/main.c index 04b58762646..a2f833711ab 100644 --- a/src/vlib/main.c +++ b/src/vlib/main.c @@ -569,6 +569,9 @@ vlib_node_sync_stats (vlib_main_t * vm, vlib_node_t * n) uword i; for (i = 0; i < rt->n_next_nodes; i++) { + if (n->flags & VLIB_NODE_FLAG_ALLOW_LAZY_NEXT_NODES && + n->next_nodes[i] == VLIB_INVALID_NODE_INDEX) + continue; nf = vlib_node_runtime_get_next_frame (vm, rt, i); vec_elt (n->n_vectors_by_next_node, i) += nf->vectors_since_last_overflow; @@ -655,7 +658,7 @@ elog_save_buffer (vlib_main_t * vm, } /* It's fairly hard to get "../oopsie" through unformat; just in case */ - if (strstr (file, "..") || index (file, '/')) + if (strstr (file, "..") || strchr (file, '/')) { vlib_cli_output (vm, "illegal characters in filename '%s'", file); return 0; @@ -1240,9 +1243,12 @@ static_always_inline uword vlib_process_resume (vlib_main_t * vm, vlib_process_t * p) { uword r; - p->flags &= ~(VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK - | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT - | VLIB_PROCESS_RESUME_PENDING); + + if (p->state == VLIB_PROCESS_STATE_WAIT_FOR_EVENT || + p->state == VLIB_PROCESS_STATE_WAIT_FOR_EVENT_OR_CLOCK) + p->event_resume_pending = 0; + + p->state = VLIB_PROCESS_STATE_RUNNING; r = clib_setjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_RETURN); if (r == VLIB_PROCESS_RETURN_LONGJMP_RETURN) { @@ -1265,12 +1271,13 @@ dispatch_process (vlib_main_t * vm, u64 t; uword n_vectors, is_suspend; - if (node->state != VLIB_NODE_STATE_POLLING - || (p->flags & (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK - | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT))) + if (node->state != VLIB_NODE_STATE_POLLING) + return last_time_stamp; + + if (p->state != VLIB_PROCESS_STATE_NOT_STARTED) return last_time_stamp; - p->flags |= VLIB_PROCESS_IS_RUNNING; + p->state = VLIB_PROCESS_STATE_RUNNING; t = last_time_stamp; vlib_elog_main_loop_event (vm, node_runtime->node_index, t, @@ -1302,20 +1309,18 @@ dispatch_process (vlib_main_t * vm, p->n_suspends += 1; p->suspended_process_frame_index = pf - nm->suspended_process_frames; - if (p->flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK) + if (p->resume_clock_interval) { - TWT (tw_timer_wheel) * tw = - (TWT (tw_timer_wheel) *) nm->timing_wheel; + TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) nm->timing_wheel; p->stop_timer_handle = TW (tw_timer_start) (tw, - vlib_timing_wheel_data_set_suspended_process - (node->runtime_index) /* [sic] pool idex */ , - 0 /* timer_id */ , - p->resume_clock_interval); + vlib_timing_wheel_data_set_suspended_process ( + node->runtime_index) /* [sic] pool idex */, + 0 /* timer_id */, p->resume_clock_interval); } } else - p->flags &= ~VLIB_PROCESS_IS_RUNNING; + p->state = VLIB_PROCESS_STATE_NOT_STARTED; t = clib_cpu_time_now (); @@ -1343,8 +1348,8 @@ vlib_start_process (vlib_main_t * vm, uword process_index) } static u64 -dispatch_suspended_process (vlib_main_t * vm, - uword process_index, u64 last_time_stamp) +dispatch_suspended_process (vlib_main_t *vm, vlib_process_restore_t *r, + u64 last_time_stamp) { vlib_node_main_t *nm = &vm->node_main; vlib_node_runtime_t *node_runtime; @@ -1353,15 +1358,39 @@ dispatch_suspended_process (vlib_main_t * vm, vlib_process_t *p; vlib_pending_frame_t *pf; u64 t, n_vectors, is_suspend; + uword process_index = r->runtime_index; + + u8 resume_permissons[VLIB_PROCRSS_N_RESTORE_REASON][VLIB_PROCESS_N_STATES] = { + [VLIB_PROCESS_RESTORE_REASON_YIELD] = { + [VLIB_PROCESS_STATE_YIELD] = 1, + }, + [VLIB_PROCESS_RESTORE_REASON_CLOCK] = { + [VLIB_PROCESS_STATE_WAIT_FOR_CLOCK] = 1, + [VLIB_PROCESS_STATE_WAIT_FOR_EVENT_OR_CLOCK] = 1, + [VLIB_PROCESS_STATE_SUSPENDED] = 1, + }, + [VLIB_PROCESS_RESTORE_REASON_EVENT] = { + [VLIB_PROCESS_STATE_WAIT_FOR_EVENT] = 1, + [VLIB_PROCESS_STATE_WAIT_FOR_EVENT_OR_CLOCK] = 1, + }, + [VLIB_PROCESS_RESTORE_REASON_TIMED_EVENT] = { + [VLIB_PROCESS_STATE_WAIT_FOR_EVENT] = 1, + [VLIB_PROCESS_STATE_WAIT_FOR_EVENT_OR_CLOCK] = 1, + }, + }; t = last_time_stamp; p = vec_elt (nm->processes, process_index); - if (PREDICT_FALSE (!(p->flags & VLIB_PROCESS_IS_RUNNING))) + + if (PREDICT_FALSE (p->state == VLIB_PROCESS_STATE_NOT_STARTED)) return last_time_stamp; - ASSERT (p->flags & (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK - | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT)); + if (resume_permissons[r->reason][p->state] == 0) + { + vec_add1 (nm->process_restore_next, *r); + return last_time_stamp; + } pf = pool_elt_at_index (nm->suspended_process_frames, p->suspended_process_frame_index); @@ -1390,7 +1419,7 @@ dispatch_suspended_process (vlib_main_t * vm, /* Suspend it again. */ n_vectors = 0; p->n_suspends += 1; - if (p->flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK) + if (p->resume_clock_interval) { p->stop_timer_handle = TW (tw_timer_start) ((TWT (tw_timer_wheel) *) nm->timing_wheel, @@ -1402,7 +1431,7 @@ dispatch_suspended_process (vlib_main_t * vm, } else { - p->flags &= ~VLIB_PROCESS_IS_RUNNING; + p->state = VLIB_PROCESS_STATE_NOT_STARTED; pool_put_index (nm->suspended_process_frames, p->suspended_process_frame_index); p->suspended_process_frame_index = ~0; @@ -1598,7 +1627,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) } *ed; /* Check if process nodes have expired from timing wheel. */ - ASSERT (nm->data_from_advancing_timing_wheel != 0); + ASSERT (nm->process_restore_current != 0); if (PREDICT_FALSE (vm->elog_trace_graph_dispatch)) ed = ELOG_DATA (&vlib_global_main.elog_main, es); @@ -1606,61 +1635,60 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) TW (tw_timer_expire_timers) ((TWT (tw_timer_wheel) *) nm->timing_wheel, vlib_time_now (vm)); - ASSERT (nm->data_from_advancing_timing_wheel != 0); + ASSERT (nm->process_restore_current != 0); if (PREDICT_FALSE (vm->elog_trace_graph_dispatch)) { ed = ELOG_DATA (&vlib_global_main.elog_main, ee); - ed->nready_procs = - _vec_len (nm->data_from_advancing_timing_wheel); + ed->nready_procs = _vec_len (nm->process_restore_current); } - if (PREDICT_FALSE - (_vec_len (nm->data_from_advancing_timing_wheel) > 0)) + if (PREDICT_FALSE (_vec_len (nm->process_restore_current) > 0)) { uword i; - for (i = 0; i < _vec_len (nm->data_from_advancing_timing_wheel); - i++) + for (i = 0; i < _vec_len (nm->process_restore_current); i++) { - u32 d = nm->data_from_advancing_timing_wheel[i]; - u32 di = vlib_timing_wheel_data_get_index (d); + vlib_process_restore_t *res = + nm->process_restore_current + i; - if (vlib_timing_wheel_data_is_timed_event (d)) - { - vlib_signal_timed_event_data_t *te = - pool_elt_at_index (nm->signal_timed_event_data_pool, - di); - vlib_node_t *n = - vlib_get_node (vm, te->process_node_index); - vlib_process_t *p = - vec_elt (nm->processes, n->runtime_index); - p->stop_timer_handle = ~0; - void *data; - data = - vlib_process_signal_event_helper (nm, n, p, - te->event_type_index, - te->n_data_elts, - te->n_data_elt_bytes); - if (te->n_data_bytes < sizeof (te->inline_event_data)) - clib_memcpy_fast (data, te->inline_event_data, + if (res->reason == VLIB_PROCESS_RESTORE_REASON_TIMED_EVENT) + { + u32 di = res->timed_event_data_pool_index; + vlib_signal_timed_event_data_t *te = + pool_elt_at_index (nm->signal_timed_event_data_pool, di); + vlib_node_t *n = + vlib_get_node (vm, te->process_node_index); + vlib_process_t *p = + vec_elt (nm->processes, n->runtime_index); + p->stop_timer_handle = ~0; + void *data; + data = vlib_process_signal_event_helper ( + nm, n, p, te->event_type_index, te->n_data_elts, + te->n_data_elt_bytes); + if (te->n_data_bytes < sizeof (te->inline_event_data)) + clib_memcpy_fast (data, te->inline_event_data, + te->n_data_bytes); + else + { + clib_memcpy_fast (data, te->event_data_as_vector, te->n_data_bytes); - else - { - clib_memcpy_fast (data, te->event_data_as_vector, - te->n_data_bytes); - vec_free (te->event_data_as_vector); - } - pool_put (nm->signal_timed_event_data_pool, te); - } + vec_free (te->event_data_as_vector); + } + pool_put (nm->signal_timed_event_data_pool, te); + } else { cpu_time_now = clib_cpu_time_now (); cpu_time_now = - dispatch_suspended_process (vm, di, cpu_time_now); + dispatch_suspended_process (vm, res, cpu_time_now); } } - vec_set_len (nm->data_from_advancing_timing_wheel, 0); + + vec_set_len (nm->process_restore_current, 0); + + CLIB_SWAP (nm->process_restore_current, + nm->process_restore_next); } } vlib_increment_main_loop_counter (vm); @@ -1843,12 +1871,23 @@ process_expired_timer_cb (u32 *expired_timer_handles) vec_foreach (handle, expired_timer_handles) { - u32 pi = vlib_timing_wheel_data_get_index (*handle); - vlib_process_t *p = vec_elt (nm->processes, pi); + u32 index = vlib_timing_wheel_data_get_index (*handle); + vlib_process_restore_t restore = {}; - p->stop_timer_handle = ~0; + if (vlib_timing_wheel_data_is_timed_event (*handle)) + { + restore.reason = VLIB_PROCESS_RESTORE_REASON_TIMED_EVENT; + restore.timed_event_data_pool_index = index; + } + else + { + vlib_process_t *p = vec_elt (nm->processes, index); + p->stop_timer_handle = ~0; + restore.reason = VLIB_PROCESS_RESTORE_REASON_CLOCK; + restore.runtime_index = index; + } + vec_add1 (nm->process_restore_current, restore); } - vec_append (nm->data_from_advancing_timing_wheel, expired_timer_handles); } /* Main function. */ @@ -1952,8 +1991,10 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input) nm->timing_wheel = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)), CLIB_CACHE_LINE_BYTES); - vec_validate (nm->data_from_advancing_timing_wheel, 10); - vec_set_len (nm->data_from_advancing_timing_wheel, 0); + vec_validate (nm->process_restore_current, 10); + vec_validate (nm->process_restore_next, 10); + vec_set_len (nm->process_restore_current, 0); + vec_set_len (nm->process_restore_next, 0); /* Create the process timing wheel */ TW (tw_timer_wheel_init) diff --git a/src/vlib/node.c b/src/vlib/node.c index 8f6c852188b..c0572f3cf83 100644 --- a/src/vlib/node.c +++ b/src/vlib/node.c @@ -805,7 +805,8 @@ vlib_node_main_init (vlib_main_t * vm) if (!a) continue; - if (~0 == vlib_node_add_named_next_with_slot (vm, n->index, a, i)) + if (~0 == vlib_node_add_named_next_with_slot (vm, n->index, a, i) && + !(n->flags & VLIB_NODE_FLAG_ALLOW_LAZY_NEXT_NODES)) { error = clib_error_create ("node `%v' refers to unknown node `%s'", n->name, a); @@ -813,7 +814,8 @@ vlib_node_main_init (vlib_main_t * vm) } } - vec_free (n->next_node_names); + if (!(n->flags & VLIB_NODE_FLAG_ALLOW_LAZY_NEXT_NODES)) + vec_free (n->next_node_names); } /* Set previous node pointers. */ @@ -851,14 +853,18 @@ vlib_node_main_init (vlib_main_t * vm) for (i = 0; i < vec_len (n->next_nodes); i++) { - next = vlib_get_node (vm, n->next_nodes[i]); + if (n->flags & VLIB_NODE_FLAG_ALLOW_LAZY_NEXT_NODES && + n->next_nodes[i] >= vec_len (nm->nodes)) + continue; - /* Validate node runtime indices are correctly initialized. */ - ASSERT (nf[i].node_runtime_index == next->runtime_index); + next = vlib_get_node (vm, n->next_nodes[i]); - nf[i].flags = 0; - if (next->flags & VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH) - nf[i].flags |= VLIB_FRAME_NO_FREE_AFTER_DISPATCH; + /* Validate node runtime indices are correctly initialized. */ + ASSERT (nf[i].node_runtime_index == next->runtime_index); + + nf[i].flags = 0; + if (next->flags & VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH) + nf[i].flags |= VLIB_FRAME_NO_FREE_AFTER_DISPATCH; } } } @@ -927,6 +933,33 @@ vlib_node_set_march_variant (vlib_main_t *vm, u32 node_index, } return -1; } + +clib_error_t * +vlib_node_main_lazy_next_update (vlib_main_t *vm) +{ + vlib_node_main_t *nm = &vm->node_main; + uword ni; + vlib_node_t *n; + for (ni = 0; ni < vec_len (nm->nodes); ni++) + { + uword nni; + n = vec_elt (nm->nodes, ni); + + if (!(n->flags & VLIB_NODE_FLAG_ALLOW_LAZY_NEXT_NODES)) + continue; + + for (nni = 0; nni < vec_len (n->next_node_names); nni++) + { + char *a = n->next_node_names[nni]; + + if (!a) + continue; + + vlib_node_add_named_next_with_slot (vm, n->index, a, nni); + } + } + return 0; +} /* * fd.io coding-style-patch-verification: ON * diff --git a/src/vlib/node.h b/src/vlib/node.h index 68813c2c3e1..bb6d8f818a8 100644 --- a/src/vlib/node.h +++ b/src/vlib/node.h @@ -297,6 +297,7 @@ typedef struct vlib_node_t #define VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE (1 << 7) #define VLIB_NODE_FLAG_TRACE_SUPPORTED (1 << 8) #define VLIB_NODE_FLAG_ADAPTIVE_MODE (1 << 9) +#define VLIB_NODE_FLAG_ALLOW_LAZY_NEXT_NODES (1 << 10) /* State for input nodes. */ u8 state; @@ -542,6 +543,45 @@ typedef struct uword opaque; } vlib_process_event_type_t; +#define foreach_vlib_process_state \ + _ (NOT_STARTED, "not started") \ + _ (RUNNING, "running") \ + _ (SUSPENDED, "suspended") \ + _ (WAIT_FOR_EVENT, "event wait") \ + _ (WAIT_FOR_CLOCK, "clock wait") \ + _ (WAIT_FOR_EVENT_OR_CLOCK, "any wait") \ + _ (WAIT_FOR_ONE_TIME_EVENT, "one time event wait") \ + _ (YIELD, "yield") + +typedef enum +{ +#define _(n, s) VLIB_PROCESS_STATE_##n, + foreach_vlib_process_state VLIB_PROCESS_N_STATES +#undef _ +} __clib_packed vlib_process_state_t; + +STATIC_ASSERT (VLIB_PROCESS_STATE_NOT_STARTED == 0, ""); + +typedef enum +{ + VLIB_PROCESS_RESTORE_REASON_UNKNOWN = 0, + VLIB_PROCESS_RESTORE_REASON_EVENT, + VLIB_PROCESS_RESTORE_REASON_CLOCK, + VLIB_PROCESS_RESTORE_REASON_TIMED_EVENT, + VLIB_PROCESS_RESTORE_REASON_YIELD, + VLIB_PROCRSS_N_RESTORE_REASON +} __clib_packed vlib_process_restore_reason_t; + +typedef struct +{ + vlib_process_restore_reason_t reason; + union + { + u32 runtime_index; + u32 timed_event_data_pool_index; + }; +} __clib_packed vlib_process_restore_t; + typedef struct { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); @@ -559,14 +599,11 @@ typedef struct #define VLIB_PROCESS_RESUME_LONGJMP_SUSPEND 0 #define VLIB_PROCESS_RESUME_LONGJMP_RESUME 1 - u16 flags; -#define VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK (1 << 0) -#define VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT (1 << 1) - /* Set to indicate that this process has been added to resume vector. */ -#define VLIB_PROCESS_RESUME_PENDING (1 << 2) + /* Process state. */ + vlib_process_state_t state; - /* Process function is currently running. */ -#define VLIB_PROCESS_IS_RUNNING (1 << 3) + /* Process is added to resume list due to pending event */ + u8 event_resume_pending : 1; /* Size of process stack. */ u16 log2_n_stack_bytes; @@ -710,8 +747,12 @@ typedef struct vlib_signal_timed_event_data_t *signal_timed_event_data_pool; - /* Opaque data vector added via timing_wheel_advance. */ - u32 *data_from_advancing_timing_wheel; + /* Vector of process nodes waiting for restore */ + vlib_process_restore_t *process_restore_current; + + /* Vector of process nodes waiting for restore in next greaph scheduler run + */ + vlib_process_restore_t *process_restore_next; /* CPU time of next process to be ready on timing wheel. */ f64 time_next_process_ready; diff --git a/src/vlib/node_cli.c b/src/vlib/node_cli.c index d0bdf5b9097..16e904e8433 100644 --- a/src/vlib/node_cli.c +++ b/src/vlib/node_cli.c @@ -339,29 +339,16 @@ format_vlib_node_state (u8 * s, va_list * va) state = "active"; if (n->type == VLIB_NODE_TYPE_PROCESS) { + char *state_str[] = { +#define _(n, s) [VLIB_PROCESS_STATE_##n] = #s, + foreach_vlib_process_state + }; vlib_process_t *p = vlib_get_process_from_node (vm, n); - switch (p->flags & (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK - | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT)) - { - default: - if (!(p->flags & VLIB_PROCESS_IS_RUNNING)) - state = "done"; - break; - - case VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK: - state = "time wait"; - break; - - case VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT: - state = "event wait"; - break; - - case (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK): - state = - "any wait"; - break; - } + if (p->state >= ARRAY_LEN (state_str) || state_str[p->state] == 0) + state = "unknown"; + else + state = state_str[p->state]; } else if (n->type != VLIB_NODE_TYPE_INTERNAL) { diff --git a/src/vlib/node_funcs.h b/src/vlib/node_funcs.h index 1beac33cf9b..ffa17ba7bb1 100644 --- a/src/vlib/node_funcs.h +++ b/src/vlib/node_funcs.h @@ -187,10 +187,7 @@ vlib_node_set_state (vlib_main_t * vm, u32 node_index, vlib_process_t *p = vec_elt (nm->processes, n->runtime_index); r = &p->node_runtime; - /* When disabling make sure flags are cleared. */ - p->flags &= ~(VLIB_PROCESS_RESUME_PENDING - | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK - | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT); + p->event_resume_pending = 0; } else r = vec_elt_at_index (nm->nodes_by_type[n->type], n->runtime_index); @@ -599,7 +596,7 @@ vlib_process_suspend (vlib_main_t * vm, f64 dt) if (vlib_process_suspend_time_is_zero (dt)) return VLIB_PROCESS_RESUME_LONGJMP_RESUME; - p->flags |= VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK; + p->state = VLIB_PROCESS_STATE_SUSPENDED; r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND); if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND) { @@ -614,6 +611,38 @@ vlib_process_suspend (vlib_main_t * vm, f64 dt) return r; } +/** Suspend a vlib cooperative multi-tasking thread and put it at the end of + * resume queue + @param vm - vlib_main_t * + @returns VLIB_PROCESS_RESUME_LONGJMP_RESUME, routinely ignored +*/ + +always_inline uword +vlib_process_yield (vlib_main_t *vm) +{ + uword r; + vlib_node_main_t *nm = &vm->node_main; + vlib_process_t *p = vec_elt (nm->processes, nm->current_process_index); + + p->state = VLIB_PROCESS_STATE_YIELD; + r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND); + if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND) + { + vlib_process_restore_t r = { + .reason = VLIB_PROCESS_RESTORE_REASON_YIELD, + .runtime_index = nm->current_process_index, + }; + p->resume_clock_interval = 0; + vec_add1 (nm->process_restore_next, r); + vlib_process_start_switch_stack (vm, 0); + clib_longjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_SUSPEND); + } + else + vlib_process_finish_switch_stack (vm); + + return r; +} + always_inline void vlib_process_free_event_type (vlib_process_t * p, uword t, uword is_one_time_event) @@ -773,11 +802,12 @@ vlib_process_wait_for_event (vlib_main_t * vm) p = vec_elt (nm->processes, nm->current_process_index); if (clib_bitmap_is_zero (p->non_empty_event_type_bitmap)) { - p->flags |= VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT; + p->state = VLIB_PROCESS_STATE_WAIT_FOR_EVENT; r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND); if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND) { + p->resume_clock_interval = 0; vlib_process_start_switch_stack (vm, 0); clib_longjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_SUSPEND); @@ -802,11 +832,12 @@ vlib_process_wait_for_one_time_event (vlib_main_t * vm, ASSERT (!pool_is_free_index (p->event_type_pool, with_type_index)); while (!clib_bitmap_get (p->non_empty_event_type_bitmap, with_type_index)) { - p->flags |= VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT; + p->state = VLIB_PROCESS_STATE_WAIT_FOR_ONE_TIME_EVENT; r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND); if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND) { + p->resume_clock_interval = 0; vlib_process_start_switch_stack (vm, 0); clib_longjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_SUSPEND); @@ -831,11 +862,12 @@ vlib_process_wait_for_event_with_type (vlib_main_t * vm, h = hash_get (p->event_type_index_by_type_opaque, with_type_opaque); while (!h || !clib_bitmap_get (p->non_empty_event_type_bitmap, h[0])) { - p->flags |= VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT; + p->state = VLIB_PROCESS_STATE_WAIT_FOR_EVENT; r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND); if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND) { + p->resume_clock_interval = 0; vlib_process_start_switch_stack (vm, 0); clib_longjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_SUSPEND); @@ -875,8 +907,7 @@ vlib_process_wait_for_event_or_clock (vlib_main_t * vm, f64 dt) wakeup_time = vlib_time_now (vm) + dt; /* Suspend waiting for both clock and event to occur. */ - p->flags |= (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT - | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK); + p->state = VLIB_PROCESS_STATE_WAIT_FOR_EVENT_OR_CLOCK; r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND); if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND) @@ -938,7 +969,7 @@ vlib_process_signal_event_helper (vlib_node_main_t * nm, uword t, uword n_data_elts, uword n_data_elt_bytes) { - uword p_flags, add_to_pending, delete_from_wheel; + uword add_to_pending = 0, delete_from_wheel = 0; u8 *data_to_be_written_by_caller; vec_attr_t va = { .elt_sz = n_data_elt_bytes }; @@ -970,47 +1001,44 @@ vlib_process_signal_event_helper (vlib_node_main_t * nm, p->non_empty_event_type_bitmap = clib_bitmap_ori (p->non_empty_event_type_bitmap, t); - p_flags = p->flags; + switch (p->state) + { + case VLIB_PROCESS_STATE_WAIT_FOR_EVENT: + add_to_pending = 1; + break; - /* Event was already signalled? */ - add_to_pending = (p_flags & VLIB_PROCESS_RESUME_PENDING) == 0; + case VLIB_PROCESS_STATE_WAIT_FOR_EVENT_OR_CLOCK: + add_to_pending = 1; + delete_from_wheel = 1; + break; - /* Process will resume when suspend time elapses? */ - delete_from_wheel = 0; - if (p_flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK) - { - /* Waiting for both event and clock? */ - if (p_flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT) - { - if (!TW (tw_timer_handle_is_free) - ((TWT (tw_timer_wheel) *) nm->timing_wheel, - p->stop_timer_handle)) - delete_from_wheel = 1; - else - /* timer just popped so process should already be on the list */ - add_to_pending = 0; - } - else - /* Waiting only for clock. Event will be queue and may be - handled when timer expires. */ - add_to_pending = 0; + default: + break; } + if (TW (tw_timer_handle_is_free) ((TWT (tw_timer_wheel) *) nm->timing_wheel, + p->stop_timer_handle)) + delete_from_wheel = 0; + /* Never add current process to pending vector since current process is already running. */ add_to_pending &= nm->current_process_index != n->runtime_index; - if (add_to_pending) + if (add_to_pending && p->event_resume_pending == 0) { - u32 x = vlib_timing_wheel_data_set_suspended_process (n->runtime_index); - p->flags = p_flags | VLIB_PROCESS_RESUME_PENDING; - vec_add1 (nm->data_from_advancing_timing_wheel, x); - if (delete_from_wheel) - { - TW (tw_timer_stop) - ((TWT (tw_timer_wheel) *) nm->timing_wheel, p->stop_timer_handle); - p->stop_timer_handle = ~0; - } + vlib_process_restore_t restore = { + .runtime_index = n->runtime_index, + .reason = VLIB_PROCESS_RESTORE_REASON_EVENT, + }; + p->event_resume_pending = 1; + vec_add1 (nm->process_restore_current, restore); + } + + if (delete_from_wheel) + { + TW (tw_timer_stop) + ((TWT (tw_timer_wheel) *) nm->timing_wheel, p->stop_timer_handle); + p->stop_timer_handle = ~0; } return data_to_be_written_by_caller; @@ -1338,6 +1366,10 @@ void vlib_node_runtime_sync_stats_node (vlib_node_t *n, vlib_node_runtime_t *r, /* Node graph initialization function. */ clib_error_t *vlib_node_main_init (vlib_main_t * vm); +/* Refresh graph after the creation of a node that was potentially mentionned + * as a named next for a node with VLIB_NODE_FLAG_ALLOW_LAZY_NEXT_NODES */ +clib_error_t *vlib_node_main_lazy_next_update (vlib_main_t *vm); + format_function_t format_vlib_node_graph; format_function_t format_vlib_node_name; format_function_t format_vlib_next_node_name; diff --git a/src/vlib/threads.c b/src/vlib/threads.c index 854d69448ac..ef2c5616f21 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -205,6 +205,10 @@ vlib_thread_init (vlib_main_t * vm) avail_cpu = clib_bitmap_set (avail_cpu, c, 0); } + /* if main thread affinity is unspecified, set to current running cpu */ + if (tm->main_lcore == ~0) + tm->main_lcore = sched_getcpu (); + /* grab cpu for main thread */ if (tm->main_lcore != ~0) { @@ -1134,6 +1138,7 @@ cpu_config (vlib_main_t * vm, unformat_input_t * input) u8 *name; uword *bitmap; u32 count; + int use_corelist = 0; tm->thread_registrations_by_name = hash_create_string (0, sizeof (uword)); @@ -1185,6 +1190,7 @@ cpu_config (vlib_main_t * vm, unformat_input_t * input) tr->coremask = bitmap; tr->count = clib_bitmap_count_set_bits (tr->coremask); + use_corelist = 1; } else if (unformat @@ -1214,6 +1220,9 @@ cpu_config (vlib_main_t * vm, unformat_input_t * input) break; } + if (use_corelist && tm->main_lcore == ~0) + return clib_error_return (0, "main-core must be specified when using " + "corelist-* or coremask-* attribute"); if (tm->sched_priority != ~0) { if (tm->sched_policy == SCHED_FIFO || tm->sched_policy == SCHED_RR) diff --git a/src/vlib/threads.h b/src/vlib/threads.h index ac0c1d5d266..c671aa78c39 100644 --- a/src/vlib/threads.h +++ b/src/vlib/threads.h @@ -200,7 +200,7 @@ vlib_smp_unsafe_warning (void) } always_inline int -__foreach_vlib_main_helper (vlib_main_t *ii, vlib_main_t **p) +__foreach_vlib_main_helper (vlib_main_t *ii, vlib_main_t **p, int checks) { vlib_main_t *vm; u32 index = ii - (vlib_main_t *) 0; @@ -209,15 +209,17 @@ __foreach_vlib_main_helper (vlib_main_t *ii, vlib_main_t **p) return 0; *p = vm = vlib_global_main.vlib_mains[index]; - ASSERT (index == 0 || vm->parked_at_barrier == 1); + ASSERT (!checks || index == 0 || vm->parked_at_barrier == 1); return 1; } -#define foreach_vlib_main() \ +#define foreach_vlib_main__(checks) \ for (vlib_main_t *ii = 0, *this_vlib_main; \ - __foreach_vlib_main_helper (ii, &this_vlib_main); ii++) \ + __foreach_vlib_main_helper (ii, &this_vlib_main, checks); ii++) \ if (this_vlib_main) +#define foreach_vlib_main() foreach_vlib_main__ (1) + #define foreach_sched_policy_posix \ _ (SCHED_OTHER, OTHER, "other") \ _ (SCHED_FIFO, FIFO, "fifo") \ diff --git a/src/vlib/unix/cli.c b/src/vlib/unix/cli.c index 90cf61d811d..051c5730aed 100644 --- a/src/vlib/unix/cli.c +++ b/src/vlib/unix/cli.c @@ -3329,6 +3329,12 @@ VLIB_CLI_COMMAND (unix_cli_q_command, static) = { .function = unix_cli_quit, }; +VLIB_CLI_COMMAND (unix_cli_exit_command, static) = { + .path = "exit", + .short_help = "Exit CLI", + .function = unix_cli_quit, +}; + /** CLI command to execute a VPP command script. */ static clib_error_t * unix_cli_exec (vlib_main_t * vm, diff --git a/src/vlib/unix/main.c b/src/vlib/unix/main.c index ee28ca8f1aa..11d0cb1160c 100644 --- a/src/vlib/unix/main.c +++ b/src/vlib/unix/main.c @@ -40,6 +40,8 @@ #include <vlib/unix/unix.h> #include <vlib/unix/plugin.h> #include <vppinfra/unix.h> +#include <vppinfra/stack.h> +#include <vppinfra/format_ansi.h> #include <limits.h> #include <signal.h> @@ -97,20 +99,42 @@ int vlib_last_signum = 0; uword vlib_last_faulting_address = 0; static void +log_one_line () +{ + vec_terminate_c_string (syslog_msg); + if (unix_main.flags & (UNIX_FLAG_INTERACTIVE | UNIX_FLAG_NOSYSLOG)) + fprintf (stderr, "%s\n", syslog_msg); + else + syslog (LOG_ERR | LOG_DAEMON, "%s", syslog_msg); + vec_reset_length (syslog_msg); +} + +static void unix_signal_handler (int signum, siginfo_t * si, ucontext_t * uc) { uword fatal = 0; + int color = + (unix_main.flags & (UNIX_FLAG_INTERACTIVE | UNIX_FLAG_NOSYSLOG)) && + (unix_main.flags & UNIX_FLAG_NOCOLOR) == 0; /* These come in handy when looking at core files from optimized images */ vlib_last_signum = signum; vlib_last_faulting_address = (uword) si->si_addr; + if (color) + syslog_msg = format (syslog_msg, ANSI_FG_BR_RED); + syslog_msg = format (syslog_msg, "received signal %U, PC %U", format_signal, signum, format_ucontext_pc, uc); - if (signum == SIGSEGV) + if (signum == SIGSEGV || signum == SIGBUS) syslog_msg = format (syslog_msg, ", faulting address %p", si->si_addr); + if (color) + syslog_msg = format (syslog_msg, ANSI_FG_DEFAULT); + + log_one_line (); + switch (signum) { /* these (caught) signals cause the application to exit */ @@ -120,11 +144,17 @@ unix_signal_handler (int signum, siginfo_t * si, ucontext_t * uc) */ if (unix_main.vlib_main && unix_main.vlib_main->main_loop_exit_set) { - syslog (LOG_ERR | LOG_DAEMON, "received SIGTERM, exiting..."); + syslog_msg = format ( + syslog_msg, "received SIGTERM from PID %d UID %d, exiting...", + si->si_pid, si->si_uid); + log_one_line (); unix_main.vlib_main->main_loop_exit_now = 1; } else - syslog (LOG_ERR | LOG_DAEMON, "IGNORE early SIGTERM..."); + { + syslog_msg = format (syslog_msg, "IGNORE early SIGTERM..."); + log_one_line (); + } break; /* fall through */ case SIGQUIT: @@ -144,26 +174,75 @@ unix_signal_handler (int signum, siginfo_t * si, ucontext_t * uc) break; } - /* Null terminate. */ - vec_add1 (syslog_msg, 0); if (fatal) { - syslog (LOG_ERR | LOG_DAEMON, "%s", syslog_msg); + int skip = 1, index = 0; - /* Address of callers: outer first, inner last. */ - uword callers[15]; - uword n_callers = clib_backtrace (callers, ARRAY_LEN (callers), 0); - int i; - for (i = 0; i < n_callers; i++) + foreach_clib_stack_frame (sf) { - vec_reset_length (syslog_msg); + if (sf->is_signal_frame) + { + int pipefd[2]; + const int n_bytes = 20; + u8 *ip = (void *) sf->ip; + + if (pipe (pipefd) == 0) + { + /* check PC points to valid memory */ + if (write (pipefd[1], ip, n_bytes) == n_bytes) + { + syslog_msg = format (syslog_msg, "Code: "); + if (color) + syslog_msg = format (syslog_msg, ANSI_FG_CYAN); + for (int i = 0; i < n_bytes; i++) + syslog_msg = format (syslog_msg, " %02x", ip[i]); + if (color) + syslog_msg = format (syslog_msg, ANSI_FG_DEFAULT); + } + else + { + syslog_msg = format ( + syslog_msg, "PC contains invalid memory address"); + } + log_one_line (); + foreach_int (i, 0, 1) + close (pipefd[i]); + } + skip = 0; + } + + if (skip) + continue; + + syslog_msg = format (syslog_msg, "#%-2d ", index++); + if (color) + syslog_msg = format (syslog_msg, ANSI_FG_BLUE); + syslog_msg = format (syslog_msg, "0x%016lx", sf->ip); + if (color) + syslog_msg = format (syslog_msg, ANSI_FG_DEFAULT); + + if (sf->name[0]) + { + if (color) + syslog_msg = format (syslog_msg, ANSI_FG_YELLOW); + syslog_msg = + format (syslog_msg, " %s + 0x%x", sf->name, sf->offset); + if (color) + syslog_msg = format (syslog_msg, ANSI_FG_DEFAULT); + } - syslog_msg = - format (syslog_msg, "#%-2d 0x%016lx %U%c", i, callers[i], - format_clib_elf_symbol_with_address, callers[i], 0); + log_one_line (); - syslog (LOG_ERR | LOG_DAEMON, "%s", syslog_msg); + if (sf->file_name) + { + if (color) + syslog_msg = format (syslog_msg, ANSI_FG_GREEN); + syslog_msg = format (syslog_msg, " from %s", sf->file_name); + if (color) + syslog_msg = format (syslog_msg, ANSI_FG_DEFAULT); + log_one_line (); + } } /* have to remove SIGABRT to avoid recursive - os_exit calling abort() */ @@ -175,9 +254,6 @@ unix_signal_handler (int signum, siginfo_t * si, ucontext_t * uc) else os_exit (1); } - else - clib_warning ("%s", syslog_msg); - } static clib_error_t * diff --git a/src/vlib/unix/plugin.c b/src/vlib/unix/plugin.c index 5cac9abc8fe..77e4633e14a 100644 --- a/src/vlib/unix/plugin.c +++ b/src/vlib/unix/plugin.c @@ -88,19 +88,14 @@ extract (u8 * sp, u8 * ep) */ static clib_error_t * -r2_to_reg (elf_main_t * em, vlib_plugin_r2_t * r2, - vlib_plugin_registration_t * reg) +r2_to_reg (elf_main_t *em, vlib_plugin_r2_t *r2, + vlib_plugin_registration_t *reg, elf_section_t *data_section) { - clib_error_t *error; - elf_section_t *section; uword data_segment_offset; u8 *data; /* It turns out that the strings land in the ".data" section */ - error = elf_get_section_by_name (em, ".data", §ion); - if (error) - return error; - data = elf_get_section_contents (em, section->index, 1); + data = elf_get_section_contents (em, data_section->index, 1); /* * Offsets in the ".vlib_plugin_r2" section @@ -177,13 +172,52 @@ load_one_plugin (plugin_main_t * pm, plugin_info_t * pi, int from_early_init) error = elf_get_section_by_name (&em, ".vlib_plugin_r2", §ion); if (error == 0) { + elf_section_t *data_section; + elf_relocation_table_t *rt; + elf_relocation_with_addend_t *r; + elf_symbol_table_t *st; + elf64_symbol_t *sym, *symok = 0; + data = elf_get_section_contents (&em, section->index, 1); r2 = (vlib_plugin_r2_t *) data; + + elf_get_section_by_name (&em, ".data", &data_section); + + // Find first symbol in .vlib_plugin_r2 section. + vec_foreach (st, em.symbol_tables) + { + vec_foreach (sym, st->symbols) + { + if (sym->section_index == section->index) + { + symok = sym; + break; + } + } + } + + // Relocate section data as per relocation tables. + if (symok != 0) + { + vec_foreach (rt, em.relocation_tables) + { + vec_foreach (r, rt->relocations) + { + if (r->address >= symok->value && + r->address < symok->value + symok->size) + { + *(uword *) ((void *) data + r->address - symok->value) += + r->addend - data_section->header.exec_address; + } + } + } + } + reg = clib_mem_alloc (sizeof (*reg)); memset (reg, 0, sizeof (*reg)); reg->default_disabled = r2->default_disabled != 0; - error = r2_to_reg (&em, r2, reg); + error = r2_to_reg (&em, r2, reg, data_section); if (error) { PLUGIN_LOG_ERR ("Bad r2 registration: %s\n", (char *) pi->name); diff --git a/src/vlibapi/api_shared.c b/src/vlibapi/api_shared.c index 79064b292c9..1f02aefe88a 100644 --- a/src/vlibapi/api_shared.c +++ b/src/vlibapi/api_shared.c @@ -230,7 +230,7 @@ vl_msg_api_trace_write_one (api_main_t *am, u8 *msg, FILE *fp) if (m && m->endian_handler) { - m->endian_handler (tmpmem, 1); + m->endian_handler (tmpmem, 0 /* host endian */); } if (m && m->tojson_handler) diff --git a/src/vlibapi/node_serialize.c b/src/vlibapi/node_serialize.c index 09b59247eab..3f921fb0450 100644 --- a/src/vlibapi/node_serialize.c +++ b/src/vlibapi/node_serialize.c @@ -84,26 +84,27 @@ vlib_node_serialize (vlib_main_t * vm, vlib_node_t *** node_dups, u8 * vector, { vlib_process_t *p = vlib_get_process_from_node (vm, n); - switch (p->flags - & (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK - | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT)) + switch (p->state) { default: - if (!(p->flags & VLIB_PROCESS_IS_RUNNING)) - state_code = STATE_DONE; break; - case VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK: + case VLIB_PROCESS_STATE_WAIT_FOR_CLOCK: + case VLIB_PROCESS_STATE_SUSPENDED: state_code = STATE_TIME_WAIT; break; - case VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT: + case VLIB_PROCESS_STATE_WAIT_FOR_EVENT: + case VLIB_PROCESS_STATE_WAIT_FOR_ONE_TIME_EVENT: state_code = STATE_EVENT_WAIT; break; - case (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK): - state_code = - STATE_ANY_WAIT; + case VLIB_PROCESS_STATE_WAIT_FOR_EVENT_OR_CLOCK: + state_code = STATE_ANY_WAIT; + break; + + case VLIB_PROCESS_STATE_NOT_STARTED: + state_code = STATE_DONE; break; } } diff --git a/src/vnet/CMakeLists.txt b/src/vnet/CMakeLists.txt index ad5f44846cc..254849cbc1b 100644 --- a/src/vnet/CMakeLists.txt +++ b/src/vnet/CMakeLists.txt @@ -28,6 +28,7 @@ list(APPEND VNET_SOURCES devices/netlink.c dev/api.c dev/args.c + dev/bus/pci.c dev/cli.c dev/config.c dev/counters.c @@ -36,7 +37,6 @@ list(APPEND VNET_SOURCES dev/error.c dev/format.c dev/handlers.c - dev/pci.c dev/port.c dev/process.c dev/queue.c @@ -74,7 +74,17 @@ list(APPEND VNET_HEADERS config.h devices/devices.h devices/netlink.h + dev/api.h + dev/args.h + dev/bus/pci.h + dev/counters.h + dev/dev_funcs.h dev/dev.h + dev/errors.h + dev/log.h + dev/mgmt.h + dev/process.h + dev/types.h flow/flow.h global_funcs.h interface/rx_queue_funcs.h @@ -94,6 +104,7 @@ list(APPEND VNET_HEADERS format_fns.h ip/ip_format_fns.h ip/ip_sas.h + ip/vtep.h ethernet/ethernet_format_fns.h ethernet/ethernet_types_api.h ) @@ -596,19 +607,6 @@ list(APPEND VNET_HEADERS ) ############################################################################## -# Layer 3 protocol: osi -############################################################################## -list(APPEND VNET_SOURCES - osi/node.c - osi/osi.c - osi/pg.c -) - -list(APPEND VNET_HEADERS - osi/osi.h -) - -############################################################################## # Layer 4 protocol: tcp ############################################################################## list(APPEND VNET_SOURCES @@ -962,12 +960,14 @@ list(APPEND VNET_API_FILES if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") list(APPEND VNET_SOURCES + dev/bus/platform.c devices/tap/cli.c devices/tap/tap.c devices/tap/tapv2_api.c ) list(APPEND VNET_HEADERS + dev/bus/platform.h devices/tap/tap.h ) @@ -1012,6 +1012,7 @@ list(APPEND VNET_SOURCES session/application_namespace.c session/segment_manager.c session/session_api.c + session/session_sdl.c ) list(APPEND VNET_HEADERS @@ -1042,10 +1043,13 @@ list(APPEND VNET_API_FILES session/session.api) list(APPEND VNET_SOURCES tls/tls.c + tls/tls_record.c ) list(APPEND VNET_HEADERS tls/tls.h + tls/tls_inlines.h + tls/tls_record.h tls/tls_test.h ) @@ -1144,6 +1148,8 @@ list(APPEND VNET_HEADERS fib/fib_path_list.h fib/fib_sas.h fib/fib_source.h + fib/fib_urpf_list.h + fib/fib_walk.h ) list(APPEND VNET_API_FILES @@ -1391,6 +1397,7 @@ list (APPEND VNET_SOURCES list(APPEND VNET_HEADERS arp/arp.h + arp/arp_packet.h ) list(APPEND VNET_API_FILES arp/arp.api) diff --git a/src/vnet/dev/pci.c b/src/vnet/dev/bus/pci.c index 3cc0cba5003..4bb8660f4b4 100644 --- a/src/vnet/dev/pci.c +++ b/src/vnet/dev/bus/pci.c @@ -4,7 +4,7 @@ #include <vnet/vnet.h> #include <vnet/dev/dev.h> -#include <vnet/dev/pci.h> +#include <vnet/dev/bus/pci.h> #include <vnet/dev/log.h> #include <vlib/unix/unix.h> diff --git a/src/vnet/dev/pci.h b/src/vnet/dev/bus/pci.h index ce9a53aa273..ce9a53aa273 100644 --- a/src/vnet/dev/pci.h +++ b/src/vnet/dev/bus/pci.h diff --git a/src/vnet/dev/bus/platform.c b/src/vnet/dev/bus/platform.c new file mode 100644 index 00000000000..56a1f9c762c --- /dev/null +++ b/src/vnet/dev/bus/platform.c @@ -0,0 +1,172 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Cisco Systems, Inc. + */ + +#include <vnet/vnet.h> +#include <vnet/dev/dev.h> +#include <vnet/dev/bus/platform.h> + +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <dirent.h> + +VLIB_REGISTER_LOG_CLASS (dev_log, static) = { + .class_name = "dev", + .subclass_name = "platform", +}; + +#define log_debug(dev, f, ...) \ + vlib_log (VLIB_LOG_LEVEL_DEBUG, dev_log.class, "%U" f, format_vnet_dev_log, \ + dev, \ + clib_string_skip_prefix (__func__, "vnet_dev_bus_platform_dt_"), \ + ##__VA_ARGS__) +#define log_err(dev, f, ...) \ + vlib_log (VLIB_LOG_LEVEL_ERR, dev_log.class, "%U" f, format_vnet_dev_log, \ + dev, 0, ##__VA_ARGS__) + +#define PLATFORM_DEV_PATH "/sys/bus/platform/devices" + +clib_dt_main_t vnet_dev_bus_platform_dt_main; + +vnet_dev_rv_t +vnet_dev_bus_platform_dt_node_from_device_id (clib_dt_node_t **nodep, + char *device_id) +{ + clib_dt_main_t *dm = &vnet_dev_bus_platform_dt_main; + clib_dt_node_t *n; + char *name = device_id + sizeof (PLATFORM_BUS_NAME); + char path[PATH_MAX]; + int r; + u8 *link; + + if (dm->root == 0) + { + clib_error_t *err; + err = clib_dt_read_from_sysfs (&vnet_dev_bus_platform_dt_main); + if (err) + { + log_err (0, "cannot read devicetree: %U", format_clib_error, err); + clib_error_free (err); + return VNET_DEV_ERR_NOT_FOUND; + } + } + + link = format (0, PLATFORM_DEV_PATH "/%s/of_node%c", name, 0); + r = readlink ((char *) link, path, sizeof (path) - 1); + + if (r < 1) + { + log_err (0, "of_node doesn't exist for '%s'", name); + vec_free (link); + return VNET_DEV_ERR_NOT_FOUND; + } + + path[r] = 0; + vec_reset_length (link); + link = format (link, PLATFORM_DEV_PATH "/%s/%s%c", name, path, 0); + if (!realpath ((char *) link, path)) + { + log_err (0, "cannot find realpath for '%s'", link); + vec_free (link); + return VNET_DEV_ERR_NOT_FOUND; + } + + vec_free (link); + + if (strncmp (CLIB_DT_LINUX_PREFIX, path, + sizeof (CLIB_DT_LINUX_PREFIX) - 1) != 0) + return VNET_DEV_ERR_BUG; + + n = clib_dt_get_node_with_path (dm, "%s", + path + sizeof (CLIB_DT_LINUX_PREFIX) - 1); + + if (n) + { + *nodep = n; + return VNET_DEV_OK; + } + + return VNET_DEV_ERR_NOT_FOUND; +} + +static void * +vnet_dev_bus_platform_get_device_info (vlib_main_t *vm, char *device_id) +{ + clib_dt_node_t *n = 0; + vnet_dev_bus_platform_device_info_t *di; + + vnet_dev_bus_platform_dt_node_from_device_id (&n, device_id); + + if (n) + { + clib_dt_property_t *compatible; + compatible = clib_dt_get_node_property_by_name (n, "compatible"); + log_debug (0, "node found, is compatible %U", + format_clib_dt_property_data, compatible); + di = clib_mem_alloc (sizeof (*di)); + di->node = n; + return di; + } + + return 0; +} + +static void +vnet_dev_bus_platform_free_device_info (vlib_main_t *vm, void *p) +{ + clib_mem_free (p); +} + +static void +vnet_dev_bus_platform_close (vlib_main_t *vm, vnet_dev_t *dev) +{ + log_debug (dev, ""); +} + +static vnet_dev_rv_t +vnet_dev_bus_platform_open (vlib_main_t *vm, vnet_dev_t *dev) +{ + clib_dt_node_t *n = 0; + vnet_dev_bus_platform_device_data_t *dd = vnet_dev_get_bus_data (dev); + vnet_dev_rv_t rv; + + log_debug (dev, ""); + + rv = vnet_dev_bus_platform_dt_node_from_device_id (&n, dev->device_id); + if (rv != VNET_DEV_OK) + return rv; + + dd->node = n; + return VNET_DEV_OK; +} + +static u8 * +format_dev_bus_platform_device_info (u8 *s, va_list *args) +{ + vnet_dev_format_args_t __clib_unused *a = + va_arg (*args, vnet_dev_format_args_t *); + vnet_dev_t *dev = va_arg (*args, vnet_dev_t *); + vnet_dev_bus_platform_device_data_t *dd = vnet_dev_get_bus_data (dev); + return format (s, "device-tree path is '%v'", dd->node->path); +} + +static u8 * +format_dev_bus_platform_device_addr (u8 *s, va_list *args) +{ + vnet_dev_t *dev = va_arg (*args, vnet_dev_t *); + return format (s, "%s", dev->device_id + sizeof (PLATFORM_BUS_NAME)); +} + +VNET_DEV_REGISTER_BUS (pp2) = { + .name = PLATFORM_BUS_NAME, + .device_data_size = sizeof (vnet_dev_bus_platform_device_info_t), + .ops = { + .get_device_info = vnet_dev_bus_platform_get_device_info, + .free_device_info = vnet_dev_bus_platform_free_device_info, + .device_open = vnet_dev_bus_platform_open, + .device_close = vnet_dev_bus_platform_close, + .format_device_info = format_dev_bus_platform_device_info, + .format_device_addr = format_dev_bus_platform_device_addr, + }, +}; diff --git a/src/vnet/dev/bus/platform.h b/src/vnet/dev/bus/platform.h new file mode 100644 index 00000000000..3492aad57ed --- /dev/null +++ b/src/vnet/dev/bus/platform.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2023 Cisco Systems, Inc. + */ + +#ifndef _VNET_DEV_BUS_PLATFORM_H_ +#define _VNET_DEV_BUS_PLATFORM_H_ + +#include <vppinfra/clib.h> +#include <vppinfra/devicetree.h> +#include <vlib/vlib.h> +#include <vnet/dev/dev.h> + +#define PLATFORM_BUS_NAME "platform" + +extern clib_dt_main_t vnet_dev_bus_platform_dt_main; + +typedef struct +{ + clib_dt_node_t *node; +} vnet_dev_bus_platform_device_info_t; + +typedef struct +{ + clib_dt_node_t *node; +} vnet_dev_bus_platform_device_data_t; + +#endif /* _VNET_DEV_BUS_PLATFORM_H_ */ diff --git a/src/vnet/dev/counters.c b/src/vnet/dev/counters.c index 0a1e0a7419d..d02839d664f 100644 --- a/src/vnet/dev/counters.c +++ b/src/vnet/dev/counters.c @@ -54,7 +54,7 @@ vnet_dev_counters_clear (vlib_main_t *vm, vnet_dev_counter_main_t *cm) { for (int i = 0; i < cm->n_counters; i++) { - cm->counter_start[i] = cm->counter_data[i]; + cm->counter_start[i] += cm->counter_data[i]; cm->counter_data[i] = 0; } } diff --git a/src/vnet/dev/dev.h b/src/vnet/dev/dev.h index bbf2f9dff21..eb06eeba34e 100644 --- a/src/vnet/dev/dev.h +++ b/src/vnet/dev/dev.h @@ -115,6 +115,7 @@ typedef struct vnet_dev_rx_queue_op_t *start; vnet_dev_rx_queue_op_no_rv_t *stop; vnet_dev_rx_queue_op_no_rv_t *free; + vnet_dev_rx_queue_op_no_rv_t *clear_counters; format_function_t *format_info; } vnet_dev_rx_queue_ops_t; @@ -124,6 +125,7 @@ typedef struct vnet_dev_tx_queue_op_t *start; vnet_dev_tx_queue_op_no_rv_t *stop; vnet_dev_tx_queue_op_no_rv_t *free; + vnet_dev_tx_queue_op_no_rv_t *clear_counters; format_function_t *format_info; } vnet_dev_tx_queue_ops_t; @@ -245,6 +247,7 @@ typedef struct vnet_dev_port_op_no_rv_t *stop; vnet_dev_port_op_no_rv_t *deinit; vnet_dev_port_op_no_rv_t *free; + vnet_dev_port_op_no_rv_t *clear_counters; format_function_t *format_status; format_function_t *format_flow; } vnet_dev_port_ops_t; diff --git a/src/vnet/dev/errors.h b/src/vnet/dev/errors.h index 430a6aef282..6ececad12ec 100644 --- a/src/vnet/dev/errors.h +++ b/src/vnet/dev/errors.h @@ -41,6 +41,7 @@ _ (UNSUPPORTED_DEVICE, "unsupported device") \ _ (UNSUPPORTED_DEVICE_VER, "unsupported device version") \ _ (ALREADY_DONE, "already done") \ - _ (NO_SUCH_INTERFACE, "no such interface") + _ (NO_SUCH_INTERFACE, "no such interface") \ + _ (INIT_FAILED, "init failed") #endif /* _VNET_DEV_ERRORS_H_ */ diff --git a/src/vnet/dev/port.c b/src/vnet/dev/port.c index 8a6df54cbc8..df7805c1ff2 100644 --- a/src/vnet/dev/port.c +++ b/src/vnet/dev/port.c @@ -305,7 +305,8 @@ vnet_dev_port_cfg_change_req_validate (vlib_main_t *vm, vnet_dev_port_t *port, switch (req->type) { case VNET_DEV_PORT_CFG_MAX_RX_FRAME_SIZE: - if (req->max_rx_frame_size > port->attr.max_supported_rx_frame_size) + if ((req->max_rx_frame_size > port->attr.max_supported_rx_frame_size) || + (req->max_rx_frame_size < ETHERNET_MIN_PACKET_BYTES)) return VNET_DEV_ERR_INVALID_VALUE; if (req->max_rx_frame_size == port->max_rx_frame_size) return VNET_DEV_ERR_NO_CHANGE; @@ -733,16 +734,26 @@ vnet_dev_port_if_remove (vlib_main_t *vm, vnet_dev_port_t *port) void vnet_dev_port_clear_counters (vlib_main_t *vm, vnet_dev_port_t *port) { - if (port->counter_main) + if (port->port_ops.clear_counters) + port->port_ops.clear_counters (vm, port); + else if (port->counter_main) vnet_dev_counters_clear (vm, port->counter_main); foreach_vnet_dev_port_rx_queue (q, port) - if (q->counter_main) - vnet_dev_counters_clear (vm, q->counter_main); + { + if (port->rx_queue_ops.clear_counters) + port->rx_queue_ops.clear_counters (vm, q); + else if (q->counter_main) + vnet_dev_counters_clear (vm, q->counter_main); + } foreach_vnet_dev_port_tx_queue (q, port) - if (q->counter_main) - vnet_dev_counters_clear (vm, q->counter_main); + { + if (port->tx_queue_ops.clear_counters) + port->tx_queue_ops.clear_counters (vm, q); + else if (q->counter_main) + vnet_dev_counters_clear (vm, q->counter_main); + } log_notice (port->dev, "counters cleared on port %u", port->port_id); } diff --git a/src/vnet/dev/process.c b/src/vnet/dev/process.c index 3c1f0b8d2d8..3d3b49c01b6 100644 --- a/src/vnet/dev/process.c +++ b/src/vnet/dev/process.c @@ -31,6 +31,8 @@ typedef struct { vnet_dev_event_t event; u8 reply_needed : 1; + u8 completed : 1; + vnet_dev_rv_t rv; u32 calling_process_index; union { @@ -69,7 +71,32 @@ typedef struct }; } vnet_dev_event_data_t; -static vnet_dev_rv_t +vnet_dev_event_data_t * +vnet_dev_event_data_alloc (vlib_main_t *vm, vnet_dev_t *dev) +{ + vnet_dev_event_data_t *ed = clib_mem_alloc (sizeof (vnet_dev_event_data_t)); + *ed = (vnet_dev_event_data_t){}; + + return ed; +} + +void +vnet_dev_event_data_free (vlib_main_t *vm, vnet_dev_t *dev, + vnet_dev_event_data_t *ed) +{ + clib_mem_free (ed); +} + +static void +ev_log_debug (vlib_main_t *vm, vnet_dev_t *dev, vnet_dev_event_data_t *ed, + char *str) +{ + log_debug (dev, "%s requested received from process node %U (%u)", str, + format_vlib_node_name, vm, ed->calling_process_index, + ed->calling_process_index); +} + +static void vnet_dev_process_one_event (vlib_main_t *vm, vnet_dev_t *dev, vnet_dev_event_data_t *ed) { @@ -81,50 +108,56 @@ vnet_dev_process_one_event (vlib_main_t *vm, vnet_dev_t *dev, case VNET_DEV_EVENT_CLOCK: break; case VNET_DEV_EVENT_PROCESS_QUIT: - log_debug (dev, "quit requested"); + ev_log_debug (vm, dev, ed, "quit"); dev->process_node_quit = 1; break; case VNET_DEV_EVENT_PERIODIC_START: - log_debug (dev, "periodic start"); + ev_log_debug (vm, dev, ed, "periodic start"); dev->process_node_periodic = 1; break; case VNET_DEV_EVENT_PERIODIC_STOP: - log_debug (dev, "periodic stop"); + ev_log_debug (vm, dev, ed, "periodic stop"); dev->process_node_periodic = 0; break; case VNET_DEV_EVENT_PORT_CONFIG_CHANGE_REQ: - log_debug (dev, "port config change"); + ev_log_debug (vm, dev, ed, "port config change"); p = ed->port_cfg_change.port; rv = vnet_dev_port_cfg_change (vm, p, ed->port_cfg_change.change_req); break; case VNET_DEV_EVENT_CALL_OP: - log_debug (dev, "call op"); + ev_log_debug (vm, dev, ed, "call op"); rv = ed->call_op.op (vm, dev); break; case VNET_DEV_EVENT_CALL_OP_NO_RV: - log_debug (dev, "call op no rv"); + ev_log_debug (vm, dev, ed, "call op no rv"); ed->call_op_no_rv.op (vm, dev); break; case VNET_DEV_EVENT_CALL_OP_NO_WAIT: - log_debug (dev, "call op no wait"); + ev_log_debug (vm, dev, ed, "call op no wait"); ed->call_op_no_wait.op (vm, dev); break; case VNET_DEV_EVENT_CALL_PORT_OP: - log_debug (dev, "call port op"); + ev_log_debug (vm, dev, ed, "call port op"); rv = ed->call_port_op.op (vm, ed->call_port_op.port); break; case VNET_DEV_EVENT_CALL_PORT_OP_NO_RV: - log_debug (dev, "call port op no rv"); + ev_log_debug (vm, dev, ed, "call port op no rv"); ed->call_port_op_no_rv.op (vm, ed->call_port_op_no_rv.port); break; case VNET_DEV_EVENT_CALL_PORT_OP_NO_WAIT: - log_debug (dev, "call port op no wait"); + ev_log_debug (vm, dev, ed, "call port op no wait"); ed->call_port_op_no_wait.op (vm, ed->call_port_op_no_wait.port); break; default: ASSERT (0); } - return rv; + if (ed->reply_needed) + { + ed->rv = rv; + ed->completed = 1; + } + else + vnet_dev_event_data_free (vm, dev, ed); } static uword @@ -133,7 +166,7 @@ vnet_dev_process (vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f) vnet_dev_main_t *dm = &vnet_dev_main; vnet_dev_periodic_op_t *pop, *pops = 0; f64 next = CLIB_F64_MAX; - vnet_dev_event_data_t *event_data = 0, *new_event_data, *ed; + vnet_dev_event_data_t **event_data = 0, **new_event_data, **edp; vnet_dev_t *dev = *((vnet_dev_t **) vlib_node_get_runtime_data (vm, rt->node_index)); @@ -160,14 +193,8 @@ vnet_dev_process (vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f) ASSERT (event_type == 0); - vec_foreach (ed, event_data) - { - vnet_dev_rv_t rv; - rv = vnet_dev_process_one_event (vm, dev, ed); - if (ed->reply_needed) - vlib_process_signal_event (vm, ed->calling_process_index, - ed->event, rv); - } + vec_foreach (edp, event_data) + vnet_dev_process_one_event (vm, dev, *edp); vec_reset_length (event_data); } @@ -260,47 +287,67 @@ vnet_dev_process_create (vlib_main_t *vm, vnet_dev_t *dev) static void vnet_dev_process_event_send (vlib_main_t *vm, vnet_dev_t *dev, - vnet_dev_event_data_t ed) + vnet_dev_event_data_t *ed) { - vnet_dev_event_data_t *edp = vlib_process_signal_event_data ( - vm, dev->process_node_index, 0, 1, sizeof (ed)); - *edp = ed; + void *p = vlib_process_signal_event_data (vm, dev->process_node_index, 0, 1, + sizeof (void *)); + *(vnet_dev_event_data_t **) p = ed; } static vnet_dev_rv_t vnet_dev_process_event_send_and_wait (vlib_main_t *vm, vnet_dev_t *dev, - vnet_dev_event_data_t ed) + vnet_dev_event_data_t *ed) { - uword event, *event_data = 0; - vnet_dev_rv_t rv; + ed->calling_process_index = vlib_get_current_process_node_index (vm); + vnet_dev_rv_t rv = VNET_DEV_ERR_PROCESS_REPLY; - ed.calling_process_index = vlib_get_current_process_node_index (vm); + ed->reply_needed = 1; - if (ed.calling_process_index == dev->process_node_index) - return vnet_dev_process_one_event (vm, dev, &ed); + if (ed->calling_process_index == dev->process_node_index) + { + vnet_dev_process_one_event (vm, dev, ed); + rv = ed->rv; + goto done; + } - ed.reply_needed = 1; vnet_dev_process_event_send (vm, dev, ed); - vlib_process_wait_for_event_or_clock (vm, 5.0); - event = vlib_process_get_events (vm, &event_data); - if (event != ed.event) + vlib_process_yield (vm); + + if (!ed->completed) { - log_err (dev, "%s", - event == VNET_DEV_EVENT_CLOCK ? - "timeout waiting for process node to respond" : - "unexpected event received"); - rv = VNET_DEV_ERR_PROCESS_REPLY; + f64 t0, interval = 25e-6, max_interval = 50e-3; + t0 = vlib_time_now (vm); + + do + { + vlib_process_suspend (vm, interval); + if (ed->completed) + { + rv = ed->rv; + goto done; + } + if (interval < max_interval) + { + interval *= 2; + if (interval > max_interval) + interval = max_interval; + } + } + while (vlib_time_now (vm) - t0 < 5.0); } else - rv = event_data[0]; - vec_free (event_data); + rv = ed->rv; + +done: + vnet_dev_event_data_free (vm, dev, ed); return rv; } void vnet_dev_process_quit (vlib_main_t *vm, vnet_dev_t *dev) { - vnet_dev_event_data_t ed = { .event = VNET_DEV_EVENT_PROCESS_QUIT }; + vnet_dev_event_data_t *ed = vnet_dev_event_data_alloc (vm, dev); + *ed = (vnet_dev_event_data_t){ .event = VNET_DEV_EVENT_PROCESS_QUIT }; vnet_dev_process_event_send_and_wait (vm, dev, ed); } @@ -308,7 +355,8 @@ static int _vnet_dev_poll_add (vlib_main_t *vm, vnet_dev_t *dev, vnet_dev_periodic_op_t pop) { - const vnet_dev_event_data_t ed = { .event = VNET_DEV_EVENT_PERIODIC_START }; + vnet_dev_event_data_t *ed = vnet_dev_event_data_alloc (vm, dev); + *ed = (vnet_dev_event_data_t){ .event = VNET_DEV_EVENT_PERIODIC_START }; vnet_dev_periodic_op_t *p; pool_foreach (p, dev->periodic_ops) @@ -325,7 +373,8 @@ _vnet_dev_poll_add (vlib_main_t *vm, vnet_dev_t *dev, static int _vnet_dev_poll_remove (vlib_main_t *vm, vnet_dev_t *dev, void *op, void *arg) { - const vnet_dev_event_data_t ed = { .event = VNET_DEV_EVENT_PERIODIC_STOP }; + vnet_dev_event_data_t *ed = vnet_dev_event_data_alloc (vm, dev); + *ed = (vnet_dev_event_data_t){ .event = VNET_DEV_EVENT_PERIODIC_STOP }; vnet_dev_periodic_op_t *pop; pool_foreach (pop, dev->periodic_ops) @@ -391,7 +440,8 @@ vnet_dev_rv_t vnet_dev_process_port_cfg_change_req (vlib_main_t *vm, vnet_dev_port_t *port, vnet_dev_port_cfg_change_req_t *pccr) { - const vnet_dev_event_data_t ed = { + vnet_dev_event_data_t *ed = vnet_dev_event_data_alloc (vm, port->dev); + *ed = (vnet_dev_event_data_t) { .event = VNET_DEV_EVENT_PORT_CONFIG_CHANGE_REQ, .port_cfg_change = { .port = port, @@ -405,7 +455,8 @@ vnet_dev_process_port_cfg_change_req (vlib_main_t *vm, vnet_dev_port_t *port, vnet_dev_rv_t vnet_dev_process_call_op (vlib_main_t *vm, vnet_dev_t *dev, vnet_dev_op_t *op) { - const vnet_dev_event_data_t ed = { + vnet_dev_event_data_t *ed = vnet_dev_event_data_alloc (vm, dev); + *ed = (vnet_dev_event_data_t){ .event = VNET_DEV_EVENT_CALL_OP, .call_op.op = op, }; @@ -417,7 +468,8 @@ vnet_dev_rv_t vnet_dev_process_call_op_no_rv (vlib_main_t *vm, vnet_dev_t *dev, vnet_dev_op_no_rv_t *op) { - const vnet_dev_event_data_t ed = { + vnet_dev_event_data_t *ed = vnet_dev_event_data_alloc (vm, dev); + *ed = (vnet_dev_event_data_t){ .event = VNET_DEV_EVENT_CALL_OP_NO_RV, .call_op_no_rv.op = op, }; @@ -429,7 +481,8 @@ void vnet_dev_process_call_op_no_wait (vlib_main_t *vm, vnet_dev_t *dev, vnet_dev_op_no_rv_t *op) { - const vnet_dev_event_data_t ed = { + vnet_dev_event_data_t *ed = vnet_dev_event_data_alloc (vm, dev); + *ed = (vnet_dev_event_data_t){ .event = VNET_DEV_EVENT_CALL_OP_NO_WAIT, .call_op_no_rv.op = op, }; @@ -441,7 +494,8 @@ vnet_dev_rv_t vnet_dev_process_call_port_op (vlib_main_t *vm, vnet_dev_port_t *port, vnet_dev_port_op_t *op) { - const vnet_dev_event_data_t ed = { + vnet_dev_event_data_t *ed = vnet_dev_event_data_alloc (vm, port->dev); + *ed = (vnet_dev_event_data_t){ .event = VNET_DEV_EVENT_CALL_PORT_OP, .call_port_op = { .op = op, .port = port }, }; @@ -453,7 +507,8 @@ vnet_dev_rv_t vnet_dev_process_call_port_op_no_rv (vlib_main_t *vm, vnet_dev_port_t *port, vnet_dev_port_op_no_rv_t *op) { - const vnet_dev_event_data_t ed = { + vnet_dev_event_data_t *ed = vnet_dev_event_data_alloc (vm, port->dev); + *ed = (vnet_dev_event_data_t){ .event = VNET_DEV_EVENT_CALL_PORT_OP_NO_RV, .call_port_op_no_rv = { .op = op, .port = port }, }; @@ -465,7 +520,8 @@ void vnet_dev_process_call_port_op_no_wait (vlib_main_t *vm, vnet_dev_port_t *port, vnet_dev_port_op_no_rv_t *op) { - const vnet_dev_event_data_t ed = { + vnet_dev_event_data_t *ed = vnet_dev_event_data_alloc (vm, port->dev); + *ed = (vnet_dev_event_data_t){ .event = VNET_DEV_EVENT_CALL_PORT_OP_NO_WAIT, .call_port_op_no_wait = { .op = op, .port = port }, }; diff --git a/src/vnet/devices/tap/tap.c b/src/vnet/devices/tap/tap.c index 1e2ee87041d..b0b0a3af13f 100644 --- a/src/vnet/devices/tap/tap.c +++ b/src/vnet/devices/tap/tap.c @@ -85,7 +85,7 @@ virtio_eth_set_max_frame_size (vnet_main_t *vnm, vnet_hw_interface_t *hi, return 0; } -#define TAP_MAX_INSTANCE 1024 +#define TAP_MAX_INSTANCE 8192 static void tap_free (vlib_main_t * vm, virtio_if_t * vif) @@ -162,7 +162,8 @@ tap_create_if (vlib_main_t * vm, tap_create_if_args_t * args) } else { - args->id = clib_bitmap_first_clear (tm->tap_ids); + args->id = clib_bitmap_next_clear (tm->tap_ids, args->auto_id_offset % + TAP_MAX_INSTANCE); } if (args->id > TAP_MAX_INSTANCE) diff --git a/src/vnet/devices/tap/tap.h b/src/vnet/devices/tap/tap.h index 6b88c34fe41..1df2fb7e1ad 100644 --- a/src/vnet/devices/tap/tap.h +++ b/src/vnet/devices/tap/tap.h @@ -42,6 +42,7 @@ typedef enum typedef struct { u32 id; + u32 auto_id_offset; u8 mac_addr_set; mac_address_t mac_addr; u16 num_rx_queues; diff --git a/src/vnet/devices/virtio/FEATURE.yaml b/src/vnet/devices/virtio/FEATURE.yaml index 446a45b61a3..870023861de 100644 --- a/src/vnet/devices/virtio/FEATURE.yaml +++ b/src/vnet/devices/virtio/FEATURE.yaml @@ -13,6 +13,7 @@ features: - Support virtio 1.1 packed ring in vhost - Support for tx queue size configuration (tested on host kernel 5.15 and qemu version 6.2.0) + - Support RSS (tested on ubuntu 23.10) description: "Virtio implementation" missing: - API dump filtering by sw_if_index diff --git a/src/vnet/devices/virtio/cli.c b/src/vnet/devices/virtio/cli.c index c1b6c8be065..c4364600722 100644 --- a/src/vnet/devices/virtio/cli.c +++ b/src/vnet/devices/virtio/cli.c @@ -62,6 +62,8 @@ virtio_pci_create_command_fn (vlib_main_t * vm, unformat_input_t * input, args.bind = VIRTIO_BIND_FORCE; else if (unformat (line_input, "bind")) args.bind = VIRTIO_BIND_DEFAULT; + else if (unformat (line_input, "rss-enabled")) + args.rss_enabled = 1; else return clib_error_return (0, "unknown input `%U'", format_unformat_error, input); @@ -77,7 +79,7 @@ VLIB_CLI_COMMAND (virtio_pci_create_command, static) = { .path = "create interface virtio", .short_help = "create interface virtio <pci-address> " "[feature-mask <hex-mask>] [tx-queue-size <size>] " - "[gso-enabled] [csum-enabled] " + "[gso-enabled] [csum-enabled] [rss-enabled] " "[buffering [size <buffering-szie>]] [packed] [bind [force]]", .function = virtio_pci_create_command_fn, }; diff --git a/src/vnet/devices/virtio/pci.c b/src/vnet/devices/virtio/pci.c index 6234f64fcfb..140cdb94153 100644 --- a/src/vnet/devices/virtio/pci.c +++ b/src/vnet/devices/virtio/pci.c @@ -37,6 +37,13 @@ #define PCI_MSIX_ENABLE 0x8000 +static const u8 virtio_rss_key[VIRTIO_NET_RSS_MAX_KEY_SIZE] = { + 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67, + 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb, + 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, + 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa, +}; + static pci_device_id_t virtio_pci_device_ids[] = { { .vendor_id = PCI_VENDOR_ID_VIRTIO, @@ -584,6 +591,35 @@ virtio_pci_enable_multiqueue (vlib_main_t * vm, virtio_if_t * vif, return status; } +static int +virtio_pci_enable_multiqueue_rss (vlib_main_t *vm, virtio_if_t *vif, + u16 num_queues) +{ + virtio_ctrl_msg_t mq_hdr; + virtio_net_rss_config *rss = (virtio_net_rss_config *) mq_hdr.data; + virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR; + + STATIC_ASSERT (sizeof (*rss) <= sizeof (mq_hdr.data), + "virtio_net_rss_config size too big"); + mq_hdr.ctrl.class = VIRTIO_NET_CTRL_MQ; + mq_hdr.ctrl.cmd = VIRTIO_NET_CTRL_MQ_RSS_CONFIG; + mq_hdr.status = VIRTIO_NET_ERR; + + rss->hash_types = VIRTIO_NET_HASH_TYPE_SUPPORTED; + rss->indirection_table_mask = VIRTIO_NET_RSS_MAX_TABLE_LEN - 1; + rss->unclassified_queue = 0; + for (int i = 0; i < VIRTIO_NET_RSS_MAX_TABLE_LEN; i++) + rss->indirection_table[i] = i % num_queues; + rss->max_tx_vq = num_queues; + rss->hash_key_length = VIRTIO_NET_RSS_MAX_KEY_SIZE; + clib_memcpy (rss->hash_key_data, virtio_rss_key, + VIRTIO_NET_RSS_MAX_KEY_SIZE); + + status = virtio_pci_send_ctrl_msg (vm, vif, &mq_hdr, sizeof (*rss)); + virtio_log_debug (vif, "multi-queue with rss enable %u queues", num_queues); + return status; +} + static u8 virtio_pci_queue_size_valid (u16 qsz) { @@ -933,6 +969,9 @@ virtio_negotiate_features (vlib_main_t * vm, virtio_if_t * vif, | VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT) | VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC); + if (vif->rss_enabled) + supported_features |= VIRTIO_FEATURE (VIRTIO_NET_F_RSS); + if (vif->is_modern) supported_features |= VIRTIO_FEATURE (VIRTIO_F_VERSION_1); @@ -1375,6 +1414,7 @@ virtio_pci_create_if (vlib_main_t * vm, virtio_pci_create_if_args_t * args) vif->dev_instance = vif - vim->interfaces; vif->per_interface_next_index = ~0; vif->pci_addr.as_u32 = args->addr; + vif->rss_enabled = args->rss_enabled; if (args->virtio_flags & VIRTIO_FLAG_PACKED) vif->is_packed = 1; @@ -1536,8 +1576,16 @@ virtio_pci_create_if (vlib_main_t * vm, virtio_pci_create_if_args_t * args) if ((vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)) && (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ))) { - if (virtio_pci_enable_multiqueue (vm, vif, vif->max_queue_pairs)) - virtio_log_warning (vif, "multiqueue is not set"); + if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_RSS)) + { + if (virtio_pci_enable_multiqueue_rss (vm, vif, vif->max_queue_pairs)) + virtio_log_warning (vif, "multiqueue with rss is not set"); + } + else + { + if (virtio_pci_enable_multiqueue (vm, vif, vif->max_queue_pairs)) + virtio_log_warning (vif, "multiqueue is not set"); + } } return; diff --git a/src/vnet/devices/virtio/pci.h b/src/vnet/devices/virtio/pci.h index 5eb80f823be..745ad6fce87 100644 --- a/src/vnet/devices/virtio/pci.h +++ b/src/vnet/devices/virtio/pci.h @@ -60,6 +60,9 @@ typedef enum /* If multiqueue is provided by host, then we support it. */ #define VIRTIO_NET_CTRL_MQ 4 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0 +#define VIRTIO_NET_CTRL_MQ_RSS_CONFIG 1 +#define VIRTIO_NET_CTRL_MQ_HASH_CONFIG 2 + #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000 @@ -139,14 +142,77 @@ typedef struct u64 queue_device; /* read-write */ } virtio_pci_common_cfg_t; +#define foreach_virtio_net_hash_report_type \ + _ (NONE, 0) \ + _ (IPV4, 1) \ + _ (TCPV4, 2) \ + _ (UDPV4, 3) \ + _ (IPV6, 4) \ + _ (TCPV6, 5) \ + _ (UDPV6, 6) \ + _ (IPV6_EX, 7) \ + _ (TCPV6_EX, 8) \ + _ (UDPV6_EX, 9) + +typedef enum +{ +#define _(n, i) VIRTIO_NET_HASH_REPORT_##n = i, + foreach_virtio_net_hash_report_type +#undef _ +} virtio_net_hash_report_type_t; + typedef struct { u8 mac[6]; u16 status; u16 max_virtqueue_pairs; u16 mtu; + u32 speed; + u8 duplex; + u8 rss_max_key_size; + u16 rss_max_indirection_table_length; + u32 supported_hash_types; } virtio_net_config_t; +#define VIRTIO_NET_RSS_MAX_TABLE_LEN 128 +#define VIRTIO_NET_RSS_MAX_KEY_SIZE 40 + +#define foreach_virtio_net_hash_type \ + _ (IPV4, 0) \ + _ (TCPV4, 1) \ + _ (UDPV4, 2) \ + _ (IPV6, 3) \ + _ (TCPV6, 4) \ + _ (UDPV6, 5) \ + _ (IPV6_EX, 6) \ + _ (TCPV6_EX, 7) \ + _ (UDPV6_EX, 8) + +typedef enum +{ +#define _(n, i) VIRTIO_NET_HASH_TYPE_##n = (1 << i), + foreach_virtio_net_hash_type +#undef _ +} virtio_net_hash_type_t; + +#define VIRTIO_NET_HASH_TYPE_SUPPORTED \ + (VIRTIO_NET_HASH_TYPE_IPV4 | VIRTIO_NET_HASH_TYPE_TCPV4 | \ + VIRTIO_NET_HASH_TYPE_UDPV4 | VIRTIO_NET_HASH_TYPE_IPV6 | \ + VIRTIO_NET_HASH_TYPE_TCPV6 | VIRTIO_NET_HASH_TYPE_UDPV6 | \ + VIRTIO_NET_HASH_TYPE_IPV6_EX | VIRTIO_NET_HASH_TYPE_TCPV6_EX | \ + VIRTIO_NET_HASH_TYPE_UDPV6_EX) + +typedef struct +{ + u32 hash_types; + u16 indirection_table_mask; + u16 unclassified_queue; + u16 indirection_table[VIRTIO_NET_RSS_MAX_TABLE_LEN]; + u16 max_tx_vq; + u8 hash_key_length; + u8 hash_key_data[VIRTIO_NET_RSS_MAX_KEY_SIZE]; +} virtio_net_rss_config; + /* * Control virtqueue data structures * @@ -210,13 +276,14 @@ typedef struct _virtio_pci_func void (*device_debug_config_space) (vlib_main_t * vm, virtio_if_t * vif); } virtio_pci_func_t; -#define foreach_virtio_flags \ - _ (GSO, 0) \ - _ (CSUM_OFFLOAD, 1) \ - _ (GRO_COALESCE, 2) \ - _ (PACKED, 3) \ - _ (IN_ORDER, 4) \ - _ (BUFFERING, 5) +#define foreach_virtio_flags \ + _ (GSO, 0) \ + _ (CSUM_OFFLOAD, 1) \ + _ (GRO_COALESCE, 2) \ + _ (PACKED, 3) \ + _ (IN_ORDER, 4) \ + _ (BUFFERING, 5) \ + _ (RSS, 6) typedef enum { @@ -243,6 +310,7 @@ typedef struct u64 features; u8 gso_enabled; u8 checksum_offload_enabled; + u8 rss_enabled; u32 tx_queue_size; virtio_bind_t bind; u32 buffering_size; diff --git a/src/vnet/devices/virtio/virtio.api b/src/vnet/devices/virtio/virtio.api index a11492ec258..14a58491282 100644 --- a/src/vnet/devices/virtio/virtio.api +++ b/src/vnet/devices/virtio/virtio.api @@ -63,6 +63,7 @@ enumflag virtio_flags { VIRTIO_API_FLAG_PACKED = 8, /* enable packed ring support, provided it is available from backend */ VIRTIO_API_FLAG_IN_ORDER = 16, /* enable in order support, provided it is available from backend */ VIRTIO_API_FLAG_BUFFERING = 32 [backwards_compatible], /* enable buffering to handle backend jitter/delays */ + VIRTIO_API_FLAG_RSS = 64 [backwards_compatible], /* enable rss support */ }; /** \brief Initialize a new virtio pci interface with the given parameters diff --git a/src/vnet/devices/virtio/virtio.c b/src/vnet/devices/virtio/virtio.c index d2302fa1dc4..840936a43ff 100644 --- a/src/vnet/devices/virtio/virtio.c +++ b/src/vnet/devices/virtio/virtio.c @@ -404,6 +404,7 @@ virtio_show (vlib_main_t *vm, u32 *hw_if_indices, u8 show_descr, vlib_cli_output (vm, " csum-enabled %d", vif->csum_offload_enabled); vlib_cli_output (vm, " packet-coalesce %d", vif->packet_coalesce); vlib_cli_output (vm, " packet-buffering %d", vif->packet_buffering); + vlib_cli_output (vm, " rss-enabled %d", vif->rss_enabled); if (type & (VIRTIO_IF_TYPE_TAP | VIRTIO_IF_TYPE_PCI)) vlib_cli_output (vm, " Mac Address: %U", format_ethernet_address, vif->mac_addr); diff --git a/src/vnet/devices/virtio/virtio.h b/src/vnet/devices/virtio/virtio.h index 431b1d25c26..a8e258884a4 100644 --- a/src/vnet/devices/virtio/virtio.h +++ b/src/vnet/devices/virtio/virtio.h @@ -140,6 +140,7 @@ typedef struct vnet_virtio_vring_t *txq_vrings; int gso_enabled; int csum_offload_enabled; + int rss_enabled; union { int *tap_fds; diff --git a/src/vnet/devices/virtio/virtio_api.c b/src/vnet/devices/virtio/virtio_api.c index 3197a2fab6d..2e25efff29e 100644 --- a/src/vnet/devices/virtio/virtio_api.c +++ b/src/vnet/devices/virtio/virtio_api.c @@ -111,18 +111,18 @@ vl_api_virtio_pci_create_v2_t_handler (vl_api_virtio_pci_create_v2_t * mp) STATIC_ASSERT (((int) VIRTIO_API_FLAG_BUFFERING == (int) VIRTIO_FLAG_BUFFERING), "virtio buffering api flag mismatch"); + STATIC_ASSERT (((int) VIRTIO_API_FLAG_RSS == (int) VIRTIO_FLAG_RSS), + "virtio rss api flag mismatch"); ap->virtio_flags = clib_net_to_host_u32 (mp->virtio_flags); ap->features = clib_net_to_host_u64 (mp->features); if (ap->virtio_flags & VIRTIO_API_FLAG_GSO) ap->gso_enabled = 1; - else - ap->gso_enabled = 0; if (ap->virtio_flags & VIRTIO_API_FLAG_CSUM_OFFLOAD) ap->checksum_offload_enabled = 1; - else - ap->checksum_offload_enabled = 0; + if (ap->virtio_flags & VIRTIO_API_FLAG_RSS) + ap->rss_enabled = 1; virtio_pci_create_if (vm, ap); diff --git a/src/vnet/devices/virtio/virtio_std.h b/src/vnet/devices/virtio/virtio_std.h index ec988c08dbb..66b8bac092c 100644 --- a/src/vnet/devices/virtio/virtio_std.h +++ b/src/vnet/devices/virtio/virtio_std.h @@ -15,49 +15,61 @@ #ifndef __VIRTIO_STD_H__ #define __VIRTIO_STD_H__ -#define foreach_virtio_net_features \ - _ (VIRTIO_NET_F_CSUM, 0) /* Host handles pkts w/ partial csum */ \ - _ (VIRTIO_NET_F_GUEST_CSUM, 1) /* Guest handles pkts w/ partial csum */ \ - _ (VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, 2) /* Dynamic offload configuration. */ \ - _ (VIRTIO_NET_F_MTU, 3) /* Initial MTU advice. */ \ - _ (VIRTIO_NET_F_MAC, 5) /* Host has given MAC address. */ \ - _ (VIRTIO_NET_F_GSO, 6) /* Host handles pkts w/ any GSO. */ \ - _ (VIRTIO_NET_F_GUEST_TSO4, 7) /* Guest can handle TSOv4 in. */ \ - _ (VIRTIO_NET_F_GUEST_TSO6, 8) /* Guest can handle TSOv6 in. */ \ - _ (VIRTIO_NET_F_GUEST_ECN, 9) /* Guest can handle TSO[6] w/ ECN in. */ \ - _ (VIRTIO_NET_F_GUEST_UFO, 10) /* Guest can handle UFO in. */ \ - _ (VIRTIO_NET_F_HOST_TSO4, 11) /* Host can handle TSOv4 in. */ \ - _ (VIRTIO_NET_F_HOST_TSO6, 12) /* Host can handle TSOv6 in. */ \ - _ (VIRTIO_NET_F_HOST_ECN, 13) /* Host can handle TSO[6] w/ ECN in. */ \ - _ (VIRTIO_NET_F_HOST_UFO, 14) /* Host can handle UFO in. */ \ - _ (VIRTIO_NET_F_MRG_RXBUF, 15) /* Host can merge receive buffers. */ \ - _ (VIRTIO_NET_F_STATUS, 16) /* virtio_net_config.status available */ \ - _ (VIRTIO_NET_F_CTRL_VQ, 17) /* Control channel available */ \ - _ (VIRTIO_NET_F_CTRL_RX, 18) /* Control channel RX mode support */ \ - _ (VIRTIO_NET_F_CTRL_VLAN, 19) /* Control channel VLAN filtering */ \ - _ (VIRTIO_NET_F_CTRL_RX_EXTRA, 20) /* Extra RX mode control support */ \ - _ (VIRTIO_NET_F_GUEST_ANNOUNCE, 21) /* Guest can announce device on the network */ \ - _ (VIRTIO_NET_F_MQ, 22) /* Device supports Receive Flow Steering */ \ - _ (VIRTIO_NET_F_CTRL_MAC_ADDR, 23) /* Set MAC address */ \ - _ (VIRTIO_F_NOTIFY_ON_EMPTY, 24) \ - _ (VHOST_F_LOG_ALL, 26) /* Log all write descriptors */ \ - _ (VIRTIO_F_ANY_LAYOUT, 27) /* Can the device handle any descriptor layout */ \ - _ (VIRTIO_RING_F_INDIRECT_DESC, 28) /* Support indirect buffer descriptors */ \ - _ (VIRTIO_RING_F_EVENT_IDX, 29) /* The Guest publishes the used index for which it expects an interrupt \ - * at the end of the avail ring. Host should ignore the avail->flags field. */ \ -/* The Host publishes the avail index for which it expects a kick \ - * at the end of the used ring. Guest should ignore the used->flags field. */ \ - _ (VHOST_USER_F_PROTOCOL_FEATURES, 30) \ - _ (VIRTIO_F_VERSION_1, 32) /* v1.0 compliant. */ \ - _ (VIRTIO_F_IOMMU_PLATFORM, 33) \ - _ (VIRTIO_F_RING_PACKED, 34) \ - _ (VIRTIO_F_IN_ORDER, 35) /* all buffers are used by the device in the */ \ - /* same order in which they have been made available */ \ +#define foreach_virtio_net_features \ + _ (VIRTIO_NET_F_CSUM, 0) /* Host handles pkts w/ partial csum */ \ + _ (VIRTIO_NET_F_GUEST_CSUM, 1) /* Guest handles pkts w/ partial csum */ \ + _ (VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \ + 2) /* Dynamic offload configuration. */ \ + _ (VIRTIO_NET_F_MTU, 3) /* Initial MTU advice. */ \ + _ (VIRTIO_NET_F_MAC, 5) /* Host has given MAC address. */ \ + _ (VIRTIO_NET_F_GSO, 6) /* Host handles pkts w/ any GSO. */ \ + _ (VIRTIO_NET_F_GUEST_TSO4, 7) /* Guest can handle TSOv4 in. */ \ + _ (VIRTIO_NET_F_GUEST_TSO6, 8) /* Guest can handle TSOv6 in. */ \ + _ (VIRTIO_NET_F_GUEST_ECN, 9) /* Guest can handle TSO[6] w/ ECN in. */ \ + _ (VIRTIO_NET_F_GUEST_UFO, 10) /* Guest can handle UFO in. */ \ + _ (VIRTIO_NET_F_HOST_TSO4, 11) /* Host can handle TSOv4 in. */ \ + _ (VIRTIO_NET_F_HOST_TSO6, 12) /* Host can handle TSOv6 in. */ \ + _ (VIRTIO_NET_F_HOST_ECN, 13) /* Host can handle TSO[6] w/ ECN in. */ \ + _ (VIRTIO_NET_F_HOST_UFO, 14) /* Host can handle UFO in. */ \ + _ (VIRTIO_NET_F_MRG_RXBUF, 15) /* Host can merge receive buffers. */ \ + _ (VIRTIO_NET_F_STATUS, 16) /* virtio_net_config.status available */ \ + _ (VIRTIO_NET_F_CTRL_VQ, 17) /* Control channel available */ \ + _ (VIRTIO_NET_F_CTRL_RX, 18) /* Control channel RX mode support */ \ + _ (VIRTIO_NET_F_CTRL_VLAN, 19) /* Control channel VLAN filtering */ \ + _ (VIRTIO_NET_F_CTRL_RX_EXTRA, 20) /* Extra RX mode control support */ \ + _ (VIRTIO_NET_F_GUEST_ANNOUNCE, \ + 21) /* Guest can announce device on the network */ \ + _ (VIRTIO_NET_F_MQ, 22) /* Device supports Receive Flow Steering */ \ + _ (VIRTIO_NET_F_CTRL_MAC_ADDR, 23) /* Set MAC address */ \ + _ (VIRTIO_F_NOTIFY_ON_EMPTY, 24) \ + _ (VHOST_F_LOG_ALL, 26) /* Log all write descriptors */ \ + _ (VIRTIO_F_ANY_LAYOUT, \ + 27) /* Can the device handle any descriptor layout */ \ + _ (VIRTIO_RING_F_INDIRECT_DESC, \ + 28) /* Support indirect buffer descriptors */ \ + _ (VIRTIO_RING_F_EVENT_IDX, \ + 29) /* The Guest publishes the used index for which it expects an \ + * interrupt at the end of the avail ring. Host should ignore the \ + * avail->flags field. */ \ + /* The Host publishes the avail index for which it expects a kick \ + * at the end of the used ring. Guest should ignore the used->flags field. \ + */ \ + _ (VHOST_USER_F_PROTOCOL_FEATURES, 30) \ + _ (VIRTIO_F_VERSION_1, 32) /* v1.0 compliant. */ \ + _ (VIRTIO_F_IOMMU_PLATFORM, 33) \ + _ (VIRTIO_F_RING_PACKED, 34) \ + _ (VIRTIO_F_IN_ORDER, 35) /* all buffers are used by the device in the */ \ + /* same order in which they have been made available */ \ _ (VIRTIO_F_ORDER_PLATFORM, 36) /* memory accesses by the driver and the */ \ - /* device are ordered in a way described by the platfor */ \ - _ (VIRTIO_F_NOTIFICATION_DATA, 38) /* the driver passes extra data (besides */ \ - /* identifying the virtqueue) in its device notifications. */ \ - _ (VIRTIO_NET_F_SPEED_DUPLEX, 63) /* Device set linkspeed and duplex */ + /* device are ordered in a way described by the platfor */ \ + _ (VIRTIO_F_NOTIFICATION_DATA, \ + 38) /* the driver passes extra data (besides */ \ + /* identifying the virtqueue) in its device notifications. */ \ + _ (VIRTIO_NET_F_RING_RESET, 40) /* Device supports individual ring reset */ \ + _ (VIRTIO_NET_F_HASH_REPORT, \ + 57) /* Device supports per packet hash value */ \ + _ (VIRTIO_NET_F_RSS, 60) /* Device supports RSS */ \ + _ (VIRTIO_NET_F_SPEED_DUPLEX, 63) /* Device set linkspeed and duplex */ typedef enum { diff --git a/src/vnet/error.c b/src/vnet/error.c index 473d11135f1..7ae32962132 100644 --- a/src/vnet/error.c +++ b/src/vnet/error.c @@ -37,6 +37,10 @@ u8 * format_vnet_api_errno (u8 *s, va_list *args) { vnet_api_error_t api_error = va_arg (*args, vnet_api_error_t); + + if (0 == api_error) + return format (s, "Success"); + #ifdef _ #undef _ #endif diff --git a/src/vnet/fib/fib_api.c b/src/vnet/fib/fib_api.c index 07d6699d87a..426161b0670 100644 --- a/src/vnet/fib/fib_api.c +++ b/src/vnet/fib/fib_api.c @@ -190,6 +190,7 @@ fib_api_path_decode (vl_api_fib_path_t *in, break; case FIB_API_PATH_TYPE_DROP: out->frp_flags |= FIB_ROUTE_PATH_DROP; + out->frp_sw_if_index = ntohl(in->sw_if_index); break; case FIB_API_PATH_TYPE_LOCAL: out->frp_flags |= FIB_ROUTE_PATH_LOCAL; @@ -493,6 +494,21 @@ fib_api_route_add_del (u8 is_add, return (0); } +u8 * +format_vl_api_address_union (u8 * s, va_list * args) +{ + const vl_api_address_union_t *addr = + va_arg (*args, vl_api_address_union_t *); + vl_api_address_family_t af = va_arg (*args, int); + + if (ADDRESS_IP6 == af) + s = format (s, "%U", format_ip6_address, addr->ip6); + else + s = format (s, "%U", format_ip4_address, addr->ip4); + + return s; +} + u8* format_vl_api_fib_path (u8 * s, va_list * args) { diff --git a/src/vnet/fib/fib_api.h b/src/vnet/fib/fib_api.h index 0c59531b438..9eac62fe1c6 100644 --- a/src/vnet/fib/fib_api.h +++ b/src/vnet/fib/fib_api.h @@ -51,6 +51,7 @@ extern int fib_api_route_add_del (u8 is_add, fib_entry_flag_t entry_flags, fib_route_path_t *rpaths); +extern u8 *format_vl_api_address_union (u8 * s, va_list * args); extern u8* format_vl_api_fib_path(u8 * s, va_list * args); diff --git a/src/vnet/fib/fib_entry.c b/src/vnet/fib/fib_entry.c index b78346ce45a..adf880b8bbb 100644 --- a/src/vnet/fib/fib_entry.c +++ b/src/vnet/fib/fib_entry.c @@ -1828,10 +1828,18 @@ fib_entry_pool_size (void) return (pool_elts(fib_entry_pool)); } -#if CLIB_DEBUG > 0 void fib_table_assert_empty (const fib_table_t *fib_table) { + if (0 == fib_table->ft_total_route_counts) + return; + + vlib_log_err (fib_entry_logger, + "BUG: %U table %d (index %d) is not empty", + format_fib_protocol, fib_table->ft_proto, + fib_table->ft_table_id, fib_table->ft_index); + +#if CLIB_DEBUG > 0 fib_node_index_t *fei, *feis = NULL; fib_entry_t *fib_entry; @@ -1848,8 +1856,8 @@ fib_table_assert_empty (const fib_table_t *fib_table) } ASSERT(0); -} #endif +} static clib_error_t * show_fib_entry_command (vlib_main_t * vm, diff --git a/src/vnet/fib/fib_entry_src.c b/src/vnet/fib/fib_entry_src.c index c79b745b5b5..d9cf82343c5 100644 --- a/src/vnet/fib/fib_entry_src.c +++ b/src/vnet/fib/fib_entry_src.c @@ -875,8 +875,12 @@ fib_entry_src_covered_inherit_add_i (fib_entry_t *fib_entry, * The covered's source data has been inherited, presumably * from this cover, i.e. this is a modify. */ - esrc = fib_entry_src_action_update_from_cover(fib_entry, cover_src); - fib_entry_source_change(fib_entry, esrc->fes_src, esrc->fes_src); + fib_source_t best_source; + + best_source = fib_entry_get_best_source( + fib_entry_get_index(fib_entry)); + fib_entry_src_action_update_from_cover(fib_entry, cover_src); + fib_entry_source_change(fib_entry, best_source, cover_src->fes_src); } else { @@ -1538,6 +1542,10 @@ fib_entry_flags_update (const fib_entry_t *fib_entry, { esrc->fes_entry_flags |= FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT; } + if (rpath->frp_flags & FIB_ROUTE_PATH_DROP) + { + esrc->fes_entry_flags |= FIB_ENTRY_FLAG_NO_ATTACHED_EXPORT; + } } if (fib_route_attached_cross_table(fib_entry, rpath) && !(esrc->fes_entry_flags & FIB_ENTRY_FLAG_NO_ATTACHED_EXPORT)) diff --git a/src/vnet/fib/fib_table.h b/src/vnet/fib/fib_table.h index 0eaaa67eea2..d78c9e619b8 100644 --- a/src/vnet/fib/fib_table.h +++ b/src/vnet/fib/fib_table.h @@ -980,9 +980,7 @@ extern u8 *format_fib_table_memory(u8 *s, va_list *args); /** * Debug function */ -#if CLIB_DEBUG > 0 extern void fib_table_assert_empty(const fib_table_t *fib_table); -#endif #endif diff --git a/src/vnet/fib/fib_types.c b/src/vnet/fib/fib_types.c index c4472c7122d..9abb89bc6a0 100644 --- a/src/vnet/fib/fib_types.c +++ b/src/vnet/fib/fib_types.c @@ -715,6 +715,10 @@ unformat_fib_route_path (unformat_input_t * input, va_list * args) rpath->frp_proto = DPO_PROTO_IP6; rpath->frp_flags = FIB_ROUTE_PATH_INTF_RX; } + else if (unformat (input, "drop")) + { + rpath->frp_flags = FIB_ROUTE_PATH_DROP; + } else if (unformat (input, "local")) { clib_memset (&rpath->frp_addr, 0, sizeof (rpath->frp_addr)); diff --git a/src/vnet/fib/ip4_fib.c b/src/vnet/fib/ip4_fib.c index 0eff8d0d485..23f70a770bf 100644 --- a/src/vnet/fib/ip4_fib.c +++ b/src/vnet/fib/ip4_fib.c @@ -201,10 +201,7 @@ ip4_fib_table_destroy (u32 fib_index) /* * validate no more routes. */ -#if CLIB_DEBUG > 0 - if (0 != fib_table->ft_total_route_counts) - fib_table_assert_empty(fib_table); -#endif + fib_table_assert_empty(fib_table); vec_foreach(n_locks, fib_table->ft_src_route_counts) { diff --git a/src/vnet/fib/ip6_fib.c b/src/vnet/fib/ip6_fib.c index d37b77e08a4..f844cfaa420 100644 --- a/src/vnet/fib/ip6_fib.c +++ b/src/vnet/fib/ip6_fib.c @@ -20,14 +20,20 @@ #include <vppinfra/bihash_24_8.h> #include <vppinfra/bihash_template.c> -ip6_fib_table_instance_t ip6_fib_table[IP6_FIB_NUM_TABLES]; +ip6_fib_fwding_table_instance_t ip6_fib_fwding_table; /* ip6 lookup table config parameters */ u32 ip6_fib_table_nbuckets; uword ip6_fib_table_size; +typedef struct ip6_fib_hash_key_t_ +{ + ip6_address_t addr; + u8 len; +} ip6_fib_hash_key_t; + static void -vnet_ip6_fib_init (u32 fib_index) +ip6_fib_hash_load_specials (u32 fib_index) { fib_prefix_t pfx = { .fp_proto = FIB_PROTOCOL_IP6, @@ -77,7 +83,7 @@ create_fib_with_table_id (u32 table_id, ASSERT((fib_table - ip6_main.fibs) == (v6_fib - ip6_main.v6_fibs)); - + fib_table->ft_proto = FIB_PROTOCOL_IP6; fib_table->ft_index = v6_fib->index = @@ -92,9 +98,15 @@ create_fib_with_table_id (u32 table_id, fib_table->ft_flags = flags; fib_table->ft_desc = desc; - vnet_ip6_fib_init(fib_table->ft_index); fib_table_lock(fib_table->ft_index, FIB_PROTOCOL_IP6, src); + v6_fib->fib_entry_by_dst_address = hash_create_mem(2, sizeof(ip6_fib_hash_key_t), sizeof(fib_node_index_t)); + + /* + * add the special entries into the new FIB + */ + ip6_fib_hash_load_specials (fib_table->ft_index); + return (fib_table->ft_index); } @@ -160,10 +172,7 @@ ip6_fib_table_destroy (u32 fib_index) /* * validate no more routes. */ -#if CLIB_DEBUG > 0 - if (0 != fib_table->ft_total_route_counts) - fib_table_assert_empty(fib_table); -#endif + fib_table_assert_empty(fib_table); vec_foreach_index(source, fib_table->ft_src_route_counts) { @@ -176,54 +185,40 @@ ip6_fib_table_destroy (u32 fib_index) } vec_free (fib_table->ft_locks); vec_free(fib_table->ft_src_route_counts); + hash_free(pool_elt_at_index(ip6_main.v6_fibs, fib_index)->fib_entry_by_dst_address); pool_put_index(ip6_main.v6_fibs, fib_table->ft_index); pool_put(ip6_main.fibs, fib_table); } +static void +ip6_fib_table_mk_key (ip6_fib_hash_key_t *key, const ip6_address_t *addr, u8 len) +{ + const ip6_address_t *mask = &ip6_main.fib_masks[len]; + key->addr.as_u64[0] = addr->as_u64[0] & mask->as_u64[0]; + key->addr.as_u64[1] = addr->as_u64[1] & mask->as_u64[1]; + key->len = len; +} + fib_node_index_t ip6_fib_table_lookup (u32 fib_index, const ip6_address_t *addr, u32 len) { - ip6_fib_table_instance_t *table; - clib_bihash_kv_24_8_t kv, value; - int i, n_p, rv; - u64 fib; - - table = &ip6_fib_table[IP6_FIB_TABLE_NON_FWDING]; - n_p = vec_len (table->prefix_lengths_in_search_order); - - kv.key[0] = addr->as_u64[0]; - kv.key[1] = addr->as_u64[1]; - fib = ((u64)((fib_index))<<32); + uword *hash = pool_elt_at_index(ip6_main.v6_fibs, fib_index)->fib_entry_by_dst_address; + ip6_fib_hash_key_t key; + i32 mask_len; + uword *result; - /* - * start search from a mask length same length or shorter. - * we don't want matches longer than the mask passed - */ - i = 0; - while (i < n_p && table->prefix_lengths_in_search_order[i] > len) + for (mask_len = len; mask_len >= 0; mask_len--) { - i++; + ip6_fib_table_mk_key (&key, addr, mask_len); + result = hash_get_mem(hash, &key); + if (result) { + return result[0]; + } } - for (; i < n_p; i++) - { - int dst_address_length = table->prefix_lengths_in_search_order[i]; - ip6_address_t * mask = &ip6_main.fib_masks[dst_address_length]; - - ASSERT(dst_address_length >= 0 && dst_address_length <= 128); - //As lengths are decreasing, masks are increasingly specific. - kv.key[0] &= mask->as_u64[0]; - kv.key[1] &= mask->as_u64[1]; - kv.key[2] = fib | dst_address_length; - - rv = clib_bihash_search_inline_2_24_8(&table->ip6_hash, &kv, &value); - if (rv == 0) - return value.value; - } - - return (FIB_NODE_INDEX_INVALID); + return FIB_NODE_INDEX_INVALID; } fib_node_index_t @@ -231,53 +226,11 @@ ip6_fib_table_lookup_exact_match (u32 fib_index, const ip6_address_t *addr, u32 len) { - ip6_fib_table_instance_t *table; - clib_bihash_kv_24_8_t kv, value; - ip6_address_t *mask; - u64 fib; - int rv; - - table = &ip6_fib_table[IP6_FIB_TABLE_NON_FWDING]; - mask = &ip6_main.fib_masks[len]; - fib = ((u64)((fib_index))<<32); - - kv.key[0] = addr->as_u64[0] & mask->as_u64[0]; - kv.key[1] = addr->as_u64[1] & mask->as_u64[1]; - kv.key[2] = fib | len; - - rv = clib_bihash_search_inline_2_24_8(&table->ip6_hash, &kv, &value); - if (rv == 0) - return value.value; - - return (FIB_NODE_INDEX_INVALID); -} - -static void -compute_prefix_lengths_in_search_order (ip6_fib_table_instance_t *table) -{ - u8 *old, *prefix_lengths_in_search_order = NULL; - int i; - - /* - * build the list in a scratch space then cutover so the workers - * can continue uninterrupted. - */ - old = table->prefix_lengths_in_search_order; - - /* Note: bitmap reversed so this is in fact a longest prefix match */ - clib_bitmap_foreach (i, table->non_empty_dst_address_length_bitmap) - { - int dst_address_length = 128 - i; - vec_add1(prefix_lengths_in_search_order, dst_address_length); - } - - table->prefix_lengths_in_search_order = prefix_lengths_in_search_order; - - /* - * let the workers go once round the track before we free the old set - */ - vlib_worker_wait_one_loop(); - vec_free(old); + uword *hash = pool_elt_at_index(ip6_main.v6_fibs, fib_index)->fib_entry_by_dst_address; + ip6_fib_hash_key_t key; + ip6_fib_table_mk_key (&key, addr, len); + uword *result = hash_get(hash, &key); + return result ? result[0] : FIB_NODE_INDEX_INVALID; } void @@ -285,30 +238,10 @@ ip6_fib_table_entry_remove (u32 fib_index, const ip6_address_t *addr, u32 len) { - ip6_fib_table_instance_t *table; - clib_bihash_kv_24_8_t kv; - ip6_address_t *mask; - u64 fib; - - table = &ip6_fib_table[IP6_FIB_TABLE_NON_FWDING]; - mask = &ip6_main.fib_masks[len]; - fib = ((u64)((fib_index))<<32); - - kv.key[0] = addr->as_u64[0] & mask->as_u64[0]; - kv.key[1] = addr->as_u64[1] & mask->as_u64[1]; - kv.key[2] = fib | len; - - clib_bihash_add_del_24_8(&table->ip6_hash, &kv, 0); - - /* refcount accounting */ - ASSERT (table->dst_address_length_refcounts[len] > 0); - if (--table->dst_address_length_refcounts[len] == 0) - { - table->non_empty_dst_address_length_bitmap = - clib_bitmap_set (table->non_empty_dst_address_length_bitmap, - 128 - len, 0); - compute_prefix_lengths_in_search_order (table); - } + uword **hash = &pool_elt_at_index(ip6_main.v6_fibs, fib_index)->fib_entry_by_dst_address; + ip6_fib_hash_key_t key; + ip6_fib_table_mk_key (&key, addr, len); + hash_unset_mem_free(hash, &key); } void @@ -317,29 +250,11 @@ ip6_fib_table_entry_insert (u32 fib_index, u32 len, fib_node_index_t fib_entry_index) { - ip6_fib_table_instance_t *table; - clib_bihash_kv_24_8_t kv; - ip6_address_t *mask; - u64 fib; - - table = &ip6_fib_table[IP6_FIB_TABLE_NON_FWDING]; - mask = &ip6_main.fib_masks[len]; - fib = ((u64)((fib_index))<<32); - - kv.key[0] = addr->as_u64[0] & mask->as_u64[0]; - kv.key[1] = addr->as_u64[1] & mask->as_u64[1]; - kv.key[2] = fib | len; - kv.value = fib_entry_index; - - clib_bihash_add_del_24_8(&table->ip6_hash, &kv, 1); - - if (0 == table->dst_address_length_refcounts[len]++) - { - table->non_empty_dst_address_length_bitmap = - clib_bitmap_set (table->non_empty_dst_address_length_bitmap, - 128 - len, 1); - compute_prefix_lengths_in_search_order (table); - } + uword **hash = &pool_elt_at_index(ip6_main.v6_fibs, fib_index)->fib_entry_by_dst_address; + ip6_fib_hash_key_t key; + ip6_fib_table_mk_key (&key, addr, len); + ASSERT (0 == hash_get(*hash, &key) && "entry already exists"); + hash_set_mem_alloc(hash, &key, fib_entry_index); } u32 ip6_fib_table_fwding_lookup_with_if_index (ip6_main_t * im, @@ -364,18 +279,46 @@ ip6_fib_table_get_index_for_sw_if_index (u32 sw_if_index) return (ip6_main.fib_index_by_sw_if_index[sw_if_index]); } +static void +compute_prefix_lengths_in_search_order (ip6_fib_fwding_table_instance_t *table) +{ + u8 *old, *prefix_lengths_in_search_order = NULL; + int i; + + /* + * build the list in a scratch space then cutover so the workers + * can continue uninterrupted. + */ + old = table->prefix_lengths_in_search_order; + + /* Note: bitmap reversed so this is in fact a longest prefix match */ + clib_bitmap_foreach (i, table->non_empty_dst_address_length_bitmap) + { + int dst_address_length = 128 - i; + vec_add1(prefix_lengths_in_search_order, dst_address_length); + } + + table->prefix_lengths_in_search_order = prefix_lengths_in_search_order; + + /* + * let the workers go once round the track before we free the old set + */ + vlib_worker_wait_one_loop(); + vec_free(old); +} + void ip6_fib_table_fwding_dpo_update (u32 fib_index, const ip6_address_t *addr, u32 len, const dpo_id_t *dpo) { - ip6_fib_table_instance_t *table; + ip6_fib_fwding_table_instance_t *table; clib_bihash_kv_24_8_t kv; ip6_address_t *mask; u64 fib; - table = &ip6_fib_table[IP6_FIB_TABLE_FWDING]; + table = &ip6_fib_fwding_table; mask = &ip6_main.fib_masks[len]; fib = ((u64)((fib_index))<<32); @@ -401,12 +344,12 @@ ip6_fib_table_fwding_dpo_remove (u32 fib_index, u32 len, const dpo_id_t *dpo) { - ip6_fib_table_instance_t *table; + ip6_fib_fwding_table_instance_t *table; clib_bihash_kv_24_8_t kv; ip6_address_t *mask; u64 fib; - table = &ip6_fib_table[IP6_FIB_TABLE_FWDING]; + table = &ip6_fib_fwding_table; mask = &ip6_main.fib_masks[len]; fib = ((u64)((fib_index))<<32); @@ -428,101 +371,18 @@ ip6_fib_table_fwding_dpo_remove (u32 fib_index, } } -/** - * @brief Context when walking the IPv6 table. Since all VRFs are in the - * same hash table, we need to filter only those we need as we walk - */ -typedef struct ip6_fib_walk_ctx_t_ -{ - u32 i6w_fib_index; - fib_table_walk_fn_t i6w_fn; - void *i6w_ctx; - fib_prefix_t i6w_root; - fib_prefix_t *i6w_sub_trees; -} ip6_fib_walk_ctx_t; - -static int -ip6_fib_walk_cb (clib_bihash_kv_24_8_t * kvp, - void *arg) -{ - ip6_fib_walk_ctx_t *ctx = arg; - ip6_address_t key; - - if ((kvp->key[2] >> 32) == ctx->i6w_fib_index) - { - key.as_u64[0] = kvp->key[0]; - key.as_u64[1] = kvp->key[1]; - - if (ip6_destination_matches_route(&ip6_main, - &key, - &ctx->i6w_root.fp_addr.ip6, - ctx->i6w_root.fp_len)) - { - const fib_prefix_t *sub_tree; - int skip = 0; - - /* - * exclude sub-trees the walk does not want to explore - */ - vec_foreach(sub_tree, ctx->i6w_sub_trees) - { - if (ip6_destination_matches_route(&ip6_main, - &key, - &sub_tree->fp_addr.ip6, - sub_tree->fp_len)) - { - skip = 1; - break; - } - } - - if (!skip) - { - switch (ctx->i6w_fn(kvp->value, ctx->i6w_ctx)) - { - case FIB_TABLE_WALK_CONTINUE: - break; - case FIB_TABLE_WALK_SUB_TREE_STOP: { - fib_prefix_t pfx = { - .fp_proto = FIB_PROTOCOL_IP6, - .fp_len = kvp->key[2] & 0xffffffff, - .fp_addr.ip6 = key, - }; - vec_add1(ctx->i6w_sub_trees, pfx); - break; - } - case FIB_TABLE_WALK_STOP: - goto done; - } - } - } - } -done: - - return (1); -} - void ip6_fib_table_walk (u32 fib_index, fib_table_walk_fn_t fn, void *arg) { - ip6_fib_walk_ctx_t ctx = { - .i6w_fib_index = fib_index, - .i6w_fn = fn, - .i6w_ctx = arg, - .i6w_root = { - .fp_proto = FIB_PROTOCOL_IP6, - }, - .i6w_sub_trees = NULL, + const fib_prefix_t root = { + .fp_proto = FIB_PROTOCOL_IP6, + // address and length default to all 0 }; - - clib_bihash_foreach_key_value_pair_24_8( - &ip6_fib_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash, - ip6_fib_walk_cb, - &ctx); - - vec_free(ctx.i6w_sub_trees); + /* A full tree walk is the dengenerate case of a sub-tree from + * the very root */ + return (ip6_fib_table_sub_tree_walk(fib_index, &root, fn, arg)); } void @@ -531,17 +391,43 @@ ip6_fib_table_sub_tree_walk (u32 fib_index, fib_table_walk_fn_t fn, void *arg) { - ip6_fib_walk_ctx_t ctx = { - .i6w_fib_index = fib_index, - .i6w_fn = fn, - .i6w_ctx = arg, - .i6w_root = *root, - }; + uword *hash = pool_elt_at_index(ip6_main.v6_fibs, fib_index)->fib_entry_by_dst_address; + const ip6_fib_hash_key_t *key, *sub_tree; + ip6_fib_hash_key_t *sub_trees = 0; + u32 fei; - clib_bihash_foreach_key_value_pair_24_8( - &ip6_fib_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash, - ip6_fib_walk_cb, - &ctx); + /* + * There is no efficient way to walk this hash table. + * so we walk over all entries and check it is covered by the root. + */ + hash_foreach_mem(key, fei, hash, ({ + /* check if the prefix is covered by the root */ + if (!ip6_destination_matches_route(&ip6_main, &key->addr, &root->fp_addr.ip6, root->fp_len)) + continue; /* not covered by root, ignore */ + + /* exclude sub-trees the walk does not want to explore */ + vec_foreach (sub_tree, sub_trees) + { + if (ip6_destination_matches_route(&ip6_main, &key->addr, &sub_tree->addr, sub_tree->len)) + goto ignore_sub_tree; + } + + switch (fn(fei, arg)) + { + case FIB_TABLE_WALK_STOP: + goto done; + case FIB_TABLE_WALK_CONTINUE: + break; + case FIB_TABLE_WALK_SUB_TREE_STOP: + vec_add1(sub_trees, *key); + break; + } + +ignore_sub_tree:; + })); + +done: + vec_free(sub_trees); } typedef struct ip6_fib_show_ctx_t_ { @@ -602,8 +488,7 @@ format_ip6_fib_table_memory (u8 * s, va_list * args) { uword bytes_inuse; - bytes_inuse = (alloc_arena_next(&(ip6_fib_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash)) + - alloc_arena_next(&(ip6_fib_table[IP6_FIB_TABLE_FWDING].ip6_hash))); + bytes_inuse = alloc_arena_next(&ip6_fib_fwding_table.ip6_hash); s = format(s, "%=30s %=6d %=12ld\n", "IPv6 unicast", @@ -612,26 +497,60 @@ format_ip6_fib_table_memory (u8 * s, va_list * args) return (s); } -typedef struct { - u32 fib_index; - u64 count_by_prefix_length[129]; -} count_routes_in_fib_at_prefix_length_arg_t; - -static int -count_routes_in_fib_at_prefix_length (clib_bihash_kv_24_8_t * kvp, - void *arg) +void +ip6_fib_table_show (vlib_main_t *vm, fib_table_t *fib_table, int summary) { - count_routes_in_fib_at_prefix_length_arg_t * ap = arg; - int mask_width; + ip6_main_t * im6 = &ip6_main; + ip6_fib_t *fib = pool_elt_at_index(im6->v6_fibs, fib_table->ft_index); + fib_source_t source; + u8 *s = NULL; + + s = format(s, "%U, fib_index:%d, flow hash:[%U] epoch:%d flags:%U locks:[", + format_fib_table_name, fib->index, + FIB_PROTOCOL_IP6, + fib->index, + format_ip_flow_hash_config, + fib_table->ft_flow_hash_config, + fib_table->ft_epoch, + format_fib_table_flags, fib_table->ft_flags); + + vec_foreach_index(source, fib_table->ft_locks) + { + if (0 != fib_table->ft_locks[source]) + { + s = format(s, "%U:%d, ", + format_fib_source, source, + fib_table->ft_locks[source]); + } + } + s = format (s, "]"); + vlib_cli_output (vm, "%v", s); + vec_free(s); - if ((kvp->key[2]>>32) != ap->fib_index) - return (BIHASH_WALK_CONTINUE); + /* Show summary? */ + if (summary) + { + u32 count_by_prefix_length[129]; + const ip6_fib_hash_key_t *key; + u32 fei; + int len; - mask_width = kvp->key[2] & 0xFF; + vlib_cli_output (vm, "%=20s%=16s", "Prefix length", "Count"); - ap->count_by_prefix_length[mask_width]++; + clib_memset (count_by_prefix_length, 0, sizeof(count_by_prefix_length)); - return (BIHASH_WALK_CONTINUE); + hash_foreach_mem(key, fei, fib->fib_entry_by_dst_address, ({ + ASSERT(key->len <= 128); + count_by_prefix_length[key->len]++; + })); + + for (len = 128; len >= 0; len--) + { + if (count_by_prefix_length[len]) + vlib_cli_output (vm, "%=20d%=16lld", + len, count_by_prefix_length[len]); + } + } } static clib_error_t * @@ -639,7 +558,6 @@ ip6_show_fib (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { - count_routes_in_fib_at_prefix_length_arg_t _ca, *ca = &_ca; ip6_main_t * im6 = &ip6_main; fib_table_t *fib_table; ip6_fib_t * fib; @@ -686,22 +604,15 @@ ip6_show_fib (vlib_main_t * vm, if (hash) { - vlib_cli_output (vm, "IPv6 Non-Forwarding Hash Table:\n%U\n", - BV (format_bihash), - &ip6_fib_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash, - detail); vlib_cli_output (vm, "IPv6 Forwarding Hash Table:\n%U\n", BV (format_bihash), - &ip6_fib_table[IP6_FIB_TABLE_FWDING].ip6_hash, + &ip6_fib_fwding_table.ip6_hash, detail); return (NULL); } pool_foreach (fib_table, im6->fibs) { - fib_source_t source; - u8 *s = NULL; - fib = pool_elt_at_index(im6->v6_fibs, fib_table->ft_index); if (table_id >= 0 && table_id != (int)fib->table_id) continue; @@ -710,50 +621,9 @@ ip6_show_fib (vlib_main_t * vm, if (fib_table->ft_flags & FIB_TABLE_FLAG_IP6_LL) continue; - s = format(s, "%U, fib_index:%d, flow hash:[%U] epoch:%d flags:%U locks:[", - format_fib_table_name, fib->index, - FIB_PROTOCOL_IP6, - fib->index, - format_ip_flow_hash_config, - fib_table->ft_flow_hash_config, - fib_table->ft_epoch, - format_fib_table_flags, fib_table->ft_flags); - - vec_foreach_index(source, fib_table->ft_locks) - { - if (0 != fib_table->ft_locks[source]) - { - s = format(s, "%U:%d, ", - format_fib_source, source, - fib_table->ft_locks[source]); - } - } - s = format (s, "]"); - vlib_cli_output (vm, "%v", s); - vec_free(s); - - /* Show summary? */ - if (! verbose) - { - clib_bihash_24_8_t * h = &ip6_fib_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash; - int len; - - vlib_cli_output (vm, "%=20s%=16s", "Prefix length", "Count"); - - clib_memset (ca, 0, sizeof(*ca)); - ca->fib_index = fib->index; - - clib_bihash_foreach_key_value_pair_24_8 - (h, count_routes_in_fib_at_prefix_length, ca); - - for (len = 128; len >= 0; len--) - { - if (ca->count_by_prefix_length[len]) - vlib_cli_output (vm, "%=20d%=16lld", - len, ca->count_by_prefix_length[len]); - } - continue; - } + ip6_fib_table_show(vm, fib_table, !verbose); + if (!verbose) + continue; if (!matching) { @@ -909,12 +779,9 @@ ip6_fib_init (vlib_main_t * vm) if (ip6_fib_table_size == 0) ip6_fib_table_size = IP6_FIB_DEFAULT_HASH_MEMORY_SIZE; - clib_bihash_init_24_8 (&(ip6_fib_table[IP6_FIB_TABLE_FWDING].ip6_hash), + clib_bihash_init_24_8 (&(ip6_fib_fwding_table.ip6_hash), "ip6 FIB fwding table", ip6_fib_table_nbuckets, ip6_fib_table_size); - clib_bihash_init_24_8 (&ip6_fib_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash, - "ip6 FIB non-fwding table", - ip6_fib_table_nbuckets, ip6_fib_table_size); return (NULL); } diff --git a/src/vnet/fib/ip6_fib.h b/src/vnet/fib/ip6_fib.h index 706bebbce09..80d56b1a7ba 100644 --- a/src/vnet/fib/ip6_fib.h +++ b/src/vnet/fib/ip6_fib.h @@ -26,36 +26,15 @@ #include <vppinfra/bihash_template.h> /* - * Default size of the ip6 fib hash table + * Default size of the ip6 fib forwarding hash table */ #define IP6_FIB_DEFAULT_HASH_NUM_BUCKETS (64 * 1024) #define IP6_FIB_DEFAULT_HASH_MEMORY_SIZE (32<<20) /** - * Enumeration of the FIB table instance types + * A representation the forwarding IP6 table */ -typedef enum ip6_fib_table_instance_type_t_ -{ - /** - * This table stores the routes that are used to forward traffic. - * The key is the prefix, the result the adjacency to forward on. - */ - IP6_FIB_TABLE_FWDING, - /** - * The table that stores ALL routes learned by the DP. - * Some of these routes may not be ready to install in forwarding - * at a given time. - * The key in this table is the prefix, the result is the fib_entry_t - */ - IP6_FIB_TABLE_NON_FWDING, -} ip6_fib_table_instance_type_t; - -#define IP6_FIB_NUM_TABLES (IP6_FIB_TABLE_NON_FWDING+1) - -/** - * A representation of a single IP6 table - */ -typedef struct ip6_fib_table_instance_t_ +typedef struct ip6_fib_fwding_table_instance_t_ { /* The hash table */ clib_bihash_24_8_t ip6_hash; @@ -64,12 +43,12 @@ typedef struct ip6_fib_table_instance_t_ uword *non_empty_dst_address_length_bitmap; u8 *prefix_lengths_in_search_order; i32 dst_address_length_refcounts[129]; -} ip6_fib_table_instance_t; +} ip6_fib_fwding_table_instance_t; /** * The two FIB tables; fwding and non-fwding */ -extern ip6_fib_table_instance_t ip6_fib_table[IP6_FIB_NUM_TABLES]; +extern ip6_fib_fwding_table_instance_t ip6_fib_fwding_table; extern fib_node_index_t ip6_fib_table_lookup(u32 fib_index, const ip6_address_t *addr, @@ -115,13 +94,13 @@ always_inline u32 ip6_fib_table_fwding_lookup (u32 fib_index, const ip6_address_t * dst) { - ip6_fib_table_instance_t *table; + ip6_fib_fwding_table_instance_t *table; clib_bihash_kv_24_8_t kv, value; int i, len; int rv; u64 fib; - table = &ip6_fib_table[IP6_FIB_TABLE_FWDING]; + table = &ip6_fib_fwding_table; len = vec_len (table->prefix_lengths_in_search_order); kv.key[0] = dst->as_u64[0]; @@ -230,6 +209,7 @@ u32 ip6_fib_index_from_table_id (u32 table_id) } extern u32 ip6_fib_table_get_index_for_sw_if_index(u32 sw_if_index); +extern void ip6_fib_table_show (vlib_main_t *vm, fib_table_t *fib_table, int summary); #endif diff --git a/src/vnet/interface/runtime.c b/src/vnet/interface/runtime.c index a88a23bd4c9..6428d5ab75b 100644 --- a/src/vnet/interface/runtime.c +++ b/src/vnet/interface/runtime.c @@ -72,6 +72,9 @@ vnet_hw_if_update_runtime_data (vnet_main_t *vnm, u32 hw_if_index) clib_bitmap_t *pending_int = 0; int last_int = -1; + if (node_index == 0) + return; + log_debug ("update node '%U' triggered by interface %v", format_vlib_node_name, vm, node_index, hi->name); diff --git a/src/vnet/interface_api.c b/src/vnet/interface_api.c index c727e519138..69bf4b72ba4 100644 --- a/src/vnet/interface_api.c +++ b/src/vnet/interface_api.c @@ -579,7 +579,7 @@ ip_table_bind (fib_protocol_t fproto, u32 sw_if_index, u32 table_id) fib_index = fib_table_find (fproto, table_id); mfib_index = mfib_table_find (fproto, table_id); - if (~0 == fib_index || ~0 == mfib_index) + if (~0 == fib_index) { return (VNET_API_ERROR_NO_SUCH_FIB); } @@ -601,7 +601,8 @@ ip_table_bind (fib_protocol_t fproto, u32 sw_if_index, u32 table_id) /* clang-format on */ fib_table_bind (fproto, sw_if_index, fib_index); - mfib_table_bind (fproto, sw_if_index, mfib_index); + if (mfib_index != ~0) + mfib_table_bind (fproto, sw_if_index, mfib_index); return (0); } diff --git a/src/vnet/ip/icmp6.c b/src/vnet/ip/icmp6.c index b095f679cc8..f93ebce4bf1 100644 --- a/src/vnet/ip/icmp6.c +++ b/src/vnet/ip/icmp6.c @@ -338,7 +338,7 @@ ip6_icmp_error (vlib_main_t * vm, if (throttle_check (&icmp_throttle, thread_index, r0, seed)) { - vlib_error_count (vm, node->node_index, ICMP4_ERROR_DROP, 1); + vlib_error_count (vm, node->node_index, ICMP6_ERROR_DROP, 1); from += 1; n_left_from -= 1; continue; diff --git a/src/vnet/ip/ip.api b/src/vnet/ip/ip.api index 967f56cf917..fc7d7582dec 100644 --- a/src/vnet/ip/ip.api +++ b/src/vnet/ip/ip.api @@ -57,6 +57,23 @@ autoreply define ip_table_add_del vl_api_ip_table_t table; }; +/** \brief Add / del table request - version 2 + A table can be added multiple times, but need be deleted only once. + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param table - the FIB table to add or del + @param create_mfib - whether to create mfib or not + @param is_add - add or del +*/ +autoreply define ip_table_add_del_v2 +{ + u32 client_index; + u32 context; + vl_api_ip_table_t table; + bool create_mfib [default=true]; + bool is_add [default=true]; +}; + /** \brief Allocate an unused table A table can be added multiple times. If a large number of tables are in use (millions), this API might diff --git a/src/vnet/ip/ip.h b/src/vnet/ip/ip.h index 9ebefa0cf5d..084243dccfa 100644 --- a/src/vnet/ip/ip.h +++ b/src/vnet/ip/ip.h @@ -262,7 +262,7 @@ extern vlib_node_registration_t ip4_inacl_node; extern vlib_node_registration_t ip6_inacl_node; void ip_table_create (fib_protocol_t fproto, u32 table_id, u8 is_api, - const u8 * name); + u8 create_mfib, const u8 *name); void ip_table_delete (fib_protocol_t fproto, u32 table_id, u8 is_api); diff --git a/src/vnet/ip/ip6.h b/src/vnet/ip/ip6.h index 56eec523d5b..f8462a5cbff 100644 --- a/src/vnet/ip/ip6.h +++ b/src/vnet/ip/ip6.h @@ -68,6 +68,11 @@ typedef struct /* Index into FIB vector. */ u32 index; + + /** + * The hash table DB + */ + uword *fib_entry_by_dst_address; } ip6_fib_t; typedef struct ip6_mfib_t diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c index 48fb633fd32..31adc90ecab 100644 --- a/src/vnet/ip/ip6_forward.c +++ b/src/vnet/ip/ip6_forward.c @@ -51,6 +51,7 @@ #include <vnet/dpo/receive_dpo.h> #include <vnet/dpo/classify_dpo.h> #include <vnet/classify/vnet_classify.h> +#include <vnet/adj/adj_dp.h> #include <vnet/pg/pg.h> #ifndef CLIB_MARCH_VARIANT @@ -1897,18 +1898,6 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm, vnet_buffer (p0)->ip.save_rewrite_length = rw_len0; vnet_buffer (p1)->ip.save_rewrite_length = rw_len1; - if (do_counters) - { - vlib_increment_combined_counter - (&adjacency_counters, - thread_index, adj_index0, 1, - vlib_buffer_length_in_chain (vm, p0) + rw_len0); - vlib_increment_combined_counter - (&adjacency_counters, - thread_index, adj_index1, 1, - vlib_buffer_length_in_chain (vm, p1) + rw_len1); - } - /* Check MTU of outgoing interface. */ u16 ip0_len = clib_net_to_host_u16 (ip0->payload_length) + @@ -1933,16 +1922,15 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm, * wants to see the IP header */ if (PREDICT_TRUE (error0 == IP6_ERROR_NONE)) { - p0->current_data -= rw_len0; - p0->current_length += rw_len0; + vlib_buffer_advance (p0, -(word) rw_len0); tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index; vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0; next0 = adj0[0].rewrite_header.next_index; if (PREDICT_FALSE (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES)) - vnet_feature_arc_start_w_cfg_index - (lm->output_feature_arc_index, tx_sw_if_index0, &next0, p0, - adj0->ia_cfg_index); + vnet_feature_arc_start_w_cfg_index ( + lm->output_feature_arc_index, tx_sw_if_index0, &next0, p0, + adj0->ia_cfg_index); } else { @@ -1950,18 +1938,16 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm, } if (PREDICT_TRUE (error1 == IP6_ERROR_NONE)) { - p1->current_data -= rw_len1; - p1->current_length += rw_len1; - + vlib_buffer_advance (p1, -(word) rw_len1); tx_sw_if_index1 = adj1[0].rewrite_header.sw_if_index; vnet_buffer (p1)->sw_if_index[VLIB_TX] = tx_sw_if_index1; next1 = adj1[0].rewrite_header.next_index; if (PREDICT_FALSE (adj1[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES)) - vnet_feature_arc_start_w_cfg_index - (lm->output_feature_arc_index, tx_sw_if_index1, &next1, p1, - adj1->ia_cfg_index); + vnet_feature_arc_start_w_cfg_index ( + lm->output_feature_arc_index, tx_sw_if_index1, &next1, p1, + adj1->ia_cfg_index); } else { @@ -1969,40 +1955,46 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm, } if (is_midchain) - { - /* Guess we are only writing on ipv6 header. */ - vnet_rewrite_two_headers (adj0[0], adj1[0], - ip0, ip1, sizeof (ip6_header_t)); - } + /* Guess we are only writing on ipv6 header. */ + vnet_rewrite_two_headers (adj0[0], adj1[0], ip0, ip1, + sizeof (ip6_header_t)); else /* Guess we are only writing on simple Ethernet header. */ vnet_rewrite_two_headers (adj0[0], adj1[0], ip0, ip1, sizeof (ethernet_header_t)); + if (do_counters) + { + if (error0 == IP6_ERROR_NONE) + vlib_increment_combined_counter ( + &adjacency_counters, thread_index, adj_index0, 1, + vlib_buffer_length_in_chain (vm, p0) + rw_len0); + if (error1 == IP6_ERROR_NONE) + vlib_increment_combined_counter ( + &adjacency_counters, thread_index, adj_index1, 1, + vlib_buffer_length_in_chain (vm, p1) + rw_len1); + } + if (is_midchain) { - if (adj0->sub_type.midchain.fixup_func) - adj0->sub_type.midchain.fixup_func - (vm, adj0, p0, adj0->sub_type.midchain.fixup_data); - if (adj1->sub_type.midchain.fixup_func) - adj1->sub_type.midchain.fixup_func - (vm, adj1, p1, adj1->sub_type.midchain.fixup_data); + if (error0 == IP6_ERROR_NONE) + adj_midchain_fixup (vm, adj0, p0, VNET_LINK_IP6); + if (error1 == IP6_ERROR_NONE) + adj_midchain_fixup (vm, adj1, p1, VNET_LINK_IP6); } if (is_mcast) { /* * copy bytes from the IP address into the MAC rewrite */ - vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK, - adj0-> - rewrite_header.dst_mcast_offset, - &ip0->dst_address.as_u32[3], - (u8 *) ip0); - vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK, - adj1-> - rewrite_header.dst_mcast_offset, - &ip1->dst_address.as_u32[3], - (u8 *) ip1); + if (error0 == IP6_ERROR_NONE) + vnet_ip_mcast_fixup_header ( + IP6_MCAST_ADDR_MASK, adj0->rewrite_header.dst_mcast_offset, + &ip0->dst_address.as_u32[3], (u8 *) ip0); + if (error1 == IP6_ERROR_NONE) + vnet_ip_mcast_fixup_header ( + IP6_MCAST_ADDR_MASK, adj1->rewrite_header.dst_mcast_offset, + &ip1->dst_address.as_u32[3], (u8 *) ip1); } vlib_validate_buffer_enqueue_x2 (vm, node, next_index, @@ -2061,28 +2053,10 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm, } } - if (is_midchain) - { - /* Guess we are only writing on ip6 header. */ - vnet_rewrite_one_header (adj0[0], ip0, sizeof (ip6_header_t)); - } - else - /* Guess we are only writing on simple Ethernet header. */ - vnet_rewrite_one_header (adj0[0], ip0, - sizeof (ethernet_header_t)); - /* Update packet buffer attributes/set output interface. */ rw_len0 = adj0[0].rewrite_header.data_bytes; vnet_buffer (p0)->ip.save_rewrite_length = rw_len0; - if (do_counters) - { - vlib_increment_combined_counter - (&adjacency_counters, - thread_index, adj_index0, 1, - vlib_buffer_length_in_chain (vm, p0) + rw_len0); - } - /* Check MTU of outgoing interface. */ u16 ip0_len = clib_net_to_host_u16 (ip0->payload_length) + @@ -2098,9 +2072,7 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm, * wants to see the IP header */ if (PREDICT_TRUE (error0 == IP6_ERROR_NONE)) { - p0->current_data -= rw_len0; - p0->current_length += rw_len0; - + vlib_buffer_advance (p0, -(word) rw_len0); tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index; vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0; @@ -2108,30 +2080,37 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm, if (PREDICT_FALSE (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES)) - vnet_feature_arc_start_w_cfg_index - (lm->output_feature_arc_index, tx_sw_if_index0, &next0, p0, - adj0->ia_cfg_index); + vnet_feature_arc_start_w_cfg_index ( + lm->output_feature_arc_index, tx_sw_if_index0, &next0, p0, + adj0->ia_cfg_index); + + if (is_midchain) + /* Guess we are only writing on ip6 header. */ + vnet_rewrite_one_header (adj0[0], ip0, sizeof (ip6_header_t)); + else + /* Guess we are only writing on simple Ethernet header. */ + vnet_rewrite_one_header (adj0[0], ip0, + sizeof (ethernet_header_t)); + + if (do_counters) + { + vlib_increment_combined_counter ( + &adjacency_counters, thread_index, adj_index0, 1, + vlib_buffer_length_in_chain (vm, p0) + rw_len0); + } + + if (is_midchain && adj0->sub_type.midchain.fixup_func) + adj_midchain_fixup (vm, adj0, p0, VNET_LINK_IP6); + if (is_mcast) + vnet_ip_mcast_fixup_header ( + IP6_MCAST_ADDR_MASK, adj0->rewrite_header.dst_mcast_offset, + &ip0->dst_address.as_u32[3], (u8 *) ip0); } else { p0->error = error_node->errors[error0]; } - if (is_midchain) - { - if (adj0->sub_type.midchain.fixup_func) - adj0->sub_type.midchain.fixup_func - (vm, adj0, p0, adj0->sub_type.midchain.fixup_data); - } - if (is_mcast) - { - vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK, - adj0-> - rewrite_header.dst_mcast_offset, - &ip0->dst_address.as_u32[3], - (u8 *) ip0); - } - from += 1; n_left_from -= 1; to_next += 1; @@ -2215,7 +2194,7 @@ VLIB_NODE_FN (ip6_mcast_midchain_node) (vlib_main_t * vm, VLIB_REGISTER_NODE (ip6_midchain_node) = { .name = "ip6-midchain", .vector_size = sizeof (u32), - .format_trace = format_ip6_forward_next_trace, + .format_trace = format_ip6_rewrite_trace, .sibling_of = "ip6-rewrite", }; diff --git a/src/vnet/ip/ip6_input.h b/src/vnet/ip/ip6_input.h index 49e37ec1808..25eae62723d 100644 --- a/src/vnet/ip/ip6_input.h +++ b/src/vnet/ip/ip6_input.h @@ -53,11 +53,9 @@ typedef enum } ip6_input_next_t; always_inline void -ip6_input_check_x2 (vlib_main_t * vm, - vlib_node_runtime_t * error_node, - vlib_buffer_t * p0, vlib_buffer_t * p1, - ip6_header_t * ip0, ip6_header_t * ip1, - u32 * next0, u32 * next1) +ip6_input_check_x2 (vlib_main_t *vm, vlib_node_runtime_t *error_node, + vlib_buffer_t *p0, vlib_buffer_t *p1, ip6_header_t *ip0, + ip6_header_t *ip1, u32 *next0, u32 *next1) { u8 error0, error1; @@ -65,13 +63,15 @@ ip6_input_check_x2 (vlib_main_t * vm, /* Version != 6? Drop it. */ error0 = - (clib_net_to_host_u32 - (ip0->ip_version_traffic_class_and_flow_label) >> 28) != - 6 ? IP6_ERROR_VERSION : error0; + (clib_net_to_host_u32 (ip0->ip_version_traffic_class_and_flow_label) >> + 28) != 6 ? + IP6_ERROR_VERSION : + error0; error1 = - (clib_net_to_host_u32 - (ip1->ip_version_traffic_class_and_flow_label) >> 28) != - 6 ? IP6_ERROR_VERSION : error1; + (clib_net_to_host_u32 (ip1->ip_version_traffic_class_and_flow_label) >> + 28) != 6 ? + IP6_ERROR_VERSION : + error1; /* hop limit < 1? Drop it. for link-local broadcast packets, * like dhcpv6 packets from client has hop-limit 1, which should not @@ -81,18 +81,18 @@ ip6_input_check_x2 (vlib_main_t * vm, error1 = ip1->hop_limit < 1 ? IP6_ERROR_TIME_EXPIRED : error1; /* L2 length must be at least minimal IP header. */ - error0 = - p0->current_length < sizeof (ip0[0]) ? IP6_ERROR_TOO_SHORT : error0; - error1 = - p1->current_length < sizeof (ip1[0]) ? IP6_ERROR_TOO_SHORT : error1; + error0 = p0->current_length < sizeof (ip0[0]) ? IP6_ERROR_TOO_SHORT : error0; + error1 = p1->current_length < sizeof (ip1[0]) ? IP6_ERROR_TOO_SHORT : error1; if (PREDICT_FALSE (error0 != IP6_ERROR_NONE)) { + p0->error = error_node->errors[error0]; + if (error0 == IP6_ERROR_TIME_EXPIRED) { - icmp6_error_set_vnet_buffer (p0, ICMP6_time_exceeded, - ICMP6_time_exceeded_ttl_exceeded_in_transit, - 0); + icmp6_error_set_vnet_buffer ( + p0, ICMP6_time_exceeded, + ICMP6_time_exceeded_ttl_exceeded_in_transit, 0); *next0 = IP6_INPUT_NEXT_ICMP_ERROR; } else @@ -102,11 +102,13 @@ ip6_input_check_x2 (vlib_main_t * vm, } if (PREDICT_FALSE (error1 != IP6_ERROR_NONE)) { + p1->error = error_node->errors[error1]; + if (error1 == IP6_ERROR_TIME_EXPIRED) { - icmp6_error_set_vnet_buffer (p1, ICMP6_time_exceeded, - ICMP6_time_exceeded_ttl_exceeded_in_transit, - 0); + icmp6_error_set_vnet_buffer ( + p1, ICMP6_time_exceeded, + ICMP6_time_exceeded_ttl_exceeded_in_transit, 0); *next1 = IP6_INPUT_NEXT_ICMP_ERROR; } else @@ -117,9 +119,8 @@ ip6_input_check_x2 (vlib_main_t * vm, } always_inline void -ip6_input_check_x1 (vlib_main_t * vm, - vlib_node_runtime_t * error_node, - vlib_buffer_t * p0, ip6_header_t * ip0, u32 * next0) +ip6_input_check_x1 (vlib_main_t *vm, vlib_node_runtime_t *error_node, + vlib_buffer_t *p0, ip6_header_t *ip0, u32 *next0) { u8 error0; @@ -127,9 +128,10 @@ ip6_input_check_x1 (vlib_main_t * vm, /* Version != 6? Drop it. */ error0 = - (clib_net_to_host_u32 - (ip0->ip_version_traffic_class_and_flow_label) >> 28) != - 6 ? IP6_ERROR_VERSION : error0; + (clib_net_to_host_u32 (ip0->ip_version_traffic_class_and_flow_label) >> + 28) != 6 ? + IP6_ERROR_VERSION : + error0; /* hop limit < 1? Drop it. for link-local broadcast packets, * like dhcpv6 packets from client has hop-limit 1, which should not @@ -138,16 +140,16 @@ ip6_input_check_x1 (vlib_main_t * vm, error0 = ip0->hop_limit < 1 ? IP6_ERROR_TIME_EXPIRED : error0; /* L2 length must be at least minimal IP header. */ - error0 = - p0->current_length < sizeof (ip0[0]) ? IP6_ERROR_TOO_SHORT : error0; + error0 = p0->current_length < sizeof (ip0[0]) ? IP6_ERROR_TOO_SHORT : error0; if (PREDICT_FALSE (error0 != IP6_ERROR_NONE)) { + p0->error = error_node->errors[error0]; if (error0 == IP6_ERROR_TIME_EXPIRED) { - icmp6_error_set_vnet_buffer (p0, ICMP6_time_exceeded, - ICMP6_time_exceeded_ttl_exceeded_in_transit, - 0); + icmp6_error_set_vnet_buffer ( + p0, ICMP6_time_exceeded, + ICMP6_time_exceeded_ttl_exceeded_in_transit, 0); *next0 = IP6_INPUT_NEXT_ICMP_ERROR; } else diff --git a/src/vnet/ip/ip6_ll_table.c b/src/vnet/ip/ip6_ll_table.c index f9172f6c50c..2234ea9df37 100644 --- a/src/vnet/ip/ip6_ll_table.c +++ b/src/vnet/ip/ip6_ll_table.c @@ -144,17 +144,20 @@ ip6_ll_table_entry_delete (const ip6_ll_prefix_t * ilp) fib_node_index_t ip6_ll_entry_index; u32 fib_index; + fib_index = ip6_ll_fib_get (ilp->ilp_sw_if_index); + if (~0 == fib_index) + return; + ip6_ll_entry_index = ip6_ll_table_lookup_exact_match (ilp); + if (FIB_NODE_INDEX_INVALID == ip6_ll_entry_index) + return; - if (FIB_NODE_INDEX_INVALID != ip6_ll_entry_index) - fib_table_entry_delete_index (ip6_ll_entry_index, FIB_SOURCE_IP6_ND); + fib_table_entry_delete_index (ip6_ll_entry_index, FIB_SOURCE_IP6_ND); /* * if there are no ND sourced prefixes left, then we can clean up this FIB */ - fib_index = ip6_ll_fib_get (ilp->ilp_sw_if_index); - if (~0 != fib_index && - 0 == fib_table_get_num_entries (fib_index, FIB_PROTOCOL_IP6, + if (0 == fib_table_get_num_entries (fib_index, FIB_PROTOCOL_IP6, FIB_SOURCE_IP6_ND)) { fib_table_unlock (fib_index, FIB_PROTOCOL_IP6, FIB_SOURCE_IP6_ND); @@ -208,33 +211,10 @@ ip6_ll_table_show_all (vlib_main_t * vm, u32 fib_index) vec_free (ctx.entries); } -typedef struct -{ - u32 fib_index; - u64 count_by_prefix_length[129]; -} count_routes_in_fib_at_prefix_length_arg_t; - -static int -count_routes_in_fib_at_prefix_length (clib_bihash_kv_24_8_t * kvp, void *arg) -{ - count_routes_in_fib_at_prefix_length_arg_t *ap = arg; - int mask_width; - - if ((kvp->key[2] >> 32) != ap->fib_index) - return (BIHASH_WALK_CONTINUE); - - mask_width = kvp->key[2] & 0xFF; - - ap->count_by_prefix_length[mask_width]++; - - return (BIHASH_WALK_CONTINUE); -} - static clib_error_t * ip6_ll_show_fib (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { - count_routes_in_fib_at_prefix_length_arg_t _ca, *ca = &_ca; fib_table_t *fib_table; int verbose, matching; ip6_address_t matching_address; @@ -272,9 +252,6 @@ ip6_ll_show_fib (vlib_main_t * vm, vec_foreach_index (sw_if_index, ip6_ll_table.ilt_fibs) { - fib_source_t source; - u8 *s = NULL; - fib_index = ip6_ll_table.ilt_fibs[sw_if_index]; if (~0 == fib_index) continue; @@ -284,44 +261,9 @@ ip6_ll_show_fib (vlib_main_t * vm, if (!(fib_table->ft_flags & FIB_TABLE_FLAG_IP6_LL)) continue; - s = format (s, "%U, fib_index:%d, locks:[", - format_fib_table_name, fib_index, - FIB_PROTOCOL_IP6, fib_index); - vec_foreach_index (source, fib_table->ft_locks) - { - if (0 != fib_table->ft_locks[source]) - { - s = format (s, "%U:%d, ", - format_fib_source, source, fib_table->ft_locks[source]); - } - } - s = format (s, "]"); - vlib_cli_output (vm, "%v", s); - vec_free (s); - - /* Show summary? */ + ip6_fib_table_show (vm, fib_table, !verbose); if (!verbose) - { - clib_bihash_24_8_t *h = - &ip6_fib_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash; - int len; - - vlib_cli_output (vm, "%=20s%=16s", "Prefix length", "Count"); - - clib_memset (ca, 0, sizeof (*ca)); - ca->fib_index = fib_index; - - clib_bihash_foreach_key_value_pair_24_8 - (h, count_routes_in_fib_at_prefix_length, ca); - - for (len = 128; len >= 0; len--) - { - if (ca->count_by_prefix_length[len]) - vlib_cli_output (vm, "%=20d%=16lld", - len, ca->count_by_prefix_length[len]); - } - continue; - } + continue; if (!matching) { diff --git a/src/vnet/ip/ip_api.c b/src/vnet/ip/ip_api.c index 644b4988abc..5ced88fec2e 100644 --- a/src/vnet/ip/ip_api.c +++ b/src/vnet/ip/ip_api.c @@ -636,7 +636,8 @@ vl_api_ip_table_add_del_t_handler (vl_api_ip_table_add_del_t * mp) if (mp->is_add) { - ip_table_create (fproto, table_id, 1, mp->table.name); + ip_table_create (fproto, table_id, 1 /* is_api */, 1 /* create_mfib */, + mp->table.name); } else { @@ -647,6 +648,28 @@ vl_api_ip_table_add_del_t_handler (vl_api_ip_table_add_del_t * mp) } void +vl_api_ip_table_add_del_v2_t_handler (vl_api_ip_table_add_del_v2_t *mp) +{ + vl_api_ip_table_add_del_v2_reply_t *rmp; + fib_protocol_t fproto = + (mp->table.is_ip6 ? FIB_PROTOCOL_IP6 : FIB_PROTOCOL_IP4); + u32 table_id = ntohl (mp->table.table_id); + int rv = 0; + + if (mp->is_add) + { + ip_table_create (fproto, table_id, 1 /* is_api */, mp->create_mfib, + mp->table.name); + } + else + { + ip_table_delete (fproto, table_id, 1); + } + + REPLY_MACRO (VL_API_IP_TABLE_ADD_DEL_V2_REPLY); +} + +void vl_api_ip_table_allocate_t_handler (vl_api_ip_table_allocate_t *mp) { vl_api_ip_table_allocate_reply_t *rmp; @@ -661,7 +684,8 @@ vl_api_ip_table_allocate_t_handler (vl_api_ip_table_allocate_t *mp) if (~0 == table_id) rv = VNET_API_ERROR_EAGAIN; else - ip_table_create (fproto, table_id, 1, mp->table.name); + ip_table_create (fproto, table_id, 1 /* is_api */, 1 /* create_mfib */, + mp->table.name); REPLY_MACRO2 (VL_API_IP_TABLE_ALLOCATE_REPLY, { clib_memcpy_fast (&rmp->table, &mp->table, sizeof (mp->table)); @@ -915,8 +939,8 @@ vl_api_ip_route_lookup_v2_t_handler (vl_api_ip_route_lookup_v2_t *mp) } void -ip_table_create (fib_protocol_t fproto, - u32 table_id, u8 is_api, const u8 * name) +ip_table_create (fib_protocol_t fproto, u32 table_id, u8 is_api, + u8 create_mfib, const u8 *name) { u32 fib_index, mfib_index; vnet_main_t *vnm = vnet_get_main (); @@ -936,16 +960,23 @@ ip_table_create (fib_protocol_t fproto, * their own unicast tables. */ fib_index = fib_table_find (fproto, table_id); - mfib_index = mfib_table_find (fproto, table_id); - /* * Always try to re-lock in case the fib was deleted by an API call * but was not yet freed because some other locks were held */ fib_table_find_or_create_and_lock_w_name ( fproto, table_id, (is_api ? FIB_SOURCE_API : FIB_SOURCE_CLI), name); - mfib_table_find_or_create_and_lock_w_name ( - fproto, table_id, (is_api ? MFIB_SOURCE_API : MFIB_SOURCE_CLI), name); + + if (create_mfib) + { + /* same for mfib, if needs be */ + mfib_index = mfib_table_find (fproto, table_id); + mfib_table_find_or_create_and_lock_w_name ( + fproto, table_id, (is_api ? MFIB_SOURCE_API : MFIB_SOURCE_CLI), + name); + } + else + mfib_index = 0; if ((~0 == fib_index) || (~0 == mfib_index)) call_elf_section_ip_table_callbacks (vnm, table_id, 1 /* is_add */ , @@ -1655,9 +1686,10 @@ vl_api_ip_table_replace_begin_t_handler (vl_api_ip_table_replace_begin_t * mp) rv = VNET_API_ERROR_NO_SUCH_FIB; else { + u32 mfib_index = mfib_table_find (fproto, ntohl (mp->table.table_id)); fib_table_mark (fib_index, fproto, FIB_SOURCE_API); - mfib_table_mark (mfib_table_find (fproto, ntohl (mp->table.table_id)), - fproto, MFIB_SOURCE_API); + if (mfib_index != INDEX_INVALID) + mfib_table_mark (mfib_index, fproto, MFIB_SOURCE_API); } REPLY_MACRO (VL_API_IP_TABLE_REPLACE_BEGIN_REPLY); } @@ -1677,10 +1709,10 @@ vl_api_ip_table_replace_end_t_handler (vl_api_ip_table_replace_end_t * mp) rv = VNET_API_ERROR_NO_SUCH_FIB; else { + u32 mfib_index = mfib_table_find (fproto, ntohl (mp->table.table_id)); fib_table_sweep (fib_index, fproto, FIB_SOURCE_API); - mfib_table_sweep (mfib_table_find - (fproto, ntohl (mp->table.table_id)), fproto, - MFIB_SOURCE_API); + if (mfib_index != INDEX_INVALID) + mfib_table_sweep (mfib_index, fproto, MFIB_SOURCE_API); } REPLY_MACRO (VL_API_IP_TABLE_REPLACE_END_REPLY); } @@ -1703,6 +1735,7 @@ vl_api_ip_table_flush_t_handler (vl_api_ip_table_flush_t * mp) vnet_main_t *vnm = vnet_get_main (); vnet_interface_main_t *im = &vnm->interface_main; vnet_sw_interface_t *si; + u32 mfib_index; /* Shut down interfaces in this FIB / clean out intfc routes */ pool_foreach (si, im->sw_interfaces) @@ -1717,8 +1750,10 @@ vl_api_ip_table_flush_t_handler (vl_api_ip_table_flush_t * mp) } fib_table_flush (fib_index, fproto, FIB_SOURCE_API); - mfib_table_flush (mfib_table_find (fproto, ntohl (mp->table.table_id)), - fproto, MFIB_SOURCE_API); + + mfib_index = mfib_table_find (fproto, ntohl (mp->table.table_id)); + if (mfib_index != INDEX_INVALID) + mfib_table_flush (mfib_index, fproto, MFIB_SOURCE_API); } REPLY_MACRO (VL_API_IP_TABLE_FLUSH_REPLY); @@ -2128,6 +2163,8 @@ ip_api_hookup (vlib_main_t * vm) am, REPLY_MSG_ID_BASE + VL_API_IP_ROUTE_ADD_DEL_V2, 1); vl_api_set_msg_thread_safe ( am, REPLY_MSG_ID_BASE + VL_API_IP_ROUTE_ADD_DEL_V2_REPLY, 1); + vl_api_set_msg_thread_safe (am, REPLY_MSG_ID_BASE + VL_API_IP_ADDRESS_DUMP, + 1); return 0; } diff --git a/src/vnet/ip/ip_sas.c b/src/vnet/ip/ip_sas.c index 0fc261724f1..01f6c90baf8 100644 --- a/src/vnet/ip/ip_sas.c +++ b/src/vnet/ip/ip_sas.c @@ -54,6 +54,8 @@ ip6_sas_commonlen (const ip6_address_t *a1, const ip6_address_t *a2) static int ip4_sas_commonlen (const ip4_address_t *a1, const ip4_address_t *a2) { + if (!a1 || !a2) + return 0; u64 a = clib_net_to_host_u32 (a1->as_u32) ^ clib_net_to_host_u32 (a2->as_u32); if (a == 0) diff --git a/src/vnet/ip/ip_test.c b/src/vnet/ip/ip_test.c index 727afba67f4..0d1c71063ae 100644 --- a/src/vnet/ip/ip_test.c +++ b/src/vnet/ip/ip_test.c @@ -464,6 +464,60 @@ api_ip_table_add_del (vat_main_t *vam) } static int +api_ip_table_add_del_v2 (vat_main_t *vam) +{ + unformat_input_t *i = vam->input; + vl_api_ip_table_add_del_v2_t *mp; + u8 create_mfib = 1; + u32 table_id = ~0; + u8 is_ipv6 = 0; + u8 is_add = 1; + int ret = 0; + + /* Parse args required to build the message */ + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "ipv6")) + is_ipv6 = 1; + else if (unformat (i, "del")) + is_add = 0; + else if (unformat (i, "add")) + is_add = 1; + else if (unformat (i, "table %d", &table_id)) + ; + else if (unformat (i, "no-mfib")) + create_mfib = 0; + else + { + clib_warning ("parse error '%U'", format_unformat_error, i); + return -99; + } + } + + if (~0 == table_id) + { + errmsg ("missing table-ID"); + return -99; + } + + /* Construct the API message */ + M (IP_TABLE_ADD_DEL_V2, mp); + + mp->table.table_id = ntohl (table_id); + mp->table.is_ip6 = is_ipv6; + mp->is_add = is_add; + mp->create_mfib = create_mfib; + + /* send it... */ + S (mp); + + /* Wait for a reply... */ + W (ret); + + return ret; +} + +static int api_ip_table_replace_begin (vat_main_t *vam) { unformat_input_t *i = vam->input; diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c index c225c222a38..b978bd79742 100644 --- a/src/vnet/ip/lookup.c +++ b/src/vnet/ip/lookup.c @@ -419,10 +419,12 @@ vnet_ip_table_cmd (vlib_main_t * vm, unformat_input_t _line_input, *line_input = &_line_input; clib_error_t *error = NULL; u32 table_id, is_add; + u8 create_mfib; u8 *name = NULL; is_add = 1; table_id = ~0; + create_mfib = 1; /* Get a line of input. */ if (!unformat_user (main_input, unformat_line_input, line_input)) @@ -438,6 +440,8 @@ vnet_ip_table_cmd (vlib_main_t * vm, is_add = 1; else if (unformat (line_input, "name %s", &name)) ; + else if (unformat (line_input, "no-mfib")) + create_mfib = 0; else { error = unformat_parse_error (line_input); @@ -459,7 +463,8 @@ vnet_ip_table_cmd (vlib_main_t * vm, table_id = ip_table_get_unused_id (fproto); vlib_cli_output (vm, "%u\n", table_id); } - ip_table_create (fproto, table_id, 0, name); + ip_table_create (fproto, table_id, 0 /* is_api */, create_mfib, + name); } else { @@ -603,6 +608,8 @@ VLIB_CLI_COMMAND (vlib_cli_show_ip6_command, static) = { * @cliexcmd{ip route add 7.0.0.1/32 via 6.0.0.2 GigabitEthernet2/0/0 weight 3} * To add a route to a particular FIB table (VRF), use: * @cliexcmd{ip route add 172.16.24.0/24 table 7 via GigabitEthernet2/0/0} + * To add a route to drop the traffic: + * @cliexcmd{ip route add 172.16.24.0/24 table 100 via 127.0.0.1 drop} ?*/ VLIB_CLI_COMMAND (ip_route_command, static) = { .path = "ip route", @@ -612,7 +619,7 @@ VLIB_CLI_COMMAND (ip_route_command, static) = { "<value>] [udp-encap <value>] [ip4-lookup-in-table <value>] " "[ip6-lookup-in-table <value>] [mpls-lookup-in-table <value>] " "[resolve-via-host] [resolve-via-connected] [rx-ip4|rx-ip6 " - "<interface>] [out-labels <value value value>]", + "<interface>] [out-labels <value value value>] [drop]", .function = vnet_ip_route_cmd, .is_mp_safe = 1, }; diff --git a/src/vnet/ipsec/ah_decrypt.c b/src/vnet/ipsec/ah_decrypt.c index 918ebf03f67..ec4db0fed57 100644 --- a/src/vnet/ipsec/ah_decrypt.c +++ b/src/vnet/ipsec/ah_decrypt.c @@ -500,10 +500,10 @@ ah_decrypt_init (vlib_main_t *vm) { ipsec_main_t *im = &ipsec_main; - im->ah4_dec_fq_index = - vlib_frame_queue_main_init (ah4_decrypt_node.index, 0); - im->ah6_dec_fq_index = - vlib_frame_queue_main_init (ah6_decrypt_node.index, 0); + im->ah4_dec_fq_index = vlib_frame_queue_main_init (ah4_decrypt_node.index, + im->handoff_queue_size); + im->ah6_dec_fq_index = vlib_frame_queue_main_init (ah6_decrypt_node.index, + im->handoff_queue_size); return 0; } diff --git a/src/vnet/ipsec/ah_encrypt.c b/src/vnet/ipsec/ah_encrypt.c index 960327f071d..86694660878 100644 --- a/src/vnet/ipsec/ah_encrypt.c +++ b/src/vnet/ipsec/ah_encrypt.c @@ -490,10 +490,10 @@ ah_encrypt_init (vlib_main_t *vm) { ipsec_main_t *im = &ipsec_main; - im->ah4_enc_fq_index = - vlib_frame_queue_main_init (ah4_encrypt_node.index, 0); - im->ah6_enc_fq_index = - vlib_frame_queue_main_init (ah6_encrypt_node.index, 0); + im->ah4_enc_fq_index = vlib_frame_queue_main_init (ah4_encrypt_node.index, + im->handoff_queue_size); + im->ah6_enc_fq_index = vlib_frame_queue_main_init (ah6_encrypt_node.index, + im->handoff_queue_size); return 0; } diff --git a/src/vnet/ipsec/esp_decrypt.c b/src/vnet/ipsec/esp_decrypt.c index 26d8ca1deee..01b2d2971b0 100644 --- a/src/vnet/ipsec/esp_decrypt.c +++ b/src/vnet/ipsec/esp_decrypt.c @@ -1675,14 +1675,14 @@ esp_decrypt_init (vlib_main_t *vm) { ipsec_main_t *im = &ipsec_main; - im->esp4_dec_fq_index = - vlib_frame_queue_main_init (esp4_decrypt_node.index, 0); - im->esp6_dec_fq_index = - vlib_frame_queue_main_init (esp6_decrypt_node.index, 0); - im->esp4_dec_tun_fq_index = - vlib_frame_queue_main_init (esp4_decrypt_tun_node.index, 0); - im->esp6_dec_tun_fq_index = - vlib_frame_queue_main_init (esp6_decrypt_tun_node.index, 0); + im->esp4_dec_fq_index = vlib_frame_queue_main_init (esp4_decrypt_node.index, + im->handoff_queue_size); + im->esp6_dec_fq_index = vlib_frame_queue_main_init (esp6_decrypt_node.index, + im->handoff_queue_size); + im->esp4_dec_tun_fq_index = vlib_frame_queue_main_init ( + esp4_decrypt_tun_node.index, im->handoff_queue_size); + im->esp6_dec_tun_fq_index = vlib_frame_queue_main_init ( + esp6_decrypt_tun_node.index, im->handoff_queue_size); return 0; } diff --git a/src/vnet/ipsec/esp_encrypt.c b/src/vnet/ipsec/esp_encrypt.c index dd47053874c..f6d1ecaed24 100644 --- a/src/vnet/ipsec/esp_encrypt.c +++ b/src/vnet/ipsec/esp_encrypt.c @@ -589,6 +589,22 @@ esp_prepare_async_frame (vlib_main_t *vm, ipsec_per_thread_data_t *ptd, async_next, iv, tag, aad, flag); } +/* Per RFC6935 section 5, the UDP checksum must be computed when originating + * an IPv6 UDP packet. The default behavior may be overridden when conditions + * defined by RFC6936 are satisfied. This implementation does not satisfy all + * the conditions so the checksum must be computed. + */ +static_always_inline void +set_ip6_udp_cksum_offload (vlib_buffer_t *b, i16 l3_hdr_offset, + i16 l4_hdr_offset) +{ + vnet_buffer (b)->l3_hdr_offset = l3_hdr_offset; + vnet_buffer (b)->l4_hdr_offset = l4_hdr_offset; + vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM); + b->flags |= (VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_L3_HDR_OFFSET_VALID | + VNET_BUFFER_F_L4_HDR_OFFSET_VALID); +} + always_inline uword esp_encrypt_inline (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, vnet_link_t lt, int is_tun, @@ -869,6 +885,15 @@ esp_encrypt_inline (vlib_main_t *vm, vlib_node_runtime_t *node, esp_update_ip4_hdr (ip4, len, /* is_transport */ 0, 0); } + if (ipsec_sa_is_set_UDP_ENCAP (sa0) && + ipsec_sa_is_set_IS_TUNNEL_V6 (sa0)) + { + i16 l3_off = b[0]->current_data - hdr_len; + i16 l4_off = l3_off + sizeof (ip6_header_t); + + set_ip6_udp_cksum_offload (b[0], l3_off, l4_off); + } + dpo = &sa0->dpo; if (!is_tun) { @@ -988,6 +1013,14 @@ esp_encrypt_inline (vlib_main_t *vm, vlib_node_runtime_t *node, esp_fill_udp_hdr (sa0, udp, udp_len); } + if (udp && (VNET_LINK_IP6 == lt)) + { + i16 l3_off = b[0]->current_data - hdr_len + l2_len; + i16 l4_off = l3_off + ip_len; + + set_ip6_udp_cksum_offload (b[0], l3_off, l4_off); + } + sync_next[0] = ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT; } @@ -1445,16 +1478,16 @@ esp_encrypt_init (vlib_main_t *vm) { ipsec_main_t *im = &ipsec_main; - im->esp4_enc_fq_index = - vlib_frame_queue_main_init (esp4_encrypt_node.index, 0); - im->esp6_enc_fq_index = - vlib_frame_queue_main_init (esp6_encrypt_node.index, 0); - im->esp4_enc_tun_fq_index = - vlib_frame_queue_main_init (esp4_encrypt_tun_node.index, 0); - im->esp6_enc_tun_fq_index = - vlib_frame_queue_main_init (esp6_encrypt_tun_node.index, 0); - im->esp_mpls_enc_tun_fq_index = - vlib_frame_queue_main_init (esp_mpls_encrypt_tun_node.index, 0); + im->esp4_enc_fq_index = vlib_frame_queue_main_init (esp4_encrypt_node.index, + im->handoff_queue_size); + im->esp6_enc_fq_index = vlib_frame_queue_main_init (esp6_encrypt_node.index, + im->handoff_queue_size); + im->esp4_enc_tun_fq_index = vlib_frame_queue_main_init ( + esp4_encrypt_tun_node.index, im->handoff_queue_size); + im->esp6_enc_tun_fq_index = vlib_frame_queue_main_init ( + esp6_encrypt_tun_node.index, im->handoff_queue_size); + im->esp_mpls_enc_tun_fq_index = vlib_frame_queue_main_init ( + esp_mpls_encrypt_tun_node.index, im->handoff_queue_size); return 0; } diff --git a/src/vnet/ipsec/ipsec.c b/src/vnet/ipsec/ipsec.c index f8c39c327ed..8b43dd23cc8 100644 --- a/src/vnet/ipsec/ipsec.c +++ b/src/vnet/ipsec/ipsec.c @@ -663,6 +663,7 @@ ipsec_config (vlib_main_t *vm, unformat_input_t *input) u32 ipsec_spd_fp_num_buckets; bool fp_spd_ip4_enabled = false; bool fp_spd_ip6_enabled = false; + u32 handoff_queue_size; while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { @@ -758,6 +759,11 @@ ipsec_config (vlib_main_t *vm, unformat_input_t *input) ipsec_tun_table_init (AF_IP6, table_size, n_buckets); } + else if (unformat (input, "async-handoff-queue-size %d", + &handoff_queue_size)) + { + im->handoff_queue_size = handoff_queue_size; + } else return clib_error_return (0, "unknown input `%U'", format_unformat_error, input); diff --git a/src/vnet/ipsec/ipsec.h b/src/vnet/ipsec/ipsec.h index 4aa09d7560e..9ab054cf2a9 100644 --- a/src/vnet/ipsec/ipsec.h +++ b/src/vnet/ipsec/ipsec.h @@ -248,6 +248,8 @@ typedef struct u32 esp4_dec_tun_fq_index; u32 esp6_dec_tun_fq_index; + u32 handoff_queue_size; + /* Number of buckets for flow cache */ u32 ipsec4_out_spd_hash_num_buckets; u32 ipsec4_out_spd_flow_cache_entries; diff --git a/src/vnet/ipsec/ipsec_input.c b/src/vnet/ipsec/ipsec_input.c index 6ccc0be2622..48f7deadda3 100644 --- a/src/vnet/ipsec/ipsec_input.c +++ b/src/vnet/ipsec/ipsec_input.c @@ -274,6 +274,159 @@ ip6_addr_match_range (ip6_address_t * a, ip6_address_t * la, } always_inline void +ipsec_collect_ah_trace (vlib_buffer_t **b, vlib_node_runtime_t *node, + vlib_main_t *vm, ip4_header_t *ip0, ah_header_t *ah0, + u8 has_space0, ipsec_spd_t *spd0, ipsec_policy_t *p0, + u32 pi0) +{ + if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE) && + PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED)) + { + ipsec_input_trace_t *tr = vlib_add_trace (vm, node, b[0], sizeof (*tr)); + + tr->proto = ip0->protocol; + tr->sa_id = p0 ? p0->sa_id : ~0; + tr->spi = has_space0 ? clib_net_to_host_u32 (ah0->spi) : ~0; + tr->seq = has_space0 ? clib_net_to_host_u32 (ah0->seq_no) : ~0; + tr->spd = spd0->id; + tr->policy_index = pi0; + } +} + +always_inline void +ipsec_ah_packet_process (vlib_main_t *vm, ipsec_main_t *im, ip4_header_t *ip0, + ah_header_t *ah0, u32 thread_index, ipsec_spd_t *spd0, + vlib_buffer_t **b, vlib_node_runtime_t *node, + u64 *ipsec_bypassed, u64 *ipsec_dropped, + u64 *ipsec_matched, u64 *ipsec_unprocessed, u16 *next) + +{ + ipsec_policy_t *p0 = NULL; + u32 pi0 = ~0; + u8 has_space0; + /* if flow cache is enabled, first search through flow cache for a + * policy match and revert back to linear search on failure */ + bool search_flow_cache = im->input_flow_cache_flag; + + while (1) + { + if (search_flow_cache) + { + p0 = ipsec4_input_spd_find_flow_cache_entry ( + im, ip0->src_address.as_u32, ip0->dst_address.as_u32, + IPSEC_SPD_POLICY_IP4_INBOUND_PROTECT); + } + else + { + p0 = ipsec_input_protect_policy_match ( + spd0, clib_net_to_host_u32 (ip0->src_address.as_u32), + clib_net_to_host_u32 (ip0->dst_address.as_u32), + clib_net_to_host_u32 (ah0->spi)); + } + + has_space0 = vlib_buffer_has_space (b[0], (clib_address_t) (ah0 + 1) - + (clib_address_t) ip0); + + if (PREDICT_TRUE ((p0 != NULL) & (has_space0))) + { + *ipsec_matched += 1; + pi0 = p0 - im->policies; + vlib_increment_combined_counter (&ipsec_spd_policy_counters, + thread_index, pi0, 1, + clib_net_to_host_u16 (ip0->length)); + + vnet_buffer (b[0])->ipsec.sad_index = p0->sa_index; + next[0] = im->ah4_decrypt_next_index; + ipsec_collect_ah_trace (b, node, vm, ip0, ah0, has_space0, spd0, p0, + pi0); + return; + } + else + { + p0 = 0; + pi0 = ~0; + } + if (search_flow_cache) + { + p0 = ipsec4_input_spd_find_flow_cache_entry ( + im, ip0->src_address.as_u32, ip0->dst_address.as_u32, + IPSEC_SPD_POLICY_IP4_INBOUND_BYPASS); + } + + else + { + p0 = ipsec_input_policy_match ( + spd0, clib_net_to_host_u32 (ip0->src_address.as_u32), + clib_net_to_host_u32 (ip0->dst_address.as_u32), + IPSEC_SPD_POLICY_IP4_INBOUND_BYPASS); + } + + if (PREDICT_TRUE ((p0 != NULL))) + { + *ipsec_bypassed += 1; + pi0 = p0 - im->policies; + vlib_increment_combined_counter (&ipsec_spd_policy_counters, + thread_index, pi0, 1, + clib_net_to_host_u16 (ip0->length)); + ipsec_collect_ah_trace (b, node, vm, ip0, ah0, has_space0, spd0, p0, + pi0); + return; + } + else + { + p0 = 0; + pi0 = ~0; + }; + + if (search_flow_cache) + { + p0 = ipsec4_input_spd_find_flow_cache_entry ( + im, ip0->src_address.as_u32, ip0->dst_address.as_u32, + IPSEC_SPD_POLICY_IP4_INBOUND_DISCARD); + } + + else + { + p0 = ipsec_input_policy_match ( + spd0, clib_net_to_host_u32 (ip0->src_address.as_u32), + clib_net_to_host_u32 (ip0->dst_address.as_u32), + IPSEC_SPD_POLICY_IP4_INBOUND_DISCARD); + } + + if (PREDICT_TRUE ((p0 != NULL))) + { + *ipsec_dropped += 1; + pi0 = p0 - im->policies; + vlib_increment_combined_counter (&ipsec_spd_policy_counters, + thread_index, pi0, 1, + clib_net_to_host_u16 (ip0->length)); + + next[0] = IPSEC_INPUT_NEXT_DROP; + ipsec_collect_ah_trace (b, node, vm, ip0, ah0, has_space0, spd0, p0, + pi0); + return; + } + else + { + p0 = 0; + pi0 = ~0; + }; + /* flow cache search failed, retry with linear search */ + if (search_flow_cache && p0 == NULL) + { + search_flow_cache = false; + } + else if (search_flow_cache == false && p0 == NULL) + { + /* Drop by default if no match on PROTECT, BYPASS or DISCARD */ + *ipsec_unprocessed += 1; + next[0] = IPSEC_INPUT_NEXT_DROP; + return; + } + } +} + +always_inline void ipsec_esp_packet_process (vlib_main_t *vm, ipsec_main_t *im, ip4_header_t *ip0, esp_header_t *esp0, u32 thread_index, ipsec_spd_t *spd0, vlib_buffer_t **b, @@ -299,10 +452,11 @@ ipsec_esp_packet_process (vlib_main_t *vm, ipsec_main_t *im, ip4_header_t *ip0, search_flow_cache = im->input_flow_cache_flag; udp_or_esp: - /* SPI ID field in the ESP header MUST NOT be a zero value */ if (esp0->spi == 0) { - /* Drop the packet if SPI ID is zero */ + /* RFC 4303, section 2.1: The SPI value of zero (0 is reserved for + * local, implementation-specific use and MUST NOT be sent on the wire. + */ *ipsec_unprocessed += 1; next[0] = IPSEC_INPUT_NEXT_DROP; return; @@ -523,15 +677,12 @@ VLIB_NODE_FN (ipsec4_input_node) (vlib_main_t * vm, while (n_left_from > 0) { - u32 next32, pi0; + u32 next32; ip4_header_t *ip0; esp_header_t *esp0 = NULL; ah_header_t *ah0; ip4_ipsec_config_t *c0; ipsec_spd_t *spd0; - ipsec_policy_t *p0 = NULL; - u8 has_space0; - bool search_flow_cache = false; if (n_left_from > 2) { @@ -552,12 +703,10 @@ VLIB_NODE_FN (ipsec4_input_node) (vlib_main_t * vm, udp_header_t *udp0 = NULL; udp0 = (udp_header_t *) ((u8 *) ip0 + ip4_header_bytes (ip0)); - /* As per rfc3948 in UDP Encapsulated Header, UDP checksum must be - * Zero, and receivers must not depen upon UPD checksum. - * inside ESP header , SPI ID value MUST NOT be a zero value - * */ - - if (udp0->checksum == 0) + /* RFC5996 Section 2.23 "Port 4500 is reserved for + * UDP-encapsulated ESP and IKE." + */ + if (clib_host_to_net_u16 (4500) == udp0->dst_port) { esp0 = (esp_header_t *) ((u8 *) udp0 + sizeof (udp_header_t)); @@ -582,140 +731,9 @@ VLIB_NODE_FN (ipsec4_input_node) (vlib_main_t * vm, { ah0 = (ah_header_t *) ((u8 *) ip0 + ip4_header_bytes (ip0)); - // if flow cache is enabled, first search through flow cache for a - // policy match and revert back to linear search on failure - search_flow_cache = im->input_flow_cache_flag; - - ah: - if (search_flow_cache) - { - p0 = ipsec4_input_spd_find_flow_cache_entry ( - im, ip0->src_address.as_u32, ip0->dst_address.as_u32, - IPSEC_SPD_POLICY_IP4_INBOUND_PROTECT); - } - - else - { - p0 = ipsec_input_protect_policy_match ( - spd0, clib_net_to_host_u32 (ip0->src_address.as_u32), - clib_net_to_host_u32 (ip0->dst_address.as_u32), - clib_net_to_host_u32 (ah0->spi)); - } - - has_space0 = - vlib_buffer_has_space (b[0], - (clib_address_t) (ah0 + 1) - - (clib_address_t) ip0); - - if (PREDICT_TRUE ((p0 != NULL) & (has_space0))) - { - ipsec_matched += 1; - - pi0 = p0 - im->policies; - vlib_increment_combined_counter - (&ipsec_spd_policy_counters, - thread_index, pi0, 1, clib_net_to_host_u16 (ip0->length)); - - vnet_buffer (b[0])->ipsec.sad_index = p0->sa_index; - next[0] = im->ah4_decrypt_next_index; - goto trace1; - } - else - { - p0 = 0; - pi0 = ~0; - } - - if (search_flow_cache) - { - p0 = ipsec4_input_spd_find_flow_cache_entry ( - im, ip0->src_address.as_u32, ip0->dst_address.as_u32, - IPSEC_SPD_POLICY_IP4_INBOUND_BYPASS); - } - - else - { - p0 = ipsec_input_policy_match ( - spd0, clib_net_to_host_u32 (ip0->src_address.as_u32), - clib_net_to_host_u32 (ip0->dst_address.as_u32), - IPSEC_SPD_POLICY_IP4_INBOUND_BYPASS); - } - - if (PREDICT_TRUE ((p0 != NULL))) - { - ipsec_bypassed += 1; - - pi0 = p0 - im->policies; - vlib_increment_combined_counter ( - &ipsec_spd_policy_counters, thread_index, pi0, 1, - clib_net_to_host_u16 (ip0->length)); - - goto trace1; - } - else - { - p0 = 0; - pi0 = ~0; - }; - - if (search_flow_cache) - { - p0 = ipsec4_input_spd_find_flow_cache_entry ( - im, ip0->src_address.as_u32, ip0->dst_address.as_u32, - IPSEC_SPD_POLICY_IP4_INBOUND_DISCARD); - } - - else - { - p0 = ipsec_input_policy_match ( - spd0, clib_net_to_host_u32 (ip0->src_address.as_u32), - clib_net_to_host_u32 (ip0->dst_address.as_u32), - IPSEC_SPD_POLICY_IP4_INBOUND_DISCARD); - } - - if (PREDICT_TRUE ((p0 != NULL))) - { - ipsec_dropped += 1; - - pi0 = p0 - im->policies; - vlib_increment_combined_counter ( - &ipsec_spd_policy_counters, thread_index, pi0, 1, - clib_net_to_host_u16 (ip0->length)); - - next[0] = IPSEC_INPUT_NEXT_DROP; - goto trace1; - } - else - { - p0 = 0; - pi0 = ~0; - }; - - // flow cache search failed, retry with linear search - if (search_flow_cache && p0 == NULL) - { - search_flow_cache = false; - goto ah; - } - - /* Drop by default if no match on PROTECT, BYPASS or DISCARD */ - ipsec_unprocessed += 1; - next[0] = IPSEC_INPUT_NEXT_DROP; - - trace1: - if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE) && - PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED)) - { - ipsec_input_trace_t *tr = - vlib_add_trace (vm, node, b[0], sizeof (*tr)); - - tr->proto = ip0->protocol; - tr->sa_id = p0 ? p0->sa_id : ~0; - tr->spi = has_space0 ? clib_net_to_host_u32 (ah0->spi) : ~0; - tr->seq = has_space0 ? clib_net_to_host_u32 (ah0->seq_no) : ~0; - tr->spd = spd0->id; - tr->policy_index = pi0; - } + ipsec_ah_packet_process (vm, im, ip0, ah0, thread_index, spd0, b, + node, &ipsec_bypassed, &ipsec_dropped, + &ipsec_matched, &ipsec_unprocessed, next); } else { diff --git a/src/vnet/ipsec/ipsec_sa.h b/src/vnet/ipsec/ipsec_sa.h index 4f73f1eab0f..640d9288a42 100644 --- a/src/vnet/ipsec/ipsec_sa.h +++ b/src/vnet/ipsec/ipsec_sa.h @@ -486,7 +486,7 @@ ipsec_sa_anti_replay_and_sn_advance (const ipsec_sa_t *sa, u32 seq, return 0; } - if (PREDICT_TRUE (sa->seq >= window_size - 1)) + if (PREDICT_TRUE (window_size > 0 && sa->seq >= window_size - 1)) { /* * the last sequence number VPP received is more than one diff --git a/src/vnet/ipsec/ipsec_tun.c b/src/vnet/ipsec/ipsec_tun.c index ecda291e985..5fb07b3ba09 100644 --- a/src/vnet/ipsec/ipsec_tun.c +++ b/src/vnet/ipsec/ipsec_tun.c @@ -23,6 +23,7 @@ #include <vnet/adj/adj_midchain.h> #include <vnet/teib/teib.h> #include <vnet/mpls/mpls.h> +#include <vnet/interface.h> /* instantiate the bihash functions */ #include <vppinfra/bihash_8_16.h> @@ -704,22 +705,13 @@ out: return (rv); } -int -ipsec_tun_protect_del (u32 sw_if_index, const ip_address_t * nh) +static int +ipsec_tun_protect_del_by_idx (index_t itpi) { ipsec_tun_protect_t *itp; ipsec_main_t *im; - index_t itpi; - - ITP_DBG2 ("delete: %U/%U", - format_vnet_sw_if_index_name, vnet_get_main (), sw_if_index, - format_ip_address, nh); im = &ipsec_main; - if (NULL == nh) - nh = &IP_ADDR_ALL_0; - - itpi = ipsec_tun_protect_find (sw_if_index, nh); if (INDEX_INVALID == itpi) return (VNET_API_ERROR_NO_SUCH_ENTRY); @@ -736,6 +728,22 @@ ipsec_tun_protect_del (u32 sw_if_index, const ip_address_t * nh) return (0); } +int +ipsec_tun_protect_del (u32 sw_if_index, const ip_address_t *nh) +{ + index_t itpi; + + ITP_DBG2 ("delete: %U/%U", format_vnet_sw_if_index_name, vnet_get_main (), + sw_if_index, format_ip_address, nh); + + if (NULL == nh) + nh = &IP_ADDR_ALL_0; + + itpi = ipsec_tun_protect_find (sw_if_index, nh); + + return ipsec_tun_protect_del_by_idx (itpi); +} + void ipsec_tun_protect_walk (ipsec_tun_protect_walk_cb_t fn, void *ctx) { @@ -747,6 +755,26 @@ ipsec_tun_protect_walk (ipsec_tun_protect_walk_cb_t fn, void *ctx) } } +walk_rc_t +ipsec_tun_interface_cleanup (index_t itpi, void *ctx) +{ + ipsec_tun_protect_del_by_idx (itpi); + return WALK_CONTINUE; +} + +static clib_error_t * +ipsec_tun_interface_add_del (vnet_main_t *vnm, u32 sw_if_index, u32 is_add) +{ + if (is_add) + return 0; + + ipsec_tun_protect_walk_itf (sw_if_index, ipsec_tun_interface_cleanup, 0); + + return 0; +} + +VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ipsec_tun_interface_add_del); + void ipsec_tun_protect_walk_itf (u32 sw_if_index, ipsec_tun_protect_walk_cb_t fn, void *ctx) diff --git a/src/vnet/ipsec/ipsec_tun_in.c b/src/vnet/ipsec/ipsec_tun_in.c index c82de3ebaff..3dde084cb24 100644 --- a/src/vnet/ipsec/ipsec_tun_in.c +++ b/src/vnet/ipsec/ipsec_tun_in.c @@ -60,8 +60,8 @@ format_ipsec_tun_protect_input_trace (u8 * s, va_list * args) s = format (s, "IPSec: %U seq %u", format_ipsec6_tunnel_kv, &t->kv6, t->seq); else - s = format (s, "IPSec: %U seq %u sa %d", - format_ipsec4_tunnel_kv, &t->kv4, t->seq); + s = + format (s, "IPSec: %U seq %u", format_ipsec4_tunnel_kv, &t->kv4, t->seq); return s; } diff --git a/src/vnet/l2/l2_vtr.h b/src/vnet/l2/l2_vtr.h index 1cd9209973b..40285873273 100644 --- a/src/vnet/l2/l2_vtr.h +++ b/src/vnet/l2/l2_vtr.h @@ -75,15 +75,13 @@ typedef struct always_inline u32 l2_vtr_process (vlib_buffer_t * b0, vtr_config_t * config) { - u64 temp_8; - u32 temp_4; u8 *eth; + u8 save_macs[12]; eth = vlib_buffer_get_current (b0); /* copy the 12B dmac and smac to a temporary location */ - temp_8 = *((u64 *) eth); - temp_4 = *((u32 *) (eth + 8)); + clib_memcpy_fast (save_macs, eth, sizeof (save_macs)); /* adjust for popped tags */ eth += config->pop_bytes; @@ -95,7 +93,8 @@ l2_vtr_process (vlib_buffer_t * b0, vtr_config_t * config) } /* copy the 2 new tags to the start of the packet */ - *((u64 *) (eth + 12 - 8)) = config->raw_tags; + clib_memcpy_fast (eth + 12 - 8, &config->raw_tags, + sizeof (config->raw_tags)); /* TODO: set cos bits */ @@ -103,8 +102,7 @@ l2_vtr_process (vlib_buffer_t * b0, vtr_config_t * config) eth -= config->push_bytes; /* copy the 12 dmac and smac back to the packet */ - *((u64 *) eth) = temp_8; - *((u32 *) (eth + 8)) = temp_4; + clib_memcpy_fast (eth, save_macs, sizeof (save_macs)); /* Update l2 parameters */ vnet_buffer (b0)->l2.l2_len += @@ -124,7 +122,6 @@ l2_vtr_process (vlib_buffer_t * b0, vtr_config_t * config) return 0; } - /* * Perform the egress pre-vlan tag rewrite EFP Filter check. * The post-vlan tag rewrite check is a separate graph node. diff --git a/src/vnet/llc/llc.c b/src/vnet/llc/llc.c index 4cbf17d48df..e17eaa6fd25 100644 --- a/src/vnet/llc/llc.c +++ b/src/vnet/llc/llc.c @@ -208,7 +208,6 @@ add_protocol (llc_main_t * pm, llc_protocol_t protocol, char *protocol_name) static clib_error_t * llc_init (vlib_main_t * vm) { - clib_error_t *error; llc_main_t *pm = &llc_main; clib_memset (pm, 0, sizeof (pm[0])); @@ -221,9 +220,6 @@ llc_init (vlib_main_t * vm) foreach_llc_protocol; #undef _ - if ((error = vlib_call_init_function (vm, snap_init))) - return error; - return vlib_call_init_function (vm, llc_input_init); } diff --git a/src/vnet/llc/node.c b/src/vnet/llc/node.c index d1ee6948269..dee0e060460 100644 --- a/src/vnet/llc/node.c +++ b/src/vnet/llc/node.c @@ -313,10 +313,6 @@ llc_register_input_protocol (vlib_main_t * vm, clib_error_t *error = vlib_call_init_function (vm, llc_input_init); if (error) clib_error_report (error); - /* Otherwise, osi_input_init will wipe out e.g. the snap init */ - error = vlib_call_init_function (vm, osi_input_init); - if (error) - clib_error_report (error); } pi = llc_get_protocol_info (lm, protocol); diff --git a/src/vnet/pg/output.c b/src/vnet/pg/output.c index 042591a7709..fa1a14cc4af 100644 --- a/src/vnet/pg/output.c +++ b/src/vnet/pg/output.c @@ -88,7 +88,13 @@ pg_output (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) pcap_add_buffer (&pif->pcap_main, vm, bi0, ETHERNET_MAX_PACKET_BYTES); } if (pif->pcap_file_name != 0) - pcap_write (&pif->pcap_main); + { + pcap_packet_type_t pm_pt = pif->pcap_main.packet_type; + pif->pcap_main.packet_type = + pg_intf_mode_to_pcap_packet_type (pif->mode); + pcap_write (&pif->pcap_main); + pif->pcap_main.packet_type = pm_pt; + } if ((pif->pcap_main.flags & PCAP_MAIN_INIT_DONE) && pif->pcap_main.n_packets_captured >= pif->pcap_main.n_packets_to_capture) diff --git a/src/vnet/pg/pg.h b/src/vnet/pg/pg.h index 6d5b25ba25a..bede747428c 100644 --- a/src/vnet/pg/pg.h +++ b/src/vnet/pg/pg.h @@ -306,6 +306,15 @@ typedef enum pg_interface_mode_t_ PG_MODE_IP6, } pg_interface_mode_t; +always_inline pcap_packet_type_t +pg_intf_mode_to_pcap_packet_type (pg_interface_mode_t mode) +{ + if ((mode == PG_MODE_IP4) || (mode == PG_MODE_IP6)) + return PCAP_PACKET_TYPE_ip; + else + return PCAP_PACKET_TYPE_ethernet; +} + typedef struct { /* TX lock */ diff --git a/src/vnet/session/application.c b/src/vnet/session/application.c index c66548507e5..7c63ada2774 100644 --- a/src/vnet/session/application.c +++ b/src/vnet/session/application.c @@ -968,7 +968,7 @@ application_namespace_cleanup (app_namespace_t *app_ns) ns_index = app_namespace_index (app_ns); pool_foreach (app, app_main.app_pool) if (app->ns_index == ns_index) - vec_add1 (app_indices, app->ns_index); + vec_add1 (app_indices, app->app_index); vec_foreach (app_index, app_indices) { diff --git a/src/vnet/session/application_local.c b/src/vnet/session/application_local.c index 3cb743d10e0..3ac2ba4cfbc 100644 --- a/src/vnet/session/application_local.c +++ b/src/vnet/session/application_local.c @@ -70,6 +70,7 @@ typedef struct ct_main_ u32 **fwrk_pending_connects; /**< First wrk pending half-opens */ u32 fwrk_thread; /**< First worker thread */ u8 fwrk_have_flush; /**< Flag for connect flush rpc */ + u8 is_init; } ct_main_t; static ct_main_t ct_main; @@ -1350,14 +1351,22 @@ ct_enable_disable (vlib_main_t * vm, u8 is_en) ct_main_t *cm = &ct_main; ct_worker_t *wrk; + if (is_en == 0) + return 0; + + if (cm->is_init) + return 0; + cm->n_workers = vlib_num_workers (); cm->fwrk_thread = transport_cl_thread (); vec_validate (cm->wrk, vtm->n_vlib_mains); + vec_validate (cm->fwrk_pending_connects, cm->n_workers); vec_foreach (wrk, cm->wrk) clib_spinlock_init (&wrk->pending_connects_lock); clib_spinlock_init (&cm->ho_reuseable_lock); clib_rwlock_init (&cm->app_segs_lock); - vec_validate (cm->fwrk_pending_connects, cm->n_workers); + cm->is_init = 1; + return 0; } diff --git a/src/vnet/session/application_namespace.c b/src/vnet/session/application_namespace.c index f547dcfc031..8b06331d803 100644 --- a/src/vnet/session/application_namespace.c +++ b/src/vnet/session/application_namespace.c @@ -34,6 +34,17 @@ static app_namespace_t *app_namespace_pool; static u8 app_sapi_enabled; +void +app_namespace_walk (app_namespace_walk_fn_t fn, void *ctx) +{ + app_namespace_t *app_ns; + + pool_foreach (app_ns, app_namespace_pool) + { + fn (app_ns, ctx); + } +} + app_namespace_t * app_namespace_get (u32 index) { @@ -136,12 +147,10 @@ vnet_app_namespace_add_del (vnet_app_namespace_add_del_args_t *a) app_ns->ns_secret = a->secret; app_ns->sw_if_index = a->sw_if_index; - app_ns->ip4_fib_index = - fib_table_find (FIB_PROTOCOL_IP4, a->ip4_fib_id); - app_ns->ip6_fib_index = - fib_table_find (FIB_PROTOCOL_IP6, a->ip6_fib_id); - session_lookup_set_tables_appns (app_ns); + app_ns->ip4_fib_index = fib_table_find (FIB_PROTOCOL_IP4, a->ip4_fib_id); + app_ns->ip6_fib_index = fib_table_find (FIB_PROTOCOL_IP6, a->ip6_fib_id); + session_lookup_set_tables_appns (app_ns); } else { @@ -164,6 +173,9 @@ vnet_app_namespace_add_del (vnet_app_namespace_add_del_args_t *a) if (app_ns->sock_name) vec_free (app_ns->sock_name); + session_lookup_table_cleanup (FIB_PROTOCOL_IP4, app_ns->ip4_fib_index); + session_lookup_table_cleanup (FIB_PROTOCOL_IP6, app_ns->ip6_fib_index); + app_namespace_free (app_ns); } diff --git a/src/vnet/session/application_namespace.h b/src/vnet/session/application_namespace.h index 261325cbe0e..b441e3c48f2 100644 --- a/src/vnet/session/application_namespace.h +++ b/src/vnet/session/application_namespace.h @@ -15,6 +15,7 @@ #include <vppinfra/socket.h> #include <vnet/vnet.h> +#include <vnet/session/session_types.h> #include <vnet/session/session_table.h> #ifndef SRC_VNET_SESSION_APPLICATION_NAMESPACE_H_ @@ -87,6 +88,9 @@ vnet_app_namespace_add_del (vnet_app_namespace_add_del_args_t *a); u32 app_namespace_get_fib_index (app_namespace_t * app_ns, u8 fib_proto); session_table_t *app_namespace_get_local_table (app_namespace_t * app_ns); +typedef void (*app_namespace_walk_fn_t) (app_namespace_t *app_ns, void *ctx); +extern void app_namespace_walk (app_namespace_walk_fn_t fn, void *ctx); + always_inline app_namespace_t * app_namespace_get_default (void) { diff --git a/src/vnet/session/application_worker.c b/src/vnet/session/application_worker.c index befdb7c7002..43007ad76ed 100644 --- a/src/vnet/session/application_worker.c +++ b/src/vnet/session/application_worker.c @@ -836,9 +836,12 @@ app_worker_mq_wrk_is_congested (app_worker_t *app_wrk, u32 thread_index) void app_worker_set_mq_wrk_congested (app_worker_t *app_wrk, u32 thread_index) { - clib_atomic_fetch_add_relax (&app_wrk->mq_congested, 1); ASSERT (thread_index == vlib_get_thread_index ()); - app_wrk->wrk_mq_congested[thread_index] = 1; + if (!app_wrk->wrk_mq_congested[thread_index]) + { + clib_atomic_fetch_add_relax (&app_wrk->mq_congested, 1); + app_wrk->wrk_mq_congested[thread_index] = 1; + } } void diff --git a/src/vnet/session/mma_template.c b/src/vnet/session/mma_template.c index 4b2770bb756..26a23b2d270 100644 --- a/src/vnet/session/mma_template.c +++ b/src/vnet/session/mma_template.c @@ -67,6 +67,9 @@ RT (mma_rule_free) (RTT (mma_rules_table) * srt, RTT (mma_rule) * rule) void RT (mma_rules_table_free) (RTT (mma_rules_table) * srt) { + RTT (mma_rule) * rule; + + pool_flush (rule, srt->rules, ({})); pool_free (srt->rules); } diff --git a/src/vnet/session/session.api b/src/vnet/session/session.api index 6affae4112d..26346eaa466 100644 --- a/src/vnet/session/session.api +++ b/src/vnet/session/session.api @@ -13,12 +13,19 @@ * limitations under the License. */ -option version = "4.0.0"; +option version = "4.0.1"; import "vnet/interface_types.api"; import "vnet/ip/ip_types.api"; +typedef sdl_rule +{ + vl_api_prefix_t lcl; + u32 action_index; + string tag[64]; +}; + enum transport_proto : u8 { TRANSPORT_PROTO_API_TCP, @@ -28,6 +35,14 @@ enum transport_proto : u8 TRANSPORT_PROTO_API_QUIC, }; +enum rt_backend_engine : u8 +{ + RT_BACKEND_ENGINE_API_DISABLE = 0, + RT_BACKEND_ENGINE_API_RULE_TABLE, + RT_BACKEND_ENGINE_API_NONE, + RT_BACKEND_ENGINE_API_SDL, +}; + /** \brief Application attach to session layer @param client_index - opaque cookie to identify the sender @param context - sender context, to match reply w/ request @@ -166,11 +181,18 @@ define app_worker_add_del_reply @param is_enable - disable session layer if 0, enable otherwise */ autoreply define session_enable_disable { + option deprecated; u32 client_index; u32 context; bool is_enable [default=true]; }; +autoreply define session_enable_disable_v2 { + u32 client_index; + u32 context; + vl_api_rt_backend_engine_t rt_engine_type; +}; + /** \brief enable/disable session layer socket api @param client_index - opaque cookie to identify the sender client to vpp direction only @@ -417,6 +439,42 @@ define session_rules_details string tag[64]; }; +autoreply define session_sdl_add_del { + u32 client_index; + u32 context; + u32 appns_index; + bool is_add; + u32 count; + vl_api_sdl_rule_t r[count]; +}; + +/** \brief Dump session sdl + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + */ +define session_sdl_dump +{ + u32 client_index; + u32 context; +}; + +/** \brief Session sdl details + @param context - sender context, to match reply w/ request + @param lcl - local prefix + @param action_index - the only action defined now is forward to + application with index action_index + @param appns_index - application namespace where rule is to be applied to + @param tag - tag + */ +define session_sdl_details +{ + u32 context; + vl_api_prefix_t lcl; + u32 action_index; + u32 appns_index; + string tag[64]; +}; + /* * Local Variables: * eval: (c-set-style "gnu") diff --git a/src/vnet/session/session.c b/src/vnet/session/session.c index 67e7ee39001..ac02281cf5c 100644 --- a/src/vnet/session/session.c +++ b/src/vnet/session/session.c @@ -24,6 +24,7 @@ #include <vnet/fib/ip4_fib.h> #include <vlib/stats/stats.h> #include <vlib/dma/dma.h> +#include <vnet/session/session_rules_table.h> session_main_t session_main; @@ -104,6 +105,22 @@ session_program_tx_io_evt (session_handle_tu_t sh, session_evt_type_t evt_type) } int +session_program_rx_io_evt (session_handle_tu_t sh) +{ + if (sh.thread_index == vlib_get_thread_index ()) + { + session_t *s = session_get_from_handle (sh); + return session_enqueue_notify (s); + } + else + { + return session_send_evt_to_thread ((void *) &sh.session_index, 0, + (u32) sh.thread_index, + SESSION_IO_EVT_BUILTIN_RX); + } +} + +int session_send_ctrl_evt_to_thread (session_t * s, session_evt_type_t evt_type) { /* only events supported are disconnect, shutdown and reset */ @@ -1659,8 +1676,10 @@ session_transport_close (session_t * s) { if (s->session_state == SESSION_STATE_TRANSPORT_CLOSED) session_set_state (s, SESSION_STATE_CLOSED); - /* If transport is already deleted, just free the session */ - else if (s->session_state >= SESSION_STATE_TRANSPORT_DELETED) + /* If transport is already deleted, just free the session. Half-opens + * expected to be already cleaning up at this point */ + else if (s->session_state >= SESSION_STATE_TRANSPORT_DELETED && + !(s->flags & SESSION_F_HALF_OPEN)) session_program_cleanup (s); return; } @@ -1687,7 +1706,8 @@ session_transport_reset (session_t * s) { if (s->session_state == SESSION_STATE_TRANSPORT_CLOSED) session_set_state (s, SESSION_STATE_CLOSED); - else if (s->session_state >= SESSION_STATE_TRANSPORT_DELETED) + else if (s->session_state >= SESSION_STATE_TRANSPORT_DELETED && + !(s->flags & SESSION_F_HALF_OPEN)) session_program_cleanup (s); return; } @@ -1866,7 +1886,8 @@ session_register_update_time_fn (session_update_time_fn fn, u8 is_add) } else { - vec_del1 (smm->update_time_fns, fi_pos); + if (found) + vec_del1 (smm->update_time_fns, fi_pos); } } @@ -1976,7 +1997,8 @@ session_stats_collector_init (void) } static clib_error_t * -session_manager_main_enable (vlib_main_t * vm) +session_manager_main_enable (vlib_main_t *vm, + session_rt_engine_type_t rt_engine_type) { session_main_t *smm = &session_main; vlib_thread_main_t *vtm = vlib_get_thread_main (); @@ -1984,6 +2006,9 @@ session_manager_main_enable (vlib_main_t * vm) session_worker_t *wrk; int i; + if (session_rt_backend_enable_disable (rt_engine_type)) + return clib_error_return (0, "error on enable backend engine"); + /* We only initialize once and do not de-initialized on disable */ if (smm->is_initialized) goto done; @@ -2062,9 +2087,11 @@ done: } static void -session_manager_main_disable (vlib_main_t * vm) +session_manager_main_disable (vlib_main_t *vm, + session_rt_engine_type_t rt_engine_type) { transport_enable_disable (vm, 0 /* is_en */ ); + session_rt_backend_enable_disable (rt_engine_type); } /* in this new callback, cookie hint the index */ @@ -2148,6 +2175,16 @@ session_node_enable_dma (u8 is_en, int n_vlibs) } } +static void +session_main_start_q_process (vlib_main_t *vm, vlib_node_state_t state) +{ + vlib_node_t *n; + + vlib_node_set_state (vm, session_queue_process_node.index, state); + n = vlib_get_node (vm, session_queue_process_node.index); + vlib_start_process (vm, n->runtime_index); +} + void session_node_enable_disable (u8 is_en) { @@ -2155,7 +2192,6 @@ session_node_enable_disable (u8 is_en) u8 state = is_en ? VLIB_NODE_STATE_POLLING : VLIB_NODE_STATE_DISABLED; session_main_t *sm = &session_main; vlib_main_t *vm; - vlib_node_t *n; int n_vlibs, i; n_vlibs = vlib_get_n_threads (); @@ -2169,10 +2205,7 @@ session_node_enable_disable (u8 is_en) if (is_en) { session_main_get_worker (0)->state = SESSION_WRK_INTERRUPT; - vlib_node_set_state (vm, session_queue_process_node.index, - state); - n = vlib_get_node (vm, session_queue_process_node.index); - vlib_start_process (vm, n->runtime_index); + session_main_start_q_process (vm, state); } else { @@ -2195,22 +2228,24 @@ session_node_enable_disable (u8 is_en) } clib_error_t * -vnet_session_enable_disable (vlib_main_t * vm, u8 is_en) +vnet_session_enable_disable (vlib_main_t *vm, + session_enable_disable_args_t *args) { clib_error_t *error = 0; - if (is_en) + + if (args->is_en) { if (session_main.is_enabled) return 0; - error = session_manager_main_enable (vm); - session_node_enable_disable (is_en); + error = session_manager_main_enable (vm, args->rt_engine_type); + session_node_enable_disable (1); } else { session_main.is_enabled = 0; - session_manager_main_disable (vm); - session_node_enable_disable (is_en); + session_manager_main_disable (vm, args->rt_engine_type); + session_node_enable_disable (0); } return error; @@ -2237,10 +2272,15 @@ static clib_error_t * session_main_loop_init (vlib_main_t * vm) { session_main_t *smm = &session_main; + if (smm->session_enable_asap) { + session_enable_disable_args_t args = { .is_en = 1, + .rt_engine_type = + smm->rt_engine_type }; + vlib_worker_thread_barrier_sync (vm); - vnet_session_enable_disable (vm, 1 /* is_en */ ); + vnet_session_enable_disable (vm, &args); vlib_worker_thread_barrier_release (vm); } return 0; @@ -2330,8 +2370,22 @@ session_config_fn (vlib_main_t * vm, unformat_input_t * input) smm->port_allocator_min_src_port = tmp; else if (unformat (input, "max-src-port %d", &tmp)) smm->port_allocator_max_src_port = tmp; + else if (unformat (input, "enable rt-backend rule-table")) + { + smm->rt_engine_type = RT_BACKEND_ENGINE_RULE_TABLE; + smm->session_enable_asap = 1; + } + else if (unformat (input, "enable rt-backend sdl")) + { + smm->rt_engine_type = RT_BACKEND_ENGINE_SDL; + smm->session_enable_asap = 1; + } else if (unformat (input, "enable")) - smm->session_enable_asap = 1; + { + /* enable session without rt-backend */ + smm->rt_engine_type = RT_BACKEND_ENGINE_NONE; + smm->session_enable_asap = 1; + } else if (unformat (input, "use-app-socket-api")) (void) appns_sapi_enable_disable (1 /* is_enable */); else if (unformat (input, "poll-main")) diff --git a/src/vnet/session/session.h b/src/vnet/session/session.h index a5604bf8725..24150fbbcd1 100644 --- a/src/vnet/session/session.h +++ b/src/vnet/session/session.h @@ -184,6 +184,19 @@ typedef void (*nat44_original_dst_lookup_fn) ( u16 i2o_dst_port, ip_protocol_t proto, u32 *original_dst, u16 *original_dst_port); +#define foreach_rt_engine \ + _ (DISABLE, "disable") \ + _ (RULE_TABLE, "enable with rt-backend rule table") \ + _ (NONE, "enable without rt-backend") \ + _ (SDL, "enable with rt-backend sdl") + +typedef enum +{ +#define _(v, s) RT_BACKEND_ENGINE_##v, + foreach_rt_engine +#undef _ +} session_rt_engine_type_t; + typedef struct session_main_ { /** Worker contexts */ @@ -235,6 +248,9 @@ typedef struct session_main_ /** Enable session manager at startup */ u8 session_enable_asap; + /** Session engine type */ + session_rt_engine_type_t rt_engine_type; + /** Poll session node in main thread */ u8 poll_main; @@ -292,6 +308,12 @@ typedef enum session_q_process_evt_ SESSION_Q_PROCESS_STOP } session_q_process_evt_t; +typedef struct _session_enable_disable_args_t +{ + session_rt_engine_type_t rt_engine_type; + u8 is_en; +} session_enable_disable_args_t; + #define TRANSPORT_PROTO_INVALID (session_main.last_transport_proto_type + 1) #define TRANSPORT_N_PROTOS (session_main.last_transport_proto_type + 1) @@ -461,6 +483,7 @@ int session_send_io_evt_to_thread_custom (void *data, u32 thread_index, session_evt_type_t evt_type); int session_program_tx_io_evt (session_handle_tu_t sh, session_evt_type_t evt_type); +int session_program_rx_io_evt (session_handle_tu_t sh); void session_send_rpc_evt_to_thread (u32 thread_index, void *fp, void *rpc_args); void session_send_rpc_evt_to_thread_force (u32 thread_index, void *fp, @@ -811,7 +834,9 @@ session_wrk_update_time (session_worker_t *wrk, f64 now) void session_wrk_enable_adaptive_mode (session_worker_t *wrk); fifo_segment_t *session_main_get_wrk_mqs_segment (void); void session_node_enable_disable (u8 is_en); -clib_error_t *vnet_session_enable_disable (vlib_main_t * vm, u8 is_en); +clib_error_t * +vnet_session_enable_disable (vlib_main_t *vm, + session_enable_disable_args_t *args); void session_wrk_handle_evts_main_rpc (void *); void session_wrk_program_app_wrk_evts (session_worker_t *wrk, u32 app_wrk_index); @@ -920,6 +945,30 @@ pool_program_safe_realloc (void **p, u32 elt_size, u32 align) } \ while (0) +always_inline u8 +session_is_enabled_without_rt_backend (void) +{ + session_main_t *smm = vnet_get_session_main (); + + return (smm->rt_engine_type == RT_BACKEND_ENGINE_NONE); +} + +always_inline u8 +session_sdl_is_enabled (void) +{ + session_main_t *smm = vnet_get_session_main (); + + return (smm->rt_engine_type == RT_BACKEND_ENGINE_SDL); +} + +always_inline u8 +session_rule_table_is_enabled (void) +{ + session_main_t *smm = vnet_get_session_main (); + + return (smm->rt_engine_type == RT_BACKEND_ENGINE_RULE_TABLE); +} + #endif /* __included_session_h__ */ /* diff --git a/src/vnet/session/session_api.c b/src/vnet/session/session_api.c index 48eb932a2c9..a729baaae29 100644 --- a/src/vnet/session/session_api.c +++ b/src/vnet/session/session_api.c @@ -18,9 +18,10 @@ #include <vnet/session/application.h> #include <vnet/session/application_interface.h> #include <vnet/session/application_local.h> -#include <vnet/session/session_rules_table.h> -#include <vnet/session/session_table.h> #include <vnet/session/session.h> +#include <vnet/session/session_table.h> +#include <vnet/session/session_rules_table.h> +#include <vnet/session/session_sdl.h> #include <vnet/ip/ip_types_api.h> #include <vnet/format_fns.h> @@ -30,6 +31,105 @@ #define REPLY_MSG_ID_BASE session_main.msg_id_base #include <vlibapi/api_helper_macros.h> +VLIB_REGISTER_LOG_CLASS (session_api_log, static) = { .class_name = "session", + .subclass_name = "api" }; + +#define log_debug(fmt, ...) \ + vlib_log_debug (session_api_log.class, "%s: " fmt, __func__, __VA_ARGS__) +#define log_warn(fmt, ...) \ + vlib_log_warn (session_api_log.class, fmt, __VA_ARGS__) +#define log_err(fmt, ...) \ + vlib_log_err (session_api_log.class, fmt, __VA_ARGS__) + +static int +verify_message_len (void *mp, u64 expected_len, char *where) +{ + u32 supplied_len = vl_msg_api_get_msg_length (mp); + + if (supplied_len < expected_len) + { + log_err ("%s: Supplied message length %d is less than expected %d", + where, supplied_len, expected_len); + return 0; + } + else + { + return 1; + } +} + +static void +vl_api_session_sdl_add_del_t_handler (vl_api_session_sdl_add_del_t *mp) +{ + vl_api_session_sdl_add_del_reply_t *rmp; + session_rule_add_del_args_t args; + session_rule_table_add_del_args_t *table_args = &args.table_args; + int rv = 0; + u32 count = clib_net_to_host_u32 (mp->count); + u64 expected_len = sizeof (*mp) + count * sizeof (mp->r[0]); + + if ((session_main.is_enabled == 0) || (session_sdl_is_enabled () == 0)) + { + rv = VNET_API_ERROR_FEATURE_DISABLED; + goto done; + } + + if (!verify_message_len (mp, expected_len, "session_sdl_add_del")) + { + rv = VNET_API_ERROR_INVALID_VALUE; + goto done; + } + + clib_memset (&args, 0, sizeof (args)); + table_args->is_add = mp->is_add; + args.scope = SESSION_RULE_SCOPE_GLOBAL; + args.appns_index = clib_net_to_host_u32 (mp->appns_index); + for (int i = 0; i < count; i++) + { + mp->r[i].tag[sizeof (mp->r[i].tag) - 1] = 0; + table_args->tag = format (0, "%s", mp->r[i].tag); + ip_prefix_decode (&mp->r[i].lcl, &table_args->lcl); + /* + * Need to set fp_proto for vnet_session_rule_add_del to find the + * correct table + */ + table_args->rmt.fp_proto = table_args->lcl.fp_proto; + table_args->action_index = clib_net_to_host_u32 (mp->r[i].action_index); + + rv = vnet_session_rule_add_del (&args); + vec_free (table_args->tag); + if (rv) + { + log_err ("session_sdl add del returned on %U @index %d: %U", + format_ip46_address, &table_args->lcl.fp_addr, + IP46_TYPE_ANY, i, format_session_error, rv); + + /* roll back */ + table_args->is_add = !mp->is_add; + for (int j = i - 1; j >= 0; j--) + { + mp->r[j].tag[sizeof (mp->r[j].tag) - 1] = 0; + table_args->tag = format (0, "%s", mp->r[j].tag); + ip_prefix_decode (&mp->r[j].lcl, &table_args->lcl); + table_args->rmt.fp_proto = table_args->lcl.fp_proto; + table_args->action_index = + clib_net_to_host_u32 (mp->r[j].action_index); + int rv2 = vnet_session_rule_add_del (&args); + vec_free (table_args->tag); + if (rv2) + log_err ("rollback session_sdl add del returned on %U " + "@index %d: %U", + format_ip46_address, &table_args->lcl.fp_addr, + IP46_TYPE_ANY, j, format_session_error, rv2); + } + break; + } + } + +done: + REPLY_MACRO (VL_API_SESSION_SDL_ADD_DEL_REPLY); +} + static transport_proto_t api_session_transport_proto_decode (const vl_api_transport_proto_t * api_tp) { @@ -520,12 +620,54 @@ vl_api_session_enable_disable_t_handler (vl_api_session_enable_disable_t * mp) vl_api_session_enable_disable_reply_t *rmp; vlib_main_t *vm = vlib_get_main (); int rv = 0; + session_enable_disable_args_t args; - vnet_session_enable_disable (vm, mp->is_enable); + args.is_en = mp->is_enable; + if (mp->is_enable) + args.rt_engine_type = RT_BACKEND_ENGINE_RULE_TABLE; + else + args.rt_engine_type = RT_BACKEND_ENGINE_DISABLE; + + if (vnet_session_enable_disable (vm, &args)) + rv = VNET_API_ERROR_INVALID_ARGUMENT; REPLY_MACRO (VL_API_SESSION_ENABLE_DISABLE_REPLY); } static void +vl_api_session_enable_disable_v2_t_handler ( + vl_api_session_enable_disable_v2_t *mp) +{ + vl_api_session_enable_disable_v2_reply_t *rmp; + vlib_main_t *vm = vlib_get_main (); + int rv = 0; + session_enable_disable_args_t args; + + STATIC_ASSERT ((session_rt_engine_type_t) RT_BACKEND_ENGINE_API_DISABLE == + RT_BACKEND_ENGINE_DISABLE, + "API value mismatch"); + STATIC_ASSERT ((session_rt_engine_type_t) RT_BACKEND_ENGINE_API_NONE == + RT_BACKEND_ENGINE_NONE, + "API value mismatch"); + STATIC_ASSERT ((session_rt_engine_type_t) RT_BACKEND_ENGINE_API_RULE_TABLE == + RT_BACKEND_ENGINE_RULE_TABLE, + "API value mismatch"); + STATIC_ASSERT ((session_rt_engine_type_t) RT_BACKEND_ENGINE_API_SDL == + RT_BACKEND_ENGINE_SDL, + "API value mismatch"); + + args.rt_engine_type = (session_rt_engine_type_t) mp->rt_engine_type; + if (args.rt_engine_type == RT_BACKEND_ENGINE_DISABLE) + args.is_en = 0; + else + args.is_en = 1; + + if (vnet_session_enable_disable (vm, &args)) + rv = VNET_API_ERROR_INVALID_VALUE; + + REPLY_MACRO (VL_API_SESSION_ENABLE_DISABLE_V2_REPLY); +} + +static void vl_api_session_sapi_enable_disable_t_handler ( vl_api_session_sapi_enable_disable_t *mp) { @@ -962,6 +1104,12 @@ vl_api_session_rule_add_del_t_handler (vl_api_session_rule_add_del_t * mp) session_rule_table_add_del_args_t *table_args = &args.table_args; int rv = 0; + if (session_main_is_enabled () == 0) + { + rv = VNET_API_ERROR_FEATURE_DISABLED; + goto done; + } + clib_memset (&args, 0, sizeof (args)); ip_prefix_decode (&mp->lcl, &table_args->lcl); @@ -986,6 +1134,7 @@ vl_api_session_rule_add_del_t_handler (vl_api_session_rule_add_del_t * mp) rv = VNET_API_ERROR_UNSPECIFIED; } vec_free (table_args->tag); +done: REPLY_MACRO (VL_API_SESSION_RULE_ADD_DEL_REPLY); } @@ -1012,6 +1161,8 @@ send_session_rule_details4 (mma_rule_16_t * rule, u8 is_local, ip_set (&rmt.fp_addr, &match->rmt_ip, 1); lcl.fp_len = ip4_mask_to_preflen (&mask->lcl_ip); rmt.fp_len = ip4_mask_to_preflen (&mask->rmt_ip); + lcl.fp_proto = FIB_PROTOCOL_IP4; + rmt.fp_proto = FIB_PROTOCOL_IP4; ip_prefix_encode (&lcl, &rmp->lcl); ip_prefix_encode (&rmt, &rmp->rmt); @@ -1054,6 +1205,8 @@ send_session_rule_details6 (mma_rule_40_t * rule, u8 is_local, ip_set (&rmt.fp_addr, &match->rmt_ip, 0); lcl.fp_len = ip6_mask_to_preflen (&mask->lcl_ip); rmt.fp_len = ip6_mask_to_preflen (&mask->rmt_ip); + lcl.fp_proto = FIB_PROTOCOL_IP6; + rmt.fp_proto = FIB_PROTOCOL_IP6; ip_prefix_encode (&lcl, &rmp->lcl); ip_prefix_encode (&rmt, &rmp->rmt); @@ -1120,14 +1273,89 @@ vl_api_session_rules_dump_t_handler (vl_api_session_rules_dump_t * mp) return; session_table_foreach (st, ({ - for (tp = 0; tp < TRANSPORT_N_PROTOS; tp++) - { - send_session_rules_table_details (&st->session_rules[tp], - st->active_fib_proto, tp, - st->is_local, st->appns_index, reg, - mp->context); - } - })); + if (st->srtg_handle != SESSION_SRTG_HANDLE_INVALID) + for (tp = 0; tp < TRANSPORT_N_PROTOS; tp++) + { + session_rules_table_t *srt = + srtg_handle_to_srt (st->srtg_handle, tp); + send_session_rules_table_details ( + srt, st->active_fib_proto, tp, st->is_local, + st->appns_index, reg, mp->context); + } + })); +} + +typedef struct session_sdl_table_walk_ctx_ +{ + vl_api_registration_t *reg; + u32 mp_context; + u32 appns_index; +} session_sdl_table_walk_ctx; + +static void +send_session_sdl_details (u32 fei, ip46_address_t *lcl_ip, u16 fp_len, + u32 action_index, u32 fp_proto, u8 *tag, void *args) +{ + session_sdl_table_walk_ctx *ctx = args; + vl_api_registration_t *reg = ctx->reg; + u32 appns_index = ctx->appns_index; + u32 context = ctx->mp_context; + vl_api_session_sdl_details_t *rmp = 0; + fib_prefix_t lcl; + + rmp = vl_msg_api_alloc (sizeof (*rmp)); + clib_memset (rmp, 0, sizeof (*rmp)); + rmp->_vl_msg_id = ntohs (REPLY_MSG_ID_BASE + VL_API_SESSION_SDL_DETAILS); + rmp->context = context; + + clib_memset (&lcl, 0, sizeof (lcl)); + if (fp_proto == FIB_PROTOCOL_IP4) + ip_set (&lcl.fp_addr, &lcl_ip->ip4, 1); + else + ip_set (&lcl.fp_addr, &lcl_ip->ip6, 0); + lcl.fp_len = fp_len; + lcl.fp_proto = fp_proto, + + ip_prefix_encode (&lcl, &rmp->lcl); + rmp->action_index = clib_host_to_net_u32 (action_index); + rmp->appns_index = clib_host_to_net_u32 (appns_index); + if (tag) + { + clib_memcpy_fast (rmp->tag, tag, vec_len (tag)); + rmp->tag[vec_len (tag)] = 0; + } + + vl_api_send_msg (reg, (u8 *) rmp); +} + +static void +vl_api_session_sdl_dump_t_handler (vl_api_session_rules_dump_t *mp) +{ + vl_api_registration_t *reg; + session_table_t *st; + session_sdl_table_walk_ctx ctx; + + reg = vl_api_client_index_to_registration (mp->client_index); + if (!reg) + return; + + ctx.reg = reg; + ctx.mp_context = mp->context; + + session_table_foreach ( + st, ({ + if (st->srtg_handle != SESSION_SRTG_HANDLE_INVALID) + { + ctx.appns_index = st->appns_index; + + if (st->active_fib_proto == FIB_PROTOCOL_IP4) + session_sdl_table_walk4 (st->srtg_handle, send_session_sdl_details, + &ctx); + else + session_sdl_table_walk6 (st->srtg_handle, send_session_sdl_details, + &ctx); + } + })); } static void @@ -1825,11 +2053,21 @@ appns_sapi_add_ns_socket (app_namespace_t * app_ns) static clib_error_t * session_api_hookup (vlib_main_t *vm) { + api_main_t *am = vlibapi_get_main (); + /* * Set up the (msg_name, crc, message-id) table */ REPLY_MSG_ID_BASE = setup_message_id_table (); + vl_api_set_msg_thread_safe ( + am, REPLY_MSG_ID_BASE + VL_API_SESSION_SDL_ADD_DEL, 1); + vl_api_set_msg_thread_safe ( + am, REPLY_MSG_ID_BASE + VL_API_SESSION_SDL_ADD_DEL_REPLY, 1); + vl_api_set_msg_thread_safe (am, REPLY_MSG_ID_BASE + VL_API_SESSION_SDL_DUMP, + 1); + vl_api_set_msg_thread_safe ( + am, REPLY_MSG_ID_BASE + VL_API_SESSION_SDL_DETAILS, 1); return 0; } diff --git a/src/vnet/session/session_cli.c b/src/vnet/session/session_cli.c index 569a77bccc1..c29a465d056 100644 --- a/src/vnet/session/session_cli.c +++ b/src/vnet/session/session_cli.c @@ -237,6 +237,27 @@ done: return rv; } +static uword +unformat_ip_port (unformat_input_t *input, va_list *args) +{ + ip46_address_t *ip = va_arg (*args, ip46_address_t *); + u16 *port = va_arg (*args, u16 *); + + if (unformat (input, "%U:%d", unformat_ip46_address, ip, IP46_TYPE_ANY, + port)) + ; + else if (unformat (input, "%U", unformat_ip46_address, ip, IP46_TYPE_ANY)) + { + *port = 0; + } + else + { + return 0; + } + + return 1; +} + uword unformat_session (unformat_input_t * input, va_list * args) { @@ -360,30 +381,80 @@ session_cli_show_all_sessions (vlib_main_t * vm, int verbose) } } +typedef enum +{ + SESSION_CLI_FILTER_FORCE_PRINT = 1 << 0, +} session_cli_filter_flags_t; + +typedef enum +{ + SESSION_CLI_FILTER_ENDPT_LOCAL = 1 << 0, + SESSION_CLI_FILTER_ENDPT_REMOTE = 1 << 1, +} session_cli_endpt_flags_t; + +typedef struct session_cli_filter_ +{ + session_cli_filter_flags_t flags; + struct + { + u32 start; + u32 end; + } range; + transport_endpoint_t endpt; + session_cli_endpt_flags_t endpt_flags; + session_state_t *states; + transport_proto_t transport_proto; + u32 thread_index; + u32 verbose; +} session_cli_filter_t; + static int -session_cli_filter_check (session_t * s, session_state_t * states, - transport_proto_t tp) +session_cli_filter_check (session_t *s, session_cli_filter_t *sf) { - if (states) + transport_connection_t *tc; + + if (sf->states) { session_state_t *state; - vec_foreach (state, states) if (s->session_state == *state) + vec_foreach (state, sf->states) + if (s->session_state == *state) goto check_transport; return 0; } check_transport: - if (tp != TRANSPORT_PROTO_INVALID && session_get_transport_proto (s) != tp) + if (sf->transport_proto != TRANSPORT_PROTO_INVALID && + session_get_transport_proto (s) != sf->transport_proto) return 0; - return 1; + if (s->session_state >= SESSION_STATE_TRANSPORT_DELETED) + return 0; + + /* No explicit ip:port match requested */ + if (!sf->endpt_flags) + return 1; + + tc = session_get_transport (s); + if (sf->endpt_flags & SESSION_CLI_FILTER_ENDPT_LOCAL) + { + if (!ip46_address_cmp (&sf->endpt.ip, &tc->lcl_ip) && + (sf->endpt.port == 0 || + sf->endpt.port == clib_net_to_host_u16 (tc->lcl_port))) + return 1; + } + if (sf->endpt_flags & SESSION_CLI_FILTER_ENDPT_REMOTE) + { + if (!ip46_address_cmp (&sf->endpt.ip, &tc->rmt_ip) && + (sf->endpt.port == 0 || + sf->endpt.port == clib_net_to_host_u16 (tc->rmt_port))) + return 1; + } + return 0; } static void -session_cli_show_session_filter (vlib_main_t * vm, u32 thread_index, - u32 start, u32 end, session_state_t * states, - transport_proto_t tp, int verbose) +session_cli_show_session_filter (vlib_main_t *vm, session_cli_filter_t *sf) { u8 output_suppressed = 0; session_worker_t *wrk; @@ -391,54 +462,62 @@ session_cli_show_session_filter (vlib_main_t * vm, u32 thread_index, u32 count = 0, max_index; int i; - wrk = session_main_get_worker_if_valid (thread_index); + if (sf->range.end < sf->range.start) + { + vlib_cli_output (vm, "invalid range start: %u end: %u", sf->range.start, + sf->range.end); + return; + } + + wrk = session_main_get_worker_if_valid (sf->thread_index); if (!wrk) { - vlib_cli_output (vm, "invalid thread index %u", thread_index); + vlib_cli_output (vm, "invalid thread index %u", sf->thread_index); return; } pool = wrk->sessions; - if (tp == TRANSPORT_PROTO_INVALID && states == 0 && !verbose - && (start == 0 && end == ~0)) + if (sf->transport_proto == TRANSPORT_PROTO_INVALID && sf->states == 0 && + !sf->verbose && (sf->range.start == 0 && sf->range.end == ~0)) { - vlib_cli_output (vm, "Thread %d: %u sessions", thread_index, + vlib_cli_output (vm, "Thread %d: %u sessions", sf->thread_index, pool_elts (pool)); return; } max_index = pool_len (pool) ? pool_len (pool) - 1 : 0; - for (i = start; i <= clib_min (end, max_index); i++) + for (i = sf->range.start; i <= clib_min (sf->range.end, max_index); i++) { if (pool_is_free_index (pool, i)) continue; s = pool_elt_at_index (pool, i); - if (session_cli_filter_check (s, states, tp)) + if (!session_cli_filter_check (s, sf)) + continue; + + count += 1; + if (sf->verbose) { - count += 1; - if (verbose) - { - if (count > 50 || (verbose > 1 && count > 10)) - { - output_suppressed = 1; - continue; - } - if (s->session_state < SESSION_STATE_TRANSPORT_DELETED) - vlib_cli_output (vm, "%U", format_session, s, verbose); - } + if (!(sf->flags & SESSION_CLI_FILTER_FORCE_PRINT) && + (count > 50 || (sf->verbose > 1 && count > 10))) + { + output_suppressed = 1; + continue; + } + vlib_cli_output (vm, "%U", format_session, s, sf->verbose); } } if (!output_suppressed) vlib_cli_output (vm, "Thread %d: %u sessions matched filter", - thread_index, count); + sf->thread_index, count); else - vlib_cli_output (vm, "Thread %d: %u sessions matched filter. Not all" - " shown. Use finer grained filter.", thread_index, - count); + vlib_cli_output (vm, + "Thread %d: %u sessions matched filter. Not all" + " shown. Use finer grained filter.", + sf->thread_index, count); } void @@ -482,14 +561,33 @@ session_cli_print_session_states (vlib_main_t * vm) #undef _ } +static u8 * +format_rt_backend (u8 *s, va_list *args) +{ + u32 i = va_arg (*args, u32); + u8 *t = 0; + + switch (i) + { +#define _(v, s) \ + case RT_BACKEND_ENGINE_##v: \ + t = (u8 *) s; \ + break; + foreach_rt_engine +#undef _ + default : return format (s, "unknown"); + } + return format (s, "%s", t); +} + static clib_error_t * show_session_command_fn (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { u8 one_session = 0, do_listeners = 0, sst, do_elog = 0, do_filter = 0; - u32 track_index, thread_index = 0, start = 0, end = ~0, session_index; + u32 track_index, thread_index = 0, session_index; transport_proto_t transport_proto = TRANSPORT_PROTO_INVALID; - session_state_t state = SESSION_N_STATES, *states = 0; + session_state_t state = SESSION_N_STATES; session_main_t *smm = &session_main; clib_error_t *error = 0; app_worker_t *app_wrk; @@ -498,18 +596,51 @@ show_session_command_fn (vlib_main_t * vm, unformat_input_t * input, u8 do_events = 0; int verbose = 0; session_t *s; + session_cli_filter_t sf = { + .transport_proto = TRANSPORT_PROTO_INVALID, + .range = { 0, ~0 }, + }; session_cli_return_if_not_enabled (); while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { - if (unformat (input, "verbose %d", &verbose)) + /* + * helpers + */ + if (unformat (input, "protos")) + { + vlib_cli_output (vm, "%U", format_transport_protos); + goto done; + } + else if (unformat (input, "rt-backend")) + { + vlib_cli_output (vm, "%U", format_rt_backend, smm->rt_engine_type); + goto done; + } + else if (unformat (input, "states")) + { + session_cli_print_session_states (vm); + goto done; + } + else if (unformat (input, "verbose %d", &verbose)) ; else if (unformat (input, "verbose")) verbose = 1; + /* + * listeners + */ else if (unformat (input, "listeners %U", unformat_transport_proto, &transport_proto)) do_listeners = 1; + /* + * session events + */ + else if (unformat (input, "events")) + do_events = 1; + /* + * single session filter + */ else if (unformat (input, "%U", unformat_session, &s)) { one_session = 1; @@ -525,17 +656,9 @@ show_session_command_fn (vlib_main_t * vm, unformat_input_t * input, } one_session = 1; } - else if (unformat (input, "thread %u", &thread_index)) - { - do_filter = 1; - } - else if (unformat (input, "state %U", unformat_session_state, &state)) - { - vec_add1 (states, state); - do_filter = 1; - } - else if (unformat (input, "proto %U index %u", unformat_transport_proto, - &transport_proto, &transport_index)) + else if (unformat (input, "thread %u proto %U index %u", &thread_index, + unformat_transport_proto, &transport_proto, + &transport_index)) { transport_connection_t *tc; tc = transport_get_connection (transport_proto, transport_index, @@ -556,30 +679,54 @@ show_session_command_fn (vlib_main_t * vm, unformat_input_t * input, } one_session = 1; } + else if (unformat (input, "elog")) + do_elog = 1; + /* + * session filter + */ + else if (unformat (input, "thread %u", &sf.thread_index)) + { + do_filter = 1; + } + else if (unformat (input, "state %U", unformat_session_state, &state)) + { + vec_add1 (sf.states, state); + do_filter = 1; + } else if (unformat (input, "proto %U", unformat_transport_proto, - &transport_proto)) + &sf.transport_proto)) do_filter = 1; - else if (unformat (input, "range %u %u", &start, &end)) + else if (unformat (input, "range %u %u", &sf.range.start, &sf.range.end)) do_filter = 1; - else if (unformat (input, "range %u", &start)) + else if (unformat (input, "range %u", &sf.range.start)) { - end = start + 50; + sf.range.end = sf.range.start + 50; do_filter = 1; } - else if (unformat (input, "elog")) - do_elog = 1; - else if (unformat (input, "protos")) + else if (unformat (input, "lcl %U", unformat_ip_port, &sf.endpt.ip, + &sf.endpt.port)) { - vlib_cli_output (vm, "%U", format_transport_protos); - goto done; + sf.endpt_flags |= SESSION_CLI_FILTER_ENDPT_LOCAL; + do_filter = 1; } - else if (unformat (input, "states")) + else if (unformat (input, "rmt %U", unformat_ip_port, &sf.endpt.ip, + &sf.endpt.port)) { - session_cli_print_session_states (vm); - goto done; + sf.endpt_flags |= SESSION_CLI_FILTER_ENDPT_REMOTE; + do_filter = 1; + } + else if (unformat (input, "ep %U", unformat_ip_port, &sf.endpt.ip, + &sf.endpt.port)) + { + sf.endpt_flags |= + SESSION_CLI_FILTER_ENDPT_REMOTE | SESSION_CLI_FILTER_ENDPT_LOCAL; + do_filter = 1; + } + else if (unformat (input, "force-print")) + { + sf.flags |= SESSION_CLI_FILTER_FORCE_PRINT; + do_filter = 1; } - else if (unformat (input, "events")) - do_events = 1; else { error = clib_error_return (0, "unknown input `%U'", @@ -636,31 +783,26 @@ show_session_command_fn (vlib_main_t * vm, unformat_input_t * input, if (do_filter) { - if (end < start) - { - error = clib_error_return (0, "invalid range start: %u end: %u", - start, end); - goto done; - } - session_cli_show_session_filter (vm, thread_index, start, end, states, - transport_proto, verbose); + sf.verbose = verbose; + session_cli_show_session_filter (vm, &sf); goto done; } session_cli_show_all_sessions (vm, verbose); done: - vec_free (states); + vec_free (sf.states); return error; } -VLIB_CLI_COMMAND (vlib_cli_show_session_command) = -{ +VLIB_CLI_COMMAND (vlib_cli_show_session_command) = { .path = "show session", - .short_help = "show session [verbose [n]] [listeners <proto>] " - "[<session-id> [elog]] [thread <n> [index <n>] " - "[proto <proto>] [state <state>] [range <min> [<max>]] " - "[protos] [states] ", + .short_help = + "show session [protos][states][rt-backend][verbose [n]] " + "[events][listeners <proto>] " + "[<session-id>][thread <n> [[proto <p>] index <n>]][elog] " + "[thread <n>][proto <proto>][state <state>][range <min> [<max>]] " + "[lcl|rmt|ep <ip>[:<port>]][force-print]", .function = show_session_command_fn, }; @@ -829,29 +971,47 @@ static clib_error_t * session_enable_disable_fn (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { - u8 is_en = 2; + session_enable_disable_args_t args; + session_main_t *smm = &session_main; while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { if (unformat (input, "enable")) - is_en = 1; + { + args.is_en = 1; + if (unformat (input, "rt-backend")) + if (unformat (input, "sdl")) + args.rt_engine_type = RT_BACKEND_ENGINE_SDL; + else if (unformat (input, "rule-table")) + args.rt_engine_type = RT_BACKEND_ENGINE_RULE_TABLE; + else + return clib_error_return (0, "unknown input `%U'", + format_unformat_error, input); + else + args.rt_engine_type = RT_BACKEND_ENGINE_NONE; + } else if (unformat (input, "disable")) - is_en = 0; + { + args.rt_engine_type = RT_BACKEND_ENGINE_DISABLE; + args.is_en = 0; + } else return clib_error_return (0, "unknown input `%U'", format_unformat_error, input); } - if (is_en > 1) - return clib_error_return (0, "expected enable | disable"); + if (smm->is_enabled && args.is_en) + if (args.rt_engine_type != smm->rt_engine_type) + return clib_error_return ( + 0, "session is already enable. Must disable first"); - return vnet_session_enable_disable (vm, is_en); + return vnet_session_enable_disable (vm, &args); } -VLIB_CLI_COMMAND (session_enable_disable_command, static) = -{ +VLIB_CLI_COMMAND (session_enable_disable_command, static) = { .path = "session", - .short_help = "session [enable|disable]", + .short_help = + "session { enable [ rt-backend sdl | rule-table ] } | { disable }", .function = session_enable_disable_fn, }; diff --git a/src/vnet/session/session_lookup.c b/src/vnet/session/session_lookup.c index ff20bc2d835..0d580ba35c6 100644 --- a/src/vnet/session/session_lookup.c +++ b/src/vnet/session/session_lookup.c @@ -28,6 +28,7 @@ #include <vnet/session/session_lookup.h> #include <vnet/session/session.h> #include <vnet/session/application.h> +#include <vnet/session/session_rules_table.h> static session_lookup_main_t sl_main; @@ -37,6 +38,8 @@ static session_lookup_main_t sl_main; */ static u32 *fib_index_to_table_index[2]; +static u32 *fib_index_to_lock_count[2]; + /* 16 octets */ typedef CLIB_PACKED (struct { union @@ -239,7 +242,7 @@ session_table_get_for_connection (transport_connection_t * tc) session_table_get (fib_index_to_table_index[fib_proto][tc->fib_index]); } -static session_table_t * +session_table_t * session_table_get_for_fib_index (u32 fib_proto, u32 fib_index) { if (vec_len (fib_index_to_table_index[fib_proto]) <= fib_index) @@ -476,10 +479,10 @@ session_lookup_rules_table_session4 (session_table_t * st, u8 proto, ip4_address_t * lcl, u16 lcl_port, ip4_address_t * rmt, u16 rmt_port) { - session_rules_table_t *srt = &st->session_rules[proto]; u32 action_index, app_index; - action_index = session_rules_table_lookup4 (srt, lcl, rmt, lcl_port, - rmt_port); + + action_index = session_rules_table_lookup4 (st->srtg_handle, proto, lcl, rmt, + lcl_port, rmt_port); app_index = session_lookup_action_to_handle (action_index); /* Nothing sophisticated for now, action index is app index */ return session_lookup_app_listen_session (app_index, FIB_PROTOCOL_IP4, @@ -492,10 +495,10 @@ session_lookup_rules_table_session6 (session_table_t * st, u8 proto, ip6_address_t * lcl, u16 lcl_port, ip6_address_t * rmt, u16 rmt_port) { - session_rules_table_t *srt = &st->session_rules[proto]; u32 action_index, app_index; - action_index = session_rules_table_lookup6 (srt, lcl, rmt, lcl_port, - rmt_port); + + action_index = session_rules_table_lookup6 (st->srtg_handle, proto, lcl, rmt, + lcl_port, rmt_port); app_index = session_lookup_action_to_handle (action_index); return session_lookup_app_listen_session (app_index, FIB_PROTOCOL_IP6, proto); @@ -515,7 +518,6 @@ u64 session_lookup_endpoint_listener (u32 table_index, session_endpoint_t * sep, u8 use_rules) { - session_rules_table_t *srt; session_table_t *st; u32 ai; int rv; @@ -535,10 +537,12 @@ session_lookup_endpoint_listener (u32 table_index, session_endpoint_t * sep, return kv4.value; if (use_rules) { + if (st->srtg_handle == SESSION_SRTG_HANDLE_INVALID) + return SESSION_INVALID_HANDLE; clib_memset (&lcl4, 0, sizeof (lcl4)); - srt = &st->session_rules[sep->transport_proto]; - ai = session_rules_table_lookup4 (srt, &lcl4, &sep->ip.ip4, 0, - sep->port); + ai = + session_rules_table_lookup4 (st->srtg_handle, sep->transport_proto, + &lcl4, &sep->ip.ip4, 0, sep->port); if (session_lookup_action_index_is_valid (ai)) return session_lookup_action_to_handle (ai); } @@ -556,10 +560,12 @@ session_lookup_endpoint_listener (u32 table_index, session_endpoint_t * sep, if (use_rules) { + if (st->srtg_handle == SESSION_SRTG_HANDLE_INVALID) + return SESSION_INVALID_HANDLE; clib_memset (&lcl6, 0, sizeof (lcl6)); - srt = &st->session_rules[sep->transport_proto]; - ai = session_rules_table_lookup6 (srt, &lcl6, &sep->ip.ip6, 0, - sep->port); + ai = + session_rules_table_lookup6 (st->srtg_handle, sep->transport_proto, + &lcl6, &sep->ip.ip6, 0, sep->port); if (session_lookup_action_index_is_valid (ai)) return session_lookup_action_to_handle (ai); } @@ -586,14 +592,13 @@ session_lookup_endpoint_listener (u32 table_index, session_endpoint_t * sep, u64 session_lookup_local_endpoint (u32 table_index, session_endpoint_t * sep) { - session_rules_table_t *srt; session_table_t *st; u32 ai; int rv; st = session_table_get (table_index); if (!st) - return SESSION_INVALID_INDEX; + return SESSION_INVALID_HANDLE; ASSERT (st->is_local); if (sep->is_ip4) @@ -601,15 +606,18 @@ session_lookup_local_endpoint (u32 table_index, session_endpoint_t * sep) session_kv4_t kv4; ip4_address_t lcl4; - /* - * Check if endpoint has special rules associated - */ - clib_memset (&lcl4, 0, sizeof (lcl4)); - srt = &st->session_rules[sep->transport_proto]; - ai = session_rules_table_lookup4 (srt, &lcl4, &sep->ip.ip4, 0, - sep->port); - if (session_lookup_action_index_is_valid (ai)) - return session_lookup_action_to_handle (ai); + if (st->srtg_handle != SESSION_SRTG_HANDLE_INVALID) + { + /* + * Check if endpoint has special rules associated + */ + clib_memset (&lcl4, 0, sizeof (lcl4)); + ai = + session_rules_table_lookup4 (st->srtg_handle, sep->transport_proto, + &lcl4, &sep->ip.ip4, 0, sep->port); + if (session_lookup_action_index_is_valid (ai)) + return session_lookup_action_to_handle (ai); + } /* * Check if session endpoint is a listener @@ -649,12 +657,15 @@ session_lookup_local_endpoint (u32 table_index, session_endpoint_t * sep) session_kv6_t kv6; ip6_address_t lcl6; - clib_memset (&lcl6, 0, sizeof (lcl6)); - srt = &st->session_rules[sep->transport_proto]; - ai = session_rules_table_lookup6 (srt, &lcl6, &sep->ip.ip6, 0, - sep->port); - if (session_lookup_action_index_is_valid (ai)) - return session_lookup_action_to_handle (ai); + if (st->srtg_handle != SESSION_SRTG_HANDLE_INVALID) + { + clib_memset (&lcl6, 0, sizeof (lcl6)); + ai = + session_rules_table_lookup6 (st->srtg_handle, sep->transport_proto, + &lcl6, &sep->ip.ip6, 0, sep->port); + if (session_lookup_action_index_is_valid (ai)) + return session_lookup_action_to_handle (ai); + } make_v6_listener_kv (&kv6, &sep->ip.ip6, sep->port, sep->transport_proto); @@ -982,22 +993,25 @@ session_lookup_connection_wt4 (u32 fib_index, ip4_address_t * lcl, if (rv == 0) return transport_get_half_open (proto, kv4.value & 0xFFFFFFFF); - /* - * Check the session rules table - */ - action_index = session_rules_table_lookup4 (&st->session_rules[proto], lcl, - rmt, lcl_port, rmt_port); - if (session_lookup_action_index_is_valid (action_index)) + if (st->srtg_handle != SESSION_SRTG_HANDLE_INVALID) { - if (action_index == SESSION_RULES_TABLE_ACTION_DROP) + /* + * Check the session rules table + */ + action_index = session_rules_table_lookup4 (st->srtg_handle, proto, lcl, + rmt, lcl_port, rmt_port); + if (session_lookup_action_index_is_valid (action_index)) { - *result = SESSION_LOOKUP_RESULT_FILTERED; + if (action_index == SESSION_RULES_TABLE_ACTION_DROP) + { + *result = SESSION_LOOKUP_RESULT_FILTERED; + return 0; + } + if ((s = session_lookup_action_to_session (action_index, + FIB_PROTOCOL_IP4, proto))) + return transport_get_listener (proto, s->connection_index); return 0; } - if ((s = session_lookup_action_to_session (action_index, - FIB_PROTOCOL_IP4, proto))) - return transport_get_listener (proto, s->connection_index); - return 0; } /* @@ -1059,19 +1073,22 @@ session_lookup_connection4 (u32 fib_index, ip4_address_t * lcl, if (rv == 0) return transport_get_half_open (proto, kv4.value & 0xFFFFFFFF); - /* - * Check the session rules table - */ - action_index = session_rules_table_lookup4 (&st->session_rules[proto], lcl, - rmt, lcl_port, rmt_port); - if (session_lookup_action_index_is_valid (action_index)) + if (st->srtg_handle != SESSION_SRTG_HANDLE_INVALID) { - if (action_index == SESSION_RULES_TABLE_ACTION_DROP) - return 0; - if ((s = session_lookup_action_to_session (action_index, - FIB_PROTOCOL_IP4, proto))) - return transport_get_listener (proto, s->connection_index); - return 0; + /* + * Check the session rules table + */ + action_index = session_rules_table_lookup4 (st->srtg_handle, proto, lcl, + rmt, lcl_port, rmt_port); + if (session_lookup_action_index_is_valid (action_index)) + { + if (action_index == SESSION_RULES_TABLE_ACTION_DROP) + return 0; + if ((s = session_lookup_action_to_session (action_index, + FIB_PROTOCOL_IP4, proto))) + return transport_get_listener (proto, s->connection_index); + return 0; + } } /* @@ -1117,17 +1134,20 @@ session_lookup_safe4 (u32 fib_index, ip4_address_t * lcl, ip4_address_t * rmt, if (rv == 0) return session_get_from_handle_safe (kv4.value); - /* - * Check the session rules table - */ - action_index = session_rules_table_lookup4 (&st->session_rules[proto], lcl, - rmt, lcl_port, rmt_port); - if (session_lookup_action_index_is_valid (action_index)) + if (st->srtg_handle != SESSION_SRTG_HANDLE_INVALID) { - if (action_index == SESSION_RULES_TABLE_ACTION_DROP) - return 0; - return session_lookup_action_to_session (action_index, FIB_PROTOCOL_IP4, - proto); + /* + * Check the session rules table + */ + action_index = session_rules_table_lookup4 (st->srtg_handle, proto, lcl, + rmt, lcl_port, rmt_port); + if (session_lookup_action_index_is_valid (action_index)) + { + if (action_index == SESSION_RULES_TABLE_ACTION_DROP) + return 0; + return session_lookup_action_to_session (action_index, + FIB_PROTOCOL_IP4, proto); + } } /* @@ -1199,20 +1219,23 @@ session_lookup_connection_wt6 (u32 fib_index, ip6_address_t * lcl, if (rv == 0) return transport_get_half_open (proto, kv6.value & 0xFFFFFFFF); - /* Check the session rules table */ - action_index = session_rules_table_lookup6 (&st->session_rules[proto], lcl, - rmt, lcl_port, rmt_port); - if (session_lookup_action_index_is_valid (action_index)) + if (st->srtg_handle != SESSION_SRTG_HANDLE_INVALID) { - if (action_index == SESSION_RULES_TABLE_ACTION_DROP) + /* Check the session rules table */ + action_index = session_rules_table_lookup6 (st->srtg_handle, proto, lcl, + rmt, lcl_port, rmt_port); + if (session_lookup_action_index_is_valid (action_index)) { - *result = SESSION_LOOKUP_RESULT_FILTERED; + if (action_index == SESSION_RULES_TABLE_ACTION_DROP) + { + *result = SESSION_LOOKUP_RESULT_FILTERED; + return 0; + } + if ((s = session_lookup_action_to_session (action_index, + FIB_PROTOCOL_IP6, proto))) + return transport_get_listener (proto, s->connection_index); return 0; } - if ((s = session_lookup_action_to_session (action_index, - FIB_PROTOCOL_IP6, proto))) - return transport_get_listener (proto, s->connection_index); - return 0; } /* If nothing is found, check if any listener is available */ @@ -1268,17 +1291,20 @@ session_lookup_connection6 (u32 fib_index, ip6_address_t * lcl, if (rv == 0) return transport_get_half_open (proto, kv6.value & 0xFFFFFFFF); - /* Check the session rules table */ - action_index = session_rules_table_lookup6 (&st->session_rules[proto], lcl, - rmt, lcl_port, rmt_port); - if (session_lookup_action_index_is_valid (action_index)) + if (st->srtg_handle != SESSION_SRTG_HANDLE_INVALID) { - if (action_index == SESSION_RULES_TABLE_ACTION_DROP) - return 0; - if ((s = session_lookup_action_to_session (action_index, - FIB_PROTOCOL_IP6, proto))) - return transport_get_listener (proto, s->connection_index); - return 0; + /* Check the session rules table */ + action_index = session_rules_table_lookup6 (st->srtg_handle, proto, lcl, + rmt, lcl_port, rmt_port); + if (session_lookup_action_index_is_valid (action_index)) + { + if (action_index == SESSION_RULES_TABLE_ACTION_DROP) + return 0; + if ((s = session_lookup_action_to_session (action_index, + FIB_PROTOCOL_IP6, proto))) + return transport_get_listener (proto, s->connection_index); + return 0; + } } /* If nothing is found, check if any listener is available */ @@ -1321,15 +1347,18 @@ session_lookup_safe6 (u32 fib_index, ip6_address_t * lcl, ip6_address_t * rmt, if (rv == 0) return session_get_from_handle_safe (kv6.value); - /* Check the session rules table */ - action_index = session_rules_table_lookup6 (&st->session_rules[proto], lcl, - rmt, lcl_port, rmt_port); - if (session_lookup_action_index_is_valid (action_index)) + if (st->srtg_handle != SESSION_SRTG_HANDLE_INVALID) { - if (action_index == SESSION_RULES_TABLE_ACTION_DROP) - return 0; - return session_lookup_action_to_session (action_index, FIB_PROTOCOL_IP6, - proto); + /* Check the session rules table */ + action_index = session_rules_table_lookup6 (st->srtg_handle, proto, lcl, + rmt, lcl_port, rmt_port); + if (session_lookup_action_index_is_valid (action_index)) + { + if (action_index == SESSION_RULES_TABLE_ACTION_DROP) + return 0; + return session_lookup_action_to_session (action_index, + FIB_PROTOCOL_IP6, proto); + } } /* If nothing is found, check if any listener is available */ @@ -1355,7 +1384,6 @@ session_error_t vnet_session_rule_add_del (session_rule_add_del_args_t *args) { app_namespace_t *app_ns = app_namespace_get (args->appns_index); - session_rules_table_t *srt; session_table_t *st; u32 fib_index; u8 fib_proto; @@ -1376,8 +1404,9 @@ vnet_session_rule_add_del (session_rule_add_del_args_t *args) fib_proto = args->table_args.rmt.fp_proto; fib_index = app_namespace_get_fib_index (app_ns, fib_proto); st = session_table_get_for_fib_index (fib_proto, fib_index); - srt = &st->session_rules[args->transport_proto]; - if ((rv = session_rules_table_add_del (srt, &args->table_args))) + session_rules_table_init (st, fib_proto); + if ((rv = session_rules_table_add_del ( + st->srtg_handle, args->transport_proto, &args->table_args))) return rv; } if (args->scope & SESSION_RULE_SCOPE_LOCAL) @@ -1386,12 +1415,30 @@ vnet_session_rule_add_del (session_rule_add_del_args_t *args) args->table_args.lcl.fp_proto = args->table_args.rmt.fp_proto; args->table_args.lcl_port = 0; st = app_namespace_get_local_table (app_ns); - srt = &st->session_rules[args->transport_proto]; - rv = session_rules_table_add_del (srt, &args->table_args); + session_rules_table_init (st, args->table_args.rmt.fp_proto); + rv = session_rules_table_add_del (st->srtg_handle, args->transport_proto, + &args->table_args); } return rv; } +static void +session_lookup_fib_table_lock (u32 fib_index, u32 protocol) +{ + fib_table_lock (fib_index, protocol, sl_main.fib_src); + vec_validate (fib_index_to_lock_count[protocol], fib_index); + fib_index_to_lock_count[protocol][fib_index]++; + ASSERT (fib_index_to_lock_count[protocol][fib_index] > 0); +} + +static void +session_lookup_fib_table_unlock (u32 fib_index, u32 protocol) +{ + fib_table_unlock (fib_index, protocol, sl_main.fib_src); + ASSERT (fib_index_to_lock_count[protocol][fib_index] > 0); + fib_index_to_lock_count[protocol][fib_index]--; +} + /** * Mark (global) tables as pertaining to app ns */ @@ -1407,7 +1454,10 @@ session_lookup_set_tables_appns (app_namespace_t * app_ns) fib_index = app_namespace_get_fib_index (app_ns, fp); st = session_table_get_or_alloc (fp, fib_index); if (st) - st->appns_index = app_namespace_index (app_ns); + { + st->appns_index = app_namespace_index (app_ns); + session_lookup_fib_table_lock (fib_index, fp); + } } } @@ -1501,6 +1551,9 @@ session_rule_command_fn (vlib_main_t * vm, unformat_input_t * input, session_cli_return_if_not_enabled (); + if (session_rule_table_is_enabled () == 0) + return clib_error_return (0, "session rule table engine is not enabled"); + clib_memset (&lcl_ip, 0, sizeof (lcl_ip)); clib_memset (&rmt_ip, 0, sizeof (rmt_ip)); while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) @@ -1622,11 +1675,12 @@ session_lookup_dump_rules_table (u32 fib_index, u8 fib_proto, u8 transport_proto) { vlib_main_t *vm = vlib_get_main (); - session_rules_table_t *srt; session_table_t *st; st = session_table_get_for_fib_index (fib_index, fib_proto); - srt = &st->session_rules[transport_proto]; - session_rules_table_cli_dump (vm, srt, fib_proto); + if (st == 0) + return; + session_rules_table_cli_dump (vm, st->srtg_handle, transport_proto, + fib_proto); } void @@ -1634,11 +1688,12 @@ session_lookup_dump_local_rules_table (u32 table_index, u8 fib_proto, u8 transport_proto) { vlib_main_t *vm = vlib_get_main (); - session_rules_table_t *srt; session_table_t *st; st = session_table_get (table_index); - srt = &st->session_rules[transport_proto]; - session_rules_table_cli_dump (vm, srt, fib_proto); + if (st == 0) + return; + session_rules_table_cli_dump (vm, st->srtg_handle, transport_proto, + fib_proto); } static clib_error_t * @@ -1650,7 +1705,6 @@ show_session_rules_command_fn (vlib_main_t * vm, unformat_input_t * input, ip46_address_t lcl_ip, rmt_ip; u8 is_ip4 = 1, show_one = 0; app_namespace_t *app_ns; - session_rules_table_t *srt; session_table_t *st; u8 *ns_id = 0, fib_proto; @@ -1665,9 +1719,9 @@ show_session_rules_command_fn (vlib_main_t * vm, unformat_input_t * input, else if (unformat (input, "appns %_%v%_", &ns_id)) ; else if (unformat (input, "scope global")) - scope = 1; + scope = SESSION_RULE_SCOPE_GLOBAL; else if (unformat (input, "scope local")) - scope = 2; + scope = SESSION_RULE_SCOPE_LOCAL; else if (unformat (input, "%U/%d %d %U/%d %d", unformat_ip4_address, &lcl_ip.ip4, &lcl_plen, &lcl_port, unformat_ip4_address, &rmt_ip.ip4, &rmt_plen, @@ -1685,14 +1739,17 @@ show_session_rules_command_fn (vlib_main_t * vm, unformat_input_t * input, show_one = 1; } else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + { + vec_free (ns_id); + return clib_error_return (0, "unknown input `%U'", + format_unformat_error, input); + } } if (transport_proto == ~0) { vlib_cli_output (vm, "transport proto must be set"); - return 0; + goto done; } if (ns_id) @@ -1701,7 +1758,7 @@ show_session_rules_command_fn (vlib_main_t * vm, unformat_input_t * input, if (!app_ns) { vlib_cli_output (vm, "appns %v doesn't exist", ns_id); - return 0; + goto done; } } else @@ -1709,7 +1766,7 @@ show_session_rules_command_fn (vlib_main_t * vm, unformat_input_t * input, app_ns = app_namespace_get_default (); } - if (scope == 1 || scope == 0) + if (scope == SESSION_RULE_SCOPE_GLOBAL || scope == 0) { fib_proto = is_ip4 ? FIB_PROTOCOL_IP4 : FIB_PROTOCOL_IP6; fib_index = is_ip4 ? app_ns->ip4_fib_index : app_ns->ip6_fib_index; @@ -1720,20 +1777,51 @@ show_session_rules_command_fn (vlib_main_t * vm, unformat_input_t * input, st = app_namespace_get_local_table (app_ns); } + if (session_rule_table_is_enabled () == 0) + { + vlib_cli_output (vm, "session rule table engine is not enabled"); + goto done; + } + if (show_one) { - srt = &st->session_rules[transport_proto]; - session_rules_table_show_rule (vm, srt, &lcl_ip, lcl_port, &rmt_ip, - rmt_port, is_ip4); - return 0; + if (st) + session_rules_table_show_rule (vm, st->srtg_handle, transport_proto, + &lcl_ip, lcl_port, &rmt_ip, rmt_port, + is_ip4); + goto done; } vlib_cli_output (vm, "%U rules table", format_transport_proto, transport_proto); - srt = &st->session_rules[transport_proto]; - session_rules_table_cli_dump (vm, srt, FIB_PROTOCOL_IP4); - session_rules_table_cli_dump (vm, srt, FIB_PROTOCOL_IP6); + if (scope == SESSION_RULE_SCOPE_LOCAL) + { + if (st) + { + session_rules_table_cli_dump (vm, st->srtg_handle, transport_proto, + FIB_PROTOCOL_IP4); + session_rules_table_cli_dump (vm, st->srtg_handle, transport_proto, + FIB_PROTOCOL_IP6); + } + } + else + { + /* + * 2 separate session tables for global entries, 1 for ip4 and 1 for ip6 + */ + st = session_table_get_for_fib_index (FIB_PROTOCOL_IP4, + app_ns->ip4_fib_index); + if (st) + session_rules_table_cli_dump (vm, st->srtg_handle, transport_proto, + FIB_PROTOCOL_IP4); + st = session_table_get_for_fib_index (FIB_PROTOCOL_IP6, + app_ns->ip6_fib_index); + if (st) + session_rules_table_cli_dump (vm, st->srtg_handle, transport_proto, + FIB_PROTOCOL_IP6); + } +done: vec_free (ns_id); return 0; } @@ -1832,6 +1920,14 @@ session_lookup_init (void) clib_spinlock_init (&slm->st_alloc_lock); + /* We are not contributing any route to the fib. But we allocate a fib source + * so that when we lock the fib table, we can view that we have a lock on the + * particular fib table in case we wonder why the fib table is not free after + * "ip table del" + */ + slm->fib_src = fib_source_allocate ( + "session lookup", FIB_SOURCE_PRIORITY_LOW, FIB_SOURCE_BH_SIMPLE); + /* * Allocate default table and map it to fib_index 0 */ @@ -1847,6 +1943,26 @@ session_lookup_init (void) session_table_init (st, FIB_PROTOCOL_IP6); } +void +session_lookup_table_cleanup (u32 fib_proto, u32 fib_index) +{ + session_table_t *st; + u32 table_index; + + session_lookup_fib_table_unlock (fib_index, fib_proto); + if (fib_index_to_lock_count[fib_proto][fib_index] == 0) + { + table_index = session_lookup_get_index_for_fib (fib_proto, fib_index); + st = session_table_get (table_index); + if (st) + { + session_table_free (st, fib_proto); + if (vec_len (fib_index_to_table_index[fib_proto]) > fib_index) + fib_index_to_table_index[fib_proto][fib_index] = ~0; + } + } +} + /* * fd.io coding-style-patch-verification: ON * diff --git a/src/vnet/session/session_lookup.h b/src/vnet/session/session_lookup.h index f9ffc15165a..9f56af20a87 100644 --- a/src/vnet/session/session_lookup.h +++ b/src/vnet/session/session_lookup.h @@ -19,6 +19,8 @@ #include <vnet/session/session_table.h> #include <vnet/session/session_types.h> #include <vnet/session/application_namespace.h> +#include <vnet/fib/fib_table.h> +#include <vnet/fib/fib_source.h> #define HALF_OPEN_LOOKUP_INVALID_VALUE ((u64)~0) @@ -32,6 +34,7 @@ typedef enum session_lookup_result_ typedef struct session_lookup_main_ { clib_spinlock_t st_alloc_lock; + fib_source_t fib_src; } session_lookup_main_t; session_t *session_lookup_safe4 (u32 fib_index, ip4_address_t * lcl, @@ -114,6 +117,17 @@ typedef enum _session_rule_scope SESSION_RULE_SCOPE_LOCAL = 2, } session_rule_scope_e; +typedef struct _session_rules_table_add_del_args +{ + fib_prefix_t lcl; + fib_prefix_t rmt; + u16 lcl_port; + u16 rmt_port; + u32 action_index; + u8 *tag; + u8 is_add; +} session_rule_table_add_del_args_t; + typedef struct _session_rule_add_del_args { /** @@ -139,6 +153,8 @@ session_error_t vnet_session_rule_add_del (session_rule_add_del_args_t *args); void session_lookup_set_tables_appns (app_namespace_t * app_ns); void session_lookup_init (void); +session_table_t *session_table_get_for_fib_index (u32 fib_proto, + u32 fib_index); #endif /* SRC_VNET_SESSION_SESSION_LOOKUP_H_ */ diff --git a/src/vnet/session/session_node.c b/src/vnet/session/session_node.c index 0ec158fb429..14b8005d5d0 100644 --- a/src/vnet/session/session_node.c +++ b/src/vnet/session/session_node.c @@ -1249,6 +1249,7 @@ session_tx_set_dequeue_params (vlib_main_t * vm, session_tx_context_t * ctx, } ASSERT (ctx->hdr.data_length > ctx->hdr.data_offset); len = ctx->hdr.data_length - ctx->hdr.data_offset; + ctx->sp.snd_mss = clib_min (ctx->sp.snd_mss, len); if (ctx->hdr.gso_size) { @@ -2163,6 +2164,8 @@ session_queue_process (vlib_main_t * vm, vlib_node_runtime_t * rt, session_queue_run_on_main (vm); break; case SESSION_Q_PROCESS_STOP: + /* Free event_data, the node will be restarted if needed */ + vec_free (event_data); vlib_node_set_state (vm, session_queue_process_node.index, VLIB_NODE_STATE_DISABLED); timeout = 100000.0; diff --git a/src/vnet/session/session_rules_table.c b/src/vnet/session/session_rules_table.c index 70a702cf55c..acb575b0f7c 100644 --- a/src/vnet/session/session_rules_table.c +++ b/src/vnet/session/session_rules_table.c @@ -17,8 +17,23 @@ #include <vnet/session/mma_template.c> #include <vnet/session/mma_40.h> #include <vnet/session/mma_template.c> -#include <vnet/session/session_rules_table.h> #include <vnet/session/transport.h> +#include <vnet/session/session.h> +#include <vnet/session/session_table.h> +#include <vnet/session/session_rules_table.h> +#include <vnet/session/session_sdl.h> + +VLIB_REGISTER_LOG_CLASS (session_rt_log, static) = { .class_name = "session", + .subclass_name = "rt" }; + +#define log_debug(fmt, ...) \ + vlib_log_debug (session_rt_log.class, "%s: " fmt, __func__, __VA_ARGS__) +#define log_warn(fmt, ...) \ + vlib_log_warn (session_rt_log.class, fmt, __VA_ARGS__) +#define log_err(fmt, ...) vlib_log_err (session_rt_log.class, fmt, __VA_ARGS__) + +static session_rules_table_group_t *srt_instances; +const session_rt_engine_vft_t *session_rt_engine_vft; u32 session_rule_tag_key_index (u32 ri, u8 is_ip4) @@ -75,6 +90,7 @@ session_rules_table_del_tag (session_rules_table_t * srt, u8 * tag, u8 is_ip4) ASSERT (rt); hash_unset_mem (srt->rules_by_tag, tag); hash_unset (srt->tags_by_rules, rti_key); + vec_free (rt->tag); pool_put (srt->rule_tags, rt); } @@ -118,13 +134,25 @@ fib_pref_normalize (fib_prefix_t * pref) } u8 * +format_session_rule_tag (u8 *s, va_list *args) +{ + static u8 *null_tag = 0; + u8 *tag = va_arg (*args, u8 *); + + if (!null_tag) + null_tag = format (0, "none"); + s = format (s, "%v", (tag != 0) ? tag : null_tag); + return s; +} + +u8 * format_session_rule4 (u8 * s, va_list * args) { session_rules_table_t *srt = va_arg (*args, session_rules_table_t *); mma_rule_16_t *sr = va_arg (*args, mma_rule_16_t *); session_mask_or_match_4_t *mask, *match; mma_rules_table_16_t *srt4; - u8 *tag = 0, *null_tag = format (0, "none"); + u8 *tag = 0; u32 ri; int i; @@ -134,20 +162,18 @@ format_session_rule4 (u8 * s, va_list * args) match = (session_mask_or_match_4_t *) & sr->match; mask = (session_mask_or_match_4_t *) & sr->mask; - s = format (s, "[%d] rule: %U/%d %d %U/%d %d action: %d tag: %v", ri, + s = format (s, "[%d] rule: %U/%d %d %U/%d %d action: %d tag: %U", ri, format_ip4_address, &match->lcl_ip, - ip4_mask_to_preflen (&mask->lcl_ip), - clib_net_to_host_u16 (match->lcl_port), format_ip4_address, - &match->rmt_ip, ip4_mask_to_preflen (&mask->rmt_ip), - clib_net_to_host_u16 (match->rmt_port), sr->action_index, - tag ? tag : null_tag); + ip4_mask_to_preflen (&mask->lcl_ip), match->lcl_port, + format_ip4_address, &match->rmt_ip, + ip4_mask_to_preflen (&mask->rmt_ip), match->rmt_port, + sr->action_index, format_session_rule_tag, tag); if (vec_len (sr->next_indices)) { s = format (s, "\n children: "); for (i = 0; i < vec_len (sr->next_indices); i++) s = format (s, "%d ", sr->next_indices[i]); } - vec_free (null_tag); return s; } @@ -158,7 +184,7 @@ format_session_rule6 (u8 * s, va_list * args) mma_rule_40_t *sr = va_arg (*args, mma_rule_40_t *); session_mask_or_match_6_t *mask, *match; mma_rules_table_40_t *srt6; - u8 *tag = 0, *null_tag = format (0, "none"); + u8 *tag = 0; u32 ri; int i; @@ -168,20 +194,18 @@ format_session_rule6 (u8 * s, va_list * args) match = (session_mask_or_match_6_t *) & sr->match; mask = (session_mask_or_match_6_t *) & sr->mask; - s = format (s, "[%d] rule: %U/%d %d %U/%d %d action: %d tag: %v", ri, + s = format (s, "[%d] rule: %U/%d %d %U/%d %d action: %d tag: %U", ri, format_ip6_address, &match->lcl_ip, - ip6_mask_to_preflen (&mask->lcl_ip), - clib_net_to_host_u16 (match->lcl_port), format_ip6_address, - &match->rmt_ip, ip6_mask_to_preflen (&mask->rmt_ip), - clib_net_to_host_u16 (match->rmt_port), sr->action_index, - tag ? tag : null_tag); + ip6_mask_to_preflen (&mask->lcl_ip), match->lcl_port, + format_ip6_address, &match->rmt_ip, + ip6_mask_to_preflen (&mask->rmt_ip), match->rmt_port, + sr->action_index, format_session_rule_tag, tag); if (vec_len (sr->next_indices)) { s = format (s, "\n children: "); for (i = 0; i < vec_len (sr->next_indices); i++) s = format (s, "%d ", sr->next_indices[i]); } - vec_free (null_tag); return s; } @@ -312,11 +336,10 @@ session_rules_table_alloc_rule_40 (mma_rules_table_40_t * srt, return rule; } -u32 -session_rules_table_lookup_rule4 (session_rules_table_t * srt, - ip4_address_t * lcl_ip, - ip4_address_t * rmt_ip, u16 lcl_port, - u16 rmt_port) +static u32 +session_rules_table_lookup_rule4 (session_rules_table_t *srt, + ip4_address_t *lcl_ip, ip4_address_t *rmt_ip, + u16 lcl_port, u16 rmt_port) { mma_rules_table_16_t *srt4 = &srt->session_rules_tables_16; session_mask_or_match_4_t key = { @@ -325,16 +348,20 @@ session_rules_table_lookup_rule4 (session_rules_table_t * srt, .lcl_port = lcl_port, .rmt_port = rmt_port, }; + + if (srt4->rules == 0) + return SESSION_TABLE_INVALID_INDEX; return mma_rules_table_lookup_rule_16 (srt4, (mma_mask_or_match_16_t *) & key, srt4->root_index); } u32 -session_rules_table_lookup4 (session_rules_table_t * srt, - ip4_address_t * lcl_ip, ip4_address_t * rmt_ip, - u16 lcl_port, u16 rmt_port) +session_rules_table_lookup4_ (u32 srtg_handle, u32 proto, + ip4_address_t *lcl_ip, ip4_address_t *rmt_ip, + u16 lcl_port, u16 rmt_port) { + session_rules_table_t *srt = srtg_handle_to_srt (srtg_handle, proto); mma_rules_table_16_t *srt4 = &srt->session_rules_tables_16; session_mask_or_match_4_t key = { .lcl_ip.as_u32 = lcl_ip->as_u32, @@ -342,21 +369,36 @@ session_rules_table_lookup4 (session_rules_table_t * srt, .lcl_port = lcl_port, .rmt_port = rmt_port, }; + + if (srt4->rules == 0) + return SESSION_TABLE_INVALID_INDEX; return mma_rules_table_lookup_16 (srt4, (mma_mask_or_match_16_t *) & key, srt4->root_index); } -u32 -session_rules_table_lookup_rule6 (session_rules_table_t * srt, - ip6_address_t * lcl_ip, - ip6_address_t * rmt_ip, u16 lcl_port, - u16 rmt_port) +session_rules_table_t * +srtg_handle_to_srt (u32 srtg_handle, u32 proto) +{ + session_rules_table_group_t *srtg = + pool_elt_at_index (srt_instances, srtg_handle); + session_rules_table_t *srt = &srtg->session_rules[proto]; + + return srt; +} + +static u32 +session_rules_table_lookup_rule6 (session_rules_table_t *srt, + ip6_address_t *lcl_ip, ip6_address_t *rmt_ip, + u16 lcl_port, u16 rmt_port) { mma_rules_table_40_t *srt6 = &srt->session_rules_tables_40; session_mask_or_match_6_t key = { .lcl_port = lcl_port, .rmt_port = rmt_port, }; + + if (srt6->rules == 0) + return SESSION_TABLE_INVALID_INDEX; clib_memcpy_fast (&key.lcl_ip, lcl_ip, sizeof (*lcl_ip)); clib_memcpy_fast (&key.rmt_ip, rmt_ip, sizeof (*rmt_ip)); return mma_rules_table_lookup_rule_40 (srt6, @@ -365,15 +407,19 @@ session_rules_table_lookup_rule6 (session_rules_table_t * srt, } u32 -session_rules_table_lookup6 (session_rules_table_t * srt, - ip6_address_t * lcl_ip, ip6_address_t * rmt_ip, - u16 lcl_port, u16 rmt_port) +session_rules_table_lookup6_ (u32 srtg_handle, u32 proto, + ip6_address_t *lcl_ip, ip6_address_t *rmt_ip, + u16 lcl_port, u16 rmt_port) { + session_rules_table_t *srt = srtg_handle_to_srt (srtg_handle, proto); mma_rules_table_40_t *srt6 = &srt->session_rules_tables_40; session_mask_or_match_6_t key = { .lcl_port = lcl_port, .rmt_port = rmt_port, }; + + if (srt6->rules == 0) + return SESSION_TABLE_INVALID_INDEX; clib_memcpy_fast (&key.lcl_ip, lcl_ip, sizeof (*lcl_ip)); clib_memcpy_fast (&key.rmt_ip, rmt_ip, sizeof (*rmt_ip)); return mma_rules_table_lookup_40 (srt6, (mma_mask_or_match_40_t *) & key, @@ -389,9 +435,10 @@ session_rules_table_lookup6 (session_rules_table_t * srt, * @return 0 if success, session_error_t error otherwise */ session_error_t -session_rules_table_add_del (session_rules_table_t *srt, - session_rule_table_add_del_args_t *args) +session_rules_table_add_del_ (u32 srtg_handle, u32 proto, + session_rule_table_add_del_args_t *args) { + session_rules_table_t *srt = srtg_handle_to_srt (srtg_handle, proto); u8 fib_proto = args->rmt.fp_proto, *rt; u32 ri_from_tag, ri; int rv; @@ -514,47 +561,90 @@ session_rules_table_add_del (session_rules_table_t *srt, } void -session_rules_table_free (session_rules_table_t *srt) +session_rules_table_free_ (session_table_t *st, u8 fib_proto) +{ + session_rules_table_group_t *srtg = + pool_elt_at_index (srt_instances, st->srtg_handle); + session_rules_table_t *srt; + + vec_foreach (srt, srtg->session_rules) + { + mma_rules_table_free_16 (&srt->session_rules_tables_16); + mma_rules_table_free_40 (&srt->session_rules_tables_40); + + hash_free (srt->tags_by_rules); + hash_free (srt->rules_by_tag); + } + srtg_instance_free (st); +} + +void +srtg_instance_free (session_table_t *st) { - mma_rules_table_free_16 (&srt->session_rules_tables_16); - mma_rules_table_free_40 (&srt->session_rules_tables_40); + session_rules_table_group_t *srtg = + pool_elt_at_index (srt_instances, st->srtg_handle); + + vec_free (srtg->session_rules); + pool_put (srt_instances, srtg); + st->srtg_handle = SESSION_SRTG_HANDLE_INVALID; +} + +session_rules_table_group_t * +srtg_instance_alloc (session_table_t *st, u32 n_proto) +{ + session_rules_table_group_t *srtg; + + pool_get (srt_instances, srtg); + vec_validate (srtg->session_rules, n_proto); + st->srtg_handle = srtg - srt_instances; + return (srtg); } void -session_rules_table_init (session_rules_table_t * srt) +session_rules_table_init_ (session_table_t *st, u8 fib_proto) { mma_rules_table_16_t *srt4; mma_rules_table_40_t *srt6; mma_rule_16_t *rule4; mma_rule_40_t *rule6; fib_prefix_t null_prefix; + session_rules_table_t *srt; + session_rules_table_group_t *srtg; - clib_memset (&null_prefix, 0, sizeof (null_prefix)); + srtg = srtg_instance_alloc (st, TRANSPORT_N_PROTOS - 1); - srt4 = &srt->session_rules_tables_16; - rule4 = session_rules_table_alloc_rule_16 (srt4, &null_prefix, 0, - &null_prefix, 0); - rule4->action_index = SESSION_RULES_TABLE_INVALID_INDEX; - srt4->root_index = mma_rules_table_rule_index_16 (srt4, rule4); - srt4->rule_cmp_fn = rule_cmp_16; + clib_memset (&null_prefix, 0, sizeof (null_prefix)); + vec_foreach (srt, srtg->session_rules) + { + srt4 = &srt->session_rules_tables_16; - srt6 = &srt->session_rules_tables_40; - rule6 = session_rules_table_alloc_rule_40 (srt6, &null_prefix, 0, - &null_prefix, 0); - rule6->action_index = SESSION_RULES_TABLE_INVALID_INDEX; - srt6->root_index = mma_rules_table_rule_index_40 (srt6, rule6); - srt6->rule_cmp_fn = rule_cmp_40; + ASSERT (srt4->rules == 0); + rule4 = session_rules_table_alloc_rule_16 (srt4, &null_prefix, 0, + &null_prefix, 0); + rule4->action_index = SESSION_RULES_TABLE_INVALID_INDEX; + srt4->root_index = mma_rules_table_rule_index_16 (srt4, rule4); + srt4->rule_cmp_fn = rule_cmp_16; - srt->rules_by_tag = hash_create_vec (0, sizeof (u8), sizeof (uword)); - srt->tags_by_rules = hash_create (0, sizeof (uword)); + srt6 = &srt->session_rules_tables_40; + ASSERT (srt6->rules == 0); + rule6 = session_rules_table_alloc_rule_40 (srt6, &null_prefix, 0, + &null_prefix, 0); + rule6->action_index = SESSION_RULES_TABLE_INVALID_INDEX; + srt6->root_index = mma_rules_table_rule_index_40 (srt6, rule6); + srt6->rule_cmp_fn = rule_cmp_40; + + srt->rules_by_tag = hash_create_vec (0, sizeof (u8), sizeof (uword)); + srt->tags_by_rules = hash_create (0, sizeof (uword)); + } } void -session_rules_table_show_rule (vlib_main_t * vm, session_rules_table_t * srt, - ip46_address_t * lcl_ip, u16 lcl_port, - ip46_address_t * rmt_ip, u16 rmt_port, - u8 is_ip4) +session_rules_table_show_rule_ (vlib_main_t *vm, u32 srtg_handle, u32 proto, + ip46_address_t *lcl_ip, u16 lcl_port, + ip46_address_t *rmt_ip, u16 rmt_port, + u8 is_ip4) { + session_rules_table_t *srt = srtg_handle_to_srt (srtg_handle, proto); mma_rules_table_16_t *srt4; mma_rules_table_40_t *srt6; mma_rule_16_t *sr4; @@ -595,9 +685,10 @@ session_rules_table_show_rule (vlib_main_t * vm, session_rules_table_t * srt, } void -session_rules_table_cli_dump (vlib_main_t * vm, session_rules_table_t * srt, - u8 fib_proto) +session_rules_table_cli_dump_ (vlib_main_t *vm, u32 srtg_handle, u32 proto, + u8 fib_proto) { + session_rules_table_t *srt = srtg_handle_to_srt (srtg_handle, proto); if (fib_proto == FIB_PROTOCOL_IP4) { mma_rules_table_16_t *srt4; @@ -624,6 +715,108 @@ session_rules_table_cli_dump (vlib_main_t * vm, session_rules_table_t * srt, } } +static const session_rt_engine_vft_t session_rules_table_vft = { + .backend_engine = RT_BACKEND_ENGINE_RULE_TABLE, + .table_lookup4 = session_rules_table_lookup4_, + .table_lookup6 = session_rules_table_lookup6_, + .table_cli_dump = session_rules_table_cli_dump_, + .table_show_rule = session_rules_table_show_rule_, + .table_add_del = session_rules_table_add_del_, + .table_init = session_rules_table_init_, + .table_free = session_rules_table_free_, +}; + +static void +session_rules_table_app_namespace_walk_cb (app_namespace_t *app_ns, void *ctx) +{ + u32 fib_index, table_index; + session_table_t *st; + + log_debug ("disable app_ns %s", app_ns->ns_id); + st = session_table_get (app_ns->local_table_index); + if (st) + session_rules_table_free (st, FIB_PROTOCOL_MAX); + + fib_index = app_namespace_get_fib_index (app_ns, FIB_PROTOCOL_IP4); + table_index = session_lookup_get_index_for_fib (FIB_PROTOCOL_IP4, fib_index); + st = session_table_get (table_index); + if (st) + session_rules_table_free (st, FIB_PROTOCOL_IP4); + + fib_index = app_namespace_get_fib_index (app_ns, FIB_PROTOCOL_IP6); + table_index = session_lookup_get_index_for_fib (FIB_PROTOCOL_IP6, fib_index); + st = session_table_get (table_index); + if (st) + session_rules_table_free (st, FIB_PROTOCOL_IP6); +} + +clib_error_t * +session_rules_table_enable_disable (int enable) +{ + clib_error_t *error; + + if (enable) + error = session_rule_table_register_engine (&session_rules_table_vft); + else + { + app_namespace_walk (session_rules_table_app_namespace_walk_cb, 0); + error = session_rule_table_deregister_engine (&session_rules_table_vft); + } + + return error; +} + +clib_error_t * +session_rt_backend_enable_disable (session_rt_engine_type_t rt_engine_type) +{ + session_main_t *smm = &session_main; + clib_error_t *error = 0; + + if (rt_engine_type < RT_BACKEND_ENGINE_DISABLE || + rt_engine_type > RT_BACKEND_ENGINE_SDL) + return clib_error_return (0, "invalid rt-backend %d", rt_engine_type); + + if (rt_engine_type == RT_BACKEND_ENGINE_SDL) + error = session_sdl_enable_disable (1); + else if (rt_engine_type == RT_BACKEND_ENGINE_RULE_TABLE) + error = session_rules_table_enable_disable (1); + else if (rt_engine_type == RT_BACKEND_ENGINE_DISABLE) + { + if (session_sdl_is_enabled ()) + error = session_sdl_enable_disable (0); + else if (session_rule_table_is_enabled ()) + error = session_rules_table_enable_disable (0); + } + + if (!error) + smm->rt_engine_type = rt_engine_type; + return error; +} + +clib_error_t * +session_rule_table_register_engine (const session_rt_engine_vft_t *vft) +{ + if (session_rt_engine_vft == vft) + return 0; + if (session_rt_engine_vft) + return clib_error_return (0, "session rule engine is already registered"); + + session_rt_engine_vft = vft; + return 0; +} + +clib_error_t * +session_rule_table_deregister_engine (const session_rt_engine_vft_t *vft) +{ + if (session_rt_engine_vft == vft) + session_rt_engine_vft = 0; + else + return clib_error_return ( + 0, "session rule engine is not registered to this engine"); + + return 0; +} + /* * fd.io coding-style-patch-verification: ON * diff --git a/src/vnet/session/session_rules_table.h b/src/vnet/session/session_rules_table.h index 010d50a6398..a61438bb9fd 100644 --- a/src/vnet/session/session_rules_table.h +++ b/src/vnet/session/session_rules_table.h @@ -22,6 +22,10 @@ #include <vnet/session/transport.h> #include <vnet/session/mma_16.h> #include <vnet/session/mma_40.h> +#include <vnet/session/session_lookup.h> +#include <vnet/session/session_table.h> + +#define SESSION_SRTG_HANDLE_INVALID ~0 typedef CLIB_PACKED (struct { @@ -58,29 +62,36 @@ typedef CLIB_PACKED (struct #define SESSION_RULES_TABLE_ACTION_DROP (MMA_TABLE_INVALID_INDEX - 1) #define SESSION_RULES_TABLE_ACTION_ALLOW (MMA_TABLE_INVALID_INDEX - 2) -typedef struct _session_rules_table_add_del_args -{ - fib_prefix_t lcl; - fib_prefix_t rmt; - u16 lcl_port; - u16 rmt_port; - u32 action_index; - u8 *tag; - u8 is_add; -} session_rule_table_add_del_args_t; - typedef struct _rule_tag { u8 *tag; } session_rule_tag_t; +typedef struct session_sdl_block +{ + u32 ip_table_id; + u32 ip6_table_id; + u32 ip_fib_index; + u32 ip6_fib_index; +} session_sdl_block_t; + typedef struct _session_rules_table_t { - /** - * Per fib proto session rules tables - */ - mma_rules_table_16_t session_rules_tables_16; - mma_rules_table_40_t session_rules_tables_40; + union + { + /** + * Per fib proto session rules tables + */ + struct + { + mma_rules_table_16_t session_rules_tables_16; + mma_rules_table_40_t session_rules_tables_40; + }; + /** + * sdl table + */ + struct session_sdl_block sdl_block; + }; /** * Hash table that maps tags to rules */ @@ -95,28 +106,163 @@ typedef struct _session_rules_table_t uword *tags_by_rules; } session_rules_table_t; -u32 session_rules_table_lookup4 (session_rules_table_t * srt, - ip4_address_t * lcl_ip, - ip4_address_t * rmt_ip, u16 lcl_port, - u16 rmt_port); -u32 session_rules_table_lookup6 (session_rules_table_t * srt, - ip6_address_t * lcl_ip, - ip6_address_t * rmt_ip, u16 lcl_port, - u16 rmt_port); -void session_rules_table_cli_dump (vlib_main_t * vm, - session_rules_table_t * srt, u8 fib_proto); -void session_rules_table_show_rule (vlib_main_t * vm, - session_rules_table_t * srt, - ip46_address_t * lcl_ip, u16 lcl_port, - ip46_address_t * rmt_ip, u16 rmt_port, - u8 is_ip4); +typedef struct _session_rules_table_group_t +{ + session_rules_table_t *session_rules; +} session_rules_table_group_t; + session_error_t -session_rules_table_add_del (session_rules_table_t *srt, - session_rule_table_add_del_args_t *args); +session_rules_table_add_del_ (u32 srtg_handle, u32 proto, + session_rule_table_add_del_args_t *args); u8 *session_rules_table_rule_tag (session_rules_table_t * srt, u32 ri, u8 is_ip4); -void session_rules_table_init (session_rules_table_t * srt); -void session_rules_table_free (session_rules_table_t *srt); +void session_rules_table_init_ (struct _session_lookup_table *st, + u8 fib_proto); +void session_rules_table_free_ (struct _session_lookup_table *st, + u8 fib_proto); + +typedef u32 (*rules_table_lookup4) (u32 srtg_handle, u32 proto, + ip4_address_t *lcl_ip, + ip4_address_t *rmt_ip, u16 lcl_port, + u16 rmt_port); +typedef u32 (*rules_table_lookup6) (u32 srtg_handle, u32 proto, + ip6_address_t *lcl_ip, + ip6_address_t *rmt_ip, u16 lcl_port, + u16 rmt_port); +typedef void (*rules_table_cli_dump) (vlib_main_t *vm, u32 srtg_handle, + u32 proto, u8 fib_proto); +typedef void (*rules_table_show_rule) (vlib_main_t *vm, u32 srtg_handle, + u32 proto, ip46_address_t *lcl_ip, + u16 lcl_port, ip46_address_t *rmt_ip, + u16 rmt_port, u8 is_ip4); +typedef session_error_t (*rules_table_add_del) ( + u32 srtg_handle, u32 proto, session_rule_table_add_del_args_t *args); +typedef void (*rules_table_init) (struct _session_lookup_table *st, + u8 fib_proto); +typedef void (*rules_table_free) (struct _session_lookup_table *st, + u8 fib_proto); + +#define foreach_session_rt_engine_vft_method_name \ + _ (lookup4) \ + _ (lookup6) \ + _ (cli_dump) \ + _ (show_rule) \ + _ (add_del) \ + _ (init) \ + _ (free) + +#define _(name) rules_table_##name table_##name; +typedef struct session_rt_engine_vft +{ + u32 backend_engine; + foreach_session_rt_engine_vft_method_name +} session_rt_engine_vft_t; +#undef _ + +extern u8 *format_session_rule_tag (u8 *s, va_list *args); +extern u8 *session_rules_table_rule_tag (session_rules_table_t *srt, u32 ri, + u8 is_ip4); +extern u32 session_rules_table_rule_for_tag (session_rules_table_t *srt, + u8 *tag); +extern void session_rules_table_add_tag (session_rules_table_t *srt, u8 *tag, + u32 rule_index, u8 is_ip4); +extern void session_rules_table_del_tag (session_rules_table_t *srt, u8 *tag, + u8 is_ip4); + +extern const session_rt_engine_vft_t *session_rt_engine_vft; +extern clib_error_t *session_rules_table_enable_disable (int enable); +extern clib_error_t * +session_rt_backend_enable_disable (session_rt_engine_type_t rt_engine_type); + +static_always_inline void +session_rules_table_init (struct _session_lookup_table *st, u8 fib_proto) +{ + if (!session_rt_engine_vft) + return; + if (st->srtg_handle != SESSION_SRTG_HANDLE_INVALID) + return; + session_rt_engine_vft->table_init (st, fib_proto); +} + +static_always_inline void +session_rules_table_free (struct _session_lookup_table *st, u8 fib_proto) +{ + if (!session_rt_engine_vft) + return; + if (st->srtg_handle == SESSION_SRTG_HANDLE_INVALID) + return; + session_rt_engine_vft->table_free (st, fib_proto); +} + +static_always_inline void +session_rules_table_show_rule (vlib_main_t *vm, u32 srtg_handle, u32 proto, + ip46_address_t *lcl_ip, u16 lcl_port, + ip46_address_t *rmt_ip, u16 rmt_port, u8 is_ip4) +{ + if (!session_rt_engine_vft) + return; + if (srtg_handle == SESSION_SRTG_HANDLE_INVALID) + return; + session_rt_engine_vft->table_show_rule (vm, srtg_handle, proto, lcl_ip, + lcl_port, rmt_ip, rmt_port, is_ip4); +} + +static_always_inline u32 +session_rules_table_lookup6 (u32 srtg_handle, u32 proto, ip6_address_t *lcl_ip, + ip6_address_t *rmt_ip, u16 lcl_port, u16 rmt_port) +{ + if (!session_rt_engine_vft) + return SESSION_RULES_TABLE_ACTION_ALLOW; + if (srtg_handle == SESSION_SRTG_HANDLE_INVALID) + return SESSION_RULES_TABLE_ACTION_ALLOW; + return session_rt_engine_vft->table_lookup6 (srtg_handle, proto, lcl_ip, + rmt_ip, lcl_port, rmt_port); +} + +static_always_inline void +session_rules_table_cli_dump (vlib_main_t *vm, u32 srtg_handle, u32 proto, + u8 fib_proto) +{ + if (!session_rt_engine_vft) + return; + if (srtg_handle == SESSION_SRTG_HANDLE_INVALID) + return; + session_rt_engine_vft->table_cli_dump (vm, srtg_handle, proto, fib_proto); +} + +static_always_inline u32 +session_rules_table_lookup4 (u32 srtg_handle, u32 proto, ip4_address_t *lcl_ip, + ip4_address_t *rmt_ip, u16 lcl_port, u16 rmt_port) +{ + if (!session_rt_engine_vft) + return SESSION_RULES_TABLE_ACTION_ALLOW; + if (srtg_handle == SESSION_SRTG_HANDLE_INVALID) + return SESSION_RULES_TABLE_ACTION_ALLOW; + return session_rt_engine_vft->table_lookup4 (srtg_handle, proto, lcl_ip, + rmt_ip, lcl_port, rmt_port); +} + +static_always_inline session_error_t +session_rules_table_add_del (u32 srtg_handle, u32 proto, + session_rule_table_add_del_args_t *args) +{ + if (!session_rt_engine_vft) + return SESSION_E_NOSUPPORT; + if (srtg_handle == SESSION_SRTG_HANDLE_INVALID) + return SESSION_E_NOSUPPORT; + return session_rt_engine_vft->table_add_del (srtg_handle, proto, args); +} + +clib_error_t * +session_rule_table_register_engine (const session_rt_engine_vft_t *vft); +clib_error_t * +session_rule_table_deregister_engine (const session_rt_engine_vft_t *vft); + +extern session_rules_table_t *srtg_handle_to_srt (u32 srtg_handle, u32 proto); +extern session_rules_table_group_t *srtg_instance_alloc (session_table_t *st, + u32 n_proto); +extern void srtg_instance_free (session_table_t *st); + #endif /* SRC_VNET_SESSION_SESSION_RULES_TABLE_H_ */ /* * fd.io coding-style-patch-verification: ON diff --git a/src/vnet/session/session_sdl.c b/src/vnet/session/session_sdl.c new file mode 100644 index 00000000000..45ab705fb62 --- /dev/null +++ b/src/vnet/session/session_sdl.c @@ -0,0 +1,768 @@ +/* + * Copyright (c) 2024 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <vnet/session/session.h> +#include <vnet/ip/ip4_forward.h> +#include <vnet/ip/ip6_forward.h> +#include <vnet/session/session_rules_table.h> +#include <vnet/session/session_sdl.h> + +VLIB_REGISTER_LOG_CLASS (session_sdl_log, static) = { .class_name = "session", + .subclass_name = "sdl" }; + +#define log_debug(fmt, ...) \ + vlib_log_debug (session_sdl_log.class, "%s: " fmt, __func__, __VA_ARGS__) +#define log_warn(fmt, ...) \ + vlib_log_warn (session_sdl_log.class, fmt, __VA_ARGS__) +#define log_err(fmt, ...) \ + vlib_log_err (session_sdl_log.class, fmt, __VA_ARGS__) + +static fib_source_t sdl_fib_src; +static dpo_type_t sdl_dpo_type; + +const static char *const *const session_sdl_dpo_nodes[DPO_PROTO_NUM] = { + [DPO_PROTO_IP4] = (const char *const[]){ "ip4-drop", 0 }, + [DPO_PROTO_IP6] = (const char *const[]){ "ip6-drop", 0 }, +}; + +static fib_route_path_t * +session_sdl_fib_create_route_paths (u32 fib_index, dpo_proto_t dpo_proto) +{ + fib_route_path_t *paths = 0; + fib_route_path_t path = { + .frp_proto = dpo_proto, + .frp_flags = FIB_ROUTE_PATH_EXCLUSIVE, + .frp_fib_index = fib_index, + .frp_sw_if_index = ~0, + .frp_weight = 1, + }; + vec_add1 (paths, path); + return paths; +} + +static void +session_sdl_dpo_lock (dpo_id_t *dpo) +{ +} + +static void +session_sdl_dpo_unlock (dpo_id_t *dpo) +{ +} + +static u8 * +format_session_sdl_dpo (u8 *s, va_list *va) +{ + index_t index = va_arg (*va, index_t); + + return format (s, "sdl: [index: %u, deny]", index); +} + +static const dpo_vft_t session_sdl_dpo_vft = { + .dv_lock = session_sdl_dpo_lock, + .dv_unlock = session_sdl_dpo_unlock, + .dv_format = format_session_sdl_dpo, +}; + +static u32 +session_sdl_lookup6 (u32 srtg_handle, u32 proto, ip6_address_t *lcl_ip, + ip6_address_t *rmt_ip, u16 lcl_port, u16 rmt_port) +{ + session_rules_table_t *srt = srtg_handle_to_srt (srtg_handle, 0); + session_sdl_block_t *sdlb = &srt->sdl_block; + index_t lbi; + const dpo_id_t *dpo; + + if (sdlb->ip6_fib_index == ~0) + return SESSION_TABLE_INVALID_INDEX; + lbi = ip6_fib_table_fwding_lookup (sdlb->ip6_fib_index, lcl_ip); + dpo = load_balance_get_fwd_bucket (load_balance_get (lbi), 0); + if (dpo->dpoi_type != sdl_dpo_type) + return SESSION_TABLE_INVALID_INDEX; + return (dpo->dpoi_index); +} + +static u32 +session_sdl_lookup4 (u32 srtg_handle, u32 proto, ip4_address_t *lcl_ip, + ip4_address_t *rmt_ip, u16 lcl_port, u16 rmt_port) +{ + session_rules_table_t *srt = srtg_handle_to_srt (srtg_handle, 0); + session_sdl_block_t *sdlb = &srt->sdl_block; + index_t lbi; + const dpo_id_t *dpo; + + if (sdlb->ip_fib_index == ~0) + return SESSION_TABLE_INVALID_INDEX; + lbi = ip4_fib_forwarding_lookup (sdlb->ip_fib_index, lcl_ip); + dpo = load_balance_get_fwd_bucket (load_balance_get (lbi), 0); + if (dpo->dpoi_type != sdl_dpo_type) + return SESSION_TABLE_INVALID_INDEX; + return (dpo->dpoi_index); +} + +typedef struct session_sdl4_fib_show_walk_ctx_t_ +{ + fib_node_index_t *ifsw_indicies; +} session_sdl4_fib_show_walk_ctx_t; + +static fib_table_walk_rc_t +session_sdl4_fib_show_walk_cb (fib_node_index_t fei, void *arg) +{ + session_sdl4_fib_show_walk_ctx_t *ctx = arg; + + vec_add1 (ctx->ifsw_indicies, fei); + + return (FIB_TABLE_WALK_CONTINUE); +} + +typedef struct session_sdl6_fib_show_ctx_t_ +{ + fib_node_index_t *entries; +} session_sdl6_fib_show_ctx_t; + +static fib_table_walk_rc_t +session_sdl6_fib_table_show_walk (fib_node_index_t fei, void *arg) +{ + session_sdl6_fib_show_ctx_t *ctx = arg; + + vec_add1 (ctx->entries, fei); + + return (FIB_TABLE_WALK_CONTINUE); +} + +static void +session_sdl_fib_table_show (u32 fei, ip46_address_t *lcl_ip, u16 fp_len, + u32 action_index, u32 fp_proto, u8 *tag, + void *args) +{ + vlib_main_t *vm = args; + u32 type = (fp_proto == FIB_PROTOCOL_IP4) ? IP46_TYPE_IP4 : IP46_TYPE_IP6; + + vlib_cli_output (vm, "[%d] rule: %U/%d action: %d tag %U", fei, + format_ip46_address, lcl_ip, type, fp_len, action_index, + format_session_rule_tag, tag); +} + +static void +session_sdl_cli_dump (vlib_main_t *vm, u32 srtg_handle, u32 proto, + u8 fib_proto) +{ + session_rules_table_t *srt = srtg_handle_to_srt (srtg_handle, 0); + session_sdl_block_t *sdlb = &srt->sdl_block; + + if (fib_proto == FIB_PROTOCOL_IP4) + { + vlib_cli_output (vm, "IP4 rules, fib index %d", sdlb->ip_fib_index); + session_sdl_table_walk4 (srtg_handle, session_sdl_fib_table_show, vm); + } + else if (fib_proto == FIB_PROTOCOL_IP6) + { + vlib_cli_output (vm, "IP6 rules, fib index %d", sdlb->ip6_fib_index); + session_sdl_table_walk6 (srtg_handle, session_sdl_fib_table_show, vm); + } +} + +static void +session_sdl4_fib_table_show_one (session_rules_table_t *srt, u32 fib_index, + vlib_main_t *vm, ip4_address_t *address, + u32 mask_len) +{ + ip4_fib_t *fib; + fib_node_index_t fei; + + fib = ip4_fib_get (fib_index); + fei = ip4_fib_table_lookup (fib, address, mask_len); + if (fei != FIB_NODE_INDEX_INVALID && fib_entry_is_sourced (fei, sdl_fib_src)) + { + u8 *tag = session_rules_table_rule_tag (srt, fei, 1); + fib_entry_t *fib_entry = fib_entry_get (fei); + fib_prefix_t pfx = fib_entry->fe_prefix; + index_t lbi = ip4_fib_forwarding_lookup (fib_index, &pfx.fp_addr.ip4); + const dpo_id_t *dpo = + load_balance_get_fwd_bucket (load_balance_get (lbi), 0); + + session_sdl_fib_table_show (fei, &pfx.fp_addr, pfx.fp_len, + dpo->dpoi_index, FIB_PROTOCOL_IP4, tag, vm); + } +} + +static void +session_sdl6_fib_table_show_one (session_rules_table_t *srt, u32 fib_index, + vlib_main_t *vm, ip6_address_t *address, + u32 mask_len) +{ + fib_node_index_t fei; + + fei = ip6_fib_table_lookup (fib_index, address, mask_len); + if (fei != FIB_NODE_INDEX_INVALID && fib_entry_is_sourced (fei, sdl_fib_src)) + { + u8 *tag = session_rules_table_rule_tag (srt, fei, 0); + fib_entry_t *fib_entry = fib_entry_get (fei); + fib_prefix_t pfx = fib_entry->fe_prefix; + index_t lbi = ip6_fib_table_fwding_lookup (fib_index, &pfx.fp_addr.ip6); + const dpo_id_t *dpo = + load_balance_get_fwd_bucket (load_balance_get (lbi), 0); + + session_sdl_fib_table_show (fei, &pfx.fp_addr, pfx.fp_len, + dpo->dpoi_index, FIB_PROTOCOL_IP6, tag, vm); + } +} + +static void +session_sdl_show_rule (vlib_main_t *vm, u32 srtg_handle, u32 proto, + ip46_address_t *lcl_ip, u16 lcl_port, + ip46_address_t *rmt_ip, u16 rmt_port, u8 is_ip4) +{ + session_rules_table_t *srt = srtg_handle_to_srt (srtg_handle, 0); + session_sdl_block_t *sdlb; + + sdlb = &srt->sdl_block; + if (is_ip4) + session_sdl4_fib_table_show_one (srt, sdlb->ip_fib_index, vm, &lcl_ip->ip4, + 32); + else + session_sdl6_fib_table_show_one (srt, sdlb->ip6_fib_index, vm, + &lcl_ip->ip6, 128); +} + +static void +session_sdl_table_init (session_table_t *st, u8 fib_proto) +{ + session_rules_table_t *srt; + session_sdl_block_t *sdlb; + u8 all = fib_proto > FIB_PROTOCOL_IP6 ? 1 : 0; + char name[80]; + app_namespace_t *app_ns = app_namespace_get (st->appns_index); + session_rules_table_group_t *srtg; + + /* Don't support local table */ + if (st->is_local == 1) + return; + + srtg = srtg_instance_alloc (st, 0); + srt = srtg->session_rules; + sdlb = &srt->sdl_block; + + if (fib_proto == FIB_PROTOCOL_IP4 || all) + { + snprintf (name, sizeof (name), "sdl4 %s", app_ns->ns_id); + sdlb->ip_table_id = ip_table_get_unused_id (FIB_PROTOCOL_IP4); + sdlb->ip_fib_index = fib_table_find_or_create_and_lock_w_name ( + FIB_PROTOCOL_IP4, sdlb->ip_table_id, sdl_fib_src, (const u8 *) name); + } + + if (fib_proto == FIB_PROTOCOL_IP6 || all) + { + snprintf (name, sizeof (name), "sdl6 %s", app_ns->ns_id); + sdlb->ip6_table_id = ip_table_get_unused_id (FIB_PROTOCOL_IP6); + sdlb->ip6_fib_index = fib_table_find_or_create_and_lock_w_name ( + FIB_PROTOCOL_IP6, sdlb->ip6_table_id, sdl_fib_src, (const u8 *) name); + } + + srt->rules_by_tag = hash_create_vec (0, sizeof (u8), sizeof (uword)); + srt->tags_by_rules = hash_create (0, sizeof (uword)); +} + +static void +session_sdl_table_free (session_table_t *st, u8 fib_proto) +{ + session_rules_table_t *srt = srtg_handle_to_srt (st->srtg_handle, 0); + session_sdl_block_t *sdlb; + u8 all = fib_proto > FIB_PROTOCOL_IP6 ? 1 : 0; + + ASSERT (st->is_local == 0); + sdlb = &srt->sdl_block; + if ((fib_proto == FIB_PROTOCOL_IP4 || all) && (sdlb->ip_fib_index != ~0)) + { + fib_table_flush (sdlb->ip_fib_index, FIB_PROTOCOL_IP4, sdl_fib_src); + fib_table_unlock (sdlb->ip_fib_index, FIB_PROTOCOL_IP4, sdl_fib_src); + } + if ((fib_proto == FIB_PROTOCOL_IP6 || all) && (sdlb->ip6_fib_index != ~0)) + { + fib_table_flush (sdlb->ip6_fib_index, FIB_PROTOCOL_IP6, sdl_fib_src); + fib_table_unlock (sdlb->ip6_fib_index, FIB_PROTOCOL_IP6, sdl_fib_src); + } + + hash_free (srt->tags_by_rules); + hash_free (srt->rules_by_tag); + + srtg_instance_free (st); +} + +static session_error_t +session_sdl_add_del (u32 srtg_handle, u32 proto, + session_rule_table_add_del_args_t *args) +{ + session_rules_table_t *srt = srtg_handle_to_srt (srtg_handle, 0); + session_sdl_block_t *sdlb = &srt->sdl_block; + u32 fib_index; + dpo_proto_t dpo_proto; + fib_route_path_t *paths = 0; + fib_prefix_t pfx = args->lcl; + session_error_t err = SESSION_E_NONE; + fib_node_index_t fei; + int is_ip4; + + if (!(args->lcl_port == 0 && args->rmt_port == 0 && + args->rmt.fp_addr.ip4.as_u32 == 0)) + return SESSION_E_NOSUPPORT; + + fei = session_rules_table_rule_for_tag (srt, args->tag); + if (args->is_add && fei != SESSION_RULES_TABLE_INVALID_INDEX) + return SESSION_E_INVALID; + + if (args->lcl.fp_proto == FIB_PROTOCOL_IP4) + { + fib_index = sdlb->ip_fib_index; + dpo_proto = DPO_PROTO_IP4; + is_ip4 = 1; + } + else + { + fib_index = sdlb->ip6_fib_index; + dpo_proto = DPO_PROTO_IP6; + is_ip4 = 0; + } + + paths = session_sdl_fib_create_route_paths (fib_index, dpo_proto); + if (args->is_add) + { + fei = fib_table_lookup_exact_match (fib_index, &pfx); + if (fei != FIB_NODE_INDEX_INVALID) + { + err = SESSION_E_IPINUSE; + goto done; + } + dpo_set (&paths->dpo, sdl_dpo_type, dpo_proto, args->action_index); + fei = fib_table_entry_path_add2 (fib_index, &pfx, sdl_fib_src, + FIB_ENTRY_FLAG_EXCLUSIVE, paths); + session_rules_table_add_tag (srt, args->tag, fei, is_ip4); + dpo_reset (&paths->dpo); + } + else + { + if (fei == SESSION_RULES_TABLE_INVALID_INDEX) + { + fei = fib_table_lookup_exact_match (fib_index, &pfx); + + if (fei == FIB_NODE_INDEX_INVALID) + { + err = SESSION_E_NOROUTE; + goto done; + } + } + + if (!fib_entry_is_sourced (fei, sdl_fib_src)) + { + err = SESSION_E_NOROUTE; + goto done; + } + + fib_entry_t *fib_entry = fib_entry_get (fei); + pfx = fib_entry->fe_prefix; + fib_table_entry_special_remove (fib_index, &pfx, sdl_fib_src); + session_rules_table_del_tag (srt, args->tag, is_ip4); + } +done: + vec_free (paths); + + return err; +} + +static const session_rt_engine_vft_t session_sdl_vft = { + .backend_engine = RT_BACKEND_ENGINE_SDL, + .table_lookup4 = session_sdl_lookup4, + .table_lookup6 = session_sdl_lookup6, + .table_cli_dump = session_sdl_cli_dump, + .table_show_rule = session_sdl_show_rule, + .table_add_del = session_sdl_add_del, + .table_init = session_sdl_table_init, + .table_free = session_sdl_table_free, +}; + +static void +session_sdl_fib_init (void) +{ + static u32 session_fib_inited = 0; + + if (session_fib_inited) + return; + session_fib_inited = 1; + sdl_fib_src = fib_source_allocate ("session sdl", FIB_SOURCE_PRIORITY_LOW, + FIB_SOURCE_BH_SIMPLE); + sdl_dpo_type = + dpo_register_new_type (&session_sdl_dpo_vft, session_sdl_dpo_nodes); +} + +static void +session_sdl_app_namespace_walk_cb (app_namespace_t *app_ns, void *ctx) +{ + u32 fib_index, table_index; + session_table_t *st; + + log_debug ("disable app_ns %s", app_ns->ns_id); + + fib_index = app_namespace_get_fib_index (app_ns, FIB_PROTOCOL_IP4); + table_index = session_lookup_get_index_for_fib (FIB_PROTOCOL_IP4, fib_index); + st = session_table_get (table_index); + if (st) + session_rules_table_free (st, FIB_PROTOCOL_IP4); + + fib_index = app_namespace_get_fib_index (app_ns, FIB_PROTOCOL_IP6); + table_index = session_lookup_get_index_for_fib (FIB_PROTOCOL_IP6, fib_index); + st = session_table_get (table_index); + if (st) + session_rules_table_free (st, FIB_PROTOCOL_IP6); +} + +clib_error_t * +session_sdl_enable_disable (int enable) +{ + clib_error_t *error = 0; + + if (enable) + { + error = session_rule_table_register_engine (&session_sdl_vft); + if (error) + { + log_err ("error in enabling sdl: %U", format_clib_error, error); + return error; + } + session_sdl_fib_init (); + } + else + { + app_namespace_walk (session_sdl_app_namespace_walk_cb, 0); + + error = session_rule_table_deregister_engine (&session_sdl_vft); + if (error) + log_err ("error in disabling sdl: %U", format_clib_error, error); + } + + return error; +} + +/* + * Source Deny List + */ +static clib_error_t * +session_sdl_command_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + u32 appns_index; + app_namespace_t *app_ns; + u32 lcl_plen = 0, action = 0; + clib_error_t *error = 0; + ip46_address_t lcl_ip; + u8 conn_set = 0; + u8 fib_proto = -1, is_add = 1, *ns_id = 0; + u8 *tag = 0, tag_only = 0; + int rv; + session_rule_add_del_args_t args; + + session_cli_return_if_not_enabled (); + + if (session_sdl_is_enabled () == 0) + return clib_error_return (0, "session sdl engine is not enabled"); + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "del")) + is_add = 0; + else if (unformat (input, "add")) + ; + else if (unformat (input, "appns %_%v%_", &ns_id)) + ; + else if (unformat (input, "%U/%d", unformat_ip4_address, &lcl_ip.ip4, + &lcl_plen)) + { + fib_proto = FIB_PROTOCOL_IP4; + conn_set = 1; + } + else if (unformat (input, "%U/%d", unformat_ip6_address, &lcl_ip.ip6, + &lcl_plen)) + { + fib_proto = FIB_PROTOCOL_IP6; + conn_set = 1; + } + else if (unformat (input, "action %d", &action)) + ; + else if (unformat (input, "tag %_%v%_", &tag)) + ; + else + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, input); + goto done; + } + } + + if (ns_id) + { + app_ns = app_namespace_get_from_id (ns_id); + if (!app_ns) + { + vlib_cli_output (vm, "namespace %v does not exist", ns_id); + goto done; + } + } + else + { + app_ns = app_namespace_get_default (); + } + appns_index = app_namespace_index (app_ns); + + if (is_add && !conn_set && action == 0) + { + vlib_cli_output (vm, "connection and action must be set for add"); + goto done; + } + if (!is_add && !tag && !conn_set) + { + vlib_cli_output (vm, "connection or tag must be set for delete"); + goto done; + } + if (vec_len (tag) > SESSION_RULE_TAG_MAX_LEN) + { + vlib_cli_output (vm, "tag too long (max u64)"); + goto done; + } + + /* Delete with only tag entered. Try v4 first and then v6 if failed */ + if ((is_add == 0) && (fib_proto == (u8) ~0)) + { + fib_proto = FIB_PROTOCOL_IP4; + tag_only = 1; + } + + memset (&args, 0, sizeof (args)); + args.transport_proto = TRANSPORT_PROTO_TCP; + args.table_args.lcl.fp_addr = lcl_ip; + args.table_args.lcl.fp_len = lcl_plen; + args.table_args.lcl.fp_proto = fib_proto; + args.table_args.rmt.fp_proto = fib_proto; + args.table_args.action_index = action; + args.table_args.is_add = is_add; + args.table_args.tag = tag; + args.appns_index = appns_index; + args.scope = SESSION_RULE_SCOPE_GLOBAL; + + if ((rv = vnet_session_rule_add_del (&args))) + { + /* Try tag only delete on v6 */ + if (rv && tag_only) + { + args.table_args.rmt.fp_proto = FIB_PROTOCOL_IP6; + args.table_args.lcl.fp_proto = FIB_PROTOCOL_IP6; + if ((rv = vnet_session_rule_add_del (&args))) + { + error = clib_error_return (0, "sdl add del returned %u", rv); + } + } + else + { + error = clib_error_return (0, "sdl add del returned %u", rv); + } + } + +done: + vec_free (ns_id); + vec_free (tag); + return error; +} + +VLIB_CLI_COMMAND (session_sdl_command, static) = { + .path = "session sdl", + .short_help = "session sdl [add|del] [appns <ns_id>] <lcl-ip/plen> action " + "<action> [tag <tag>]", + .function = session_sdl_command_fn, + .is_mp_safe = 1, +}; + +static clib_error_t * +show_session_sdl_command_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + u32 fib_index; + ip46_address_t lcl_ip; + u8 show_one = 0; + app_namespace_t *app_ns; + session_table_t *st; + u8 *ns_id = 0, fib_proto = FIB_PROTOCOL_IP4; + + session_cli_return_if_not_enabled (); + + clib_memset (&lcl_ip, 0, sizeof (lcl_ip)); + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "appns %_%s%_", &ns_id)) + ; + else if (unformat (input, "%U", unformat_ip4_address, &lcl_ip.ip4)) + { + fib_proto = FIB_PROTOCOL_IP4; + show_one = 1; + } + else if (unformat (input, "%U", unformat_ip6_address, &lcl_ip.ip6)) + { + fib_proto = FIB_PROTOCOL_IP6; + show_one = 1; + } + else + { + vec_free (ns_id); + return clib_error_return (0, "unknown input `%U'", + format_unformat_error, input); + } + } + + if (ns_id) + { + app_ns = app_namespace_get_from_id (ns_id); + if (!app_ns) + { + vlib_cli_output (vm, "appns %v doesn't exist", ns_id); + goto done; + } + } + else + { + app_ns = app_namespace_get_default (); + } + + if (session_sdl_is_enabled () == 0) + { + vlib_cli_output (vm, "session sdl engine is not enabled"); + goto done; + } + + if (show_one) + { + fib_index = app_namespace_get_fib_index (app_ns, fib_proto); + st = session_table_get_for_fib_index (fib_proto, fib_index); + if (st && (st->srtg_handle != SESSION_SRTG_HANDLE_INVALID)) + session_rules_table_show_rule (vm, st->srtg_handle, 0, &lcl_ip, 0, 0, + 0, (fib_proto == FIB_PROTOCOL_IP4)); + goto done; + } + + /* 2 separate session tables for global entries, 1 for ip4 and 1 for ip6 */ + fib_index = app_namespace_get_fib_index (app_ns, FIB_PROTOCOL_IP4); + st = session_table_get_for_fib_index (FIB_PROTOCOL_IP4, fib_index); + if (st && (st->srtg_handle != SESSION_SRTG_HANDLE_INVALID)) + session_rules_table_cli_dump (vm, st->srtg_handle, 0, FIB_PROTOCOL_IP4); + + fib_index = app_namespace_get_fib_index (app_ns, FIB_PROTOCOL_IP6); + st = session_table_get_for_fib_index (FIB_PROTOCOL_IP6, fib_index); + if (st && (st->srtg_handle != SESSION_SRTG_HANDLE_INVALID)) + session_rules_table_cli_dump (vm, st->srtg_handle, 0, FIB_PROTOCOL_IP6); +done: + vec_free (ns_id); + return 0; +} + +void +session_sdl_table_walk4 (u32 srtg_handle, session_sdl_table_walk_fn_t fn, + void *args) +{ + ip4_fib_t *fib; + session_sdl4_fib_show_walk_ctx_t ctx = { + .ifsw_indicies = NULL, + }; + fib_node_index_t *fei; + session_rules_table_t *srt = srtg_handle_to_srt (srtg_handle, 0); + session_sdl_block_t *sdlb = &srt->sdl_block; + u32 fib_index = sdlb->ip_fib_index; + + if (fib_index == ~0) + return; + fib = ip4_fib_get (fib_index); + ip4_fib_table_walk (fib, session_sdl4_fib_show_walk_cb, &ctx); + vec_sort_with_function (ctx.ifsw_indicies, fib_entry_cmp_for_sort); + + vec_foreach (fei, ctx.ifsw_indicies) + { + if (*fei != FIB_NODE_INDEX_INVALID && + fib_entry_is_sourced (*fei, sdl_fib_src)) + { + u8 *tag = session_rules_table_rule_tag (srt, *fei, 1); + fib_entry_t *fib_entry = fib_entry_get (*fei); + fib_prefix_t pfx = fib_entry->fe_prefix; + index_t lbi = + ip4_fib_forwarding_lookup (fib_index, &pfx.fp_addr.ip4); + const dpo_id_t *dpo = + load_balance_get_fwd_bucket (load_balance_get (lbi), 0); + + fn (*fei, &pfx.fp_addr, pfx.fp_len, dpo->dpoi_index, + FIB_PROTOCOL_IP4, tag, args); + } + } + + vec_free (ctx.ifsw_indicies); +} + +void +session_sdl_table_walk6 (u32 srtg_handle, session_sdl_table_walk_fn_t fn, + void *args) +{ + ip6_fib_t *fib; + fib_node_index_t *fei; + session_sdl6_fib_show_ctx_t ctx = { + .entries = NULL, + }; + session_rules_table_t *srt = srtg_handle_to_srt (srtg_handle, 0); + session_sdl_block_t *sdlb = &srt->sdl_block; + u32 fib_index = sdlb->ip6_fib_index; + + if (fib_index == ~0) + return; + fib = ip6_fib_get (fib_index); + ip6_fib_table_walk (fib->index, session_sdl6_fib_table_show_walk, &ctx); + vec_sort_with_function (ctx.entries, fib_entry_cmp_for_sort); + + vec_foreach (fei, ctx.entries) + { + if (*fei != FIB_NODE_INDEX_INVALID && + fib_entry_is_sourced (*fei, sdl_fib_src)) + { + u8 *tag = session_rules_table_rule_tag (srt, *fei, 0); + fib_entry_t *fib_entry = fib_entry_get (*fei); + fib_prefix_t pfx = fib_entry->fe_prefix; + index_t lbi = + ip6_fib_table_fwding_lookup (fib_index, &pfx.fp_addr.ip6); + const dpo_id_t *dpo = + load_balance_get_fwd_bucket (load_balance_get (lbi), 0); + + fn (*fei, &pfx.fp_addr, pfx.fp_len, dpo->dpoi_index, + FIB_PROTOCOL_IP6, tag, args); + } + } + + vec_free (ctx.entries); +} + +VLIB_CLI_COMMAND (show_session_sdl_command, static) = { + .path = "show session sdl", + .short_help = "show session sdl [appns <id> <lcl-ip>]", + .function = show_session_sdl_command_fn, + .is_mp_safe = 1, +}; + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vnet/session/session_sdl.h b/src/vnet/session/session_sdl.h new file mode 100644 index 00000000000..8d8b5b2d29e --- /dev/null +++ b/src/vnet/session/session_sdl.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2024 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_VNET_SESSION_SESSION_SDL_H_ +#define SRC_VNET_SESSION_SESSION_SDL_H_ + +clib_error_t *session_sdl_enable_disable (int enable); + +typedef void (*session_sdl_table_walk_fn_t) (u32 fei, ip46_address_t *lcl_ip, + u16 fp_len, u32 action_index, + u32 fb_proto, u8 *tag, void *ctx); +void session_sdl_table_walk4 (u32 srtg_handle, session_sdl_table_walk_fn_t fn, + void *args); +void session_sdl_table_walk6 (u32 srtg_handle, session_sdl_table_walk_fn_t fn, + void *args); + +#endif /* SRC_VNET_SESSION_SESSION_SDL_H_ */ +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vnet/session/session_table.c b/src/vnet/session/session_table.c index dbbe771979c..5dafe0e633c 100644 --- a/src/vnet/session/session_table.c +++ b/src/vnet/session/session_table.c @@ -15,6 +15,7 @@ #include <vnet/session/session_table.h> #include <vnet/session/session.h> +#include <vnet/session/session_rules_table.h> /** * Pool of session tables @@ -64,12 +65,8 @@ void session_table_free (session_table_t *slt, u8 fib_proto) { u8 all = fib_proto > FIB_PROTOCOL_IP6 ? 1 : 0; - int i; - for (i = 0; i < TRANSPORT_N_PROTOS; i++) - session_rules_table_free (&slt->session_rules[i]); - - vec_free (slt->session_rules); + session_rules_table_free (slt, fib_proto); if (fib_proto == FIB_PROTOCOL_IP4 || all) { @@ -92,10 +89,9 @@ session_table_free (session_table_t *slt, u8 fib_proto) * otherwise it uses defaults above. */ void -session_table_init (session_table_t * slt, u8 fib_proto) +session_table_init (session_table_t *slt, u8 fib_proto) { u8 all = fib_proto > FIB_PROTOCOL_IP6 ? 1 : 0; - int i; #define _(af,table,parm,value) \ u32 configured_##af##_##table##_table_##parm = value; @@ -109,6 +105,7 @@ session_table_init (session_table_t * slt, u8 fib_proto) foreach_hash_table_parameter; #undef _ + slt->srtg_handle = SESSION_SRTG_HANDLE_INVALID; if (fib_proto == FIB_PROTOCOL_IP4 || all) { clib_bihash_init2_args_16_8_t _a, *a = &_a; @@ -153,10 +150,6 @@ session_table_init (session_table_t * slt, u8 fib_proto) a->instantiate_immediately = 1; clib_bihash_init2_48_8 (a); } - - vec_validate (slt->session_rules, TRANSPORT_N_PROTOS - 1); - for (i = 0; i < TRANSPORT_N_PROTOS; i++) - session_rules_table_init (&slt->session_rules[i]); } typedef struct _ip4_session_table_walk_ctx_t diff --git a/src/vnet/session/session_table.h b/src/vnet/session/session_table.h index 636b8d77bee..aae4a1c2af5 100644 --- a/src/vnet/session/session_table.h +++ b/src/vnet/session/session_table.h @@ -18,7 +18,6 @@ #include <vppinfra/bihash_16_8.h> #include <vppinfra/bihash_48_8.h> -#include <vnet/session/session_rules_table.h> typedef struct _session_lookup_table { @@ -37,7 +36,7 @@ typedef struct _session_lookup_table /** * Per fib proto and transport proto session rules tables */ - session_rules_table_t *session_rules; + u32 srtg_handle; /** Flag that indicates if table has local scope */ u8 is_local; @@ -78,6 +77,8 @@ session_table_t *_get_session_tables (); #define session_table_foreach(VAR, BODY) \ pool_foreach (VAR, _get_session_tables ()) BODY +void session_lookup_table_cleanup (u32 fib_proto, u32 fib_index); + #endif /* SRC_VNET_SESSION_SESSION_TABLE_H_ */ /* * fd.io coding-style-patch-verification: ON diff --git a/src/vnet/session/session_test.c b/src/vnet/session/session_test.c index 770e7263024..1c865b877b5 100644 --- a/src/vnet/session/session_test.c +++ b/src/vnet/session/session_test.c @@ -271,6 +271,12 @@ api_session_enable_disable (vat_main_t *vat) } static int +api_session_enable_disable_v2 (vat_main_t *vat) +{ + return -1; +} + +static int api_app_worker_add_del (vat_main_t *vat) { return -1; @@ -354,6 +360,23 @@ api_session_sapi_enable_disable (vat_main_t *vat) return -1; } +static int +api_session_sdl_add_del (vat_main_t *vam) +{ + return -1; +} + +static void +vl_api_session_sdl_details_t_handler (vl_api_session_rules_details_t *mp) +{ +} + +static int +api_session_sdl_dump (vat_main_t *vam) +{ + return -1; +} + #include <vnet/session/session.api_test.c> /* diff --git a/src/vnet/snap/snap.c b/src/vnet/snap/snap.c index 9bee415390a..bf5994271a4 100644 --- a/src/vnet/snap/snap.c +++ b/src/vnet/snap/snap.c @@ -192,8 +192,9 @@ snap_init (vlib_main_t * vm) return vlib_call_init_function (vm, snap_input_init); } -VLIB_INIT_FUNCTION (snap_init); - +VLIB_INIT_FUNCTION (snap_init) = { + .runs_after = VLIB_INITS ("llc_init"), +}; /* * fd.io coding-style-patch-verification: ON diff --git a/src/vnet/srv6/sr_localsid.c b/src/vnet/srv6/sr_localsid.c index 12349bb95e8..62b1a271576 100644 --- a/src/vnet/srv6/sr_localsid.c +++ b/src/vnet/srv6/sr_localsid.c @@ -1471,6 +1471,7 @@ sr_localsid_fn (vlib_main_t * vm, vlib_node_runtime_t * node, ip6_header_t *ip0, *ip1, *ip2, *ip3; ip6_sr_header_t *sr0, *sr1, *sr2, *sr3; ip6_ext_header_t *prev0, *prev1, *prev2, *prev3; + prev0 = prev1 = prev2 = prev3 = 0; u32 next0, next1, next2, next3; next0 = next1 = next2 = next3 = SR_LOCALSID_NEXT_IP6_LOOKUP; ip6_sr_localsid_t *ls0, *ls1, *ls2, *ls3; @@ -1779,6 +1780,7 @@ sr_localsid_un_fn (vlib_main_t * vm, vlib_node_runtime_t * node, ip6_header_t *ip0, *ip1, *ip2, *ip3; ip6_sr_header_t *sr0, *sr1, *sr2, *sr3; ip6_ext_header_t *prev0, *prev1, *prev2, *prev3; + prev0 = prev1 = prev2 = prev3 = 0; u32 next0, next1, next2, next3; next0 = next1 = next2 = next3 = SR_LOCALSID_NEXT_IP6_LOOKUP; ip6_sr_localsid_t *ls0, *ls1, *ls2, *ls3; diff --git a/src/vnet/tcp/tcp.c b/src/vnet/tcp/tcp.c index efc72a227e8..1afc07918b7 100644 --- a/src/vnet/tcp/tcp.c +++ b/src/vnet/tcp/tcp.c @@ -879,6 +879,8 @@ format_tcp_listener_session (u8 * s, va_list * args) if (verbose) s = format (s, "%-" SESSION_CLI_STATE_LEN "U", format_tcp_state, tc->state); + if (verbose == 2) + s = format (s, "\n%U", format_tcp_listener_connection, tc); return s; } @@ -1512,6 +1514,10 @@ tcp_main_enable (vlib_main_t * vm) clib_error_t *error = 0; int thread; + /* Already initialized */ + if (tm->wrk_ctx) + return 0; + if ((error = vlib_call_init_function (vm, ip_main_init))) return error; if ((error = vlib_call_init_function (vm, ip4_lookup_init))) @@ -1642,6 +1648,9 @@ tcp_configuration_init (void) /* This value is seconds */ tcp_cfg.cleanup_time = 0.1; /* 100ms */ + + /* Time constants defined as tcp tick (1us) multiples */ + tcp_cfg.syn_rcvd_time = TCP_ESTABLISH_TIME; } static clib_error_t * diff --git a/src/vnet/tcp/tcp.h b/src/vnet/tcp/tcp.h index 2362a8bb857..8676db413a0 100644 --- a/src/vnet/tcp/tcp.h +++ b/src/vnet/tcp/tcp.h @@ -45,19 +45,21 @@ typedef struct _tcp_lookup_dispatch u8 next, error; } tcp_lookup_dispatch_t; -#define foreach_tcp_wrk_stat \ - _(timer_expirations, u64, "timer expirations") \ - _(rxt_segs, u64, "segments retransmitted") \ - _(tr_events, u32, "timer retransmit events") \ - _(to_closewait, u32, "timeout close-wait") \ - _(to_closewait2, u32, "timeout close-wait w/data") \ - _(to_finwait1, u32, "timeout fin-wait-1") \ - _(to_finwait2, u32, "timeout fin-wait-2") \ - _(to_lastack, u32, "timeout last-ack") \ - _(to_closing, u32, "timeout closing") \ - _(tr_abort, u32, "timer retransmit abort") \ - _(rst_unread, u32, "reset on close due to unread data") \ - _(no_buffer, u32, "out of buffers") \ +#define foreach_tcp_wrk_stat \ + _ (timer_expirations, u64, "timer expirations") \ + _ (rxt_segs, u64, "segments retransmitted") \ + _ (tr_events, u32, "timer retransmit events") \ + _ (to_establish, u32, "timeout establish") \ + _ (to_persist, u32, "timeout persist") \ + _ (to_closewait, u32, "timeout close-wait") \ + _ (to_closewait2, u32, "timeout close-wait w/data") \ + _ (to_finwait1, u32, "timeout fin-wait-1") \ + _ (to_finwait2, u32, "timeout fin-wait-2") \ + _ (to_lastack, u32, "timeout last-ack") \ + _ (to_closing, u32, "timeout closing") \ + _ (tr_abort, u32, "timer retransmit abort") \ + _ (rst_unread, u32, "reset on close due to unread data") \ + _ (no_buffer, u32, "out of buffers") typedef struct tcp_wrk_stats_ { @@ -197,6 +199,9 @@ typedef struct tcp_configuration_ /** Time to wait (sec) before cleaning up the connection */ f32 cleanup_time; + /** Time to wait (tcp ticks) for syn-rcvd connection to establish */ + u32 syn_rcvd_time; + /** Number of preallocated connections */ u32 preallocated_connections; @@ -354,6 +359,7 @@ format_function_t format_tcp_flags; format_function_t format_tcp_sacks; format_function_t format_tcp_rcv_sacks; format_function_t format_tcp_connection; +format_function_t format_tcp_listener_connection; format_function_t format_tcp_connection_id; #define tcp_validate_txf_size(_tc, _a) \ diff --git a/src/vnet/tcp/tcp_cli.c b/src/vnet/tcp/tcp_cli.c index b04c0bdc0cf..55bc5764df2 100644 --- a/src/vnet/tcp/tcp_cli.c +++ b/src/vnet/tcp/tcp_cli.c @@ -250,6 +250,21 @@ format_tcp_connection_id (u8 * s, va_list * args) } u8 * +format_tcp_listener_connection (u8 *s, va_list *args) +{ + tcp_connection_t *tc = va_arg (*args, tcp_connection_t *); + + s = format (s, " index: %u cfg_flags: %U cong_algo: %s snd_mss: %u\n", + tc->c_c_index, format_tcp_cfg_flags, tc, tc->cc_algo->name, + tc->snd_mss); + s = format (s, " next_node %u opaque 0x%x fib_index %u sw_if_index %d", + tc->next_node_index, tc->next_node_opaque, tc->c_fib_index, + tc->sw_if_index); + + return s; +} + +u8 * format_tcp_connection (u8 * s, va_list * args) { tcp_connection_t *tc = va_arg (*args, tcp_connection_t *); @@ -825,6 +840,74 @@ VLIB_CLI_COMMAND (show_tcp_punt_command, static) = .function = show_tcp_punt_fn, }; +static u8 * +format_tcp_cfg (u8 *s, va_list *args) +{ + tcp_configuration_t tm_cfg = va_arg (*args, tcp_configuration_t); + + s = format (s, "max rx fifo size: %U\n", format_memory_size, + tm_cfg.max_rx_fifo); + s = format (s, "min rx fifo size: %U\n", format_memory_size, + tm_cfg.min_rx_fifo); + s = format (s, "default mtu: %u\n", tm_cfg.default_mtu); + s = format (s, "initial cwnd multiplier: %u\n", + tm_cfg.initial_cwnd_multiplier); + s = format (s, "tx pacing: %s\n", + tm_cfg.enable_tx_pacing ? "enabled" : "disabled"); + s = format (s, "tso: %s\n", tm_cfg.allow_tso ? "allowed" : "disallowed"); + s = format (s, "checksum offload: %s\n", + tm_cfg.csum_offload ? "enabled" : "disabled"); + s = format (s, "congestion control algorithm: %s\n", + tcp_cc_algo_get (tm_cfg.cc_algo)->name); + s = format (s, "min rwnd update ack: %u\n", tm_cfg.rwnd_min_update_ack); + s = format (s, "max gso packet size: %U\n", format_memory_size, + tm_cfg.max_gso_size); + s = format (s, "close_wait time: %u sec\n", + (u32) (tm_cfg.closewait_time * TCP_TIMER_TICK)); + s = format (s, "time_wait time: %u sec\n", + (u32) (tm_cfg.timewait_time * TCP_TIMER_TICK)); + s = format (s, "fin_wait1 time: %u sec\n", + (u32) (tm_cfg.finwait1_time * TCP_TIMER_TICK)); + s = format (s, "fin_wait2 time: %u sec\n", + (u32) (tm_cfg.finwait2_time * TCP_TIMER_TICK)); + s = format (s, "last_ack time: %u sec\n", + (u32) (tm_cfg.lastack_time * TCP_TIMER_TICK)); + s = format (s, "fin_ack time: %u sec\n", + (u32) (tm_cfg.closing_time * TCP_TIMER_TICK)); + s = format (s, "syn_rcvd time: %u sec\n", + (u32) (tm_cfg.syn_rcvd_time * TCP_TICK)); + s = format (s, "tcp allocation error cleanup time: %0.2f sec\n", + (f32) (tm_cfg.alloc_err_timeout * TCP_TIMER_TICK)); + s = format (s, "connection cleanup time: %.2f sec\n", tm_cfg.cleanup_time); + s = format (s, "tcp preallocated connections: %u", + tm_cfg.preallocated_connections); + + return s; +} + +static clib_error_t * +show_tcp_cfg_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + tcp_main_t *tm = vnet_get_tcp_main (); + + if (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + return clib_error_return (0, "unknown input `%U'", format_unformat_error, + input); + vlib_cli_output (vm, "-----------"); + vlib_cli_output (vm, "tcp config"); + vlib_cli_output (vm, "-----------"); + vlib_cli_output (vm, "%U\n", format_tcp_cfg, tm->cfg); + + return 0; +} + +VLIB_CLI_COMMAND (show_tcp_cfg_command, static) = { + .path = "show tcp config", + .short_help = "show tcp config", + .function = show_tcp_cfg_fn, +}; + static clib_error_t * show_tcp_stats_fn (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) @@ -1009,6 +1092,8 @@ tcp_config_fn (vlib_main_t * vm, unformat_input_t * input) tcp_cfg.alloc_err_timeout = tmp_time / TCP_TIMER_TICK; else if (unformat (input, "cleanup-time %u", &tmp_time)) tcp_cfg.cleanup_time = tmp_time / 1000.0; + else if (unformat (input, "syn-rcvd-time %u", &tmp_time)) + tcp_cfg.syn_rcvd_time = tmp_time * THZ; else return clib_error_return (0, "unknown input `%U'", format_unformat_error, input); diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c index 78148cd5695..dd1ec555902 100644 --- a/src/vnet/tcp/tcp_output.c +++ b/src/vnet/tcp/tcp_output.c @@ -1391,7 +1391,7 @@ tcp_timer_retransmit_handler (tcp_connection_t * tc) tc->rtt_ts = 0; /* Passive open establish timeout */ - if (tc->rto > TCP_ESTABLISH_TIME >> 1) + if (tc->rto > tcp_cfg.syn_rcvd_time >> 1) { tcp_connection_set_state (tc, TCP_STATE_CLOSED); tcp_connection_timers_reset (tc); @@ -1458,6 +1458,8 @@ tcp_timer_retransmit_syn_handler (tcp_connection_t * tc) TCP_EVT (TCP_EVT_CC_EVT, tc, 2); tc->rtt_ts = 0; + tcp_worker_stats_inc (wrk, to_establish, 1); + /* Active open establish timeout */ if (tc->rto >= TCP_ESTABLISH_TIME >> 1) { @@ -1507,6 +1509,8 @@ tcp_timer_persist_handler (tcp_connection_t * tc) int n_bytes = 0; u8 *data; + tcp_worker_stats_inc (wrk, to_persist, 1); + /* Problem already solved or worse */ if (tc->state == TCP_STATE_CLOSED || tc->snd_wnd > tc->snd_mss || (tc->flags & TCP_CONN_FINSNT)) diff --git a/src/vnet/tcp/tcp_timer.c b/src/vnet/tcp/tcp_timer.c index d98d0d14b17..4d1c0624fee 100644 --- a/src/vnet/tcp/tcp_timer.c +++ b/src/vnet/tcp/tcp_timer.c @@ -20,6 +20,7 @@ void tcp_timer_initialize_wheel (tcp_timer_wheel_t * tw, void (*expired_timer_cb) (u32 *), f64 now) { + ASSERT (tw->timers == 0); tw_timer_wheel_init_tcp_twsl (tw, expired_timer_cb, TCP_TIMER_TICK, ~0); tw->last_run_time = now; } diff --git a/src/vnet/tls/tls.c b/src/vnet/tls/tls.c index 5f00e6e302d..8754b67ca31 100644 --- a/src/vnet/tls/tls.c +++ b/src/vnet/tls/tls.c @@ -16,13 +16,10 @@ #include <vnet/session/application_interface.h> #include <vppinfra/lock.h> #include <vnet/tls/tls.h> +#include <vnet/tls/tls_inlines.h> static tls_main_t tls_main; -static tls_engine_vft_t *tls_vfts; - -#define TLS_INVALID_HANDLE ~0 -#define TLS_IDX_MASK 0x00FFFFFF -#define TLS_ENGINE_TYPE_SHIFT 28 +tls_engine_vft_t *tls_vfts; void tls_disconnect (u32 ctx_handle, u32 thread_index); @@ -31,7 +28,7 @@ tls_disconnect_transport (tls_ctx_t * ctx) { vnet_disconnect_args_t a = { .handle = ctx->tls_session_handle, - .app_index = tls_main.app_index, + .app_index = ctx->ts_app_index, }; if (vnet_disconnect_session (&a)) @@ -50,6 +47,21 @@ tls_get_available_engine (void) return CRYPTO_ENGINE_NONE; } +static crypto_engine_type_t +tls_get_engine_type (crypto_engine_type_t requested, + crypto_engine_type_t preferred) +{ + if (requested != CRYPTO_ENGINE_NONE) + { + if (tls_vfts[requested].ctx_alloc) + return requested; + return CRYPTO_ENGINE_NONE; + } + if (!tls_vfts[preferred].ctx_alloc) + return tls_get_available_engine (); + return preferred; +} + int tls_add_vpp_q_rx_evt (session_t * s) { @@ -295,144 +307,10 @@ send_reply: ctx->parent_app_api_context); } -static inline void -tls_ctx_parse_handle (u32 ctx_handle, u32 * ctx_index, u32 * engine_type) -{ - *ctx_index = ctx_handle & TLS_IDX_MASK; - *engine_type = ctx_handle >> TLS_ENGINE_TYPE_SHIFT; -} - -static inline crypto_engine_type_t -tls_get_engine_type (crypto_engine_type_t requested, - crypto_engine_type_t preferred) -{ - if (requested != CRYPTO_ENGINE_NONE) - { - if (tls_vfts[requested].ctx_alloc) - return requested; - return CRYPTO_ENGINE_NONE; - } - if (!tls_vfts[preferred].ctx_alloc) - return tls_get_available_engine (); - return preferred; -} - -static inline u32 -tls_ctx_alloc (crypto_engine_type_t engine_type) -{ - u32 ctx_index; - ctx_index = tls_vfts[engine_type].ctx_alloc (); - return (((u32) engine_type << TLS_ENGINE_TYPE_SHIFT) | ctx_index); -} - -static inline u32 -tls_ctx_alloc_w_thread (crypto_engine_type_t engine_type, u32 thread_index) -{ - u32 ctx_index; - ctx_index = tls_vfts[engine_type].ctx_alloc_w_thread (thread_index); - return (((u32) engine_type << TLS_ENGINE_TYPE_SHIFT) | ctx_index); -} - -static inline u32 -tls_ctx_attach (crypto_engine_type_t engine_type, u32 thread_index, void *ctx) -{ - u32 ctx_index; - ctx_index = tls_vfts[engine_type].ctx_attach (thread_index, ctx); - return (((u32) engine_type << TLS_ENGINE_TYPE_SHIFT) | ctx_index); -} - -static inline void * -tls_ctx_detach (tls_ctx_t *ctx) -{ - return tls_vfts[ctx->tls_ctx_engine].ctx_detach (ctx); -} - -static inline tls_ctx_t * -tls_ctx_get (u32 ctx_handle) -{ - u32 ctx_index, engine_type; - tls_ctx_parse_handle (ctx_handle, &ctx_index, &engine_type); - return tls_vfts[engine_type].ctx_get (ctx_index); -} - -static inline tls_ctx_t * -tls_ctx_get_w_thread (u32 ctx_handle, u8 thread_index) -{ - u32 ctx_index, engine_type; - tls_ctx_parse_handle (ctx_handle, &ctx_index, &engine_type); - return tls_vfts[engine_type].ctx_get_w_thread (ctx_index, thread_index); -} - -static inline int -tls_ctx_init_server (tls_ctx_t * ctx) -{ - return tls_vfts[ctx->tls_ctx_engine].ctx_init_server (ctx); -} - -static inline int -tls_ctx_init_client (tls_ctx_t * ctx) -{ - return tls_vfts[ctx->tls_ctx_engine].ctx_init_client (ctx); -} - -static inline int -tls_ctx_write (tls_ctx_t * ctx, session_t * app_session, - transport_send_params_t * sp) -{ - u32 n_wrote; - - sp->max_burst_size = sp->max_burst_size * TRANSPORT_PACER_MIN_MSS; - n_wrote = tls_vfts[ctx->tls_ctx_engine].ctx_write (ctx, app_session, sp); - sp->bytes_dequeued = n_wrote; - return n_wrote > 0 ? clib_max (n_wrote / TRANSPORT_PACER_MIN_MSS, 1) : 0; -} - -static inline int -tls_ctx_read (tls_ctx_t * ctx, session_t * tls_session) -{ - return tls_vfts[ctx->tls_ctx_engine].ctx_read (ctx, tls_session); -} - -static inline int -tls_ctx_transport_close (tls_ctx_t * ctx) -{ - return tls_vfts[ctx->tls_ctx_engine].ctx_transport_close (ctx); -} - -static inline int -tls_ctx_transport_reset (tls_ctx_t *ctx) -{ - return tls_vfts[ctx->tls_ctx_engine].ctx_transport_reset (ctx); -} - -static inline int -tls_ctx_app_close (tls_ctx_t * ctx) -{ - return tls_vfts[ctx->tls_ctx_engine].ctx_app_close (ctx); -} - -void -tls_ctx_free (tls_ctx_t * ctx) -{ - tls_vfts[ctx->tls_ctx_engine].ctx_free (ctx); -} - -u8 -tls_ctx_handshake_is_over (tls_ctx_t * ctx) -{ - return tls_vfts[ctx->tls_ctx_engine].ctx_handshake_is_over (ctx); -} - -int -tls_reinit_ca_chain (crypto_engine_type_t tls_engine_id) -{ - return tls_vfts[tls_engine_id].ctx_reinit_cachain (); -} - void tls_notify_app_io_error (tls_ctx_t *ctx) { - ASSERT (tls_ctx_handshake_is_over (ctx)); + ASSERT (ctx->flags & TLS_CONN_F_HS_DONE); session_transport_reset_notify (&ctx->connection); session_transport_closed_notify (&ctx->connection); @@ -771,6 +649,7 @@ tls_connect (transport_endpoint_cfg_t * tep) ctx = tls_ctx_half_open_get (ctx_index); ctx->parent_app_wrk_index = sep->app_wrk_index; ctx->parent_app_api_context = sep->opaque; + ctx->ts_app_index = tm->app_index; ctx->tcp_is_ip4 = sep->is_ip4; ctx->tls_type = sep->transport_proto; ctx->ckpair_index = ccfg->ckpair_index; @@ -870,6 +749,7 @@ tls_start_listen (u32 app_listener_index, transport_endpoint_cfg_t *tep) lctx = tls_listener_ctx_get (lctx_index); lctx->parent_app_wrk_index = sep->app_wrk_index; + lctx->ts_app_index = tm->app_index; lctx->tls_session_handle = tls_al_handle; lctx->app_session_handle = listen_session_get_handle (app_listener); lctx->tcp_is_ip4 = sep->is_ip4; @@ -1046,24 +926,26 @@ static u8 * format_tls_ctx_state (u8 * s, va_list * args) { tls_ctx_t *ctx; - session_t *ts; + session_t *as; ctx = va_arg (*args, tls_ctx_t *); - ts = session_get (ctx->c_s_index, ctx->c_thread_index); - if (ts->session_state == SESSION_STATE_LISTENING) + as = session_get (ctx->c_s_index, ctx->c_thread_index); + if (as->session_state == SESSION_STATE_LISTENING) s = format (s, "%s", "LISTEN"); else { - if (ts->session_state >= SESSION_STATE_TRANSPORT_CLOSED) + if (as->session_state == SESSION_STATE_READY) + s = format (s, "%s", "ESTABLISHED"); + else if (as->session_state == SESSION_STATE_ACCEPTING) + s = format (s, "%s", "ACCEPTING"); + else if (as->session_state == SESSION_STATE_CONNECTING) + s = format (s, "%s", "CONNECTING"); + else if (as->session_state >= SESSION_STATE_TRANSPORT_CLOSED) s = format (s, "%s", "CLOSED"); - else if (ts->session_state == SESSION_STATE_APP_CLOSED) - s = format (s, "%s", "APP-CLOSED"); - else if (ts->session_state >= SESSION_STATE_TRANSPORT_CLOSING) + else if (as->session_state >= SESSION_STATE_TRANSPORT_CLOSING) s = format (s, "%s", "CLOSING"); - else if (tls_ctx_handshake_is_over (ctx)) - s = format (s, "%s", "ESTABLISHED"); else - s = format (s, "%s", "HANDSHAKE"); + s = format (s, "UNHANDLED %u", as->session_state); } return s; diff --git a/src/vnet/tls/tls.h b/src/vnet/tls/tls.h index 6bd1371b984..e56c4c079ac 100644 --- a/src/vnet/tls/tls.h +++ b/src/vnet/tls/tls.h @@ -28,6 +28,10 @@ #define TLS_CHUNK_SIZE (1 << 14) #define TLS_CA_CERT_PATH "/etc/ssl/certs/ca-certificates.crt" +#define TLS_INVALID_HANDLE ~0 +#define TLS_IDX_MASK 0x00FFFFFF +#define TLS_ENGINE_TYPE_SHIFT 28 + #if TLS_DEBUG #define TLS_DBG(_lvl, _fmt, _args...) \ if (_lvl <= TLS_DEBUG) \ @@ -98,6 +102,7 @@ typedef struct tls_ctx_ #define parent_app_api_context c_tls_ctx_id.parent_app_api_ctx #define migration_ctx c_tls_ctx_id.migrate_ctx + u32 ts_app_index; tls_conn_flags_t flags; u8 *srv_hostname; u32 evt_index; @@ -137,9 +142,8 @@ typedef struct tls_engine_vft_ int (*ctx_init_client) (tls_ctx_t * ctx); int (*ctx_init_server) (tls_ctx_t * ctx); int (*ctx_read) (tls_ctx_t * ctx, session_t * tls_session); - int (*ctx_write) (tls_ctx_t * ctx, session_t * app_session, - transport_send_params_t * sp); - u8 (*ctx_handshake_is_over) (tls_ctx_t * ctx); + int (*ctx_write) (tls_ctx_t *ctx, session_t *app_session, + transport_send_params_t *sp); int (*ctx_start_listen) (tls_ctx_t * ctx); int (*ctx_stop_listen) (tls_ctx_t * ctx); int (*ctx_transport_close) (tls_ctx_t * ctx); @@ -148,6 +152,8 @@ typedef struct tls_engine_vft_ int (*ctx_reinit_cachain) (void); } tls_engine_vft_t; +extern tls_engine_vft_t *tls_vfts; + tls_main_t *vnet_tls_get_main (void); void tls_register_engine (const tls_engine_vft_t * vft, crypto_engine_type_t type); @@ -160,7 +166,6 @@ int tls_notify_app_connected (tls_ctx_t * ctx, session_error_t err); void tls_notify_app_enqueue (tls_ctx_t * ctx, session_t * app_session); void tls_notify_app_io_error (tls_ctx_t *ctx); void tls_disconnect_transport (tls_ctx_t * ctx); -int tls_reinit_ca_chain (crypto_engine_type_t tls_engine_id); void tls_add_postponed_ho_cleanups (u32 ho_index); void tls_flush_postponed_ho_cleanups (); diff --git a/src/vnet/tls/tls_inlines.h b/src/vnet/tls/tls_inlines.h new file mode 100644 index 00000000000..3e3f59fcf51 --- /dev/null +++ b/src/vnet/tls/tls_inlines.h @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2024 Cisco Systems, Inc. + */ + +#ifndef SRC_VNET_TLS_TLS_INLINES_H_ +#define SRC_VNET_TLS_TLS_INLINES_H_ + +#include <vnet/tls/tls.h> + +static inline void +tls_ctx_parse_handle (u32 ctx_handle, u32 *ctx_index, u32 *engine_type) +{ + *ctx_index = ctx_handle & TLS_IDX_MASK; + *engine_type = ctx_handle >> TLS_ENGINE_TYPE_SHIFT; +} + +static inline u32 +tls_ctx_alloc (crypto_engine_type_t engine_type) +{ + u32 ctx_index; + ctx_index = tls_vfts[engine_type].ctx_alloc (); + return (((u32) engine_type << TLS_ENGINE_TYPE_SHIFT) | ctx_index); +} + +static inline u32 +tls_ctx_alloc_w_thread (crypto_engine_type_t engine_type, u32 thread_index) +{ + u32 ctx_index; + ctx_index = tls_vfts[engine_type].ctx_alloc_w_thread (thread_index); + return (((u32) engine_type << TLS_ENGINE_TYPE_SHIFT) | ctx_index); +} + +static inline tls_ctx_t * +tls_ctx_get (u32 ctx_handle) +{ + u32 ctx_index, engine_type; + tls_ctx_parse_handle (ctx_handle, &ctx_index, &engine_type); + return tls_vfts[engine_type].ctx_get (ctx_index); +} + +static inline tls_ctx_t * +tls_ctx_get_w_thread (u32 ctx_handle, u8 thread_index) +{ + u32 ctx_index, engine_type; + tls_ctx_parse_handle (ctx_handle, &ctx_index, &engine_type); + return tls_vfts[engine_type].ctx_get_w_thread (ctx_index, thread_index); +} + +static inline void +tls_ctx_free (tls_ctx_t *ctx) +{ + tls_vfts[ctx->tls_ctx_engine].ctx_free (ctx); +} + +static inline int +tls_ctx_init_server (tls_ctx_t *ctx) +{ + return tls_vfts[ctx->tls_ctx_engine].ctx_init_server (ctx); +} + +static inline int +tls_ctx_init_client (tls_ctx_t *ctx) +{ + return tls_vfts[ctx->tls_ctx_engine].ctx_init_client (ctx); +} + +static inline u32 +tls_ctx_attach (crypto_engine_type_t engine_type, u32 thread_index, void *ctx) +{ + u32 ctx_index; + ctx_index = tls_vfts[engine_type].ctx_attach (thread_index, ctx); + return (((u32) engine_type << TLS_ENGINE_TYPE_SHIFT) | ctx_index); +} + +static inline void * +tls_ctx_detach (tls_ctx_t *ctx) +{ + return tls_vfts[ctx->tls_ctx_engine].ctx_detach (ctx); +} + +static inline int +tls_ctx_write (tls_ctx_t *ctx, session_t *app_session, + transport_send_params_t *sp) +{ + u32 n_wrote; + + sp->max_burst_size = sp->max_burst_size * TRANSPORT_PACER_MIN_MSS; + n_wrote = tls_vfts[ctx->tls_ctx_engine].ctx_write (ctx, app_session, sp); + sp->bytes_dequeued = n_wrote; + return n_wrote > 0 ? clib_max (n_wrote / TRANSPORT_PACER_MIN_MSS, 1) : 0; +} + +static inline int +tls_ctx_read (tls_ctx_t *ctx, session_t *tls_session) +{ + return tls_vfts[ctx->tls_ctx_engine].ctx_read (ctx, tls_session); +} + +static inline int +tls_ctx_transport_close (tls_ctx_t *ctx) +{ + return tls_vfts[ctx->tls_ctx_engine].ctx_transport_close (ctx); +} + +static inline int +tls_ctx_transport_reset (tls_ctx_t *ctx) +{ + return tls_vfts[ctx->tls_ctx_engine].ctx_transport_reset (ctx); +} + +static inline int +tls_ctx_app_close (tls_ctx_t *ctx) +{ + return tls_vfts[ctx->tls_ctx_engine].ctx_app_close (ctx); +} + +static inline int +tls_reinit_ca_chain (crypto_engine_type_t tls_engine_id) +{ + return tls_vfts[tls_engine_id].ctx_reinit_cachain (); +} + +#endif /* SRC_VNET_TLS_TLS_INLINES_H_ */
\ No newline at end of file diff --git a/src/vnet/tls/tls_record.c b/src/vnet/tls/tls_record.c new file mode 100644 index 00000000000..af7d54c466b --- /dev/null +++ b/src/vnet/tls/tls_record.c @@ -0,0 +1,279 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2024 Cisco Systems, Inc. + */ + +#include <vnet/tls/tls_record.h> + +/* + * rfc8446#section-4.1.2 + * struct { + * ProtocolVersion legacy_version = 0x0303; // TLS v1.2 + * Random random; + * opaque legacy_session_id<0..32>; + * CipherSuite cipher_suites<2..2^16-2>; + * opaque legacy_compression_methods<1..2^8-1>; + * Extension extensions<8..2^16-1>; + * } ClientHello; + */ +tls_handshake_parse_error_t +tls_handshake_client_hello_parse (u8 *b, int len, + tls_handshake_msg_info_t *info) +{ + u8 *p = b; + + if (PREDICT_FALSE (len < 2 + 32 + 1 + 2 + 2 + 2)) + return TLS_HS_PARSE_ERR_INVALID_LEN; + /* skip legacy version and random */ + p += 2 + 32; + /* legacy_session_id */ + info->legacy_session_id_len = *p; + info->legacy_session_id = p + 1; + p = info->legacy_session_id + info->legacy_session_id_len; + if (PREDICT_FALSE (p - b >= len)) + return TLS_HS_PARSE_ERR_SESSION_ID_LEN; + /* cipher_suites */ + info->cipher_suite_len = clib_net_to_host_u16 (*(u16 *) p); + info->cipher_suites = p + 2; + p = info->cipher_suites + info->cipher_suite_len; + if (PREDICT_FALSE (p - b >= len)) + return TLS_HS_PARSE_ERR_CIPHER_SUITE_LEN; + /* legacy_compression_method, only support null */ + if (PREDICT_FALSE (*p != 1 || *(p + 1) != 0)) + return TLS_HS_PARSE_ERR_COMPRESSION_METHOD; + p += 2; + /* extensions */ + info->extensions_len = clib_net_to_host_u16 (*(u16 *) p); + info->extensions = p + 2; + if (PREDICT_FALSE (info->extensions + info->extensions_len - b > len)) + return TLS_HS_PARSE_ERR_CIPHER_SUITE_LEN; + + return TLS_HS_PARSE_ERR_OK; +} + +typedef tls_handshake_parse_error_t (*tls_handshake_msg_parser) ( + u8 *b, int len, tls_handshake_msg_info_t *info); + +static tls_handshake_msg_parser tls_handshake_msg_parsers[] = { + [TLS_HS_CLIENT_HELLO] = tls_handshake_client_hello_parse, +}; + +static inline u32 +tls_handshake_ext_requested (const tls_handshake_ext_info_t *req_exts, + u32 n_reqs, tls_handshake_ext_type_t ext_type) +{ + for (int i = 0; i < n_reqs; i++) + { + if (req_exts[i].type == ext_type) + return i; + } + + return ~0; +} + +tls_handshake_parse_error_t +tls_hanshake_extensions_parse (tls_handshake_msg_info_t *info, + tls_handshake_ext_info_t **exts) +{ + tls_handshake_ext_info_t *ext; + u16 ext_type, ext_len; + u8 *b, *b_end; + + ASSERT (info->extensions != 0); + + if (info->extensions_len < 2) + return TLS_HS_PARSE_ERR_EXTENSIONS_LEN; + + b = info->extensions; + b_end = info->extensions + info->extensions_len; + + while (b < b_end) + { + ext_type = clib_net_to_host_u16 (*(u16 *) b); + b += 2; + ext_len = clib_net_to_host_u16 (*(u16 *) b); + b += 2; + + if (b + ext_len > b_end) + return TLS_HS_PARSE_ERR_EXTENSIONS_LEN; + + vec_add2 (*exts, ext, 1); + ext->type = ext_type; + ext->len = ext_len; + ext->data = b; + + b += ext_len; + } + + return TLS_HS_PARSE_ERR_OK; +} + +tls_handshake_parse_error_t +tls_hanshake_extensions_try_parse (tls_handshake_msg_info_t *info, + tls_handshake_ext_info_t *req_exts, + u32 n_reqs) +{ + u8 *b, *b_end; + u16 ext_type, ext_len; + u32 n_found = 0, ext_pos; + + ASSERT (info->extensions != 0); + + if (info->extensions_len < 2) + return TLS_HS_PARSE_ERR_EXTENSIONS_LEN; + + b = info->extensions; + b_end = info->extensions + info->extensions_len; + + while (b < b_end && n_found < n_reqs) + { + ext_type = clib_net_to_host_u16 (*(u16 *) b); + b += 2; + ext_len = clib_net_to_host_u16 (*(u16 *) b); + b += 2; + + if (b + ext_len > b_end) + return TLS_HS_PARSE_ERR_EXTENSIONS_LEN; + + ext_pos = tls_handshake_ext_requested (req_exts, n_reqs, ext_type); + if (ext_pos == ~0) + { + b += ext_len; + continue; + } + + req_exts[ext_pos].len = ext_len; + req_exts[ext_pos].data = b; + + b += ext_len; + n_found++; + } + + return TLS_HS_PARSE_ERR_OK; +} + +tls_handshake_parse_error_t +tls_handshake_message_try_parse (u8 *msg, int len, + tls_handshake_msg_info_t *info) +{ + tls_handshake_msg_t *msg_hdr = (tls_handshake_msg_t *) msg; + u8 *b = msg_hdr->message; + + info->len = tls_handshake_message_len (msg_hdr); + if (info->len > len) + return info->len > TLS_FRAGMENT_MAX_ENC_LEN ? + TLS_HS_PARSE_ERR_INVALID_LEN : + TLS_HS_PARSE_ERR_WANT_MORE; + + if (msg_hdr->msg_type >= ARRAY_LEN (tls_handshake_msg_parsers) || + !tls_handshake_msg_parsers[msg_hdr->msg_type]) + return TLS_HS_PARSE_ERR_UNSUPPORTED; + + return tls_handshake_msg_parsers[msg_hdr->msg_type](b, info->len, info); +} + +/** + * As per rfc6066#section-3 + * struct { + * NameType name_type; + * select (name_type) { + * case host_name: HostName; + * } name; + * } ServerName; + * + * enum { + * host_name(0), (255) + * } NameType; + * + * opaque HostName<1..2^16-1>; + * + * struct { + * ServerName server_name_list<1..2^16-1> + * } ServerNameList; + */ +tls_handshake_parse_error_t +tls_handshake_ext_sni_parse (tls_handshake_ext_info_t *ext_info, + tls_handshake_ext_t *ext) +{ + tls_handshake_ext_sni_t *sni = (tls_handshake_ext_sni_t *) ext; + tls_handshake_ext_sni_sn_t *sn; + u16 n_names, sn_len; + u8 *b, *b_end; + + b = ext_info->data; + b_end = b + ext_info->len; + + sni->ext.type = ext_info->type; + sni->names = 0; + n_names = clib_net_to_host_u16 (*(u16 *) b); + b += 2; + + while (b < b_end && vec_len (sni->names) < n_names) + { + /* only host name supported */ + if (b[0] != 0) + return TLS_HS_PARSE_ERR_EXT_SNI_NAME_TYPE; + + b++; + /* server name length */ + sn_len = clib_net_to_host_u16 (*(u16 *) b); + if (sn_len > TLS_EXT_SNI_MAX_LEN) + return TLS_HS_PARSE_ERR_EXT_SNI_LEN; + + b += 2; + + vec_add2 (sni->names, sn, 1); + sn->name_type = 0; + vec_validate (sn->host_name, sn_len - 1); + clib_memcpy (sn->host_name, b, sn_len); + + b += sn_len; + } + + return TLS_HS_PARSE_ERR_OK; +} + +typedef tls_handshake_parse_error_t (*tls_handshake_ext_parser) ( + tls_handshake_ext_info_t *ext_info, tls_handshake_ext_t *ext); + +static tls_handshake_ext_parser tls_handshake_ext_parsers[] = { + [TLS_EXT_SERVER_NAME] = tls_handshake_ext_sni_parse, +}; + +tls_handshake_parse_error_t +tls_handshake_ext_parse (tls_handshake_ext_info_t *ext_info, + tls_handshake_ext_t *ext) +{ + if (ext_info->type >= ARRAY_LEN (tls_handshake_ext_parsers) || + !tls_handshake_ext_parsers[ext_info->type]) + return TLS_HS_PARSE_ERR_UNSUPPORTED; + + return tls_handshake_ext_parsers[ext_info->type](ext_info, ext); +} + +static void +tls_handshake_ext_sni_free (tls_handshake_ext_t *ext) +{ + tls_handshake_ext_sni_t *sni = (tls_handshake_ext_sni_t *) ext; + tls_handshake_ext_sni_sn_t *sn; + + vec_foreach (sn, sni->names) + vec_free (sn->host_name); + + vec_free (sni->names); +} + +typedef void (*tls_handshake_ext_free_fn) (tls_handshake_ext_t *ext); + +static tls_handshake_ext_free_fn tls_handshake_ext_free_fns[] = { + [TLS_EXT_SERVER_NAME] = tls_handshake_ext_sni_free, +}; + +void +tls_handshake_ext_free (tls_handshake_ext_t *ext) +{ + if (ext->type >= ARRAY_LEN (tls_handshake_ext_free_fns) || + !tls_handshake_ext_free_fns[ext->type]) + return; + + tls_handshake_ext_free_fns[ext->type](ext); +} diff --git a/src/vnet/tls/tls_record.h b/src/vnet/tls/tls_record.h new file mode 100644 index 00000000000..3f7723f06d5 --- /dev/null +++ b/src/vnet/tls/tls_record.h @@ -0,0 +1,250 @@ + +/* SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2024 Cisco Systems, Inc. + */ + +#ifndef SRC_VNET_TLS_TLS_RECORD_H__ +#define SRC_VNET_TLS_TLS_RECORD_H__ + +#include <vppinfra/clib.h> +#include <vppinfra/error.h> + +/** + * TLS record types as per rfc8446#appendix-B.1 + */ +#define foreach_tls_content_type \ + _ (INVALID, 0) \ + _ (CHANGE_CIPHER_SPEC, 20) \ + _ (ALERT, 21) \ + _ (HANDSHAKE, 22) \ + _ (APPLICATION_DATA, 23) \ + _ (HEARTBEAT, 24) /* RFC 6520 */ + +typedef enum tls_record_type_ +{ +#define _(sym, val) TLS_REC_##sym = val, + foreach_tls_content_type +#undef _ +} __clib_packed tls_record_type_t; + +typedef struct tls_protocol_version_ +{ + u8 major; + u8 minor; +} __clib_packed tls_protocol_version_t; + +#define TLS_MAJOR_VERSION 3 +#define TLS_MINOR_VERSION_MIN 0 /**< SSLv3 */ +#define TLS_MINOR_VERSION_MAX 4 /**< TLS1.3 */ + +typedef struct tls_record_header_ +{ + tls_record_type_t type; /**< content type */ + tls_protocol_version_t version; /**< version (deprecated) */ + u16 length; /**< fragment length */ + u8 fragment[0]; /**< fragment/payload */ +} __clib_packed tls_record_header_t; + +#define TLS_FRAGMENT_MAX_LEN (1 << 14) /**< 16KB rfc8446 */ +/** rfc5246 (TLS1.2) allows 2048 bytes of protection */ +#define TLS12_FRAGMENT_MAX_ENC_LEN (TLS_FRAGMENT_MAX_LEN + (2 << 10)) +#define TLS13_FRAGMENT_MAX_ENC_LEN (TLS_FRAGMENT_MAX_LEN + 256) +#define TLS_FRAGMENT_MAX_ENC_LEN TLS12_FRAGMENT_MAX_ENC_LEN + +/* + * Handshake message types as per rfc8446#appendix-B.3 + */ +#define foreach_tls_handshake_type \ + _ (HELLO_REQUEST, 0) \ + _ (CLIENT_HELLO, 1) \ + _ (SERVER_HELLO, 2) \ + _ (HELLO_VERIFY_REQUEST, 3) \ + _ (NEW_SESSION_TICKET, 4) \ + _ (END_OF_EARLY_DATA, 5) \ + _ (HELLO_RETRY_REQUEST, 6) \ + _ (ENCRYPTED_EXTENSIONS, 8) \ + _ (CERTIFICATE, 11) \ + _ (SERVER_KEY_EXCHANGE, 12) \ + _ (CERTIFICATE_REQUEST, 13) \ + _ (SERVER_HELLO_DONE, 14) \ + _ (CERTIFICATE_VERIFY, 15) \ + _ (CLIENT_KEY_EXCHANGE, 16) \ + _ (FINISHED, 20) \ + _ (CERTIFICATE_URL, 21) \ + _ (CERTIFICATE_STATUS, 22) \ + _ (SUPPLEMENTAL_DATA, 23) \ + _ (KEY_UPDATE, 24) \ + _ (MESSAGE_HASH, 254) + +typedef enum tls_handshake_type_ +{ +#define _(sym, val) TLS_HS_##sym = val, + foreach_tls_handshake_type +#undef _ +} tls_handshake_type_t; + +typedef struct +{ + u32 msg_type : 8; /**< message type */ + u32 length : 24; /**< message length */ + u8 message[0]; /**< message contents */ +} __clib_packed tls_handshake_msg_t; + +static inline u32 +tls_handshake_message_len (tls_handshake_msg_t *msg) +{ + u8 *p = (u8 *) msg; + return p[1] << 16 | p[2] << 8 | p[3]; +} + +/** + * https://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xhtml + */ +#define foreach_tls_hanshake_extensions \ + _ (SERVER_NAME, 0) \ + _ (MAX_FRAGMENT_LENGTH, 1) \ + _ (STATUS_REQUEST, 5) \ + _ (SUPPORTED_GROUPS, 10) \ + _ (EC_POINT_FORMATS, 11) \ + _ (SIGNATURE_ALGORITHMS, 13) \ + _ (APPLICATION_LAYER_PROTOCOL_NEGOTIATION, 16) \ + _ (SIGNED_CERTIFICATE_TIMESTAMP, 18) \ + _ (CLIENT_CERTIFICATE_TYPE, 19) \ + _ (SERVER_CERTIFICATE_TYPE, 20) \ + _ (PADDING, 21) \ + _ (TOKEN_BINDING, 24) \ + _ (RECORD_SIZE_LIMIT, 28) \ + _ (SESSION_TICKET, 35) \ + _ (PRE_SHARED_KEY, 41) \ + _ (EARLY_DATA, 42) \ + _ (SUPPORTED_VERSIONS, 43) \ + _ (COOKIE, 44) \ + _ (PSK_KEY_EXCHANGE_MODES, 45) \ + _ (CERTIFICATE_AUTHORITIES, 47) \ + _ (OID_FILTERS, 48) \ + _ (SIGNATURE_ALGORITHMS_CERT, 50) \ + _ (POST_HANDSHAKE_AUTH, 49) \ + _ (KEY_SHARE, 51) \ + _ (CONNECTION_ID, 54) \ + _ (QUIC_TRANSPORT_PARAMETERS, 57) \ + _ (TICKET_REQUEST, 58) \ + _ (DNSSEC_CHAIN, 59) + +typedef enum tls_handshake_extension_type_ +{ +#define _(sym, val) TLS_EXT_##sym = val, + foreach_tls_hanshake_extensions +#undef _ +} tls_handshake_ext_type_t; + +/* Base struct for all extensions */ +typedef struct tls_handshake_ext_ +{ + tls_handshake_ext_type_t type; + u8 extension[0]; +} tls_handshake_ext_t; + +typedef struct tls_handshake_ext_server_name_ +{ + u8 name_type; + u8 *host_name; +} tls_handshake_ext_sni_sn_t; + +typedef struct tls_handshake_ext_sni_ +{ + tls_handshake_ext_t ext; + tls_handshake_ext_sni_sn_t *names; +} tls_handshake_ext_sni_t; + +/* FQDN length as per rfc1035 */ +#define TLS_EXT_SNI_MAX_LEN 255 + +#define foreach_tls_handshake_parse_error \ + _ (OK, "ok") \ + _ (WANT_MORE, "want_more") \ + _ (UNSUPPORTED, "unsupported") \ + _ (INVALID_LEN, "invalid_len") \ + _ (SESSION_ID_LEN, "session_id_len") \ + _ (CIPHER_SUITE_LEN, "cipher_suite_len") \ + _ (COMPRESSION_METHOD, "compression_method") \ + _ (EXTENSIONS_LEN, "extensions_len") \ + _ (EXT_SNI_NAME_TYPE, "ext_sni_name_type") \ + _ (EXT_SNI_LEN, "ext_sni_len") + +typedef enum tls_handshake_parse_error_ +{ +#define _(sym, str) TLS_HS_PARSE_ERR_##sym, + foreach_tls_handshake_parse_error +#undef _ +} tls_handshake_parse_error_t; + +typedef struct tls_hanshake_ext_info_ +{ + tls_handshake_ext_type_t type; + u16 len; + u8 *data; +} tls_handshake_ext_info_t; + +typedef struct tls_handshake_msg_info_ +{ + tls_handshake_type_t type; + u32 len; + u8 legacy_session_id_len; + u8 *legacy_session_id; + u16 cipher_suite_len; + u8 *cipher_suites; + u16 extensions_len; + u8 *extensions; +} tls_handshake_msg_info_t; + +static inline u8 +tls_record_type_is_valid (tls_record_type_t type) +{ + switch (type) + { + case TLS_REC_CHANGE_CIPHER_SPEC: + case TLS_REC_ALERT: + case TLS_REC_HANDSHAKE: + case TLS_REC_APPLICATION_DATA: + case TLS_REC_HEARTBEAT: + return 1; + default: + return 0; + } +} + +static inline u8 +tls_record_hdr_is_valid (tls_record_header_t rec_hdr) +{ + u16 rec_len; + + if (!tls_record_type_is_valid (rec_hdr.type)) + return 0; + + /* Support for SSLv3 and TLS1.0 to TLS1.3 */ + if (rec_hdr.version.major != TLS_MAJOR_VERSION) + return 0; + + rec_len = clib_net_to_host_u16 (rec_hdr.length); + if (rec_len == 0 || rec_len > TLS_FRAGMENT_MAX_ENC_LEN) + return 0; + + return 1; +} + +tls_handshake_parse_error_t +tls_handshake_message_try_parse (u8 *msg, int len, + tls_handshake_msg_info_t *info); +tls_handshake_parse_error_t +tls_hanshake_extensions_parse (tls_handshake_msg_info_t *info, + tls_handshake_ext_info_t **exts); +tls_handshake_parse_error_t +tls_hanshake_extensions_try_parse (tls_handshake_msg_info_t *info, + tls_handshake_ext_info_t *req_exts, + u32 n_reqs); +tls_handshake_parse_error_t +tls_handshake_ext_parse (tls_handshake_ext_info_t *ext_info, + tls_handshake_ext_t *ext); +void tls_handshake_ext_free (tls_handshake_ext_t *ext); + +#endif /* SRC_VNET_TLS_TLS_RECORD_H__ */ diff --git a/src/vnet/udp/udp.c b/src/vnet/udp/udp.c index b3c02510232..8deeb9e41ee 100644 --- a/src/vnet/udp/udp.c +++ b/src/vnet/udp/udp.c @@ -232,18 +232,43 @@ udp_session_get_listener (u32 listener_index) return &us->connection; } +always_inline u16 +udp_compute_checksum (vlib_main_t *vm, vlib_buffer_t *b, u8 csum_offload, + u8 is_ip4) +{ + u16 csum = 0; + + if (csum_offload) + vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM); + else + { + if (is_ip4) + csum = + ip4_tcp_udp_compute_checksum (vm, b, vlib_buffer_get_current (b)); + else + { + int bogus = 0; + csum = ip6_tcp_udp_icmp_compute_checksum ( + vm, b, vlib_buffer_get_current (b), &bogus); + } + } + + return csum; +} + always_inline u32 udp_push_one_header (vlib_main_t *vm, udp_connection_t *uc, vlib_buffer_t *b, u8 is_cless) { + udp_header_t *uh; + b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED; /* reuse tcp medatada for now */ vnet_buffer (b)->tcp.connection_index = uc->c_c_index; if (!is_cless) { - vlib_buffer_push_udp (b, uc->c_lcl_port, uc->c_rmt_port, - udp_csum_offload (uc)); + uh = vlib_buffer_push_udp (b, uc->c_lcl_port, uc->c_rmt_port); if (uc->c_is_ip4) vlib_buffer_push_ip4_custom (vm, b, &uc->c_lcl_ip4, &uc->c_rmt_ip4, @@ -263,8 +288,7 @@ udp_push_one_header (vlib_main_t *vm, udp_connection_t *uc, vlib_buffer_t *b, hdr = *(session_dgram_hdr_t *) (data - sizeof (hdr)); /* Local port assumed to be bound, not overwriting it */ - vlib_buffer_push_udp (b, uc->c_lcl_port, hdr.rmt_port, - udp_csum_offload (uc)); + uh = vlib_buffer_push_udp (b, uc->c_lcl_port, hdr.rmt_port); if (uc->c_is_ip4) vlib_buffer_push_ip4_custom (vm, b, &hdr.lcl_ip.ip4, &hdr.rmt_ip.ip4, @@ -279,6 +303,9 @@ udp_push_one_header (vlib_main_t *vm, udp_connection_t *uc, vlib_buffer_t *b, vnet_buffer (b)->tcp.flags |= UDP_CONN_F_LISTEN; } + uh->checksum = + udp_compute_checksum (vm, b, udp_csum_offload (uc), uc->c_is_ip4); + return 0; } @@ -563,6 +590,9 @@ udp_enable_disable (vlib_main_t *vm, u8 is_en) { udp_main_t *um = &udp_main; + if (!is_en || um->is_init) + return 0; + /* Not ideal. The sparse vector used to map ports to next nodes assumes * only a few ports are ever used. When udp transport is enabled this does * not hold and, to make matters worse, ports are consumed in a random @@ -583,6 +613,7 @@ udp_enable_disable (vlib_main_t *vm, u8 is_en) vec_validate (um->transport_ports_refcnt[0], 65535); vec_validate (um->transport_ports_refcnt[1], 65535); + um->is_init = 1; return 0; } diff --git a/src/vnet/udp/udp.h b/src/vnet/udp/udp.h index 8e4e87f85a8..c6f867500e0 100644 --- a/src/vnet/udp/udp.h +++ b/src/vnet/udp/udp.h @@ -154,6 +154,7 @@ typedef struct u16 default_mtu; u16 msg_id_base; u8 csum_offload; + u8 is_init; u8 icmp_send_unreachable_disabled; } udp_main_t; diff --git a/src/vnet/udp/udp_error.def b/src/vnet/udp/udp_error.def index ef19970ce72..843aacfc6ef 100644 --- a/src/vnet/udp/udp_error.def +++ b/src/vnet/udp/udp_error.def @@ -28,3 +28,4 @@ udp_error (CREATE_SESSION, create_session, ERROR, "Failed to create session") udp_error (MQ_FULL, mq_full, ERROR, "Application msg queue full") udp_error (INVALID_CONNECTION, invalid_connection, ERROR, "Invalid connection") udp_error (PKTS_SENT, pkts_sent, INFO, "Packets sent") +udp_error (CONNECTED, connected, INFO, "Connected session") diff --git a/src/vnet/udp/udp_inlines.h b/src/vnet/udp/udp_inlines.h index f0dd44f48b5..ceec0b191b1 100644 --- a/src/vnet/udp/udp_inlines.h +++ b/src/vnet/udp/udp_inlines.h @@ -26,7 +26,7 @@ #include <vnet/udp/udp_encap.h> always_inline void * -vlib_buffer_push_udp (vlib_buffer_t * b, u16 sp, u16 dp, u8 offload_csum) +vlib_buffer_push_udp (vlib_buffer_t *b, u16 sp, u16 dp) { udp_header_t *uh; u16 udp_len = sizeof (udp_header_t) + b->current_length; @@ -38,8 +38,6 @@ vlib_buffer_push_udp (vlib_buffer_t * b, u16 sp, u16 dp, u8 offload_csum) uh->dst_port = dp; uh->checksum = 0; uh->length = clib_host_to_net_u16 (udp_len); - if (offload_csum) - vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM); vnet_buffer (b)->l4_hdr_offset = (u8 *) uh - b->data; b->flags |= VNET_BUFFER_F_L4_HDR_OFFSET_VALID; return uh; diff --git a/src/vnet/udp/udp_input.c b/src/vnet/udp/udp_input.c index a90461186c1..693824f9628 100644 --- a/src/vnet/udp/udp_input.c +++ b/src/vnet/udp/udp_input.c @@ -272,6 +272,7 @@ udp46_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, uc0->sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_RX]; if (uc0->flags & UDP_CONN_F_CONNECTED) { + error0 = UDP_ERROR_CONNECTED; if (s0->thread_index != thread_index) { /* diff --git a/src/vnet/unix/gdb_funcs.c b/src/vnet/unix/gdb_funcs.c index d6fdc985bd9..a89b7202400 100644 --- a/src/vnet/unix/gdb_funcs.c +++ b/src/vnet/unix/gdb_funcs.c @@ -238,44 +238,44 @@ gdb_show_traces () /* Get active traces from pool. */ - foreach_vlib_main () - { - fmt = "------------------- Start of thread %d %s -------------------\n"; - s = format (s, fmt, index, vlib_worker_threads[index].name); + foreach_vlib_main__ (0 /* no checks */) + { + fmt = "------------------- Start of thread %d %s -------------------\n"; + s = format (s, fmt, index, vlib_worker_threads[index].name); - tm = &this_vlib_main->trace_main; + tm = &this_vlib_main->trace_main; - trace_apply_filter (this_vlib_main); + trace_apply_filter (this_vlib_main); - traces = 0; - pool_foreach (h, tm->trace_buffer_pool) - { - vec_add1 (traces, h[0]); - } + traces = 0; + pool_foreach (h, tm->trace_buffer_pool) + { + vec_add1 (traces, h[0]); + } - if (vec_len (traces) == 0) - { - s = format (s, "No packets in trace buffer\n"); - goto done; - } + if (vec_len (traces) == 0) + { + s = format (s, "No packets in trace buffer\n"); + goto done; + } - /* Sort them by increasing time. */ - vec_sort_with_function (traces, trace_cmp); + /* Sort them by increasing time. */ + vec_sort_with_function (traces, trace_cmp); - for (i = 0; i < vec_len (traces); i++) - { - if (i == max) - { - fformat (stderr, - "Limiting display to %d packets." - " To display more specify max.", - max); - goto done; - } - - s = format (s, "Packet %d\n%U\n\n", i + 1, format_vlib_trace, - vlib_get_first_main (), traces[i]); - } + for (i = 0; i < vec_len (traces); i++) + { + if (i == max) + { + fformat (stderr, + "Limiting display to %d packets." + " To display more specify max.", + max); + goto done; + } + + s = format (s, "Packet %d\n%U\n\n", i + 1, format_vlib_trace, + vlib_get_first_main (), traces[i]); + } done: vec_free (traces); diff --git a/src/vpp-api/python/pyproject.toml b/src/vpp-api/python/pyproject.toml new file mode 100644 index 00000000000..638dd9c54fc --- /dev/null +++ b/src/vpp-api/python/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" diff --git a/src/vpp-api/python/vpp_papi/vpp_serializer.py b/src/vpp-api/python/vpp_papi/vpp_serializer.py index d724cb33ce9..707bb03b790 100644 --- a/src/vpp-api/python/vpp_papi/vpp_serializer.py +++ b/src/vpp-api/python/vpp_papi/vpp_serializer.py @@ -644,10 +644,15 @@ class VPPType(Packer): else: arg = data[a] kwarg = kwargs[a] if a in kwargs else None - if isinstance(self.packers[i], VPPType): - b += self.packers[i].pack(arg, kwarg) - else: - b += self.packers[i].pack(arg, kwargs) + try: + if isinstance(self.packers[i], VPPType): + b += self.packers[i].pack(arg, kwarg) + else: + b += self.packers[i].pack(arg, kwargs) + except Exception as e: + raise VPPSerializerValueError( + f"Exception while packing {data} for {self.name}.{a}." + ) from e return bytes(b) diff --git a/src/vpp-api/vapi/vapi.c b/src/vpp-api/vapi/vapi.c index 022f023aeb0..26c5708342f 100644 --- a/src/vpp-api/vapi/vapi.c +++ b/src/vpp-api/vapi/vapi.c @@ -1792,14 +1792,15 @@ vapi_verify_msg_size (vapi_msg_id_t id, void *buf, uword buf_size) } vapi_error_e -vapi_dispatch_one (vapi_ctx_t ctx) +vapi_dispatch_one_timedwait (vapi_ctx_t ctx, u32 wait_time) { VAPI_DBG ("vapi_dispatch_one()"); void *msg; uword size; svm_q_conditional_wait_t cond = - vapi_is_nonblocking (ctx) ? SVM_Q_NOWAIT : SVM_Q_WAIT; - vapi_error_e rv = vapi_recv (ctx, &msg, &size, cond, 0); + vapi_is_nonblocking (ctx) ? (wait_time ? SVM_Q_TIMEDWAIT : SVM_Q_NOWAIT) : + SVM_Q_WAIT; + vapi_error_e rv = vapi_recv (ctx, &msg, &size, cond, wait_time); if (VAPI_OK != rv) { VAPI_DBG ("vapi_recv failed with rv=%d", rv); @@ -1847,6 +1848,12 @@ done: } vapi_error_e +vapi_dispatch_one (vapi_ctx_t ctx) +{ + return vapi_dispatch_one_timedwait (ctx, 0); +} + +vapi_error_e vapi_dispatch (vapi_ctx_t ctx) { vapi_error_e rv = VAPI_OK; diff --git a/src/vpp-api/vapi/vapi.h b/src/vpp-api/vapi/vapi.h index 970c5080667..8f092eed1c2 100644 --- a/src/vpp-api/vapi/vapi.h +++ b/src/vpp-api/vapi/vapi.h @@ -225,6 +225,18 @@ vapi_error_e vapi_wait (vapi_ctx_t ctx); /** * @brief pick next message sent by vpp and call the appropriate callback * + * @note if using block mode, it will be blocked indefinitely until the next + * msg available. If using non-blocking mode, it will block for time_wait + * seconds until the next msg available if time_wait > 0, or does not block if + * time_wait == 0. + * + * @return VAPI_OK on success, other error code on error + */ +vapi_error_e vapi_dispatch_one_timedwait (vapi_ctx_t ctx, u32 wait_time); + +/** + * @brief pick next message sent by vpp and call the appropriate callback + * * @return VAPI_OK on success, other error code on error */ vapi_error_e vapi_dispatch_one (vapi_ctx_t ctx); @@ -235,11 +247,11 @@ vapi_error_e vapi_dispatch_one (vapi_ctx_t ctx); * * @note the dispatch loop is interrupted if any error is encountered or * returned from the callback, in which case this error is returned as the - * result of vapi_dispatch. In this case it might be necessary to call dispatch - * again to process the remaining messages. Returning VAPI_EUSER from - * a callback allows the user to break the dispatch loop (and distinguish - * this case in the calling code from other failures). VAPI never returns - * VAPI_EUSER on its own. + * result of vapi_dispatch. In this case it might be necessary to call + * dispatch again to process the remaining messages. Returning VAPI_EUSER + * from a callback allows the user to break the dispatch loop (and + * distinguish this case in the calling code from other failures). VAPI never + * returns VAPI_EUSER on its own. * * @return VAPI_OK on success, other error code on error */ diff --git a/src/vpp/api/types.c b/src/vpp/api/types.c index 92bbdb30ac9..8e118109492 100644 --- a/src/vpp/api/types.c +++ b/src/vpp/api/types.c @@ -49,21 +49,6 @@ format_vl_api_address (u8 * s, va_list * args) } u8 * -format_vl_api_address_union (u8 * s, va_list * args) -{ - const vl_api_address_union_t *addr = - va_arg (*args, vl_api_address_union_t *); - vl_api_address_family_t af = va_arg (*args, int); - - if (ADDRESS_IP6 == af) - s = format (s, "%U", format_ip6_address, addr->ip6); - else - s = format (s, "%U", format_ip4_address, addr->ip4); - - return s; -} - -u8 * format_vl_api_ip4_address (u8 * s, va_list * args) { const vl_api_ip4_address_t *addr = va_arg (*args, vl_api_ip4_address_t *); diff --git a/src/vpp/api/types.h b/src/vpp/api/types.h index 9864e8c38a8..7a546dbdc0d 100644 --- a/src/vpp/api/types.h +++ b/src/vpp/api/types.h @@ -40,7 +40,6 @@ extern u8 *format_vl_api_address (u8 * s, va_list * args); extern u8 *format_vl_api_address_family (u8 * s, va_list * args); extern u8 *format_vl_api_ip4_address (u8 * s, va_list * args); extern u8 *format_vl_api_ip6_address (u8 * s, va_list * args); -extern u8 *format_vl_api_address_union (u8 * s, va_list * args); extern u8 *format_vl_api_prefix (u8 * s, va_list * args); extern u8 *format_vl_api_mprefix (u8 * s, va_list * args); extern u8 *format_vl_api_mac_address (u8 * s, va_list * args); diff --git a/src/vpp/conf/startup.conf b/src/vpp/conf/startup.conf index 929106a4c8a..a30a15ab2b1 100644 --- a/src/vpp/conf/startup.conf +++ b/src/vpp/conf/startup.conf @@ -164,6 +164,11 @@ cpu { # num-rx-queues 2 # } + ## Set interface only in poll mode + # dev 0000:02:00.1 { + # no-rx-interrupts + # } + ## Change UIO driver used by VPP, Options are: igb_uio, vfio-pci, ## uio_pci_generic or auto (default) # uio-driver vfio-pci diff --git a/src/vpp/vnet/main.c b/src/vpp/vnet/main.c index c57efd59a62..dd4f4cc3353 100644 --- a/src/vpp/vnet/main.c +++ b/src/vpp/vnet/main.c @@ -329,6 +329,10 @@ defaulted: unformat_free (&input); + /* if main thread affinity is unspecified, set to current running cpu */ + if (main_core == ~0) + main_core = sched_getcpu (); + /* set process affinity for main thread */ if (main_core != ~0) { diff --git a/src/vppinfra/CMakeLists.txt b/src/vppinfra/CMakeLists.txt index a8c64a36121..83a8b2a7e57 100644 --- a/src/vppinfra/CMakeLists.txt +++ b/src/vppinfra/CMakeLists.txt @@ -14,6 +14,26 @@ enable_language(ASM) ############################################################################## +# find libdl +############################################################################## +list(APPEND VPPINFRA_LIBS ${CMAKE_DL_LIBS}) + +############################################################################## +# find libunwind +############################################################################## +vpp_find_path(LIBUNWIND_INCLUDE_DIR unwind.h) +vpp_find_library(LIBUNWIND_LIB NAMES unwind libunwind) + +if (LIBUNWIND_INCLUDE_DIR AND LIBUNWIND_LIB) + message(STATUS "libunwind found at ${LIBUNWIND_LIB}") + list(APPEND VPPINFRA_LIBS ${LIBUNWIND_LIB}) + add_definitions(-DHAVE_LIBUNWIND=1) +else() + message(WARNING "libunwind not found - stack traces disabled") + add_definitions(-DHAVE_LIBUNWIND=0) +endif() + +############################################################################## # Generate vppinfra/config.h ############################################################################## set(LOG2_CACHE_LINE_BYTES ${VPP_LOG2_CACHE_LINE_SIZE}) @@ -42,15 +62,14 @@ add_definitions(-fvisibility=hidden) set_source_files_properties( cJSON.c jsonformat.c PROPERTIES COMPILE_DEFINITIONS " CJSON_API_VISIBILITY " ) - ############################################################################## # vppinfra sources ############################################################################## set(VPPINFRA_SRCS - backtrace.c bitmap.c bihash_all_vector.c cpu.c + devicetree.c dlmalloc.c elf.c elog.c @@ -80,6 +99,7 @@ set(VPPINFRA_SRCS rbtree.c serialize.c socket.c + stack.c std-formats.c string.c time.c @@ -132,6 +152,7 @@ set(VPPINFRA_HEADERS crypto/aes_ctr.h crypto/aes_gcm.h crypto/poly1305.h + devicetree.h dlist.h dlmalloc.h elf_clib.h @@ -142,6 +163,7 @@ set(VPPINFRA_HEADERS fifo.h file.h format.h + format_ansi.h format_table.h hash.h heap.h @@ -175,6 +197,7 @@ set(VPPINFRA_HEADERS smp.h socket.h sparse_vec.h + stack.h string.h time.h time_range.h @@ -229,22 +252,9 @@ elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD") ) endif() -if("${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD") - option(VPP_USE_EXTERNAL_LIBEXECINFO "Use external libexecinfo (useful for non-glibc targets)." ON) -else() - option(VPP_USE_EXTERNAL_LIBEXECINFO "Use external libexecinfo (useful for non-glibc targets)." OFF) -endif() -option(VPP_USE_LIBUNWIND "Use libunwind for backtrace." OFF) - -if(VPP_USE_EXTERNAL_LIBEXECINFO) - set(EXECINFO_LIB execinfo) -elseif(VPP_USE_LIBUNWIND) - set(EXECINFO_LIB unwind) - add_compile_definitions(USE_LIBUNWIND) -endif() add_vpp_library(vppinfra SOURCES ${VPPINFRA_SRCS} - LINK_LIBRARIES m ${EXECINFO_LIB} + LINK_LIBRARIES m ${VPPINFRA_LIBS} INSTALL_HEADERS ${VPPINFRA_HEADERS} COMPONENT libvppinfra LTO diff --git a/src/vppinfra/asm_mips.h b/src/vppinfra/asm_mips.h deleted file mode 100644 index 7c9e69586f4..00000000000 --- a/src/vppinfra/asm_mips.h +++ /dev/null @@ -1,351 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - Copyright (c) 2004 Eliot Dresselhaus - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ - -#ifndef included_asm_mips_h -#define included_asm_mips_h - -/* Encoding of MIPS instructions. */ -/* Encoding of opcode field (op). */ -#define mips_foreach_opcode \ - _(SPECIAL) _(REGIMM) _(j) _(jal) _(beq) _(bne) _(blez) _(bgtz) \ - _(addi) _(addiu) _(slti) _(sltiu) _(andi) _(ori) _(xori) _(lui) \ - _(COP0) _(COP1) _(COP2) _(COP1X) _(beql) _(bnel) _(blezl) _(bgtzl) \ - _(daddi) _(daddiu) _(ldl) _(ldr) _(SPECIAL2) _(jalx) _(MDMX) _(O37) \ - _(lb) _(lh) _(lwl) _(lw) _(lbu) _(lhu) _(lwr) _(lwu) \ - _(sb) _(sh) _(swl) _(sw) _(sdl) _(sdr) _(swr) _(cache) \ - _(ll) _(lwc1) _(lwc2) _(pref) _(lld) _(ldc1) _(ldc2) _(ld) \ - _(sc) _(swc1) _(swc2) _(o73) _(scd) _(sdc1) _(sdc2) _(sd) - -/* Encoding of funct field. */ -#define mips_foreach_special_funct \ - _(sll) _(MOVCI) _(srl) _(sra) _(sllv) _(o05) _(srlv) _(srav) \ - _(jr) _(jalr) _(movz) _(movn) _(syscall) _(break) _(o16) _(sync) \ - _(mfhi) _(mthi) _(mflo) _(mtlo) _(dsllv) _(o25) _(dsrlv) _(dsrav) \ - _(mult) _(multu) _(div) _(divu) _(dmult) _(dmultu) _(ddiv) _(ddivu) \ - _(add) _(addu) _(sub) _(subu) _(and) _(or) _(xor) _(nor) \ - _(o50) _(o51) _(slt) _(sltu) _(dadd) _(daddu) _(dsub) _(dsubu) \ - _(tge) _(tgeu) _(tlt) _(tltu) _(teq) _(o65) _(tne) _(o67) \ - _(dsll) _(o71) _(dsrl) _(dsra) _(dsll32) _(o75) _(dsrl32) _(dsra32) - -/* SPECIAL2 encoding of funct field. */ -#define mips_foreach_special2_funct \ - _(madd) _(maddu) _(mul) _(o03) _(msub) _(msubu) _(o06) _(o07) \ - _(o10) _(o11) _(o12) _(o13) _(o14) _(o15) _(o16) _(o17) \ - _(o20) _(o21) _(o22) _(o23) _(o24) _(o25) _(o26) _(o27) \ - _(o30) _(o31) _(o32) _(o33) _(o34) _(o35) _(o36) _(o37) \ - _(clz) _(clo) _(o42) _(o43) _(dclz) _(dclo) _(o46) _(o47) \ - _(o50) _(o51) _(o52) _(o53) _(o54) _(o55) _(o56) _(o57) \ - _(o60) _(o61) _(o62) _(o63) _(o64) _(o65) _(o66) _(o67) \ - _(o70) _(o71) _(o72) _(o73) _(o74) _(o75) _(o76) _(sdbbp) - -/* REGIMM encoding of rt field. */ -#define mips_foreach_regimm_rt \ - _(bltz) _(bgez) _(bltzl) _(bgezl) _(o04) _(o05) _(o06) _(o07) \ - _(tgei) _(tgeiu) _(tltiu) _(teqi) _(o14) _(tnei) _(o16) _(o17) \ - _(bltzal) _(bgezal) _(bltzall) _(bgezall) _(o24) _(o25) _(o26) _(o27) \ - _(o30) _(o31) _(o32) _(o33) _(o34) _(o35) _(o36) _(o37) - -/* COP0 encoding of rs field. */ -#define mips_foreach_cop0_rs \ - _(mfc0) _(dmfc0) _(o02) _(o03) _(mtc0) _(dmtc0) _(o06) _(o07) \ - _(o10) _(o11) _(o12) _(o13) _(o14) _(o15) _(o16) _(o17) \ - _(C0) _(o21) _(o22) _(o23) _(o24) _(o25) _(o26) _(o27) \ - _(o30) _(o31) _(o32) _(o33) _(o34) _(o35) _(o36) _(o37) - -/* COP0 encoding of funct when rs == RS_CO */ -#define mips_foreach_cop0_funct \ - _(o00) _(tlbr) _(tlbwi) _(o03) _(o04) _(o05) _(tlbwr) _(o07) \ - _(tlbp) _(o11) _(o12) _(o13) _(o14) _(o15) _(o16) _(o17) \ - _(o20) _(o21) _(o22) _(o23) _(o24) _(o25) _(o26) _(o27) \ - _(eret) _(o31) _(o32) _(o33) _(o34) _(o35) _(o36) _(deret) \ - _(wait) _(o41) _(o42) _(o43) _(o44) _(o45) _(o46) _(o47) \ - _(o50) _(o51) _(o52) _(o53) _(o54) _(o55) _(o56) _(o57) \ - _(o60) _(o61) _(o62) _(o63) _(o64) _(o65) _(o66) _(o67) \ - _(o70) _(o71) _(o72) _(o73) _(o74) _(o75) _(o76) _(o77) - -/* COP1 encoding of rs field. */ -#define mips_foreach_cop1_rs \ - _(mfc1) _(dmfc1) _(cfc1) _(o03) _(mtc1) _(dmtc1) _(ctc1) _(o07) \ - _(BC1) _(o11) _(o12) _(o13) _(o14) _(o15) _(o16) _(o17) \ - _(S) _(D) _(o22) _(o23) _(W) _(L) _(o26) _(o27) \ - _(o30) _(o31) _(o32) _(o33) _(o34) _(o35) _(o36) _(o37) - -/* COP1 encoding of funct for S and D */ -#define mips_foreach_cop1_funct \ - _(add) _(sub) _(mul) _(div) _(sqrt) _(abs) _(mov) _(neg) \ - _(roundl) _(truncl) _(ceill) _(floorl) _(roundw) _(truncw) _(ceilw) _(floorw) \ - _(o20) _(MOVCF) _(movz) _(movn) _(o24) _(recip) _(rsqrt) _(o27) \ - _(o30) _(o31) _(o32) _(o33) _(o34) _(o35) _(o36) _(o37) \ - _(cvts) _(cvtd) _(o42) _(o43) _(cvtw) _(cvtl) _(o46) _(o47) \ - _(o50) _(o51) _(o52) _(o53) _(o54) _(o55) _(o56) _(o57) \ - _(cf) _(cun) _(ceq) _(cueq) _(colt) _(cult) _(cole) _(cule) \ - _(csf) _(cngle) _(cseq) _(cngl) _(clt) _(cnge) _(cle) _(cngt) - -/* COP1X encoding of funct */ -#define mips_foreach_cop1x_funct \ - _(lwxc1) _(ldxc1) _(o02) _(o03) _(o04) _(luxc1) _(o06) _(o07) \ - _(swxc1) _(sdxc1) _(o12) _(o13) _(o14) _(suxc1) _(o16) _(prefx) \ - _(o20) _(o21) _(o22) _(o23) _(o24) _(o25) _(o26) _(o27) \ - _(o30) _(o31) _(o32) _(o33) _(o34) _(o35) _(o36) _(o37) \ - _(madds) _(maddd) _(o42) _(o43) _(o44) _(o45) _(o46) _(o47) \ - _(msubs) _(msubd) _(o52) _(o53) _(o54) _(o55) _(o56) _(o57) \ - _(nmadds) _(nmaddd) _(o62) _(o63) _(o64) _(o65) _(o66) _(o67) \ - _(nmsubs) _(nmsubd) _(o72) _(o73) _(o74) _(o75) _(o76) _(o77) - -#define mips_foreach_mdmx_funct \ - _(msgn) _(ceq) _(pickf) _(pickt) _(clt) _(cle) _(min) _(max) \ - _(o10) _(o11) _(sub) _(add) _(and) _(xor) _(or) _(nor) \ - _(sll) _(o21) _(srl) _(sra) _(o24) _(o25) _(o26) _(o27) \ - _(alniob) _(alnvob) _(alniqh) _(alnvqh) _(o34) _(o35) _(o36) _(shfl) \ - _(rzu) _(rnau) _(rneu) _(o43) _(rzs) _(rnas) _(rnes) _(o47) \ - _(o50) _(o51) _(o52) _(o53) _(o54) _(o55) _(o56) _(o57) \ - _(mul) _(o61) _(muls) _(mula) _(o64) _(o65) _(suba) _(adda) \ - _(o70) _(o71) _(o72) _(o73) _(o74) _(o75) _(wac) _(rac) - -#define _(f) MIPS_OPCODE_##f, -typedef enum -{ - mips_foreach_opcode -} mips_insn_opcode_t; -#undef _ - -#define _(f) MIPS_SPECIAL_FUNCT_##f, -typedef enum -{ - mips_foreach_special_funct -} mips_insn_special_funct_t; -#undef _ - -#define _(f) MIPS_SPECIAL2_FUNCT_##f, -typedef enum -{ - mips_foreach_special2_funct -} mips_insn_special2_funct_t; -#undef _ - -#define _(f) MIPS_REGIMM_RT_##f, -typedef enum -{ - mips_foreach_regimm_rt -} mips_insn_regimm_rt_t; -#undef _ - -#define _(f) MIPS_COP0_RS_##f, -typedef enum -{ - mips_foreach_cop0_rs -} mips_insn_cop0_rs_t; -#undef _ - -#define _(f) MIPS_COP0_FUNCT_##f, -typedef enum -{ - mips_foreach_cop0_funct -} mips_insn_cop0_funct_t; -#undef _ - -#define _(f) MIPS_COP1_RS_##f, -typedef enum -{ - mips_foreach_cop1_rs -} mips_insn_cop1_rs_t; -#undef _ - -#define _(f) MIPS_COP1_FUNCT_##f, -typedef enum -{ - mips_foreach_cop1_funct -} mips_insn_cop1_funct_t; -#undef _ - -#define _(f) MIPS_COP1X_FUNCT_##f, -typedef enum -{ - mips_foreach_cop1x_funct -} mips_insn_cop1x_funct_t; -#undef _ - -#define _(f) MIPS_MDMX_FUNCT_##f, -typedef enum -{ - mips_foreach_mdmx_funct -} mips_insn_mdmx_funct_t; -#undef _ - -always_inline mips_insn_opcode_t -mips_insn_get_op (u32 insn) -{ - return (insn >> 26) & 0x3f; -} - -always_inline u32 -mips_insn_get_rs (u32 insn) -{ - return (insn >> 21) & 0x1f; -} - -always_inline u32 -mips_insn_get_rt (u32 insn) -{ - return (insn >> 16) & 0x1f; -} - -always_inline u32 -mips_insn_get_rd (u32 insn) -{ - return (insn >> 11) & 0x1f; -} - -always_inline u32 -mips_insn_get_sa (u32 insn) -{ - return (insn >> 6) & 0x1f; -} - -always_inline u32 -mips_insn_get_funct (u32 insn) -{ - return (insn >> 0) & 0x3f; -} - -always_inline i32 -mips_insn_get_immediate (u32 insn) -{ - return (((i32) insn) << 16) >> 16; -} - -always_inline u32 -mips_insn_encode_i_type (int op, int rs, int rt, int immediate) -{ - u32 insn; - insn = immediate; - insn |= rt << 16; - insn |= rs << 21; - insn |= op << 26; - - ASSERT (mips_insn_get_immediate (insn) == immediate); - ASSERT (mips_insn_get_rt (insn) == rt); - ASSERT (mips_insn_get_rs (insn) == rt); - ASSERT (mips_insn_get_op (insn) == op); - - return insn; -} - -always_inline u32 -mips_insn_encode_j_type (int op, u32 addr) -{ - u32 insn; - - insn = (addr & ((1 << 28) - 1)) / 4; - insn |= op << 26; - - return insn; -} - -always_inline u32 -mips_insn_encode_r_type (int op, int rs, int rt, int rd, int sa, int funct) -{ - u32 insn; - insn = funct; - insn |= sa << 6; - insn |= rd << 11; - insn |= rt << 16; - insn |= rs << 21; - insn |= op << 26; - - ASSERT (mips_insn_get_funct (insn) == funct); - ASSERT (mips_insn_get_sa (insn) == sa); - ASSERT (mips_insn_get_rd (insn) == rd); - ASSERT (mips_insn_get_rt (insn) == rt); - ASSERT (mips_insn_get_rs (insn) == rt); - ASSERT (mips_insn_get_op (insn) == op); - - return insn; -} - -#define mips_insn_r(op,funct,rd,rs,rt,sa) \ - mips_insn_encode_r_type (MIPS_OPCODE_##op, \ - (rs), (rt), (rd), (sa), \ - MIPS_##op##_FUNCT_##funct) - -#define mips_insn_i(op,rs,rt,imm) \ - mips_insn_encode_i_type (MIPS_OPCODE_##op, (rs), (rt), (imm)) - -#define mips_insn_j(op,target) \ - mips_insn_encode_i_type (MIPS_OPCODE_##op, (rs), (rt), (imm)) - -/* Generate unsigned load instructions of data of various sizes. */ -always_inline u32 -mips_insn_load (u32 rd, i32 offset, u32 base, u32 log2_bytes) -{ - int op; - - ASSERT (log2_bytes < 4); - switch (log2_bytes) - { - case 0: - op = MIPS_OPCODE_lbu; - break; - case 1: - op = MIPS_OPCODE_lhu; - break; - case 2: - op = MIPS_OPCODE_lwu; - break; - case 3: - op = MIPS_OPCODE_ld; - break; - } - - return mips_insn_encode_i_type (op, base, rd, offset); -} - -typedef enum -{ - MIPS_REG_SP = 29, - MIPS_REG_RA = 31, -} mips_reg_t; - -#endif /* included_asm_mips_h */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/vppinfra/asm_x86.c b/src/vppinfra/asm_x86.c deleted file mode 100644 index e6e00ce5543..00000000000 --- a/src/vppinfra/asm_x86.c +++ /dev/null @@ -1,1947 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* FIXME - opcode name remove to save table space; enum - x87 - 3dnow - cbw naming -*/ - -#include <vppinfra/error.h> -#include <vppinfra/byte_order.h> -#include <vppinfra/asm_x86.h> - -#define foreach_x86_gp_register \ - _ (AX) _ (CX) _ (DX) _ (BX) \ - _ (SP) _ (BP) _ (SI) _ (DI) - -typedef enum { -#define _(r) X86_INSN_GP_REG_##r, - foreach_x86_gp_register -#undef _ -} x86_insn_gp_register_t; - -typedef union { - struct { - u8 rm : 3; - u8 reg : 3; - u8 mode : 2; - }; - u8 byte; -} x86_insn_modrm_byte_t; - -typedef union { - struct { - u8 base : 3; - u8 index : 3; - u8 log2_scale : 2; - }; - u8 byte; -} x86_insn_sib_byte_t; - -always_inline uword -x86_insn_has_modrm_byte (x86_insn_t * insn) -{ - int i; - for (i = 0; i < ARRAY_LEN (insn->operands); i++) - switch (insn->operands[i].code) - { - case 'G': case 'E': case 'M': case 'R': - return 1; - } - return 0; -} - -always_inline uword -x86_insn_immediate_type (x86_insn_t * insn) -{ - int i; - for (i = 0; i < ARRAY_LEN (insn->operands); i++) - switch (insn->operands[i].code) - { - case 'J': - case 'I': - case 'O': - return insn->operands[i].type; - } - return 0; -} - -/* Opcode extension in modrm byte reg field. */ -#define foreach_x86_insn_modrm_reg_group \ - _ (1) _ (1a) _ (2) _ (3) _ (4) _ (5) _ (6) _ (7) \ - _ (8) _ (9) _ (10) _ (11) _ (12) _ (13) _ (14) \ - _ (15) _ (16) _ (p) - -#define foreach_x86_insn_sse_group \ - _ (10) _ (28) _ (50) _ (58) _ (60) _ (68) _ (70) _ (78) \ - _ (c0) _ (d0) _ (d8) _ (e0) _ (e8) _ (f0) _ (f8) - -enum { -#define _(x) X86_INSN_MODRM_REG_GROUP_##x, - foreach_x86_insn_modrm_reg_group -#undef _ -#define _(x) X86_INSN_SSE_GROUP_##x, - foreach_x86_insn_sse_group -#undef _ -}; - -enum { -#define _(x) \ - X86_INSN_FLAG_MODRM_REG_GROUP_##x \ - = X86_INSN_FLAG_SET_MODRM_REG_GROUP (1 + X86_INSN_MODRM_REG_GROUP_##x), - foreach_x86_insn_modrm_reg_group -#undef _ - -#define _(x) \ - X86_INSN_FLAG_SSE_GROUP_##x \ - = X86_INSN_FLAG_SET_SSE_GROUP (1 + X86_INSN_SSE_GROUP_##x), - foreach_x86_insn_sse_group -#undef _ -}; - -#define foreach_x86_gp_reg \ - _ (AX) _ (CX) _ (DX) _ (BX) \ - _ (SP) _ (BP) _ (SI) _ (DI) - -#define foreach_x86_condition \ - _ (o) _ (no) _ (b) _ (nb) \ - _ (z) _ (nz) _ (be) _ (nbe) \ - _ (s) _ (ns) _ (p) _ (np) \ - _ (l) _ (nl) _ (le) _ (nle) - -#define _3f(x,f,o0,o1,o2) \ -{ \ - .name = #x, \ - .flags = (f), \ - .operands[0] = { .data = #o0 }, \ - .operands[1] = { .data = #o1 }, \ - .operands[2] = { .data = #o2 }, \ -} - -#define _2f(x,f,o0,o1) _3f(x,f,o0,o1,__) -#define _1f(x,f,o0) _2f(x,f,o0,__) -#define _0f(x,f) _1f(x,f,__) - -#define _3(x,o0,o1,o2) _3f(x,0,o0,o1,o2) -#define _2(x,o0,o1) _2f(x,0,o0,o1) -#define _1(x,o0) _1f(x,0,o0) -#define _0(x) _0f(x,0) - -static x86_insn_t x86_insns_one_byte[256] = { - -#define _(x) \ - _2 (x, Eb, Gb), \ - _2 (x, Ev, Gv), \ - _2 (x, Gb, Eb), \ - _2 (x, Gv, Ev), \ - _2 (x, AL, Ib), \ - _2 (x, AX, Iz) - - /* 0x00 */ - _ (add), - _0 (push_es), - _0 (pop_es), - _ (or), - _0 (push_cs), - _0 (escape_two_byte), - - /* 0x10 */ - _ (adc), - _0 (push_ss), - _0 (pop_ss), - _ (sbb), - _0 (push_ds), - _0 (pop_ds), - - /* 0x20 */ - _ (and), - _0 (segment_es), - _0 (daa), - _ (sub), - _0 (segment_cs), - _0 (das), - - /* 0x30 */ - _ (xor), - _0 (segment_ss), - _0 (aaa), - _ (cmp), - _0 (segment_ds), - _0 (aas), - -#undef _ - - /* 0x40 */ -#define _(r) _1 (inc, r), - foreach_x86_gp_reg -#undef _ -#define _(r) _1 (dec, r), - foreach_x86_gp_reg -#undef _ - - /* 0x50 */ -#define _(r) _1f (push, X86_INSN_FLAG_DEFAULT_64_BIT, r), - foreach_x86_gp_reg -#undef _ -#define _(r) _1f (pop, X86_INSN_FLAG_DEFAULT_64_BIT, r), - foreach_x86_gp_reg -#undef _ - - /* 0x60 */ - _0 (pusha), - _0 (popa), - _2 (bound, Gv, Ma), - _2 (movsxd, Gv, Ed), - _0 (segment_fs), - _0 (segment_gs), - _0 (operand_type), - _0 (address_size), - _1f (push, X86_INSN_FLAG_DEFAULT_64_BIT, Iz), - _3 (imul, Gv, Ev, Iz), - _1f (push, X86_INSN_FLAG_DEFAULT_64_BIT, Ib), - _3 (imul, Gv, Ev, Ib), - _1 (insb, DX), - _1 (insw, DX), - _1 (outsb, DX), - _1 (outsw, DX), - - /* 0x70 */ -#define _(x) _1 (j##x, Jb), - foreach_x86_condition -#undef _ - - /* 0x80 */ - _2f (modrm_group_1, X86_INSN_FLAG_MODRM_REG_GROUP_1, Eb, Ib), - _2f (modrm_group_1, X86_INSN_FLAG_MODRM_REG_GROUP_1, Ev, Iz), - _2f (modrm_group_1, X86_INSN_FLAG_MODRM_REG_GROUP_1, Eb, Ib), - _2f (modrm_group_1, X86_INSN_FLAG_MODRM_REG_GROUP_1, Ev, Ib), - _2 (test, Eb, Gb), - _2 (test, Ev, Gv), - _2 (xchg, Eb, Gb), - _2 (xchg, Ev, Gv), - _2 (mov, Eb, Gb), - _2 (mov, Ev, Gv), - _2 (mov, Gb, Eb), - _2 (mov, Gv, Ev), - _2 (mov, Ev, Sw), - _2 (lea, Gv, Ev), - _2 (mov, Sw, Ew), - _1f (modrm_group_1a, X86_INSN_FLAG_MODRM_REG_GROUP_1a, Ev), - - /* 0x90 */ - _0 (nop), - _1 (xchg, CX), - _1 (xchg, DX), - _1 (xchg, BX), - _1 (xchg, SP), - _1 (xchg, BP), - _1 (xchg, SI), - _1 (xchg, DI), - _0 (cbw), - _0 (cwd), - _1 (call, Ap), - _0 (wait), - _0 (pushf), - _0 (popf), - _0 (sahf), - _0 (lahf), - - /* 0xa0 */ - _2 (mov, AL, Ob), - _2 (mov, AX, Ov), - _2 (mov, Ob, AL), - _2 (mov, Ov, AX), - _0 (movsb), - _0 (movsw), - _0 (cmpsb), - _0 (cmpsw), - _2 (test, AL, Ib), - _2 (test, AX, Iz), - _1 (stosb, AL), - _1 (stosw, AX), - _1 (lodsb, AL), - _1 (lodsw, AX), - _1 (scasb, AL), - _1 (scasw, AX), - - /* 0xb0 */ - _2 (mov, AL, Ib), - _2 (mov, CL, Ib), - _2 (mov, DL, Ib), - _2 (mov, BL, Ib), - _2 (mov, AH, Ib), - _2 (mov, CH, Ib), - _2 (mov, DH, Ib), - _2 (mov, BH, Ib), -#define _(r) _2 (mov, r, Iv), - foreach_x86_gp_reg -#undef _ - - /* 0xc0 */ - _2f (modrm_group_2, X86_INSN_FLAG_MODRM_REG_GROUP_2, Eb, Ib), - _2f (modrm_group_2, X86_INSN_FLAG_MODRM_REG_GROUP_2, Ev, Ib), - _1 (ret, Iw), - _0 (ret), - _2 (les, Gz, Mp), - _2 (lds, Gz, Mp), - _2f (modrm_group_11, X86_INSN_FLAG_MODRM_REG_GROUP_11, Eb, Ib), - _2f (modrm_group_11, X86_INSN_FLAG_MODRM_REG_GROUP_11, Ev, Iz), - _2 (enter, Iw, Ib), - _0 (leave), - _1 (ret, Iw), - _0 (ret), - _0 (int3), - _1 (int, Ib), - _0 (into), - _0 (iret), - - /* 0xd0 */ - _2f (modrm_group_2, X86_INSN_FLAG_MODRM_REG_GROUP_2, Eb, 1b), - _2f (modrm_group_2, X86_INSN_FLAG_MODRM_REG_GROUP_2, Ev, 1b), - _2f (modrm_group_2, X86_INSN_FLAG_MODRM_REG_GROUP_2, Eb, CL), - _2f (modrm_group_2, X86_INSN_FLAG_MODRM_REG_GROUP_2, Ev, CL), - _0 (aam), - _0 (aad), - _0 (salc), - _0 (xlat), - /* FIXME x87 */ - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - - /* 0xe0 */ - _1 (loopnz, Jb), - _1 (loopz, Jb), - _1 (loop, Jb), - _1 (jcxz, Jb), - _2 (in, AL, Ib), - _2 (in, AX, Ib), - _2 (out, Ib, AL), - _2 (out, Ib, AX), - _1f (call, X86_INSN_FLAG_DEFAULT_64_BIT, Jz), - _1f ( jmp, X86_INSN_FLAG_DEFAULT_64_BIT, Jz), - _1 (jmp, Ap), - _1 (jmp, Jb), - _2 (in, AL, DX), - _2 (in, AX, DX), - _2 (out, DX, AL), - _2 (out, DX, AX), - - /* 0xf0 */ - _0 (lock), - _0 (int1), - _0 (repne), - _0 (rep), - _0 (hlt), - _0 (cmc), - _0f (modrm_group_3, X86_INSN_FLAG_MODRM_REG_GROUP_3), - _0f (modrm_group_3, X86_INSN_FLAG_MODRM_REG_GROUP_3), - _0 (clc), - _0 (stc), - _0 (cli), - _0 (sti), - _0 (cld), - _0 (std), - _1f (modrm_group_4, X86_INSN_FLAG_MODRM_REG_GROUP_4, Eb), - _0f (modrm_group_5, X86_INSN_FLAG_MODRM_REG_GROUP_5), -}; - -static x86_insn_t x86_insns_two_byte[256] = { - /* 0x00 */ - _0f (modrm_group_6, X86_INSN_FLAG_MODRM_REG_GROUP_6), - _0f (modrm_group_7, X86_INSN_FLAG_MODRM_REG_GROUP_7), - _2 (lar, Gv, Ew), - _2 (lsl, Gv, Ew), - _0 (bad), - _0 (syscall), - _0 (clts), - _0 (sysret), - _0 (invd), - _0 (wbinvd), - _0 (bad), - _0 (ud2), - _0 (bad), - _0f (modrm_group_p, X86_INSN_FLAG_MODRM_REG_GROUP_p), - _0 (femms), - _0 (escape_3dnow), - - /* 0x10 */ - _2f (movups, X86_INSN_FLAG_SSE_GROUP_10, Gx, Ex), - _2f (movups, X86_INSN_FLAG_SSE_GROUP_10, Ex, Gx), - _2f (movlps, X86_INSN_FLAG_SSE_GROUP_10, Ex, Gx), - _2f (movlps, X86_INSN_FLAG_SSE_GROUP_10, Gx, Ex), - _2f (unpcklps, X86_INSN_FLAG_SSE_GROUP_10, Gx, Ex), - _2f (unpckhps, X86_INSN_FLAG_SSE_GROUP_10, Gx, Ex), - _2f (movhps, X86_INSN_FLAG_SSE_GROUP_10, Ex, Gx), - _2f (movhps, X86_INSN_FLAG_SSE_GROUP_10, Gx, Ex), - _0f (modrm_group_16, X86_INSN_FLAG_MODRM_REG_GROUP_16), - _0 (nop), - _0 (nop), - _0 (nop), - _0 (nop), - _0 (nop), - _0 (nop), - _0 (nop), - - /* 0x20 */ - _2 (mov, Rv, Cv), - _2 (mov, Rv, Dv), - _2 (mov, Cv, Rv), - _2 (mov, Dv, Rv), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _2f (movaps, X86_INSN_FLAG_SSE_GROUP_28, Gx, Ex), - _2f (movaps, X86_INSN_FLAG_SSE_GROUP_28, Ex, Gx), - _2f (cvtpi2ps, X86_INSN_FLAG_SSE_GROUP_28, Gx, Ex), - _2f (movntps, X86_INSN_FLAG_SSE_GROUP_28, Mx, Gx), - _2f (cvttps2pi, X86_INSN_FLAG_SSE_GROUP_28, Gx, Ex), - _2f (cvtps2pi, X86_INSN_FLAG_SSE_GROUP_28, Gx, Ex), - _2f (ucomiss, X86_INSN_FLAG_SSE_GROUP_28, Gx, Ex), - _2f (comiss, X86_INSN_FLAG_SSE_GROUP_28, Gx, Ex), - - /* 0x30 */ - _0 (wrmsr), - _0 (rdtsc), - _0 (rdmsr), - _0 (rdpmc), - _0 (sysenter), - _0 (sysexit), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - - /* 0x40 */ -#define _(x) _2 (cmov##x, Gv, Ev), - foreach_x86_condition -#undef _ - - /* 0x50 */ - _2f (movmskps, X86_INSN_FLAG_SSE_GROUP_50, Gd, Rx), - _2f (sqrtps, X86_INSN_FLAG_SSE_GROUP_50, Gx, Ex), - _2f (rsqrtps, X86_INSN_FLAG_SSE_GROUP_50, Gx, Ex), - _2f (rcpps, X86_INSN_FLAG_SSE_GROUP_50, Gx, Ex), - _2f (andps, X86_INSN_FLAG_SSE_GROUP_50, Gx, Ex), - _2f (andnps, X86_INSN_FLAG_SSE_GROUP_50, Gx, Ex), - _2f (orps, X86_INSN_FLAG_SSE_GROUP_50, Gx, Ex), - _2f (xorps, X86_INSN_FLAG_SSE_GROUP_50, Gx, Ex), - _2f (addps, X86_INSN_FLAG_SSE_GROUP_58, Gx, Ex), - _2f (mulps, X86_INSN_FLAG_SSE_GROUP_58, Gx, Ex), - _2f (cvtps2pd, X86_INSN_FLAG_SSE_GROUP_58, Gx, Ex), - _2f (cvtdq2ps, X86_INSN_FLAG_SSE_GROUP_58, Gx, Ex), - _2f (subps, X86_INSN_FLAG_SSE_GROUP_58, Gx, Ex), - _2f (minps, X86_INSN_FLAG_SSE_GROUP_58, Gx, Ex), - _2f (divps, X86_INSN_FLAG_SSE_GROUP_58, Gx, Ex), - _2f (maxps, X86_INSN_FLAG_SSE_GROUP_58, Gx, Ex), - - /* 0x60 */ - _2f (punpcklbw, X86_INSN_FLAG_SSE_GROUP_60, Gm, Em), - _2f (punpcklwd, X86_INSN_FLAG_SSE_GROUP_60, Gm, Em), - _2f (punpckldq, X86_INSN_FLAG_SSE_GROUP_60, Gm, Em), - _2f (packsswb, X86_INSN_FLAG_SSE_GROUP_60, Gm, Em), - _2f (pcmpgtb, X86_INSN_FLAG_SSE_GROUP_60, Gm, Em), - _2f (pcmpgtw, X86_INSN_FLAG_SSE_GROUP_60, Gm, Em), - _2f (pcmpgtd, X86_INSN_FLAG_SSE_GROUP_60, Gm, Em), - _2f (packuswb, X86_INSN_FLAG_SSE_GROUP_60, Gm, Em), - _2f (punpckhbw, X86_INSN_FLAG_SSE_GROUP_68, Gm, Em), - _2f (punpckhwd, X86_INSN_FLAG_SSE_GROUP_68, Gm, Em), - _2f (punpckhdq, X86_INSN_FLAG_SSE_GROUP_68, Gm, Em), - _2f (packssdw, X86_INSN_FLAG_SSE_GROUP_68, Gm, Em), - _0f (bad, X86_INSN_FLAG_SSE_GROUP_68), - _0f (bad, X86_INSN_FLAG_SSE_GROUP_68), - _2f (movd, X86_INSN_FLAG_SSE_GROUP_68, Gm, Em), - _2f (movq, X86_INSN_FLAG_SSE_GROUP_68, Gm, Em), - - /* 0x70 */ - _3f (pshufw, X86_INSN_FLAG_SSE_GROUP_70, Gm, Em, Ib), - _0f (modrm_group_12, X86_INSN_FLAG_MODRM_REG_GROUP_12), - _0f (modrm_group_13, X86_INSN_FLAG_MODRM_REG_GROUP_13), - _0f (modrm_group_14, X86_INSN_FLAG_MODRM_REG_GROUP_14), - _2f (pcmpeqb, X86_INSN_FLAG_SSE_GROUP_70, Gm, Em), - _2f (pcmpeqw, X86_INSN_FLAG_SSE_GROUP_70, Gm, Em), - _2f (pcmpeqd, X86_INSN_FLAG_SSE_GROUP_70, Gm, Em), - _0f (emms, X86_INSN_FLAG_SSE_GROUP_70), - _0f (bad, X86_INSN_FLAG_SSE_GROUP_78), - _0f (bad, X86_INSN_FLAG_SSE_GROUP_78), - _0f (bad, X86_INSN_FLAG_SSE_GROUP_78), - _0f (bad, X86_INSN_FLAG_SSE_GROUP_78), - _0f (bad, X86_INSN_FLAG_SSE_GROUP_78), - _0f (bad, X86_INSN_FLAG_SSE_GROUP_78), - _2f (movd, X86_INSN_FLAG_SSE_GROUP_78, Em, Gm), - _2f (movq, X86_INSN_FLAG_SSE_GROUP_78, Em, Gm), - - /* 0x80 */ -#define _(x) _1 (jmp##x, Jz), - foreach_x86_condition -#undef _ - - /* 0x90 */ -#define _(x) _1 (set##x, Eb), - foreach_x86_condition -#undef _ - - /* 0xa0 */ - _0 (push_fs), - _0 (pop_fs), - _0 (cpuid), - _2 (bt, Ev, Gv), - _3 (shld, Ev, Gv, Ib), - _3 (shld, Ev, Gv, CL), - _0 (bad), - _0 (bad), - _0 (push_gs), - _0 (pop_gs), - _0 (rsm), - _2 (bts, Ev, Gv), - _3 (shrd, Ev, Gv, Ib), - _3 (shrd, Ev, Gv, CL), - _0f (modrm_group_15, X86_INSN_FLAG_MODRM_REG_GROUP_15), - _2 (imul, Gv, Ev), - - /* 0xb0 */ - _2 (cmpxchg, Eb, Gb), - _2 (cmpxchg, Ev, Gv), - _2 (lss, Gz, Mp), - _2 (btr, Ev, Gv), - _2 (lfs, Gz, Mp), - _2 (lgs, Gz, Mp), - _2 (movzbl, Gv, Eb), - _2 (movzwl, Gv, Ew), - _0 (bad), - _0f (modrm_group_10, X86_INSN_FLAG_MODRM_REG_GROUP_10), - _2f (modrm_group_8, X86_INSN_FLAG_MODRM_REG_GROUP_8, Ev, Ib), - _2 (btc, Ev, Gv), - _2 (bsf, Gv, Ev), - _2 (bsr, Gv, Ev), - _2 (movsx, Gv, Eb), - _2 (movsx, Gv, Ew), - - /* 0xc0 */ - _2 (xadd, Eb, Gb), - _2 (xadd, Ev, Gv), - _3f (cmpps, X86_INSN_FLAG_SSE_GROUP_c0, Gx, Ex, Ib), - _2 (movnti, Mv, Gv), - _3f (pinsrw, X86_INSN_FLAG_SSE_GROUP_c0, Gm, Ew, Ib), - _3f (pextrw, X86_INSN_FLAG_SSE_GROUP_c0, Gd, Rm, Ib), - _3f (shufps, X86_INSN_FLAG_SSE_GROUP_c0, Gx, Ex, Ib), - _1f (modrm_group_9, X86_INSN_FLAG_MODRM_REG_GROUP_9, Mx), -#define _(r) _1 (bswap, r), - foreach_x86_gp_reg -#undef _ - - /* 0xd0 */ - _0f (bad, X86_INSN_FLAG_SSE_GROUP_d0), - _2f (psrlw, X86_INSN_FLAG_SSE_GROUP_d0, Gm, Em), - _2f (psrld, X86_INSN_FLAG_SSE_GROUP_d0, Gm, Em), - _2f (psrlq, X86_INSN_FLAG_SSE_GROUP_d0, Gm, Em), - _2f (paddq, X86_INSN_FLAG_SSE_GROUP_d0, Gm, Em), - _2f (pmullw, X86_INSN_FLAG_SSE_GROUP_d0, Gm, Em), - _0f (bad, X86_INSN_FLAG_SSE_GROUP_d0), - _2f (pmovmskb, X86_INSN_FLAG_SSE_GROUP_d0, Gd, Rm), - _2f (psubusb, X86_INSN_FLAG_SSE_GROUP_d8, Gm, Em), - _2f (psubusw, X86_INSN_FLAG_SSE_GROUP_d8, Gm, Em), - _2f (pminub, X86_INSN_FLAG_SSE_GROUP_d8, Gm, Em), - _2f (pand, X86_INSN_FLAG_SSE_GROUP_d8, Gm, Em), - _2f (paddusb, X86_INSN_FLAG_SSE_GROUP_d8, Gm, Em), - _2f (paddusw, X86_INSN_FLAG_SSE_GROUP_d8, Gm, Em), - _2f (pmaxub, X86_INSN_FLAG_SSE_GROUP_d8, Gm, Em), - _2f (pandn, X86_INSN_FLAG_SSE_GROUP_d8, Gm, Em), - - /* 0xe0 */ - _2f (pavgb, X86_INSN_FLAG_SSE_GROUP_e0, Gm, Em), - _2f (psraw, X86_INSN_FLAG_SSE_GROUP_e0, Gm, Em), - _2f (psrad, X86_INSN_FLAG_SSE_GROUP_e0, Gm, Em), - _2f (pavgw, X86_INSN_FLAG_SSE_GROUP_e0, Gm, Em), - _2f (pmulhuw, X86_INSN_FLAG_SSE_GROUP_e0, Gm, Em), - _2f (pmulhw, X86_INSN_FLAG_SSE_GROUP_e0, Gm, Em), - _2f (bad, X86_INSN_FLAG_SSE_GROUP_e0, Gm, Em), - _2f (movntq, X86_INSN_FLAG_SSE_GROUP_e0, Mm, Gm), - _2f (psubsb, X86_INSN_FLAG_SSE_GROUP_e8, Gm, Em), - _2f (psubsw, X86_INSN_FLAG_SSE_GROUP_e8, Gm, Em), - _2f (pminsw, X86_INSN_FLAG_SSE_GROUP_e8, Gm, Em), - _2f (por, X86_INSN_FLAG_SSE_GROUP_e8, Gm, Em), - _2f (paddsb, X86_INSN_FLAG_SSE_GROUP_e8, Gm, Em), - _2f (paddsw, X86_INSN_FLAG_SSE_GROUP_e8, Gm, Em), - _2f (pmaxsw, X86_INSN_FLAG_SSE_GROUP_e8, Gm, Em), - _2f (pxor, X86_INSN_FLAG_SSE_GROUP_e8, Gm, Em), - - /* 0xf0 */ - _0f (bad, X86_INSN_FLAG_SSE_GROUP_f0), - _2f (psllw, X86_INSN_FLAG_SSE_GROUP_f0, Gm, Em), - _2f (pslld, X86_INSN_FLAG_SSE_GROUP_f0, Gm, Em), - _2f (psllq, X86_INSN_FLAG_SSE_GROUP_f0, Gm, Em), - _2f (pmuludq, X86_INSN_FLAG_SSE_GROUP_f0, Gm, Em), - _2f (pmaddwd, X86_INSN_FLAG_SSE_GROUP_f0, Gm, Em), - _2f (psadbw, X86_INSN_FLAG_SSE_GROUP_f0, Gm, Em), - _2f (maskmovq, X86_INSN_FLAG_SSE_GROUP_f0, Gm, Em), - _2f (psubb, X86_INSN_FLAG_SSE_GROUP_f8, Gm, Em), - _2f (psubw, X86_INSN_FLAG_SSE_GROUP_f8, Gm, Em), - _2f (psubd, X86_INSN_FLAG_SSE_GROUP_f8, Gm, Em), - _2f (psubq, X86_INSN_FLAG_SSE_GROUP_f8, Gm, Em), - _2f (paddb, X86_INSN_FLAG_SSE_GROUP_f8, Gm, Em), - _2f (paddw, X86_INSN_FLAG_SSE_GROUP_f8, Gm, Em), - _2f (paddd, X86_INSN_FLAG_SSE_GROUP_f8, Gm, Em), - _0f (bad, X86_INSN_FLAG_SSE_GROUP_f8), -}; - -typedef struct { - x86_insn_t insns[8]; -} x86_insn_group8_t; - -/* Escape groups are indexed by modrm reg field. */ -static x86_insn_group8_t x86_insn_modrm_reg_groups[] = { - [X86_INSN_MODRM_REG_GROUP_1].insns = { - _0 (add), _0 ( or), _0 (adc), _0 (sbb), - _0 (and), _0 (sub), _0 (xor), _0 (cmp), - }, - - [X86_INSN_MODRM_REG_GROUP_1a].insns = { - _0f (pop, X86_INSN_FLAG_DEFAULT_64_BIT), - _0 (bad), _0 (bad), _0 (bad), - _0 (bad), _0 (bad), _0 (bad), _0 (bad), - }, - - [X86_INSN_MODRM_REG_GROUP_2].insns = { - _0 (rol), _0 (ror), _0 (rcl), _0 (rcr), - _0 (shl), _0 (shr), _0 (sal), _0 (sar), - }, - - [X86_INSN_MODRM_REG_GROUP_3].insns = { - _0 (test), _0 (test), _0 (not), _0 (neg), - _0 (mul), _0 (imul), _0 (div), _0 (idiv), - }, - - [X86_INSN_MODRM_REG_GROUP_4].insns = { - _0 (inc), _0 (dec), _0 (bad), _0 (bad), - _0 (bad), _0 (bad), _0 (bad), _0 (bad), - }, - - [X86_INSN_MODRM_REG_GROUP_5].insns = { - _1 (inc, Ev), - _1 (dec, Ev), - _1f (call, X86_INSN_FLAG_DEFAULT_64_BIT, Ev), - _1 (call, Mp), - _1f (jmp, X86_INSN_FLAG_DEFAULT_64_BIT, Ev), - _1 (jmp, Mp), - _1f (push, X86_INSN_FLAG_DEFAULT_64_BIT, Ev), - _0 (bad), - }, - - [X86_INSN_MODRM_REG_GROUP_6].insns = { - _1 (sldt, Ev), - _1 (str, Ev), - _1 (lldt, Ev), - _1 (ltr, Ev), - _1 (verr, Ev), - _1 (verw, Ev), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_MODRM_REG_GROUP_7].insns = { - _1 (sgdt, Mv), - _1 (sidt, Mv), - _1 (lgdt, Mv), - _1 (lidt, Mv), - _1 (smsw, Ev), - _0 (bad), - _1 (lmsw, Ew), - _1 (invlpg, Mv), - }, - - [X86_INSN_MODRM_REG_GROUP_8].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _2 (bt, Ev, Ib), - _2 (bts, Ev, Ib), - _2 (btr, Ev, Ib), - _2 (btc, Ev, Ib), - }, - - [X86_INSN_MODRM_REG_GROUP_9].insns = { - _0 (bad), - _1 (cmpxchg, Mx), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_MODRM_REG_GROUP_10].insns = { - _0 (bad), _0 (bad), _0 (bad), _0 (bad), - _0 (bad), _0 (bad), _0 (bad), _0 (bad), - }, - - [X86_INSN_MODRM_REG_GROUP_11].insns = { - _0 (mov), _0 (bad), _0 (bad), _0 (bad), - _0 (bad), _0 (bad), _0 (bad), _0 (bad), - }, - - [X86_INSN_MODRM_REG_GROUP_12].insns = { - _0 (bad), - _0 (bad), - _2 (psrlw, Rm, Ib), - _0 (bad), - _2 (psraw, Rm, Ib), - _0 (bad), - _2 (psllw, Rm, Ib), - _0 (bad), - }, - - [X86_INSN_MODRM_REG_GROUP_13].insns = { - _0 (bad), - _0 (bad), - _2 (psrld, Rm, Ib), - _0 (bad), - _2 (psrad, Rm, Ib), - _0 (bad), - _2 (pslld, Rm, Ib), - _0 (bad), - }, - - [X86_INSN_MODRM_REG_GROUP_14].insns = { - _0 (bad), - _0 (bad), - _2 (psrlq, Rm, Ib), - _0f (bad, 0), - _0 (bad), - _0 (bad), - _2 (psllq, Rm, Ib), - _0f (bad, 0), - }, - - [X86_INSN_MODRM_REG_GROUP_15].insns = { - _1 (fxsave, Mv), - _1 (fxrstor, Mv), - _1 (ldmxcsr, Mv), - _1 (stmxcsr, Mv), - _0 (bad), - _1 (lfence, Mv), - _1 (mfence, Mv), - _1 (sfence, Mv), - }, - - [X86_INSN_MODRM_REG_GROUP_16].insns = { - _1 (prefetch_nta, Mv), - _1 (prefetch_t0, Mv), - _1 (prefetch_t1, Mv), - _1 (prefetch_t2, Mv), - _1 (prefetch_nop, Mv), - _1 (prefetch_nop, Mv), - _1 (prefetch_nop, Mv), - _1 (prefetch_nop, Mv), - }, - - [X86_INSN_MODRM_REG_GROUP_p].insns = { - _1 (prefetch_exclusive, Mv), - _1 (prefetch_modified, Mv), - _1 (prefetch_nop, Mv), - _1 (prefetch_modified, Mv), - _1 (prefetch_nop, Mv), - _1 (prefetch_nop, Mv), - _1 (prefetch_nop, Mv), - _1 (prefetch_nop, Mv), - }, -}; - -static x86_insn_group8_t x86_insn_sse_groups_repz[] = { - [X86_INSN_SSE_GROUP_10].insns = { - _2 (movss, Gx, Ex), - _2 (movss, Ex, Gx), - _2 (movsldup, Gx, Ex), - _0 (bad), - _0 (bad), - _0 (bad), - _2 (movshdup, Gx, Ex), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_28].insns = { - _0 (bad), - _0 (bad), - _2 (cvtsi2ss, Gx, Ev), - _0 (bad), - _2 (cvttss2si, Gv, Ex), - _2 (cvtss2si, Gv, Ex), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_50].insns = { - _0 (bad), - _2 (sqrtss, Gx, Ex), - _2 (rsqrtps, Gx, Ex), - _2 (rcpss, Gx, Ex), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_58].insns = { - _2 (addss, Gx, Ex), - _2 (mulss, Gx, Ex), - _2 (cvtss2sd, Gx, Ex), - _2 (cvttps2dq, Gx, Ex), - _2 (subss, Gx, Ex), - _2 (minss, Gx, Ex), - _2 (divss, Gx, Ex), - _2 (maxss, Gx, Ex), - }, - - [X86_INSN_SSE_GROUP_60].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_68].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _2 (movdqu, Gx, Ex), - }, - - [X86_INSN_SSE_GROUP_70].insns = { - _3 (pshufhw, Gx, Ex, Ib), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_78].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _2 (movq, Gx, Ex), - _2 (movdqu, Ex, Gx), - }, - - [X86_INSN_SSE_GROUP_c0].insns = { - _0 (bad), - _0 (bad), - _3 (cmpss, Gx, Ex, Ib), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_d0].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _2 (movq2dq, Gx, Em), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_d8].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_e0].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _2 (cvtdq2pd, Gx, Ex), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_e8].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_f0].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_f8].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, -}; - -static x86_insn_group8_t x86_insn_sse_groups_operand_size[] = { - [X86_INSN_SSE_GROUP_10].insns = { - _2 (movupd, Gx, Ex), - _2 (movupd, Ex, Gx), - _2 (movlpd, Gx, Ex), - _2 (movlpd, Ex, Gx), - _2 (unpcklpd, Gx, Ex), - _2 (unpckhpd, Gx, Ex), - _2 (movhpd, Gx, Mx), - _2 (movhpd, Mx, Gx), - }, - - [X86_INSN_SSE_GROUP_28].insns = { - _2 (movapd, Gx, Ex), - _2 (movapd, Ex, Gx), - _2 (cvtpi2pd, Gx, Ex), - _2 (movntpd, Mx, Gx), - _2 (cvttpd2pi, Gx, Mx), - _2 (cvtpd2pi, Gx, Mx), - _2 (ucomisd, Gx, Ex), - _2 (comisd, Gx, Ex), - }, - - [X86_INSN_SSE_GROUP_50].insns = { - _2 (movmskpd, Gd, Rx), - _2 (sqrtpd, Gx, Ex), - _0 (bad), - _0 (bad), - _2 (andpd, Gx, Ex), - _2 (andnpd, Gx, Ex), - _2 (orpd, Gx, Ex), - _2 (xorpd, Gx, Ex), - }, - - [X86_INSN_SSE_GROUP_58].insns = { - _2 (addpd, Gx, Ex), - _2 (mulpd, Gx, Ex), - _2 (cvtpd2ps, Gx, Ex), - _2 (cvtps2dq, Gx, Ex), - _2 (subpd, Gx, Ex), - _2 (minpd, Gx, Ex), - _2 (divpd, Gx, Ex), - _2 (maxpd, Gx, Ex), - }, - - [X86_INSN_SSE_GROUP_60].insns = { - _2 (punpcklbw, Gx, Ex), - _2 (punpcklwd, Gx, Ex), - _2 (punpckldq, Gx, Ex), - _2 (packsswb, Gx, Ex), - _2 (pcmpgtb, Gx, Ex), - _2 (pcmpgtw, Gx, Ex), - _2 (pcmpgtd, Gx, Ex), - _2 (packuswb, Gx, Ex), - }, - - [X86_INSN_SSE_GROUP_68].insns = { - _2 (punpckhbw, Gx, Ex), - _2 (punpckhwd, Gx, Ex), - _2 (punpckhdq, Gx, Ex), - _2 (packssdw, Gx, Ex), - _2 (punpcklqdq, Gx, Ex), - _2 (punpckhqdq, Gx, Ex), - _2 (movd, Gx, Ev), - _2 (movdqa, Gx, Ex), - }, - - [X86_INSN_SSE_GROUP_70].insns = { - _3 (pshufd, Gx, Ex, Ib), - _0f (modrm_group_12, X86_INSN_FLAG_MODRM_REG_GROUP_12), - _0f (modrm_group_13, X86_INSN_FLAG_MODRM_REG_GROUP_13), - _0f (modrm_group_14, X86_INSN_FLAG_MODRM_REG_GROUP_14), - _2 (pcmpeqb, Gx, Ex), - _2 (pcmpeqw, Gx, Ex), - _2 (pcmpeqd, Gx, Ex), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_78].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _2 (haddpd, Gx, Ex), - _2 (hsubpd, Gx, Ex), - _2 (movd, Ev, Gx), - _2 (movdqa, Ex, Gx), - }, - - [X86_INSN_SSE_GROUP_c0].insns = { - _0 (bad), - _0 (bad), - _3 (cmppd, Gx, Ex, Ib), - _0 (bad), - _3 (pinsrw, Gx, Ew, Ib), - _3 (pextrw, Gd, Gx, Ib), - _3 (shufpd, Gx, Ex, Ib), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_d0].insns = { - _2 (addsubpd, Gx, Ex), - _2 (psrlw, Gx, Ex), - _2 (psrld, Gx, Ex), - _2 (psrlq, Gx, Ex), - _2 (paddq, Gx, Ex), - _2 (pmullw, Gx, Ex), - _2 (movq, Ex, Gx), - _2 (pmovmskb, Gd, Rx), - }, - - [X86_INSN_SSE_GROUP_d8].insns = { - _2 (psubusb, Gx, Ex), - _2 (psubusw, Gx, Ex), - _2 (pminub, Gx, Ex), - _2 (pand, Gx, Ex), - _2 (paddusb, Gx, Ex), - _2 (paddusw, Gx, Ex), - _2 (pmaxub, Gx, Ex), - _2 (pandn, Gx, Ex), - }, - - [X86_INSN_SSE_GROUP_e0].insns = { - _2 (pavgb, Gx, Ex), - _2 (psraw, Gx, Ex), - _2 (psrad, Gx, Ex), - _2 (pavgw, Gx, Ex), - _2 (pmulhuw, Gx, Ex), - _2 (pmulhw, Gx, Ex), - _2 (cvttpd2dq, Gx, Ex), - _2 (movntdq, Mx, Gx), - }, - - [X86_INSN_SSE_GROUP_e8].insns = { - _2 (psubsb, Gx, Ex), - _2 (psubsw, Gx, Ex), - _2 (pminsw, Gx, Ex), - _2 (por, Gx, Ex), - _2 (paddsb, Gx, Ex), - _2 (paddsw, Gx, Ex), - _2 (pmaxsw, Gx, Ex), - _2 (pxor, Gx, Ex), - }, - - [X86_INSN_SSE_GROUP_f0].insns = { - _0 (bad), - _2 (psllw, Gx, Ex), - _2 (pslld, Gx, Ex), - _2 (psllq, Gx, Ex), - _2 (pmuludq, Gx, Ex), - _2 (pmaddwd, Gx, Ex), - _2 (psadbw, Gx, Ex), - _2 (maskmovdqu, Gx, Ex), - }, - - [X86_INSN_SSE_GROUP_f8].insns = { - _2 (psubb, Gx, Ex), - _2 (psubw, Gx, Ex), - _2 (psubd, Gx, Ex), - _2 (psubq, Gx, Ex), - _2 (paddb, Gx, Ex), - _2 (paddw, Gx, Ex), - _2 (paddd, Gx, Ex), - _0 (bad), - }, -}; - -static x86_insn_group8_t x86_insn_sse_groups_repnz[] = { - [X86_INSN_SSE_GROUP_10].insns = { - _2 (movsd, Gx, Ex), - _2 (movsd, Ex, Gx), - _2 (movddup, Gx, Ex), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_28].insns = { - _0 (bad), - _0 (bad), - _2 (cvtsi2sd, Gx, Ev), - _0 (bad), - _2 (cvttsd2si, Gv, Ex), - _2 (cvtsd2si, Gv, Ex), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_50].insns = { - _0 (bad), - _2 (sqrtsd, Gx, Ex), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_58].insns = { - _2 (addsd, Gx, Ex), - _2 (mulsd, Gx, Ex), - _2 (cvtsd2ss, Gx, Ex), - _0 (bad), - _2 (subsd, Gx, Ex), - _2 (minsd, Gx, Ex), - _2 (divsd, Gx, Ex), - _2 (maxsd, Gx, Ex), - }, - - [X86_INSN_SSE_GROUP_60].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_68].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_70].insns = { - _3 (pshuflw, Gx, Ex, Ib), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_78].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _2 (haddps, Gx, Ex), - _2 (hsubps, Gx, Ex), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_c0].insns = { - _0 (bad), - _0 (bad), - _3 (cmpsd, Gx, Ex, Ib), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_d0].insns = { - _2 (addsubps, Gx, Ex), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _2 (movdq2q, Gm, Ex), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_d8].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_e0].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _2 (cvtpd2dq, Gx, Ex), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_e8].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_f0].insns = { - _2 (lddqu, Gx, Mx), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_f8].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, -}; - -#undef _ - -/* Parses memory displacements and immediates. */ -static u8 * x86_insn_parse_number (u32 log2_n_bytes, - u8 * code, u8 * code_end, - i64 * result) -{ - i64 x = 0; - - if (code + (1 << log2_n_bytes) > code_end) - return 0; - - switch (log2_n_bytes) - { - case 3: - x = clib_little_to_host_unaligned_mem_u64 ((u64 *) code); - break; - - case 2: - x = (i32) clib_little_to_host_unaligned_mem_u32 ((u32 *) code); - break; - - case 1: - x = (i16) clib_little_to_host_unaligned_mem_u16 ((u16 *) code); - break; - - case 0: - x = (i8) code[0]; - break; - - default: - ASSERT (0); - } - - *result = x; - return code + (1 << log2_n_bytes); -} - -static u32 -x86_insn_log2_immediate_bytes (x86_insn_parse_t * p, x86_insn_t * insn) -{ - u32 i = ~0; - switch (x86_insn_immediate_type (insn)) - { - case 'b': i = 0; break; - case 'w': i = 1; break; - case 'd': i = 2; break; - case 'q': i = 3; break; - - case 'z': - i = p->log2_effective_operand_bytes; - if (i > 2) i = 2; - break; - - case 'v': - i = p->log2_effective_operand_bytes; - break; - - default: - i = ~0; - break; - } - - return i; -} - -static u8 * -x86_insn_parse_modrm_byte (x86_insn_parse_t * x, - x86_insn_modrm_byte_t modrm, - u32 parse_flags, - u8 * code, - u8 * code_end) -{ - u8 effective_address_bits; - - if (parse_flags & X86_INSN_PARSE_64_BIT) - effective_address_bits = (x->flags & X86_INSN_ADDRESS_SIZE) ? 32 : 64; - else if (parse_flags & X86_INSN_PARSE_32_BIT) - effective_address_bits = (x->flags & X86_INSN_ADDRESS_SIZE) ? 16 : 32; - else - effective_address_bits = (x->flags & X86_INSN_ADDRESS_SIZE) ? 32 : 16; - - x->log2_effective_address_bytes = 1; - x->log2_effective_address_bytes += effective_address_bits > 16; - x->log2_effective_address_bytes += effective_address_bits > 32; - - x->regs[0] |= modrm.reg; - if (modrm.mode == 3) - x->regs[1] |= modrm.rm; - else - { - u32 log2_disp_bytes = ~0; - - x->flags |= X86_INSN_IS_ADDRESS; - - if (effective_address_bits != 16) - { - u8 has_sib_byte = 0; - - switch (modrm.mode) - { - case 0: - /* When base is bp displacement is present for mode 0. */ - if (modrm.rm == X86_INSN_GP_REG_BP) - { - log2_disp_bytes = x->log2_effective_address_bytes; - break; - } - else if (modrm.rm == X86_INSN_GP_REG_SP - && effective_address_bits != 16) - { - has_sib_byte = 1; - break; - } - /* fall through */ - case 1: - case 2: - x->regs[1] |= modrm.rm; - x->flags |= X86_INSN_HAS_BASE; - if (modrm.mode != 0) - { - log2_disp_bytes = (modrm.mode == 1 - ? 0 - : x->log2_effective_address_bytes); - if (log2_disp_bytes > 2) - log2_disp_bytes = 2; - } - break; - } - - if (has_sib_byte) - { - x86_insn_sib_byte_t sib; - - if (code >= code_end) - return 0; - sib.byte = *code++; - - x->log2_index_scale = 1 << sib.log2_scale; - x->regs[1] |= sib.base; - x->flags |= X86_INSN_HAS_BASE; - - if (sib.index != X86_INSN_GP_REG_SP) - { - x->regs[2] |= sib.index; - x->flags |= X86_INSN_HAS_INDEX; - } - } - } - else - { - /* effective_address_bits == 16 */ - switch (modrm.mode) - { - case 0: - if (modrm.rm == 6) - { - /* [disp16] */ - log2_disp_bytes = 1; - break; - } - /* fall through */ - case 1: - case 2: - switch (modrm.rm) - { - case 0: /* [bx + si/di] */ - case 1: - x->regs[1] = X86_INSN_GP_REG_BX; - x->regs[2] = X86_INSN_GP_REG_SI + (modrm.rm & 1); - x->flags |= X86_INSN_HAS_BASE | X86_INSN_HAS_INDEX; - break; - - case 2: /* [bp + si/di] */ - case 3: - x->regs[1] = X86_INSN_GP_REG_BP; - x->regs[2] = X86_INSN_GP_REG_SI + (modrm.rm & 1); - x->flags |= X86_INSN_HAS_BASE | X86_INSN_HAS_INDEX; - break; - - case 4: /* [si/di] */ - case 5: - x->regs[1] = X86_INSN_GP_REG_SI + (modrm.rm & 1); - x->flags |= X86_INSN_HAS_BASE; - break; - - case 6: /* [bp + disp] */ - x->regs[1] = X86_INSN_GP_REG_BP; - x->flags |= X86_INSN_HAS_BASE; - break; - - case 7: /* [bx + disp] */ - x->regs[1] = X86_INSN_GP_REG_BX; - x->flags |= X86_INSN_HAS_BASE; - break; - } - - if (modrm.mode != 0) - log2_disp_bytes = modrm.mode == 1 ? 0 : 1; - break; - } - } - - if (log2_disp_bytes != ~0) - { - i64 disp; - code = x86_insn_parse_number (log2_disp_bytes, code, code_end, - &disp); - if (code) - x->displacement = disp; - } - } - - return code; -} - -u8 * x86_insn_parse (x86_insn_parse_t * p, u8 * code_start) -{ - u8 i, * code, * code_end; - x86_insn_t * insn, * group_insn; - u8 default_operand_bits, effective_operand_bits; - u32 opcode, parse_flags; - - /* Preserve global parse flags. */ - parse_flags = p->flags & (X86_INSN_PARSE_32_BIT | X86_INSN_PARSE_64_BIT); - clib_memset (p, 0, sizeof (p[0])); - p->flags = parse_flags; - - /* 64 implies 32 bit parsing. */ - if (parse_flags & X86_INSN_PARSE_64_BIT) - parse_flags |= X86_INSN_PARSE_32_BIT; - - /* Instruction must be <= 15 bytes. */ - code = code_start; - code_end = code + 15; - - /* Parse legacy prefixes. */ - while (1) - { - if (code >= code_end) - goto insn_too_long; - i = code[0]; - code++; - switch (i) - { - default: goto prefix_done; - - /* Set flags based on prefix. */ -#define _(x,o) case o: p->flags |= X86_INSN_##x; break; - foreach_x86_legacy_prefix; -#undef _ - } - } - prefix_done: - - /* REX prefix. */ - if ((parse_flags & X86_INSN_PARSE_64_BIT) && i >= 0x40 && i <= 0x4f) - { - p->regs[0] |= ((i & (1 << 2)) != 0) << 3; /* r bit */ - p->regs[1] |= ((i & (1 << 0)) != 0) << 3; /* b bit */ - p->regs[2] |= ((i & (1 << 1)) != 0) << 3; /* x bit */ - p->flags |= ((i & (1 << 3)) /* w bit */ - ? X86_INSN_OPERAND_SIZE_64 : 0); - if (code >= code_end) - goto insn_too_long; - i = *code++; - } - - opcode = i; - if (opcode == 0x0f) - { - /* two byte opcode. */; - if (code >= code_end) - goto insn_too_long; - i = *code++; - opcode = (opcode << 8) | i; - insn = x86_insns_two_byte + i; - } - else - { - static x86_insn_t arpl = { - .name = "arpl", - .operands[0].data = "Ew", - .operands[1].data = "Gw", - }; - - if (PREDICT_FALSE (i == 0x63 - && ! (parse_flags & X86_INSN_PARSE_64_BIT))) - insn = &arpl; - else - insn = x86_insns_one_byte + i; - } - - if ((i = X86_INSN_FLAG_GET_SSE_GROUP (insn->flags)) != 0) - { - x86_insn_group8_t * g8; - - if (p->flags & X86_INSN_OPERAND_SIZE) - g8 = x86_insn_sse_groups_operand_size; - else if (p->flags & X86_INSN_REPZ) - g8 = x86_insn_sse_groups_repz; - else if (p->flags & X86_INSN_REPNZ) - g8 = x86_insn_sse_groups_repnz; - else - g8 = 0; - - /* insn flags have 1 + group so != 0 test above can work. */ - ASSERT ((i - 1) < ARRAY_LEN (x86_insn_sse_groups_operand_size)); - if (g8) - insn = g8[i - 1].insns + (opcode & 7); - } - - /* Parse modrm and displacement if present. */ - if (x86_insn_has_modrm_byte (insn)) - { - x86_insn_modrm_byte_t modrm; - - if (code >= code_end) - goto insn_too_long; - modrm.byte = *code++; - - /* Handle special 0x0f01 and 0x0fae encodings. */ - if (PREDICT_FALSE (modrm.mode == 3 - && (opcode == 0x0f01 - || opcode == 0x0fae))) - { - static x86_insn_t x86_insns_0f01_special[] = { - _0 (swapgs), _0 (rdtscp), _0 (bad), _0 (bad), - _0 (bad), _0 (bad), _0 (bad), _0 (bad), - }; - static x86_insn_t x86_insns_0fae_special[] = { - _0 (vmrun), _0 (vmmcall), _0 (vmload), _0 (vmsave), - _0 (stgi), _0 (clgi), _0 (skinit), _0 (invlpga), - }; - - if (opcode == 0x0f01) - insn = x86_insns_0f01_special; - else - insn = x86_insns_0fae_special; - insn += modrm.rm; - opcode = (opcode << 8) | modrm.byte; - } - else - { - code = x86_insn_parse_modrm_byte (p, modrm, parse_flags, - code, code_end); - if (! code) - goto insn_too_long; - } - } - - group_insn = 0; - if ((i = X86_INSN_FLAG_GET_MODRM_REG_GROUP (insn->flags)) != 0) - { - u32 g = i - 1; - ASSERT (g < ARRAY_LEN (x86_insn_modrm_reg_groups)); - group_insn = x86_insn_modrm_reg_groups[g].insns + (p->regs[0] & 7); - } - - p->insn = insn[0]; - if (group_insn) - { - u32 k; - p->insn.name = group_insn->name; - p->insn.flags |= group_insn->flags; - for (k = 0; k < ARRAY_LEN (group_insn->operands); k++) - if (x86_insn_operand_is_valid (group_insn, k)) - p->insn.operands[k] = group_insn->operands[k]; - } - - default_operand_bits - = ((((parse_flags & X86_INSN_PARSE_32_BIT) != 0) - ^ ((p->flags & X86_INSN_OPERAND_SIZE) != 0)) - ? BITS (u32) : BITS (u16)); - - if ((parse_flags & X86_INSN_PARSE_64_BIT) - && (p->insn.flags & X86_INSN_FLAG_DEFAULT_64_BIT)) - default_operand_bits = BITS (u64); - - effective_operand_bits = default_operand_bits; - if (p->flags & X86_INSN_OPERAND_SIZE_64) - effective_operand_bits = BITS (u64); - - p->log2_effective_operand_bytes = 1; - p->log2_effective_operand_bytes += effective_operand_bits > 16; - p->log2_effective_operand_bytes += effective_operand_bits > 32; - - /* Parse immediate if present. */ - { - u32 l = x86_insn_log2_immediate_bytes (p, insn); - if (l <= 3) - { - code = x86_insn_parse_number (l, code, code_end, &p->immediate); - if (! code) - goto insn_too_long; - } - } - - return code; - - insn_too_long: - return 0; -} - -static u8 * format_x86_gp_reg_operand (u8 * s, va_list * va) -{ - u32 r = va_arg (*va, u32); - u32 log2_n_bytes = va_arg (*va, u32); - - const char names8[8] = "acdbsbsd"; - const char names16[8] = "xxxxppii"; - - ASSERT (r < 16); - - /* Add % register prefix. */ - vec_add1 (s, '%'); - - switch (log2_n_bytes) - { - case 0: - { - - if (r < 8) - s = format (s, "%c%c", names8[r & 3], (r >> 2) ? 'l' : 'h'); - else - s = format (s, "r%db", r); - } - break; - - case 2: - case 3: - s = format (s, "%c", log2_n_bytes == 2 ? 'e' : 'r'); - /* fall through */ - case 1: - if (r < 8) - s = format (s, "%c%c", names8[r], names16[r]); - else - { - s = format (s, "%d", r); - if (log2_n_bytes != 3) - s = format (s, "%c", log2_n_bytes == 1 ? 'w' : 'd'); - } - break; - - default: - ASSERT (0); - } - - return s; -} - -static u8 * format_x86_reg_operand (u8 * s, va_list * va) -{ - u32 reg = va_arg (*va, u32); - u32 log2_n_bytes = va_arg (*va, u32); - u32 type = va_arg (*va, u32); - - switch (type) - { - default: - ASSERT (0); - break; - - case 'x': - ASSERT (reg < 16); - return format (s, "%%xmm%d", reg); - - case 'm': - ASSERT (reg < 8); - return format (s, "%%mm%d", reg); - - /* Explicit byte/word/double-word/quad-word */ - case 'b': log2_n_bytes = 0; break; - case 'w': log2_n_bytes = 1; break; - case 'd': log2_n_bytes = 2; break; - case 'q': log2_n_bytes = 3; break; - - /* Use effective operand size. */ - case 'v': break; - - /* word or double-word depending on effective operand size. */ - case 'z': - log2_n_bytes = clib_min (log2_n_bytes, 2); - break; - } - - s = format (s, "%U", format_x86_gp_reg_operand, reg, log2_n_bytes); - return s; -} - -static u8 * format_x86_mem_operand (u8 * s, va_list * va) -{ - x86_insn_parse_t * p = va_arg (*va, x86_insn_parse_t *); - - if (p->displacement != 0) - s = format (s, "0x%x", p->displacement); - - if (p->flags & X86_INSN_HAS_BASE) - { - s = format (s, "(%U", - format_x86_gp_reg_operand, p->regs[1], - p->log2_effective_address_bytes); - if (p->flags & X86_INSN_HAS_INDEX) - { - s = format (s, ",%U", - format_x86_gp_reg_operand, p->regs[2], - p->log2_effective_address_bytes); - if (p->log2_index_scale != 0) - s = format (s, ",%d", 1 << p->log2_index_scale); - } - s = format (s, ")"); - } - - /* [RIP+disp] PC relative addressing in 64 bit mode. */ - else if (p->flags & X86_INSN_PARSE_64_BIT) - s = format (s, "(%%rip)"); - - return s; -} - -static u8 * format_x86_insn_operand (u8 * s, va_list * va) -{ - x86_insn_parse_t * p = va_arg (*va, x86_insn_parse_t *); - x86_insn_t * insn = &p->insn; - u32 o = va_arg (*va, u32); - u8 c, t; - - ASSERT (o < ARRAY_LEN (insn->operands)); - c = insn->operands[o].code; - t = insn->operands[o].type; - - /* Register encoded in instruction. */ - if (c < 8) - return format (s, "%U", - format_x86_gp_reg_operand, c, - p->log2_effective_operand_bytes); - - switch (c) - { - /* Memory or reg field from modrm byte. */ - case 'M': - ASSERT (p->flags & X86_INSN_IS_ADDRESS); - /* FALLTHROUGH */ - case 'E': - if (p->flags & X86_INSN_IS_ADDRESS) - s = format (s, "%U", format_x86_mem_operand, p); - else - s = format (s, "%U", - format_x86_reg_operand, p->regs[1], - p->log2_effective_operand_bytes, t); - break; - - /* reg field from modrm byte. */ - case 'R': - case 'G': - s = format (s, "%U", - format_x86_reg_operand, p->regs[0], - p->log2_effective_operand_bytes, t); - break; - - case 'I': - { - u32 l = x86_insn_log2_immediate_bytes (p, insn); - i64 mask = pow2_mask (8ULL << l); - s = format (s, "$0x%Lx", p->immediate & mask); - } - break; - - case 'J': - if (p->immediate < 0) - s = format (s, "- 0x%Lx", -p->immediate); - else - s = format (s, "+ 0x%Lx", p->immediate); - break; - - case 'O': - s = format (s, "0x%Lx", p->immediate); - break; - - case 'A': - /* AX/AL */ - s = format (s, "%U", - format_x86_gp_reg_operand, X86_INSN_GP_REG_AX, - t == 'L' ? 0 : p->log2_effective_operand_bytes); - break; - - case 'B': - /* BX/BL/BP */ - s = format (s, "%U", - format_x86_gp_reg_operand, - t == 'P' ? X86_INSN_GP_REG_BP : X86_INSN_GP_REG_BX, - t == 'L' ? 0 : p->log2_effective_operand_bytes); - break; - - case 'C': - /* CX/CL */ - s = format (s, "%U", - format_x86_gp_reg_operand, X86_INSN_GP_REG_CX, - t == 'L' ? 0 : p->log2_effective_operand_bytes); - break; - - case 'D': - /* DX/DL/DI */ - s = format (s, "%U", - format_x86_gp_reg_operand, - t == 'I' ? X86_INSN_GP_REG_DI : X86_INSN_GP_REG_DX, - t == 'L' ? 0 : p->log2_effective_operand_bytes); - break; - - case 'S': - /* SI/SP */ - s = format (s, "%U", - format_x86_gp_reg_operand, - t == 'I' ? X86_INSN_GP_REG_SI : X86_INSN_GP_REG_SP, - p->log2_effective_operand_bytes); - break; - - case '1': - s = format (s, "1"); - break; - - default: - ASSERT (0); - } - - return s; -} - -u8 * format_x86_insn_parse (u8 * s, va_list * va) -{ - x86_insn_parse_t * p = va_arg (*va, x86_insn_parse_t *); - x86_insn_t * insn = &p->insn; - u32 o, i, is_src_dst; - - s = format (s, "%s", insn->name); - - if (! x86_insn_operand_is_valid (insn, 0)) - goto done; - - is_src_dst = x86_insn_operand_is_valid (insn, 1); - - /* If instruction has immediate add suffix to opcode to - indicate operand size. */ - if (is_src_dst) - { - u32 b; - - b = x86_insn_log2_immediate_bytes (p, insn); - if (b < p->log2_effective_operand_bytes - && (p->flags & X86_INSN_IS_ADDRESS)) - s = format (s, "%c", "bwlq"[b]); - } - - for (i = 0; i < ARRAY_LEN (insn->operands); i++) - { - o = is_src_dst + i; - if (! x86_insn_operand_is_valid (insn, o)) - break; - s = format (s, "%s%U", - i == 0 ? " " : ", ", - format_x86_insn_operand, p, o); - } - - if (is_src_dst) - s = format (s, ", %U", - format_x86_insn_operand, p, 0); - - done: - return s; -} diff --git a/src/vppinfra/asm_x86.h b/src/vppinfra/asm_x86.h deleted file mode 100644 index dacef61755c..00000000000 --- a/src/vppinfra/asm_x86.h +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef included_asm_x86_h -#define included_asm_x86_h - -#include <vppinfra/format.h> - -typedef union -{ - struct - { - u8 code; - u8 type; - }; - u8 data[2]; -} x86_insn_operand_t; - -typedef struct -{ - /* Instruction name. */ - char *name; - - /* X86 instructions may have up to 3 operands. */ - x86_insn_operand_t operands[3]; - - u16 flags; -#define X86_INSN_FLAG_DEFAULT_64_BIT (1 << 0) -#define X86_INSN_FLAG_SET_SSE_GROUP(n) ((n) << 5) -#define X86_INSN_FLAG_GET_SSE_GROUP(f) (((f) >> 5) & 0x1f) -#define X86_INSN_FLAG_SET_MODRM_REG_GROUP(n) (((n) & 0x3f) << 10) -#define X86_INSN_FLAG_GET_MODRM_REG_GROUP(f) (((f) >> 10) & 0x3f) -} x86_insn_t; - -always_inline uword -x86_insn_operand_is_valid (x86_insn_t * i, uword o) -{ - ASSERT (o < ARRAY_LEN (i->operands)); - return i->operands[o].code != '_'; -} - -#define foreach_x86_legacy_prefix \ - _ (OPERAND_SIZE, 0x66) \ - _ (ADDRESS_SIZE, 0x67) \ - _ (SEGMENT_CS, 0x2e) \ - _ (SEGMENT_DS, 0x3e) \ - _ (SEGMENT_ES, 0x26) \ - _ (SEGMENT_FS, 0x64) \ - _ (SEGMENT_GS, 0x65) \ - _ (SEGMENT_SS, 0x36) \ - _ (LOCK, 0xf0) \ - _ (REPZ, 0xf3) \ - _ (REPNZ, 0xf2) - -#define foreach_x86_insn_parse_flag \ - /* Parse in 32/64-bit mode. */ \ - _ (PARSE_32_BIT, 0) \ - _ (PARSE_64_BIT, 0) \ - _ (IS_ADDRESS, 0) \ - /* regs[1/2] is a valid base/index register */ \ - _ (HAS_BASE, 0) \ - _ (HAS_INDEX, 0) \ - /* rex w bit */ \ - _ (OPERAND_SIZE_64, 0) - -typedef enum -{ -#define _(f,o) X86_INSN_FLAG_BIT_##f, - foreach_x86_insn_parse_flag foreach_x86_legacy_prefix -#undef _ -} x86_insn_parse_flag_bit_t; - -typedef enum -{ -#define _(f,o) X86_INSN_##f = 1 << X86_INSN_FLAG_BIT_##f, - foreach_x86_insn_parse_flag foreach_x86_legacy_prefix -#undef _ -} x86_insn_parse_flag_t; - -typedef struct -{ - /* Registers in instruction. - [0] is modrm reg field - [1] is base reg - [2] is index reg. */ - u8 regs[3]; - - /* Scale for index register. */ - u8 log2_index_scale:2; - u8 log2_effective_operand_bytes:3; - u8 log2_effective_address_bytes:3; - - i32 displacement; - - /* Parser flags: set of x86_insn_parse_flag_t enums. */ - u32 flags; - - i64 immediate; - - x86_insn_t insn; -} x86_insn_parse_t; - -u8 *x86_insn_parse (x86_insn_parse_t * p, u8 * code_start); -format_function_t format_x86_insn_parse; - -#endif /* included_asm_x86_h */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/vppinfra/backtrace.c b/src/vppinfra/backtrace.c deleted file mode 100644 index bae563d04b0..00000000000 --- a/src/vppinfra/backtrace.c +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - Copyright (c) 2004 Eliot Dresselhaus - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ - -#include <vppinfra/clib.h> -#include <vppinfra/error.h> - -#ifdef __mips__ - -/* Let code below know we've defined _clib_backtrace */ -#define clib_backtrace_defined - -#include <vppinfra/asm_mips.h> - -__clib_export uword -clib_backtrace (uword * callers, uword max_callers, uword n_frames_to_skip) -{ - u32 *pc; - void *sp; - uword i, saved_pc; - - /* Figure current PC, saved PC and stack pointer. */ - asm volatile (".set push\n" - ".set noat\n" "move %[saved_pc], $31\n" "move %[sp], $29\n" - /* Fetches current PC. */ - "la $at, 1f\n" - "jalr %[pc], $at\n" - "nop\n" - "1:\n" - ".set pop\n":[pc] "=r" (pc), - [saved_pc] "=r" (saved_pc),[sp] "=r" (sp)); - - /* Also skip current frame. */ - n_frames_to_skip += 1; - - for (i = 0; i < max_callers + n_frames_to_skip; i++) - { - mips_insn_opcode_t op; - mips_insn_special_funct_t funct; - i32 insn, rs, rt, rd, immediate, found_saved_pc; - u32 *start_pc; - - /* Parse instructions until we reach prologue for this - stack frame. We'll need to figure out where saved - PC is and where previous stack frame lives. */ - start_pc = pc; - found_saved_pc = 0; - while (1) - { - insn = *--pc; - op = mips_insn_get_op (insn); - funct = mips_insn_get_funct (insn); - rs = mips_insn_get_rs (insn); - rt = mips_insn_get_rt (insn); - rd = mips_insn_get_rd (insn); - immediate = mips_insn_get_immediate (insn); - - switch (op) - { - default: - break; - - case MIPS_OPCODE_sd: - case MIPS_OPCODE_sw: - /* Trace stores of return address. */ - if (rt == MIPS_REG_RA) - { - void *addr = sp + immediate; - - /* If RA is stored somewhere other than in the - stack frame, give up. */ - if (rs != MIPS_REG_SP) - goto backtrace_done; - - ASSERT (immediate % 4 == 0); - if (op == MIPS_OPCODE_sw) - saved_pc = ((u32 *) addr)[0]; - else - saved_pc = ((u64 *) addr)[0]; - found_saved_pc = 1; - } - break; - - case MIPS_OPCODE_addiu: - case MIPS_OPCODE_daddiu: - case MIPS_OPCODE_addi: - case MIPS_OPCODE_daddi: - if (rt == MIPS_REG_SP) - { - if (rs != MIPS_REG_SP) - goto backtrace_done; - - ASSERT (immediate % 4 == 0); - - /* Assume positive offset is part of the epilogue. - E.g. - jr ra - add sp,sp,100 - */ - if (immediate > 0) - continue; - - /* Negative offset means allocate stack space. - This could either be the prologue or could be due to - alloca. */ - sp -= immediate; - - /* This frame will not save RA. */ - if (i == 0) - goto found_prologue; - - /* Assume that addiu sp,sp,-N without store of ra means - that we have not found the prologue yet. */ - if (found_saved_pc) - goto found_prologue; - } - break; - - case MIPS_OPCODE_slti: - case MIPS_OPCODE_sltiu: - case MIPS_OPCODE_andi: - case MIPS_OPCODE_ori: - case MIPS_OPCODE_xori: - case MIPS_OPCODE_lui: - case MIPS_OPCODE_ldl: - case MIPS_OPCODE_ldr: - case MIPS_OPCODE_lb: - case MIPS_OPCODE_lh: - case MIPS_OPCODE_lwl: - case MIPS_OPCODE_lw: - case MIPS_OPCODE_lbu: - case MIPS_OPCODE_lhu: - case MIPS_OPCODE_lwr: - case MIPS_OPCODE_lwu: - case MIPS_OPCODE_ld: - /* Give up when we find anyone setting the stack pointer. */ - if (rt == MIPS_REG_SP) - goto backtrace_done; - break; - - case MIPS_OPCODE_SPECIAL: - if (rd == MIPS_REG_SP) - switch (funct) - { - default: - /* Give up when we find anyone setting the stack pointer. */ - goto backtrace_done; - - case MIPS_SPECIAL_FUNCT_break: - case MIPS_SPECIAL_FUNCT_jr: - case MIPS_SPECIAL_FUNCT_sync: - case MIPS_SPECIAL_FUNCT_syscall: - case MIPS_SPECIAL_FUNCT_tge: - case MIPS_SPECIAL_FUNCT_tgeu: - case MIPS_SPECIAL_FUNCT_tlt: - case MIPS_SPECIAL_FUNCT_tltu: - case MIPS_SPECIAL_FUNCT_teq: - case MIPS_SPECIAL_FUNCT_tne: - /* These instructions can validly have rd == MIPS_REG_SP */ - break; - } - break; - } - } - - found_prologue: - /* Check sanity of saved pc. */ - if (saved_pc & 3) - goto backtrace_done; - if (saved_pc == 0) - goto backtrace_done; - - if (i >= n_frames_to_skip) - callers[i - n_frames_to_skip] = saved_pc; - pc = uword_to_pointer (saved_pc, u32 *); - } - -backtrace_done: - if (i < n_frames_to_skip) - return 0; - else - return i - n_frames_to_skip; -} -#endif /* __mips__ */ - -#ifndef clib_backtrace_defined -#define clib_backtrace_defined - -#ifndef USE_LIBUNWIND -/* use glibc backtrace for stack trace */ -#include <execinfo.h> -#else -#include <libunwind.h> -static int -backtrace (void **buffer, int size) -{ - return unw_backtrace (buffer, size); -} -#endif - -__clib_export uword -clib_backtrace (uword * callers, uword max_callers, uword n_frames_to_skip) -{ - int size; - void *array[20]; - /* Also skip current frame. */ - n_frames_to_skip += 1; - - size = clib_min (ARRAY_LEN (array), max_callers + n_frames_to_skip); - - size = backtrace (array, size); - - uword i; - - for (i = 0; i < max_callers + n_frames_to_skip && i < size; i++) - { - if (i >= n_frames_to_skip) - callers[i - n_frames_to_skip] = pointer_to_uword (array[i]); - } - - if (i < n_frames_to_skip) - return 0; - else - return i - n_frames_to_skip; -} - - -#endif /* clib_backtrace_defined */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/vppinfra/cache.h b/src/vppinfra/cache.h index 4229a068486..13778a423fd 100644 --- a/src/vppinfra/cache.h +++ b/src/vppinfra/cache.h @@ -68,6 +68,17 @@ #define CLIB_PREFETCH_WRITE 1 #define CLIB_PREFETCH_STORE 1 /* alias for write */ +/* locality arguments to __builtin_prefetch. */ +#define CLIB_PREFETCH_TO_STREAM 0 // NTA +#define CLIB_PREFETCH_TO_L3 1 // T2 +#define CLIB_PREFETCH_TO_L2 2 // T1 +#define CLIB_PREFETCH_TO_L1 3 // T0 + +#define _CLIB_TARGETED_PREFETCH(n, size, type, loc) \ + if ((size) > (n) *CLIB_CACHE_PREFETCH_BYTES) \ + __builtin_prefetch (_addr + (n) *CLIB_CACHE_PREFETCH_BYTES, \ + CLIB_PREFETCH_##type, CLIB_PREFETCH_TO_##loc); + #define _CLIB_PREFETCH(n, size, type) \ if ((size) > (n) *CLIB_CACHE_PREFETCH_BYTES) \ __builtin_prefetch (_addr + (n) *CLIB_CACHE_PREFETCH_BYTES, \ @@ -86,6 +97,19 @@ } \ while (0) +#define CLIB_TARGETED_PREFETCH(addr, size, type, locality) \ + do \ + { \ + void *_addr = (addr); \ + \ + ASSERT ((size) <= 4 * CLIB_CACHE_PREFETCH_BYTES); \ + _CLIB_TARGETED_PREFETCH (0, size, type, locality); \ + _CLIB_TARGETED_PREFETCH (1, size, type, locality); \ + _CLIB_TARGETED_PREFETCH (2, size, type, locality); \ + _CLIB_TARGETED_PREFETCH (3, size, type, locality); \ + } \ + while (0) + #undef _ static_always_inline void diff --git a/src/vppinfra/clib.h b/src/vppinfra/clib.h index d14582492d6..5348738ec6a 100644 --- a/src/vppinfra/clib.h +++ b/src/vppinfra/clib.h @@ -106,6 +106,13 @@ #define CLIB_STRING_ARRAY(...) \ (char *[]) { __VA_ARGS__, 0 } +#define CLIB_SWAP(a, b) \ + { \ + typeof (a) __tmp = a; \ + a = b; \ + b = __tmp; \ + } + /* sanitizers */ #ifdef __has_feature #if __has_feature(address_sanitizer) @@ -385,10 +392,6 @@ void qsort (void *base, uword n, uword size, int (*)(const void *, const void *)); #endif -/* Stack backtrace. */ -uword -clib_backtrace (uword * callers, uword max_callers, uword n_frames_to_skip); - #include <vppinfra/byte_order.h> #endif /* included_clib_h */ diff --git a/src/vppinfra/cpu.h b/src/vppinfra/cpu.h index 7a1b75fcf7d..b3743d4c26d 100644 --- a/src/vppinfra/cpu.h +++ b/src/vppinfra/cpu.h @@ -150,6 +150,7 @@ _CLIB_MARCH_FN_REGISTRATION(fn) _ (movdir64b, 7, ecx, 28) \ _ (enqcmd, 7, ecx, 29) \ _ (avx512_fp16, 7, edx, 23) \ + _ (aperfmperf, 0x00000006, ecx, 0) \ _ (invariant_tsc, 0x80000007, edx, 8) \ _ (monitorx, 0x80000001, ecx, 29) diff --git a/src/vppinfra/crypto/aes_cbc.h b/src/vppinfra/crypto/aes_cbc.h index cb3d0784051..ee9263df260 100644 --- a/src/vppinfra/crypto/aes_cbc.h +++ b/src/vppinfra/crypto/aes_cbc.h @@ -539,4 +539,207 @@ clib_aes256_cbc_decrypt (const aes_cbc_key_data_t *kd, const u8 *ciphertext, clib_aes_cbc_decrypt (kd, ciphertext, len, iv, AES_KEY_256, plaintext); } +#if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0 +#pragma GCC optimize("O3") +#endif + +#if defined(__VAES__) && defined(__AVX512F__) +#define u8xN u8x64 +#define u32xN u32x16 +#define u32xN_min_scalar u32x16_min_scalar +#define u32xN_is_all_zero u32x16_is_all_zero +#define u32xN_splat u32x16_splat +#elif defined(__VAES__) +#define u8xN u8x32 +#define u32xN u32x8 +#define u32xN_min_scalar u32x8_min_scalar +#define u32xN_is_all_zero u32x8_is_all_zero +#define u32xN_splat u32x8_splat +#else +#define u8xN u8x16 +#define u32xN u32x4 +#define u32xN_min_scalar u32x4_min_scalar +#define u32xN_is_all_zero u32x4_is_all_zero +#define u32xN_splat u32x4_splat +#endif + +static_always_inline u32 +clib_aes_cbc_encrypt_multi (aes_cbc_key_data_t **key_data, + const uword *key_indices, u8 **plaintext, + const uword *oplen, u8 **iv, aes_key_size_t ks, + u8 **ciphertext, uword n_ops) +{ + int rounds = AES_KEY_ROUNDS (ks); + u8 placeholder[8192]; + u32 i, j, count, n_left = n_ops; + u32xN placeholder_mask = {}; + u32xN len = {}; + u32 key_index[4 * N_AES_LANES]; + u8 *src[4 * N_AES_LANES] = {}; + u8 *dst[4 * N_AES_LANES] = {}; + u8xN r[4] = {}; + u8xN k[15][4] = {}; + + for (i = 0; i < 4 * N_AES_LANES; i++) + key_index[i] = ~0; + +more: + for (i = 0; i < 4 * N_AES_LANES; i++) + if (len[i] == 0) + { + if (n_left == 0) + { + /* no more work to enqueue, so we are enqueueing placeholder buffer + */ + src[i] = dst[i] = placeholder; + len[i] = sizeof (placeholder); + placeholder_mask[i] = 0; + } + else + { + u8x16 t = aes_block_load (iv[0]); + ((u8x16 *) r)[i] = t; + + src[i] = plaintext[0]; + dst[i] = ciphertext[0]; + len[i] = oplen[0]; + placeholder_mask[i] = ~0; + if (key_index[i] != key_indices[0]) + { + aes_cbc_key_data_t *kd; + key_index[i] = key_indices[0]; + kd = key_data[key_index[i]]; + for (j = 0; j < rounds + 1; j++) + ((u8x16 *) k[j])[i] = kd->encrypt_key[j]; + } + n_left--; + iv++; + ciphertext++; + plaintext++; + key_indices++; + oplen++; + } + } + + count = u32xN_min_scalar (len); + + ASSERT (count % 16 == 0); + + for (i = 0; i < count; i += 16) + { +#if defined(__VAES__) && defined(__AVX512F__) + r[0] = u8x64_xor3 (r[0], aes_block_load_x4 (src, i), k[0][0]); + r[1] = u8x64_xor3 (r[1], aes_block_load_x4 (src + 4, i), k[0][1]); + r[2] = u8x64_xor3 (r[2], aes_block_load_x4 (src + 8, i), k[0][2]); + r[3] = u8x64_xor3 (r[3], aes_block_load_x4 (src + 12, i), k[0][3]); + + for (j = 1; j < rounds; j++) + { + r[0] = aes_enc_round_x4 (r[0], k[j][0]); + r[1] = aes_enc_round_x4 (r[1], k[j][1]); + r[2] = aes_enc_round_x4 (r[2], k[j][2]); + r[3] = aes_enc_round_x4 (r[3], k[j][3]); + } + r[0] = aes_enc_last_round_x4 (r[0], k[j][0]); + r[1] = aes_enc_last_round_x4 (r[1], k[j][1]); + r[2] = aes_enc_last_round_x4 (r[2], k[j][2]); + r[3] = aes_enc_last_round_x4 (r[3], k[j][3]); + + aes_block_store_x4 (dst, i, r[0]); + aes_block_store_x4 (dst + 4, i, r[1]); + aes_block_store_x4 (dst + 8, i, r[2]); + aes_block_store_x4 (dst + 12, i, r[3]); +#elif defined(__VAES__) + r[0] = u8x32_xor3 (r[0], aes_block_load_x2 (src, i), k[0][0]); + r[1] = u8x32_xor3 (r[1], aes_block_load_x2 (src + 2, i), k[0][1]); + r[2] = u8x32_xor3 (r[2], aes_block_load_x2 (src + 4, i), k[0][2]); + r[3] = u8x32_xor3 (r[3], aes_block_load_x2 (src + 6, i), k[0][3]); + + for (j = 1; j < rounds; j++) + { + r[0] = aes_enc_round_x2 (r[0], k[j][0]); + r[1] = aes_enc_round_x2 (r[1], k[j][1]); + r[2] = aes_enc_round_x2 (r[2], k[j][2]); + r[3] = aes_enc_round_x2 (r[3], k[j][3]); + } + r[0] = aes_enc_last_round_x2 (r[0], k[j][0]); + r[1] = aes_enc_last_round_x2 (r[1], k[j][1]); + r[2] = aes_enc_last_round_x2 (r[2], k[j][2]); + r[3] = aes_enc_last_round_x2 (r[3], k[j][3]); + + aes_block_store_x2 (dst, i, r[0]); + aes_block_store_x2 (dst + 2, i, r[1]); + aes_block_store_x2 (dst + 4, i, r[2]); + aes_block_store_x2 (dst + 6, i, r[3]); +#else +#if __x86_64__ + r[0] = u8x16_xor3 (r[0], aes_block_load (src[0] + i), k[0][0]); + r[1] = u8x16_xor3 (r[1], aes_block_load (src[1] + i), k[0][1]); + r[2] = u8x16_xor3 (r[2], aes_block_load (src[2] + i), k[0][2]); + r[3] = u8x16_xor3 (r[3], aes_block_load (src[3] + i), k[0][3]); + + for (j = 1; j < rounds; j++) + { + r[0] = aes_enc_round_x1 (r[0], k[j][0]); + r[1] = aes_enc_round_x1 (r[1], k[j][1]); + r[2] = aes_enc_round_x1 (r[2], k[j][2]); + r[3] = aes_enc_round_x1 (r[3], k[j][3]); + } + + r[0] = aes_enc_last_round_x1 (r[0], k[j][0]); + r[1] = aes_enc_last_round_x1 (r[1], k[j][1]); + r[2] = aes_enc_last_round_x1 (r[2], k[j][2]); + r[3] = aes_enc_last_round_x1 (r[3], k[j][3]); + + aes_block_store (dst[0] + i, r[0]); + aes_block_store (dst[1] + i, r[1]); + aes_block_store (dst[2] + i, r[2]); + aes_block_store (dst[3] + i, r[3]); +#else + r[0] ^= aes_block_load (src[0] + i); + r[1] ^= aes_block_load (src[1] + i); + r[2] ^= aes_block_load (src[2] + i); + r[3] ^= aes_block_load (src[3] + i); + for (j = 0; j < rounds - 1; j++) + { + r[0] = vaesmcq_u8 (vaeseq_u8 (r[0], k[j][0])); + r[1] = vaesmcq_u8 (vaeseq_u8 (r[1], k[j][1])); + r[2] = vaesmcq_u8 (vaeseq_u8 (r[2], k[j][2])); + r[3] = vaesmcq_u8 (vaeseq_u8 (r[3], k[j][3])); + } + r[0] = vaeseq_u8 (r[0], k[j][0]) ^ k[rounds][0]; + r[1] = vaeseq_u8 (r[1], k[j][1]) ^ k[rounds][1]; + r[2] = vaeseq_u8 (r[2], k[j][2]) ^ k[rounds][2]; + r[3] = vaeseq_u8 (r[3], k[j][3]) ^ k[rounds][3]; + aes_block_store (dst[0] + i, r[0]); + aes_block_store (dst[1] + i, r[1]); + aes_block_store (dst[2] + i, r[2]); + aes_block_store (dst[3] + i, r[3]); +#endif +#endif + } + + len -= u32xN_splat (count); + + for (i = 0; i < 4 * N_AES_LANES; i++) + { + src[i] += count; + dst[i] += count; + } + + if (n_left > 0) + goto more; + + if (!u32xN_is_all_zero (len & placeholder_mask)) + goto more; + + return n_ops; +} + +#undef u8xN +#undef u32xN +#undef u32xN_min_scalar +#undef u32xN_is_all_zero +#undef u32xN_splat + #endif /* __crypto_aes_cbc_h__ */ diff --git a/src/vppinfra/devicetree.c b/src/vppinfra/devicetree.c new file mode 100644 index 00000000000..b725d5a0aed --- /dev/null +++ b/src/vppinfra/devicetree.c @@ -0,0 +1,346 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Cisco Systems, Inc. + */ + +#include <vppinfra/clib.h> +#include <vppinfra/devicetree.h> + +#ifdef __linux +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <dirent.h> + +static_always_inline clib_dt_node_t * +clib_dt_node_add_child (clib_dt_main_t *dm, clib_dt_node_t *n, char *name) +{ + clib_dt_node_t *cn; + + cn = clib_mem_alloc (sizeof (clib_dt_node_t)); + *cn = (clib_dt_node_t){ .parent = n, .depth = n ? n->depth + 1 : 0 }; + vec_add1 (dm->nodes, cn); + + if (n == 0) + { + ASSERT (dm->root == 0); + dm->root = cn; + return cn; + } + + vec_add1 (n->child_nodes, cn); + cn->path = format (0, "%v/%s", n->path, name); + cn->dt_main = dm; + hash_set_mem (dm->node_by_path, cn->path, cn); + if (vec_len (n->child_nodes) > 1) + { + clib_dt_node_t *prev = n->child_nodes[vec_len (n->child_nodes) - 2]; + prev->next = cn; + cn->prev = prev; + } + + return cn; +} +#endif + +void +clib_dt_main_free (clib_dt_main_t *dm) +{ + vec_foreach_pointer (n, dm->nodes) + { + vec_foreach_pointer (p, n->properties) + clib_mem_free (p); + vec_free (n->child_nodes); + vec_free (n->path); + vec_free (n->properties); + } + + vec_free (dm->nodes); + hash_free (dm->node_by_path); + hash_free (dm->node_by_phandle); +} + +#ifdef __linux +__clib_export clib_error_t * +clib_dt_read_from_sysfs (clib_dt_main_t *dm) +{ + DIR *dir, **dir_stack = 0; + struct dirent *e; + clib_dt_node_t *n; + u8 *path = 0; + u32 path_prefix_len; + clib_error_t *err = 0; + + path = format (0, CLIB_DT_LINUX_PREFIX); + path_prefix_len = vec_len (path); + vec_add1 (path, 0); + + dir = opendir ((char *) path); + if (!dir) + { + err = clib_error_return (0, "'%s' opendir failed", path); + goto done; + } + + dm->node_by_path = hash_create_vec (0, sizeof (u8), sizeof (uword)); + dm->node_by_phandle = hash_create (0, sizeof (uword)); + vec_set_len (path, path_prefix_len); + n = clib_dt_node_add_child (dm, 0, 0); + + while (1) + { + e = readdir (dir); + + if (!e) + { + closedir (dir); + if (vec_len (dir_stack) == 0) + break; + + dir = dir_stack[vec_len (dir_stack) - 1]; + vec_pop (dir_stack); + n = n->parent; + continue; + } + + if (e->d_type == DT_REG) + { + path = format (path, "%v/%s%c", n->path, e->d_name, 0); + int fd = open ((char *) path, 0); + if (fd >= 0) + { + struct stat st; + if (fstat (fd, &st) == 0) + { + u32 sz = sizeof (clib_dt_property_t) + st.st_size; + clib_dt_property_t *p = clib_mem_alloc (sz); + clib_memset (p, 0, sz); + + if (read (fd, p->data, st.st_size) == st.st_size) + { + strncpy (p->name, e->d_name, sizeof (p->name)); + p->size = st.st_size; + vec_add1 (n->properties, p); + if (strncmp ("name", p->name, 5) == 0) + n->name = p; + if ((strncmp ("phandle", p->name, 8) == 0) && + (p->size == 4)) + { + u32 phandle = + clib_net_to_host_u32 (*(u32u *) p->data); + hash_set (dm->node_by_phandle, phandle, n); + } + } + else + { + clib_mem_free (p); + err = clib_error_return (0, "'%s' read failed", path); + close (fd); + goto done; + } + } + else + { + err = clib_error_return (0, "'%s' fstat failed", path); + close (fd); + goto done; + } + close (fd); + } + else + { + err = clib_error_return (0, "'%s' open failed", path); + goto done; + } + + vec_set_len (path, path_prefix_len); + } + else if (e->d_type == DT_DIR) + { + DIR *subdir; + if (strncmp (".", e->d_name, 2) == 0 || + strncmp ("..", e->d_name, 3) == 0) + continue; + + path = format (path, "%v/%s%c", n->path, e->d_name, 0); + subdir = opendir ((char *) path); + vec_set_len (path, path_prefix_len); + if (subdir) + { + vec_add1 (dir_stack, dir); + dir = subdir; + n = clib_dt_node_add_child (dm, n, e->d_name); + } + else + { + err = clib_error_return (0, "'%s' opendir failed", path); + goto done; + } + } + else + err = + clib_error_return (0, "unknown entry %s [%u]", e->d_name, e->d_type); + } + +done: + if (err) + clib_dt_main_free (dm); + while (vec_len (dir_stack)) + closedir (vec_pop (dir_stack)); + vec_free (dir_stack); + vec_free (path); + return err; +} +#endif + +clib_dt_node_t * +clib_dt_get_child_node (clib_dt_node_t *n, char *name) +{ + vec_foreach_pointer (cn, n->child_nodes) + { + u8 *p = cn->path + vec_len (cn->path) - 1; + u32 i = 0; + + while (p > cn->path && p[-1] != '/') + p--; + + if (p[-1] != '/') + continue; + + while (p[i] == name[i] && name[i] != 0) + i++; + + if (name[i] != 0) + continue; + + return cn; + } + + return 0; +} + +__clib_export clib_dt_node_t * +clib_dt_get_node_with_path (clib_dt_main_t *dm, char *fmt, ...) +{ + u8 *s; + uword *p; + + va_list va; + va_start (va, fmt); + s = va_format (0, fmt, &va); + va_end (va); + + if (s[0] != '/') + return 0; + + p = hash_get_mem (dm->node_by_path, s); + if (p) + return (clib_dt_node_t *) p[0]; + + return 0; +} + +__clib_export clib_dt_property_t * +clib_dt_get_node_property_by_name (clib_dt_node_t *n, char *name) +{ + vec_foreach_pointer (p, n->properties) + if (strncmp (name, p->name, sizeof (p->name)) == 0) + return p; + return 0; +} + +__clib_export int +clib_dt_node_is_compatible (clib_dt_node_t *n, char *comp) +{ + clib_dt_property_t *p; + char *s; + + p = clib_dt_get_node_property_by_name (n, "compatible"); + + if (!p) + return 0; + + s = (char *) p->data; + for (u32 i = 1, len = 1; i <= p->size; i++) + { + if (p->data[i - 1] == 0) + { + if (strncmp (comp, s, len) == 0) + return 1; + s = (char *) p->data + i; + len = 1; + } + else + len++; + } + + return 0; +} + +__clib_export u8 * +format_clib_dt_property_data (u8 *s, va_list *args) +{ + clib_dt_property_t *p = va_arg (*args, clib_dt_property_t *); + u32 sz = p->size, is_printable = 0; + u32 n_nulls = 0; + + if (sz > 2 && p->data[sz - 1] == 0 && p->data[0] != 0) + { + is_printable = 1; + for (u32 i = 1; i < sz - 1; i++) + { + u8 c = p->data[i]; + if (c == 0) + { + if (p->data[i - 1] == 0) + { + is_printable = 0; + break; + } + n_nulls++; + } + else if ((c < 0x20) || (c > 0x7f)) + { + is_printable = 0; + break; + } + } + } + + if (is_printable) + { + s = format (s, "'%s'", p->data); + if (n_nulls) + { + for (u32 i = 2; i < p->size; i++) + if (((u8 *) p->data)[i - 1] == 0) + s = format (s, ", '%s'", ((u8 *) p->data) + i); + } + } + else + { + s = format (s, "< %02x", p->data[0]); + for (u32 i = 0; i < p->size; i++) + s = format (s, " %02x", p->data[i]); + s = format (s, " >"); + } + return s; +} + +__clib_export clib_dt_node_t * +clib_dt_dereference_node (clib_dt_node_t *n, char *name) +{ + clib_dt_property_t *p; + uword *h; + + p = clib_dt_get_node_property_by_name (n, name); + if (!p || (p->size != sizeof (u32))) + return 0; + + h = hash_get (n->dt_main->node_by_phandle, + clib_net_to_host_u32 (*(u32u *) p->data)); + + if (h) + return (clib_dt_node_t *) h[0]; + + return 0; +} diff --git a/src/vppinfra/devicetree.h b/src/vppinfra/devicetree.h new file mode 100644 index 00000000000..21c2e0f7006 --- /dev/null +++ b/src/vppinfra/devicetree.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Cisco Systems, Inc. + */ + +#ifndef CLIB_DEVICETREE_H_ +#define CLIB_DEVICETREE_H_ + +#include <vppinfra/clib.h> +#include <vlib/vlib.h> + +#ifdef __linux +#define CLIB_DT_LINUX_PREFIX "/sys/firmware/devicetree/base" +#endif + +typedef struct +{ + char name[32]; + u32 size; + u8 data[]; +} clib_dt_property_t; + +typedef struct clib_dt_main clib_dt_main_t; + +typedef struct clib_dt_node +{ + u8 *path; + struct clib_dt_node *parent; + struct clib_dt_node *prev; + struct clib_dt_node *next; + struct clib_dt_node **child_nodes; + u8 depth; + clib_dt_property_t *name; + clib_dt_property_t **properties; + clib_dt_main_t *dt_main; +} clib_dt_node_t; + +typedef struct clib_dt_main +{ + clib_dt_node_t **nodes; + clib_dt_node_t *root; + uword *node_by_path; + uword *node_by_phandle; +} clib_dt_main_t; + +clib_dt_node_t *clib_dt_get_node_with_path (clib_dt_main_t *dm, char *fmt, + ...); +clib_dt_property_t *clib_dt_get_node_property_by_name (clib_dt_node_t *, + char *); +int clib_dt_node_is_compatible (clib_dt_node_t *, char *); +clib_dt_node_t *clib_dt_dereference_node (clib_dt_node_t *, char *); +#ifdef __linux +clib_error_t *clib_dt_read_from_sysfs (clib_dt_main_t *dm); +#endif + +format_function_t format_clib_dt_desc; +format_function_t format_clib_dt_property_data; + +static_always_inline int +clib_dt_proprerty_is_u32 (clib_dt_property_t *p) +{ + if (p == 0 || p->size != 4) + return 0; + return 1; +} + +static_always_inline u32 +clib_dt_proprerty_get_u32 (clib_dt_property_t *p) +{ + return clib_net_to_host_u32 (*(u32u *) p->data); +} + +#endif /* CLIB_DEVICETREE_H_ */ diff --git a/src/vppinfra/format.c b/src/vppinfra/format.c index cf17b8a1acb..642d3e20654 100644 --- a/src/vppinfra/format.c +++ b/src/vppinfra/format.c @@ -833,6 +833,16 @@ done: return s; } +__clib_export char * +format_c_string (u8 *s, const char *fmt, ...) +{ + va_list args; + va_start (args, fmt); + s = va_format (s, fmt, &args); + va_end (args); + vec_add1 (s, '\0'); + return (char *) s; +} /* * fd.io coding-style-patch-verification: ON diff --git a/src/vppinfra/format.h b/src/vppinfra/format.h index a1a70a2d64f..14bac869f89 100644 --- a/src/vppinfra/format.h +++ b/src/vppinfra/format.h @@ -372,6 +372,8 @@ int test_unformat_main (unformat_input_t * input); created circular dependency problems. */ int test_vec_main (unformat_input_t * input); +char *format_c_string (u8 *s, const char *fmt, ...); + #endif /* included_format_h */ /* diff --git a/src/vppinfra/format_ansi.h b/src/vppinfra/format_ansi.h new file mode 100644 index 00000000000..c35406aacf7 --- /dev/null +++ b/src/vppinfra/format_ansi.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Cisco Systems, Inc. + */ + +#ifndef __FORMAT_ANSI_H__ +#define __FORMAT_ANSI_H__ + +#define ANSI_RESET "\x1b[0m" +#define ANSI_BOLD "\x1b[1m" +#define ANSI_ITALIC "\x1b[3m" +#define ANSI_UNDERLINE "\x1b[4m" +#define ANSI_BLINK "\x1b[5m" +#define ANSI_FG_BLACK "\x1b[30m" +#define ANSI_FG_RED "\x1b[31m" +#define ANSI_FG_GREEN "\x1b[32m" +#define ANSI_FG_YELLOW "\x1b[33m" +#define ANSI_FG_BLUE "\x1b[34m" +#define ANSI_FG_MAGENTA "\x1b[35m" +#define ANSI_FG_CYAN "\x1b[36m" +#define ANSI_FG_WHITE "\x1b[37m" +#define ANSI_FG_DEFAULT "\x1b[39m" +#define ANSI_BG_BLACK "\x1b[40m" +#define ANSI_BG_RED "\x1b[41m" +#define ANSI_BG_GREEN "\x1b[42m" +#define ANSI_BG_YELLOW "\x1b[43m" +#define ANSI_BG_BLUE "\x1b[44m" +#define ANSI_BG_MAGENTA "\x1b[45m" +#define ANSI_BG_CYAN "\x1b[46m" +#define ANSI_BG_WHITE "\x1b[47m" +#define ANSI_BG_DEFAULT "\x1b[49m" +#define ANSI_FG_BR_BLACK "\x1b[90m" +#define ANSI_FG_BR_RED "\x1b[91m" +#define ANSI_FG_BR_GREEN "\x1b[92m" +#define ANSI_FG_BR_YELLOW "\x1b[93m" +#define ANSI_FG_BR_BLUE "\x1b[94m" +#define ANSI_FG_BR_MAGENTA "\x1b[95m" +#define ANSI_FG_BR_CYAN "\x1b[96m" +#define ANSI_FG_BR_WHITE "\x1b[97m" +#define ANSI_BG_BR_BLACK "\x1b[100m" +#define ANSI_BG_BR_RED "\x1b[101m" +#define ANSI_BG_BR_GREEN "\x1b[102m" +#define ANSI_BG_BR_YELLOW "\x1b[103m" +#define ANSI_BG_BR_BLUE "\x1b[104m" +#define ANSI_BG_BR_MAGENTA "\x1b[105m" +#define ANSI_BG_BR_CYAN "\x1b[106m" +#define ANSI_BG_BR_WHITE "\x1b[107m" + +#endif /* __FORMAT_ANSI_H__ */ diff --git a/src/vppinfra/jsonformat.c b/src/vppinfra/jsonformat.c index 1aa3864be04..73cb94769d8 100644 --- a/src/vppinfra/jsonformat.c +++ b/src/vppinfra/jsonformat.c @@ -500,12 +500,13 @@ format_vl_api_mac_address_t (u8 * s, va_list * args) mac->bytes[0], mac->bytes[1], mac->bytes[2], mac->bytes[3], mac->bytes[4], mac->bytes[5]); } -#define _(T) \ - cJSON *vl_api_ ##T## _t_tojson (vl_api_ ##T## _t *a) { \ - u8 *s = format(0, "%U", format_vl_api_ ##T## _t, a); \ - cJSON *o = cJSON_CreateString((char *)s); \ - vec_free(s); \ - return o; \ +#define _(T) \ + cJSON *vl_api_##T##_t_tojson (vl_api_##T##_t *a) \ + { \ + char *s = format_c_string (0, "%U", format_vl_api_##T##_t, a, 0); \ + cJSON *o = cJSON_CreateString (s); \ + vec_free (s); \ + return o; \ } foreach_type_tojson #undef _ diff --git a/src/vppinfra/linux/mem.c b/src/vppinfra/linux/mem.c index 734f5c4788c..17b4412e6c9 100644 --- a/src/vppinfra/linux/mem.c +++ b/src/vppinfra/linux/mem.c @@ -101,11 +101,13 @@ legacy_get_log2_default_hugepage_size (void) void clib_mem_main_init (void) { + unsigned long nodemask = 0, maxnode = CLIB_MAX_NUMAS; + unsigned long flags = MPOL_F_MEMS_ALLOWED; clib_mem_main_t *mm = &clib_mem_main; long sysconf_page_size; uword page_size; - void *va; - int fd; + void *va = 0; + int fd, mode; if (mm->log2_page_sz != CLIB_MEM_PAGE_SZ_UNKNOWN) return; @@ -131,23 +133,8 @@ clib_mem_main_init (void) mm->log2_sys_default_hugepage_sz = mm->log2_default_hugepage_sz; /* numa nodes */ - va = mmap (0, page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | - MAP_ANONYMOUS, -1, 0); - if (va == MAP_FAILED) - return; - - if (mlock (va, page_size)) - goto done; - - for (int i = 0; i < CLIB_MAX_NUMAS; i++) - { - int status; - if (syscall (__NR_move_pages, 0, 1, &va, &i, &status, 0) == 0) - mm->numa_node_bitmap |= 1ULL << i; - } - -done: - munmap (va, page_size); + if (syscall (__NR_get_mempolicy, &mode, &nodemask, maxnode, va, flags) == 0) + mm->numa_node_bitmap = nodemask; } __clib_export u64 @@ -530,6 +517,7 @@ clib_mem_get_page_stats (void *start, clib_mem_page_sz_t log2_page_size, { int i, *status = 0; void **ptr = 0; + unsigned char incore; log2_page_size = clib_mem_log2_page_size_validate (log2_page_size); @@ -551,6 +539,19 @@ clib_mem_get_page_stats (void *start, clib_mem_page_sz_t log2_page_size, for (i = 0; i < n_pages; i++) { + /* move_pages() returns -ENONET in status for huge pages on 5.19+ kernel. + * Retry with get_mempolicy() to obtain NUMA node info only if the pages + * are allocated and in memory, which is checked by mincore(). */ + if (status[i] == -ENOENT && + syscall (__NR_mincore, ptr[i], 1, &incore) == 0 && (incore & 1) != 0) + { + if (syscall (__NR_get_mempolicy, &status[i], 0, 0, ptr[i], + MPOL_F_NODE | MPOL_F_ADDR) != 0) + { + /* if get_mempolicy fails, keep the original value in status */ + status[i] = -ENONET; + } + } if (status[i] >= 0 && status[i] < CLIB_MAX_NUMAS) { stats->mapped++; diff --git a/src/vppinfra/mem.h b/src/vppinfra/mem.h index 75015d59a4a..ab9c5da30ec 100644 --- a/src/vppinfra/mem.h +++ b/src/vppinfra/mem.h @@ -299,10 +299,27 @@ void *clib_mem_init_thread_safe (void *memory, uword memory_size); void clib_mem_exit (void); +typedef struct +{ + /* Address of callers: outer first, inner last. */ + uword callers[12]; + + /* Count of allocations with this traceback. */ + u32 n_allocations; + + /* Count of bytes allocated with this traceback. */ + u32 n_bytes; + + /* Offset of this item */ + uword offset; +} mheap_trace_t; + void clib_mem_trace (int enable); int clib_mem_is_traced (void); +mheap_trace_t *clib_mem_trace_dup (clib_mem_heap_t *heap); + typedef struct { /* Total number of objects allocated. */ diff --git a/src/vppinfra/mem_dlmalloc.c b/src/vppinfra/mem_dlmalloc.c index a188164a7ba..d5ff21e58c0 100644 --- a/src/vppinfra/mem_dlmalloc.c +++ b/src/vppinfra/mem_dlmalloc.c @@ -19,21 +19,7 @@ #include <vppinfra/lock.h> #include <vppinfra/hash.h> #include <vppinfra/elf_clib.h> - -typedef struct -{ - /* Address of callers: outer first, inner last. */ - uword callers[12]; - - /* Count of allocations with this traceback. */ - u32 n_allocations; - - /* Count of bytes allocated with this traceback. */ - u32 n_bytes; - - /* Offset of this item */ - uword offset; -} mheap_trace_t; +#include <vppinfra/stack.h> typedef struct { @@ -65,15 +51,13 @@ mheap_get_trace_internal (const clib_mem_heap_t *heap, uword offset, { mheap_trace_main_t *tm = &mheap_trace_main; mheap_trace_t *t; - uword i, n_callers, trace_index, *p; - mheap_trace_t trace; + uword i, trace_index, *p; + mheap_trace_t trace = {}; + int index; if (heap != tm->current_traced_mheap || mheap_trace_thread_disable) return; - /* Spurious Coverity warnings be gone. */ - clib_memset (&trace, 0, sizeof (trace)); - clib_spinlock_lock (&tm->lock); /* heap could have changed while we were waiting on the lock */ @@ -83,9 +67,19 @@ mheap_get_trace_internal (const clib_mem_heap_t *heap, uword offset, /* Turn off tracing for this thread to avoid embarrassment... */ mheap_trace_thread_disable = 1; - /* Skip our frame and mspace_get_aligned's frame */ - n_callers = clib_backtrace (trace.callers, ARRAY_LEN (trace.callers), 2); - if (n_callers == 0) + index = -2; /* skip first 2 stack frames */ + foreach_clib_stack_frame (sf) + { + if (index >= 0) + { + if (index == ARRAY_LEN (trace.callers)) + break; + trace.callers[index] = sf->ip; + } + index++; + } + + if (index < 1) goto out; if (!tm->trace_by_callers) @@ -565,6 +559,23 @@ clib_mem_trace_enable_disable (uword enable) return rv; } +__clib_export mheap_trace_t * +clib_mem_trace_dup (clib_mem_heap_t *heap) +{ + mheap_trace_main_t *tm = &mheap_trace_main; + mheap_trace_t *traces_copy = 0; + + clib_spinlock_lock (&tm->lock); + if (vec_len (tm->traces) > 0 && heap == tm->current_traced_mheap) + { + traces_copy = vec_dup (tm->traces); + qsort (traces_copy, vec_len (traces_copy), sizeof (traces_copy[0]), + mheap_trace_sort); + } + clib_spinlock_unlock (&tm->lock); + return traces_copy; +} + __clib_export clib_mem_heap_t * clib_mem_create_heap (void *base, uword size, int is_locked, char *fmt, ...) { diff --git a/src/vppinfra/stack.c b/src/vppinfra/stack.c new file mode 100644 index 00000000000..190e880c228 --- /dev/null +++ b/src/vppinfra/stack.c @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Cisco Systems, Inc. + */ + +#define _GNU_SOURCE +#include <dlfcn.h> + +#include <vppinfra/clib.h> +#include <vppinfra/stack.h> +#include <vppinfra/error.h> + +#if HAVE_LIBUNWIND == 1 + +#define UNW_LOCAL_ONLY +#include <libunwind.h> + +static __thread unw_cursor_t cursor; +static __thread unw_context_t context; + +#endif + +__clib_export clib_stack_frame_t * +clib_stack_frame_get (clib_stack_frame_t *sf) +{ +#if HAVE_LIBUNWIND == 1 + Dl_info info = {}; + + if (sf->index == 0) + { + if (unw_getcontext (&context) < 0) + { + clib_warning ("libunwind: cannot get local machine state\n"); + return 0; + } + if (unw_init_local (&cursor, &context) < 0) + { + clib_warning ( + "libunwind: cannot initialize cursor for local unwinding\n"); + return 0; + } + if (unw_step (&cursor) < 1) + return 0; + } + else if (unw_step (&cursor) < 1) + return 0; + + if (unw_get_reg (&cursor, UNW_REG_IP, &sf->ip)) + { + clib_warning ("libunwind: cannot read IP\n"); + return 0; + } + + if (unw_get_reg (&cursor, UNW_REG_SP, &sf->sp)) + { + clib_warning ("libunwind: cannot read SP\n"); + return 0; + } + + if (unw_get_proc_name (&cursor, sf->name, sizeof (sf->name), &sf->offset) < + 0) + sf->name[0] = sf->offset = 0; + + sf->is_signal_frame = unw_is_signal_frame (&cursor) ? 1 : 0; + + if (dladdr ((void *) sf->ip, &info)) + sf->file_name = info.dli_fname; + else + sf->file_name = 0; + + sf->index++; + return sf; +#else + return 0; +#endif +} diff --git a/src/vppinfra/stack.h b/src/vppinfra/stack.h new file mode 100644 index 00000000000..98a621d4176 --- /dev/null +++ b/src/vppinfra/stack.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Cisco Systems, Inc. + */ + +#ifndef __STACK_H__ +#define __STACK_H__ + +#include <vppinfra/clib.h> + +typedef struct +{ + uword ip, sp; + uword offset; + char name[64]; + const char *file_name; + u32 index; + u8 is_signal_frame; +} clib_stack_frame_t; + +clib_stack_frame_t *clib_stack_frame_get (clib_stack_frame_t *); + +#define foreach_clib_stack_frame(sf) \ + for (clib_stack_frame_t _sf = {}, *sf = clib_stack_frame_get (&_sf); sf; \ + sf = clib_stack_frame_get (sf)) + +#endif /* __STACK_H__ */ diff --git a/src/vppinfra/time.c b/src/vppinfra/time.c index 5a6aaf182e4..f1736499a0a 100644 --- a/src/vppinfra/time.c +++ b/src/vppinfra/time.c @@ -76,8 +76,11 @@ clock_frequency_from_proc_filesystem (void) f64 ppc_timebase = 0; /* warnings be gone */ unformat_input_t input; -/* $$$$ aarch64 kernel doesn't report "cpu MHz" */ -#if defined(__aarch64__) +#if defined(__x86_64__) + if (clib_cpu_supports_aperfmperf ()) + return 0.0; +#elif defined(__aarch64__) + /* $$$$ aarch64 kernel doesn't report "cpu MHz" */ return 0.0; #endif diff --git a/src/vppinfra/time_range.c b/src/vppinfra/time_range.c index 4b5e1303763..54f5629641a 100644 --- a/src/vppinfra/time_range.c +++ b/src/vppinfra/time_range.c @@ -264,11 +264,10 @@ format_clib_timebase_time (u8 * s, va_list * args) clib_timebase_time_to_components (now, cp); - s = format (s, "%s, %u %s %u %u:%02u:%02u", - day_names_epoch_order[cp->day_name_index], - cp->day, - month_short_names[cp->month], - cp->year, cp->hour, cp->minute, cp->second); + s = format (s, "%s, %02u %s %u %02u:%02u:%02u", + day_names_epoch_order[cp->day_name_index], cp->day, + month_short_names[cp->month], cp->year, cp->hour, cp->minute, + cp->second); return (s); } diff --git a/src/vppinfra/unix-misc.c b/src/vppinfra/unix-misc.c index 31c0a489e8d..05ca2f901c6 100644 --- a/src/vppinfra/unix-misc.c +++ b/src/vppinfra/unix-misc.c @@ -67,6 +67,8 @@ __clib_export __thread uword __os_thread_index = 0; __clib_export __thread uword __os_numa_index = 0; +__clib_export clib_bitmap_t *os_get_cpu_affinity_bitmap (int pid); + clib_error_t * clib_file_n_bytes (char *file, uword * result) { @@ -219,27 +221,20 @@ unix_proc_file_contents (char *file, u8 ** result) return 0; } -void os_panic (void) __attribute__ ((weak)); - -__clib_export void +__clib_export __clib_weak void os_panic (void) { abort (); } -void os_exit (int) __attribute__ ((weak)); - -void +__clib_export __clib_weak void os_exit (int code) { exit (code); } -void os_puts (u8 * string, uword string_length, uword is_error) - __attribute__ ((weak)); - -void -os_puts (u8 * string, uword string_length, uword is_error) +__clib_export __clib_weak void +os_puts (u8 *string, uword string_length, uword is_error) { int cpu = os_get_thread_index (); int nthreads = os_get_nthreads (); @@ -282,6 +277,8 @@ os_get_online_cpu_core_bitmap () { #if __linux__ return clib_sysfs_read_bitmap ("/sys/devices/system/cpu/online"); +#elif defined(__FreeBSD__) + return os_get_cpu_affinity_bitmap (0); #else return 0; #endif @@ -298,7 +295,7 @@ os_get_cpu_affinity_bitmap (int pid) clib_bitmap_alloc (affinity_cpus, sizeof (cpu_set_t)); clib_bitmap_zero (affinity_cpus); - __CPU_ZERO_S (sizeof (cpu_set_t), &cpuset); + CPU_ZERO_S (sizeof (cpu_set_t), &cpuset); ret = sched_getaffinity (0, sizeof (cpu_set_t), &cpuset); @@ -309,13 +306,16 @@ os_get_cpu_affinity_bitmap (int pid) } for (index = 0; index < sizeof (cpu_set_t); index++) - if (__CPU_ISSET_S (index, sizeof (cpu_set_t), &cpuset)) + if (CPU_ISSET_S (index, sizeof (cpu_set_t), &cpuset)) clib_bitmap_set (affinity_cpus, index, 1); return affinity_cpus; #elif defined(__FreeBSD__) cpuset_t mask; uword *r = NULL; + clib_bitmap_alloc (r, sizeof (CPU_SETSIZE)); + clib_bitmap_zero (r); + if (cpuset_getaffinity (CPU_LEVEL_CPUSET, CPU_WHICH_CPUSET, -1, sizeof (mask), &mask) != 0) { @@ -337,21 +337,6 @@ os_get_online_cpu_node_bitmap () { #if __linux__ return clib_sysfs_read_bitmap ("/sys/devices/system/node/online"); -#elif defined(__FreeBSD__) - domainset_t domain; - uword *r = NULL; - int policy; - - if (cpuset_getdomain (CPU_LEVEL_CPUSET, CPU_WHICH_CPUSET, -1, - sizeof (domain), &domain, &policy) != 0) - { - clib_bitmap_free (r); - return NULL; - } - - for (int bit = 0; bit < CPU_SETSIZE; bit++) - clib_bitmap_set (r, bit, CPU_ISSET (bit, &domain)); - return r; #else return 0; #endif |