diff options
Diffstat (limited to 'src/plugins')
63 files changed, 5879 insertions, 1688 deletions
diff --git a/src/plugins/af_xdp/af_xdp.api b/src/plugins/af_xdp/af_xdp.api index 4c2908e2037..20aa20b4d7d 100644 --- a/src/plugins/af_xdp/af_xdp.api +++ b/src/plugins/af_xdp/af_xdp.api @@ -33,96 +33,6 @@ enumflag af_xdp_flag : u8 /** \brief @param client_index - opaque cookie to identify the sender @param context - sender context, to match reply w/ request - @param host_if - Linux netdev interface name - @param name - new af_xdp interface name (optional) - @param rxq_num - number of receive queues. 65535 can be used as special value to request all available queues (optional) - @param rxq_size - receive queue size (optional) - @param txq_size - transmit queue size (optional) - @param mode - operation mode (optional) - @param flags - flags (optional) - @param prog - eBPF program path (optional) -*/ - -define af_xdp_create -{ - u32 client_index; - u32 context; - - string host_if[64]; - string name[64]; - u16 rxq_num [default=1]; - u16 rxq_size [default=0]; - u16 txq_size [default=0]; - vl_api_af_xdp_mode_t mode [default=0]; - vl_api_af_xdp_flag_t flags [default=0]; - string prog[256]; - option vat_help = "<host-if linux-ifname> [name ifname] [rx-queue-size size] [tx-queue-size size] [num-rx-queues <num|all>] [prog pathname] [zero-copy|no-zero-copy] [no-syscall-lock]"; - option deprecated; -}; - -/** \brief - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param host_if - Linux netdev interface name - @param name - new af_xdp interface name (optional) - @param rxq_num - number of receive queues. 65535 can be used as special value to request all available queues (optional) - @param rxq_size - receive queue size (optional) - @param txq_size - transmit queue size (optional) - @param mode - operation mode (optional) - @param flags - flags (optional) - @param prog - eBPF program path (optional) - @param namespace - netns of nic (optional) -*/ - -define af_xdp_create_v2 -{ - u32 client_index; - u32 context; - - string host_if[64]; - string name[64]; - u16 rxq_num [default=1]; - u16 rxq_size [default=0]; - u16 txq_size [default=0]; - vl_api_af_xdp_mode_t mode [default=0]; - vl_api_af_xdp_flag_t flags [default=0]; - string prog[256]; - string namespace[64]; - option vat_help = "<host-if linux-ifname> [name ifname] [rx-queue-size size] [tx-queue-size size] [num-rx-queues <num|all>] [prog pathname] [netns ns] [zero-copy|no-zero-copy] [no-syscall-lock]"; - option deprecated; -}; - -/** \brief - @param context - sender context, to match reply w/ request - @param retval - return value for request - @param sw_if_index - software index for the new af_xdp interface -*/ - -define af_xdp_create_reply -{ - u32 context; - i32 retval; - vl_api_interface_index_t sw_if_index; - option deprecated; -}; - -/** \brief - @param context - sender context, to match reply w/ request - @param retval - return value for request - @param sw_if_index - software index for the new af_xdp interface -*/ - -define af_xdp_create_v2_reply -{ - u32 context; - i32 retval; - vl_api_interface_index_t sw_if_index; - option deprecated; -}; - -/** \brief - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request @param sw_if_index - interface index */ diff --git a/src/plugins/af_xdp/api.c b/src/plugins/af_xdp/api.c index 3e9a3fe2578..9ead9856ff5 100644 --- a/src/plugins/af_xdp/api.c +++ b/src/plugins/af_xdp/api.c @@ -57,65 +57,6 @@ af_xdp_api_flags (vl_api_af_xdp_flag_t flags) } static void -vl_api_af_xdp_create_t_handler (vl_api_af_xdp_create_t * mp) -{ - vlib_main_t *vm = vlib_get_main (); - af_xdp_main_t *rm = &af_xdp_main; - vl_api_af_xdp_create_reply_t *rmp; - af_xdp_create_if_args_t args; - int rv; - - clib_memset (&args, 0, sizeof (af_xdp_create_if_args_t)); - - args.linux_ifname = mp->host_if[0] ? (char *) mp->host_if : 0; - args.name = mp->name[0] ? (char *) mp->name : 0; - args.prog = mp->prog[0] ? (char *) mp->prog : 0; - args.mode = af_xdp_api_mode (mp->mode); - args.flags = af_xdp_api_flags (mp->flags); - args.rxq_size = ntohs (mp->rxq_size); - args.txq_size = ntohs (mp->txq_size); - args.rxq_num = ntohs (mp->rxq_num); - - af_xdp_create_if (vm, &args); - rv = args.rv; - - REPLY_MACRO2 (VL_API_AF_XDP_CREATE_REPLY, - ({ rmp->sw_if_index = ntohl (args.sw_if_index); })); -} - -static void -vl_api_af_xdp_create_v2_t_handler (vl_api_af_xdp_create_v2_t *mp) -{ - vlib_main_t *vm = vlib_get_main (); - af_xdp_main_t *rm = &af_xdp_main; - vl_api_af_xdp_create_v2_reply_t *rmp; - af_xdp_create_if_args_t args; - int rv; - - clib_memset (&args, 0, sizeof (af_xdp_create_if_args_t)); - - args.linux_ifname = mp->host_if[0] ? (char *) mp->host_if : 0; - args.name = mp->name[0] ? (char *) mp->name : 0; - args.prog = mp->prog[0] ? (char *) mp->prog : 0; - args.netns = mp->namespace[0] ? (char *) mp->namespace : 0; - args.mode = af_xdp_api_mode (mp->mode); - args.flags = af_xdp_api_flags (mp->flags); - args.rxq_size = ntohs (mp->rxq_size); - args.txq_size = ntohs (mp->txq_size); - args.rxq_num = ntohs (mp->rxq_num); - - af_xdp_create_if (vm, &args); - rv = args.rv; - - /* clang-format off */ - REPLY_MACRO2 (VL_API_AF_XDP_CREATE_V2_REPLY, - ({ - rmp->sw_if_index = ntohl (args.sw_if_index); - })); - /* clang-format on */ -} - -static void vl_api_af_xdp_create_v3_t_handler (vl_api_af_xdp_create_v3_t *mp) { vlib_main_t *vm = vlib_get_main (); diff --git a/src/plugins/af_xdp/test_api.c b/src/plugins/af_xdp/test_api.c index 581697e341d..5f622adcb04 100644 --- a/src/plugins/af_xdp/test_api.c +++ b/src/plugins/af_xdp/test_api.c @@ -58,75 +58,7 @@ api_af_xdp_mode (af_xdp_mode_t mode) return ~0; } -/* af_xdp create API */ -static int -api_af_xdp_create (vat_main_t * vam) -{ - vl_api_af_xdp_create_t *mp; - af_xdp_create_if_args_t args; - int ret; - - if (!unformat_user (vam->input, unformat_af_xdp_create_if_args, &args)) - { - clib_warning ("unknown input `%U'", format_unformat_error, vam->input); - return -99; - } - - M (AF_XDP_CREATE, mp); - - snprintf ((char *) mp->host_if, sizeof (mp->host_if), "%s", - args.linux_ifname ? : ""); - snprintf ((char *) mp->name, sizeof (mp->name), "%s", args.name ? : ""); - mp->rxq_num = clib_host_to_net_u16 (args.rxq_num); - mp->rxq_size = clib_host_to_net_u16 (args.rxq_size); - mp->txq_size = clib_host_to_net_u16 (args.txq_size); - mp->mode = api_af_xdp_mode (args.mode); - if (args.flags & AF_XDP_CREATE_FLAGS_NO_SYSCALL_LOCK) - mp->flags |= AF_XDP_API_FLAGS_NO_SYSCALL_LOCK; - snprintf ((char *) mp->prog, sizeof (mp->prog), "%s", args.prog ? : ""); - - S (mp); - W (ret); - - return ret; -} - -/* af_xdp create v2 API */ -static int -api_af_xdp_create_v2 (vat_main_t *vam) -{ - vl_api_af_xdp_create_v2_t *mp; - af_xdp_create_if_args_t args; - int ret; - - if (!unformat_user (vam->input, unformat_af_xdp_create_if_args, &args)) - { - clib_warning ("unknown input `%U'", format_unformat_error, vam->input); - return -99; - } - - M (AF_XDP_CREATE, mp); - - snprintf ((char *) mp->host_if, sizeof (mp->host_if), "%s", - args.linux_ifname ?: ""); - snprintf ((char *) mp->name, sizeof (mp->name), "%s", args.name ?: ""); - snprintf ((char *) mp->namespace, sizeof (mp->namespace), "%s", - args.netns ?: ""); - mp->rxq_num = clib_host_to_net_u16 (args.rxq_num); - mp->rxq_size = clib_host_to_net_u16 (args.rxq_size); - mp->txq_size = clib_host_to_net_u16 (args.txq_size); - mp->mode = api_af_xdp_mode (args.mode); - if (args.flags & AF_XDP_CREATE_FLAGS_NO_SYSCALL_LOCK) - mp->flags |= AF_XDP_API_FLAGS_NO_SYSCALL_LOCK; - snprintf ((char *) mp->prog, sizeof (mp->prog), "%s", args.prog ?: ""); - - S (mp); - W (ret); - - return ret; -} - -/* af_xdp create v2 API */ +/* af_xdp create v3 API */ static int api_af_xdp_create_v3 (vat_main_t *vam) { @@ -140,7 +72,7 @@ api_af_xdp_create_v3 (vat_main_t *vam) return -99; } - M (AF_XDP_CREATE, mp); + M (AF_XDP_CREATE_V3, mp); snprintf ((char *) mp->host_if, sizeof (mp->host_if), "%s", args.linux_ifname ?: ""); @@ -160,45 +92,9 @@ api_af_xdp_create_v3 (vat_main_t *vam) return ret; } -/* af_xdp-create reply handler */ -static void -vl_api_af_xdp_create_reply_t_handler (vl_api_af_xdp_create_reply_t * mp) -{ - vat_main_t *vam = af_xdp_test_main.vat_main; - i32 retval = ntohl (mp->retval); - - if (retval == 0) - { - fformat (vam->ofp, "created af_xdp with sw_if_index %d\n", - ntohl (mp->sw_if_index)); - } - - vam->retval = retval; - vam->result_ready = 1; - vam->regenerate_interface_table = 1; -} - -/* af_xdp-create v2 reply handler */ -static void -vl_api_af_xdp_create_v2_reply_t_handler (vl_api_af_xdp_create_v2_reply_t *mp) -{ - vat_main_t *vam = af_xdp_test_main.vat_main; - i32 retval = ntohl (mp->retval); - - if (retval == 0) - { - fformat (vam->ofp, "created af_xdp with sw_if_index %d\n", - ntohl (mp->sw_if_index)); - } - - vam->retval = retval; - vam->result_ready = 1; - vam->regenerate_interface_table = 1; -} - /* af_xdp-create v3 reply handler */ static void -vl_api_af_xdp_create_v3_reply_t_handler (vl_api_af_xdp_create_v2_reply_t *mp) +vl_api_af_xdp_create_v3_reply_t_handler (vl_api_af_xdp_create_v3_reply_t *mp) { vat_main_t *vam = af_xdp_test_main.vat_main; i32 retval = mp->retval; diff --git a/src/plugins/crypto_native/FEATURE.yaml b/src/plugins/crypto_native/FEATURE.yaml index 06f26d4a8cf..d54816d673f 100644 --- a/src/plugins/crypto_native/FEATURE.yaml +++ b/src/plugins/crypto_native/FEATURE.yaml @@ -4,6 +4,9 @@ maintainer: Damjan Marion <damarion@cisco.com> features: - CBC(128, 192, 256) - GCM(128, 192, 256) + - CTR(128, 192, 256) + - SHA(224, 256) + - HMAC-SHA(224, 256) description: "An implementation of a native crypto-engine" state: production diff --git a/src/plugins/dev_armada/CMakeLists.txt b/src/plugins/dev_armada/CMakeLists.txt index f955a9baa91..e755e7bdd46 100644 --- a/src/plugins/dev_armada/CMakeLists.txt +++ b/src/plugins/dev_armada/CMakeLists.txt @@ -16,6 +16,7 @@ set(MUSDK_LINK_FLAGS "-Wl,--whole-archive,${MUSDK_LIB_DIR}/libmusdk.a,--no-whole add_vpp_plugin(dev_armada SOURCES plugin.c + pp2/counters.c pp2/init.c pp2/format.c pp2/port.c diff --git a/src/plugins/dev_armada/pp2/counters.c b/src/plugins/dev_armada/pp2/counters.c new file mode 100644 index 00000000000..a041138bc79 --- /dev/null +++ b/src/plugins/dev_armada/pp2/counters.c @@ -0,0 +1,241 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2023 Cisco Systems, Inc. + */ + +#include <vnet/vnet.h> +#include <vnet/ethernet/ethernet.h> +#include <vnet/dev/dev.h> +#include <vnet/dev/counters.h> +#include <vnet/dev/bus/platform.h> +#include <vppinfra/ring.h> +#include <dev_armada/musdk.h> +#include <dev_armada/pp2/pp2.h> + +VLIB_REGISTER_LOG_CLASS (mvpp2_log, static) = { + .class_name = "armada", + .subclass_name = "pp2-counters", +}; + +typedef enum +{ + MVPP2_PORT_CTR_RX_BYTES, + MVPP2_PORT_CTR_RX_PACKETS, + MVPP2_PORT_CTR_RX_UCAST, + MVPP2_PORT_CTR_RX_ERRORS, + MVPP2_PORT_CTR_RX_FULLQ_DROPPED, + MVPP2_PORT_CTR_RX_BM_DROPPED, + MVPP2_PORT_CTR_RX_EARLY_DROPPED, + MVPP2_PORT_CTR_RX_FIFO_DROPPED, + MVPP2_PORT_CTR_RX_CLS_DROPPED, + + MVPP2_PORT_CTR_TX_BYTES, + MVPP2_PORT_CTR_TX_PACKETS, + MVPP2_PORT_CTR_TX_UCAST, + MVPP2_PORT_CTR_TX_ERRORS, +} mvpp2_port_counter_id_t; + +typedef enum +{ + MVPP2_RXQ_CTR_ENQ_DESC, + MVPP2_RXQ_CTR_DROP_FULLQ, + MVPP2_RXQ_CTR_DROP_EARLY, + MVPP2_RXQ_CTR_DROP_BM, +} mvpp2_rxq_counter_id_t; + +typedef enum +{ + MVPP2_TXQ_CTR_ENQ_DESC, + MVPP2_TXQ_CTR_ENQ_DEC_TO_DDR, + MVPP2_TXQ_CTR_ENQ_BUF_TO_DDR, + MVPP2_TXQ_CTR_DEQ_DESC, +} mvpp2_txq_counter_id_t; + +static vnet_dev_counter_t mvpp2_port_counters[] = { + VNET_DEV_CTR_RX_BYTES (MVPP2_PORT_CTR_RX_BYTES), + VNET_DEV_CTR_RX_PACKETS (MVPP2_PORT_CTR_RX_PACKETS), + VNET_DEV_CTR_RX_DROPS (MVPP2_PORT_CTR_RX_ERRORS), + VNET_DEV_CTR_VENDOR (MVPP2_PORT_CTR_RX_UCAST, RX, PACKETS, "unicast"), + VNET_DEV_CTR_VENDOR (MVPP2_PORT_CTR_RX_FULLQ_DROPPED, RX, PACKETS, + "fullq dropped"), + VNET_DEV_CTR_VENDOR (MVPP2_PORT_CTR_RX_BM_DROPPED, RX, PACKETS, + "bm dropped"), + VNET_DEV_CTR_VENDOR (MVPP2_PORT_CTR_RX_EARLY_DROPPED, RX, PACKETS, + "early dropped"), + VNET_DEV_CTR_VENDOR (MVPP2_PORT_CTR_RX_FIFO_DROPPED, RX, PACKETS, + "fifo dropped"), + VNET_DEV_CTR_VENDOR (MVPP2_PORT_CTR_RX_CLS_DROPPED, RX, PACKETS, + "cls dropped"), + + VNET_DEV_CTR_TX_BYTES (MVPP2_PORT_CTR_TX_BYTES), + VNET_DEV_CTR_TX_PACKETS (MVPP2_PORT_CTR_TX_PACKETS), + VNET_DEV_CTR_TX_DROPS (MVPP2_PORT_CTR_TX_ERRORS), + VNET_DEV_CTR_VENDOR (MVPP2_PORT_CTR_TX_UCAST, TX, PACKETS, "unicast"), +}; + +static vnet_dev_counter_t mvpp2_rxq_counters[] = { + VNET_DEV_CTR_VENDOR (MVPP2_RXQ_CTR_ENQ_DESC, RX, DESCRIPTORS, "enqueued"), + VNET_DEV_CTR_VENDOR (MVPP2_RXQ_CTR_DROP_FULLQ, RX, PACKETS, "drop fullQ"), + VNET_DEV_CTR_VENDOR (MVPP2_RXQ_CTR_DROP_EARLY, RX, PACKETS, "drop early"), + VNET_DEV_CTR_VENDOR (MVPP2_RXQ_CTR_DROP_BM, RX, PACKETS, "drop BM"), +}; + +static vnet_dev_counter_t mvpp2_txq_counters[] = { + VNET_DEV_CTR_VENDOR (MVPP2_TXQ_CTR_ENQ_DESC, TX, DESCRIPTORS, "enqueued"), + VNET_DEV_CTR_VENDOR (MVPP2_TXQ_CTR_DEQ_DESC, TX, PACKETS, "dequeued"), + VNET_DEV_CTR_VENDOR (MVPP2_TXQ_CTR_ENQ_BUF_TO_DDR, TX, BUFFERS, + "enq to DDR"), + VNET_DEV_CTR_VENDOR (MVPP2_TXQ_CTR_ENQ_DEC_TO_DDR, TX, DESCRIPTORS, + "enq to DDR"), +}; + +void +mvpp2_port_add_counters (vlib_main_t *vm, vnet_dev_port_t *port) +{ + vnet_dev_port_add_counters (vm, port, mvpp2_port_counters, + ARRAY_LEN (mvpp2_port_counters)); + + foreach_vnet_dev_port_rx_queue (q, port) + vnet_dev_rx_queue_add_counters (vm, q, mvpp2_rxq_counters, + ARRAY_LEN (mvpp2_rxq_counters)); + + foreach_vnet_dev_port_tx_queue (q, port) + vnet_dev_tx_queue_add_counters (vm, q, mvpp2_txq_counters, + ARRAY_LEN (mvpp2_txq_counters)); +} + +void +mvpp2_port_clear_counters (vlib_main_t *vm, vnet_dev_port_t *port) +{ + mvpp2_port_t *mp = vnet_dev_get_port_data (port); + struct pp2_ppio_statistics stats; + pp2_ppio_get_statistics (mp->ppio, &stats, 1); +} + +void +mvpp2_rxq_clear_counters (vlib_main_t *vm, vnet_dev_rx_queue_t *q) +{ + mvpp2_port_t *mp = vnet_dev_get_port_data (q->port); + struct pp2_ppio_inq_statistics stats; + pp2_ppio_inq_get_statistics (mp->ppio, 0, q->queue_id, &stats, 1); +} + +void +mvpp2_txq_clear_counters (vlib_main_t *vm, vnet_dev_tx_queue_t *q) +{ + mvpp2_port_t *mp = vnet_dev_get_port_data (q->port); + struct pp2_ppio_inq_statistics stats; + pp2_ppio_inq_get_statistics (mp->ppio, 0, q->queue_id, &stats, 1); +} + +vnet_dev_rv_t +mvpp2_port_get_stats (vlib_main_t *vm, vnet_dev_port_t *port) +{ + mvpp2_port_t *mp = vnet_dev_get_port_data (port); + struct pp2_ppio_statistics stats; + pp2_ppio_get_statistics (mp->ppio, &stats, 0); + + foreach_vnet_dev_counter (c, port->counter_main) + { + switch (c->user_data) + { + case MVPP2_PORT_CTR_RX_BYTES: + vnet_dev_counter_value_update (vm, c, stats.rx_bytes); + break; + case MVPP2_PORT_CTR_RX_PACKETS: + vnet_dev_counter_value_update (vm, c, stats.rx_packets); + break; + case MVPP2_PORT_CTR_RX_UCAST: + vnet_dev_counter_value_update (vm, c, stats.rx_unicast_packets); + break; + case MVPP2_PORT_CTR_RX_ERRORS: + vnet_dev_counter_value_update (vm, c, stats.rx_errors); + break; + case MVPP2_PORT_CTR_TX_BYTES: + vnet_dev_counter_value_update (vm, c, stats.tx_bytes); + break; + case MVPP2_PORT_CTR_TX_PACKETS: + vnet_dev_counter_value_update (vm, c, stats.tx_packets); + break; + case MVPP2_PORT_CTR_TX_UCAST: + vnet_dev_counter_value_update (vm, c, stats.tx_unicast_packets); + break; + case MVPP2_PORT_CTR_TX_ERRORS: + vnet_dev_counter_value_update (vm, c, stats.tx_errors); + break; + case MVPP2_PORT_CTR_RX_FULLQ_DROPPED: + vnet_dev_counter_value_update (vm, c, stats.rx_fullq_dropped); + break; + case MVPP2_PORT_CTR_RX_BM_DROPPED: + vnet_dev_counter_value_update (vm, c, stats.rx_bm_dropped); + break; + case MVPP2_PORT_CTR_RX_EARLY_DROPPED: + vnet_dev_counter_value_update (vm, c, stats.rx_early_dropped); + break; + case MVPP2_PORT_CTR_RX_FIFO_DROPPED: + vnet_dev_counter_value_update (vm, c, stats.rx_fifo_dropped); + break; + case MVPP2_PORT_CTR_RX_CLS_DROPPED: + vnet_dev_counter_value_update (vm, c, stats.rx_cls_dropped); + break; + + default: + ASSERT (0); + } + } + + foreach_vnet_dev_port_rx_queue (q, port) + { + struct pp2_ppio_inq_statistics stats; + pp2_ppio_inq_get_statistics (mp->ppio, 0, q->queue_id, &stats, 0); + + foreach_vnet_dev_counter (c, q->counter_main) + { + switch (c->user_data) + { + case MVPP2_RXQ_CTR_ENQ_DESC: + vnet_dev_counter_value_update (vm, c, stats.enq_desc); + break; + case MVPP2_RXQ_CTR_DROP_BM: + vnet_dev_counter_value_update (vm, c, stats.drop_bm); + break; + case MVPP2_RXQ_CTR_DROP_EARLY: + vnet_dev_counter_value_update (vm, c, stats.drop_early); + break; + case MVPP2_RXQ_CTR_DROP_FULLQ: + vnet_dev_counter_value_update (vm, c, stats.drop_fullq); + break; + default: + ASSERT (0); + } + } + } + + foreach_vnet_dev_port_tx_queue (q, port) + { + struct pp2_ppio_outq_statistics stats; + pp2_ppio_outq_get_statistics (mp->ppio, q->queue_id, &stats, 0); + + foreach_vnet_dev_counter (c, q->counter_main) + { + switch (c->user_data) + { + case MVPP2_TXQ_CTR_ENQ_DESC: + vnet_dev_counter_value_update (vm, c, stats.enq_desc); + break; + case MVPP2_TXQ_CTR_DEQ_DESC: + vnet_dev_counter_value_update (vm, c, stats.deq_desc); + break; + case MVPP2_TXQ_CTR_ENQ_BUF_TO_DDR: + vnet_dev_counter_value_update (vm, c, stats.enq_buf_to_ddr); + break; + case MVPP2_TXQ_CTR_ENQ_DEC_TO_DDR: + vnet_dev_counter_value_update (vm, c, stats.enq_dec_to_ddr); + break; + default: + ASSERT (0); + } + } + } + + return VNET_DEV_OK; +} diff --git a/src/plugins/dev_armada/pp2/format.c b/src/plugins/dev_armada/pp2/format.c index 37d482b5ce8..42c4114c512 100644 --- a/src/plugins/dev_armada/pp2/format.c +++ b/src/plugins/dev_armada/pp2/format.c @@ -152,25 +152,47 @@ format_mvpp2_rx_desc (u8 *s, va_list *args) s = format (s, " "); foreach_pp2_rx_desc_field; +#undef _ return s; } u8 * +format_mv_dsa_tag (u8 *s, va_list *args) +{ + mv_dsa_tag_t *tag = va_arg (*args, mv_dsa_tag_t *); + u32 cnt = 0; + +#define _(b, n) \ + if (#n[0] != '_') \ + s = format (s, "%s" #n " %u", cnt++ ? " " : "", tag->n); + foreach_mv_dsa_tag_field +#undef _ + return s; +} + +u8 * format_mvpp2_rx_trace (u8 *s, va_list *args) { vlib_main_t *vm = va_arg (*args, vlib_main_t *); vlib_node_t *node = va_arg (*args, vlib_node_t *); mvpp2_rx_trace_t *t = va_arg (*args, mvpp2_rx_trace_t *); vnet_main_t *vnm = vnet_get_main (); - u32 hw_if_index = t->rxq->port->intf.hw_if_index; - vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index); u32 indent = format_get_indent (s); struct pp2_ppio_desc *d = &t->desc; - s = format (s, "pp2: %v (%d) next-node %U", hi->name, hw_if_index, - format_vlib_next_node_name, vm, node->index, t->rxq->next_index); + if (t->sw_if_index != CLIB_U32_MAX) + s = format (s, "pp2: %U (%d) next-node %U", format_vnet_sw_if_index_name, + vnm, t->sw_if_index, t->sw_if_index, + format_vlib_next_node_name, vm, node->index, t->next_index); + else + s = format (s, "pp2: next-node %U", format_vlib_next_node_name, vm, + node->index, t->next_index); + s = format (s, "\n%U%U", format_white_space, indent + 2, format_mvpp2_rx_desc, d); + if (t->dsa_tag.as_u32) + s = format (s, "\n%Udsa tag: %U", format_white_space, indent + 2, + format_mv_dsa_tag, &t->dsa_tag); return s; } diff --git a/src/plugins/dev_armada/pp2/init.c b/src/plugins/dev_armada/pp2/init.c index 38ff32d8f53..4333dbb352f 100644 --- a/src/plugins/dev_armada/pp2/init.c +++ b/src/plugins/dev_armada/pp2/init.c @@ -201,6 +201,7 @@ mvpp2_init (vlib_main_t *vm, vnet_dev_t *dev) vnet_dev_rv_t rv = VNET_DEV_OK; vnet_dev_bus_platform_device_data_t *dd = vnet_dev_get_bus_data (dev); clib_dt_node_t *sc; + clib_dt_node_t *sw = 0; int pp_id = -1; if (!clib_dt_node_is_compatible (dd->node, "marvell,armada-7k-pp22")) @@ -219,12 +220,55 @@ mvpp2_init (vlib_main_t *vm, vnet_dev_t *dev) if (pp_id < 0) return VNET_DEV_ERR_UNKNOWN_DEVICE; + foreach_clib_dt_tree_node (n, clib_dt_get_root_node (sc)) + if (clib_dt_node_is_compatible (n, "marvell,mv88e6190") || + clib_dt_node_is_compatible (n, "marvell,mv88e6393x")) + { + clib_dt_node_t *ports; + sw = n; + log_debug (dev, "found mv88e6190 compatible switch at %v", n->path); + ports = clib_dt_get_child_node (sw, "ports"); + foreach_clib_dt_child_node (pn, ports) + { + u32 reg = CLIB_U32_MAX; + char *label = "(no label)"; + clib_dt_property_t *p; + clib_dt_node_t *n; + + p = clib_dt_get_node_property_by_name (pn, "reg"); + if (p) + reg = clib_dt_property_get_u32 (p); + p = clib_dt_get_node_property_by_name (pn, "label"); + if (p) + label = clib_dt_property_get_string (p); + + log_debug (dev, "port %u label %s", reg, label); + + n = clib_dt_dereference_node (pn, "phy-handle"); + if (n) + log_debug (dev, " phy is %v", n->path); + + n = clib_dt_dereference_node (pn, "sfp"); + if (n) + log_debug (dev, " sfp is %v", n->path); + + n = clib_dt_dereference_node (pn, "ethernet"); + if (n) + log_debug (dev, " connected to %v", n->path); + + p = clib_dt_get_node_property_by_name (pn, "phy-mode"); + if (p) + log_debug (dev, " phy mode is %s", + clib_dt_property_get_string (p)); + } + } + if ((mvpp2_global_init (vm, dev)) != VNET_DEV_OK) return rv; md->pp_id = pp_id; - vec_foreach_pointer (cn, dd->node->child_nodes) + foreach_clib_dt_child_node (cn, dd->node) { clib_dt_property_t *p; char netdev_name[IFNAMSIZ]; @@ -234,10 +278,10 @@ mvpp2_init (vlib_main_t *vm, vnet_dev_t *dev) p = clib_dt_get_node_property_by_name (cn, "port-id"); - if (!clib_dt_proprerty_is_u32 (p)) + if (!clib_dt_property_is_u32 (p)) continue; - ppio_id = clib_dt_proprerty_get_u32 (p); + ppio_id = clib_dt_property_get_u32 (p); log_debug (dev, "found port with ppio id %u", ppio_id); if (pp2_ppio_available (md->pp_id, ppio_id) == 0) @@ -271,6 +315,28 @@ mvpp2_init (vlib_main_t *vm, vnet_dev_t *dev) .ppio_id = ppio_id, }; + if (sw) + { + clib_dt_node_t *ports = clib_dt_get_child_node (sw, "ports"); + if (ports) + foreach_clib_dt_child_node (sp, ports) + { + clib_dt_node_t *eth; + + eth = clib_dt_dereference_node (sp, "ethernet"); + + if (cn != eth) + continue; + + mvpp2_port.is_dsa = 1; + mvpp2_port.switch_node = sw; + mvpp2_port.switch_port_node = sp; + log_debug (dev, "port is connected to switch port %v", + sp->path); + break; + } + } + vnet_dev_port_add_args_t port_add_args = { .port = { .attr = { @@ -278,18 +344,26 @@ mvpp2_init (vlib_main_t *vm, vnet_dev_t *dev) .max_rx_queues = PP2_PPIO_MAX_NUM_INQS, .max_tx_queues = PP2_PPIO_MAX_NUM_OUTQS, .max_supported_rx_frame_size = 9216, + .caps.secondary_interfaces = mvpp2_port.is_dsa != 0, }, .ops = { .init = mvpp2_port_init, .deinit = mvpp2_port_deinit, .start = mvpp2_port_start, .stop = mvpp2_port_stop, + .add_sec_if = mvpp2_port_add_sec_if, + .del_sec_if = mvpp2_port_del_sec_if, .config_change = mvpp2_port_cfg_change, .config_change_validate = mvpp2_port_cfg_change_validate, .format_status = format_mvpp2_port_status, + .clear_counters = mvpp2_port_clear_counters, }, .data_size = sizeof (mvpp2_port_t), .initial_data = &mvpp2_port, + .sec_if_args = VNET_DEV_ARGS ( + VNET_DEV_ARG_UINT32 (MVPP2_SEC_IF_ARG_DSA_SWITCH, "dsa_switch", "DSA source switch ID", .max= 31), + VNET_DEV_ARG_UINT32 (MVPP2_SEC_IF_ARG_DSA_PORT, "dsa_port", "DSA source switch port ID", .max = 31) + ), }, .rx_node = &mvpp2_rx_node, .tx_node = &mvpp2_tx_node, @@ -302,6 +376,9 @@ mvpp2_init (vlib_main_t *vm, vnet_dev_t *dev) .max_size = 4096, .size_is_power_of_two = 1, }, + .ops = { + .clear_counters = mvpp2_rxq_clear_counters, + }, }, .tx_queue = { .config = { @@ -315,6 +392,7 @@ mvpp2_init (vlib_main_t *vm, vnet_dev_t *dev) .ops = { .alloc = mvpp2_txq_alloc, .free = mvpp2_txq_free, + .clear_counters = mvpp2_txq_clear_counters, }, }, }; diff --git a/src/plugins/dev_armada/pp2/port.c b/src/plugins/dev_armada/pp2/port.c index 8e785e5e0e4..63a212e80c2 100644 --- a/src/plugins/dev_armada/pp2/port.c +++ b/src/plugins/dev_armada/pp2/port.c @@ -23,6 +23,7 @@ mvpp2_port_init (vlib_main_t *vm, vnet_dev_port_t *port) mvpp2_device_t *md = vnet_dev_get_data (dev); mvpp2_port_t *mp = vnet_dev_get_port_data (port); vnet_dev_rv_t rv = VNET_DEV_OK; + vnet_dev_rx_queue_t *rxq0 = vnet_dev_get_port_rx_queue_by_id (port, 0); struct pp2_ppio_link_info li; char match[16]; int mrv; @@ -40,17 +41,17 @@ mvpp2_port_init (vlib_main_t *vm, vnet_dev_port_t *port) .tcs_params[0] = { .pkt_offset = 0, .num_in_qs = 1, - .inqs_params = &(struct pp2_ppio_inq_params) { .size = 512 }, - .pools[0][0] = md->thread[0].bpool, + .inqs_params = &(struct pp2_ppio_inq_params) { .size = rxq0->size }, + .pools[0][0] = md->thread[rxq0->rx_thread_index].bpool, }, }, }; - foreach_vnet_dev_port_rx_queue (q, port) + foreach_vnet_dev_port_tx_queue (q, port) { struct pp2_ppio_outqs_params *oqs = &ppio_params.outqs_params; - oqs->outqs_params[0].weight = 1; - oqs->outqs_params[0].size = q->size; + oqs->outqs_params[q->queue_id].weight = 1; + oqs->outqs_params[q->queue_id].size = q->size; oqs->num_outqs++; } @@ -75,6 +76,11 @@ mvpp2_port_init (vlib_main_t *vm, vnet_dev_port_t *port) log_debug (dev, "port %u %U", port->port_id, format_pp2_ppio_link_info, &li); + for (u32 i = 0; i < VLIB_FRAME_SIZE; i++) + mp->desc_ptrs[i] = mp->descs + i; + + mvpp2_port_add_counters (vm, port); + done: if (rv != VNET_DEV_OK) mvpp2_port_stop (vm, port); @@ -145,12 +151,13 @@ mvpp2_port_poll (vlib_main_t *vm, vnet_dev_port_t *port) } } - if (changes.change.any == 0) - return; - - mp->last_link_info = li; + if (changes.change.any) + { + mp->last_link_info = li; + vnet_dev_port_state_change (vm, port, changes); + } - vnet_dev_port_state_change (vm, port, changes); + mvpp2_port_get_stats (vm, port); } vnet_dev_rv_t @@ -203,6 +210,68 @@ mvpp2_port_stop (vlib_main_t *vm, vnet_dev_port_t *port) } vnet_dev_rv_t +mvpp2_port_add_sec_if (vlib_main_t *vm, vnet_dev_port_t *port, void *p) +{ + vnet_dev_port_interface_t *sif = p; + mvpp2_port_t *mp = vnet_dev_get_port_data (port); + u32 port_id = CLIB_U32_MAX, switch_id = 0, index; + + if (mp->is_dsa == 0) + return VNET_DEV_ERR_NOT_SUPPORTED; + + foreach_vnet_dev_args (a, sif) + { + switch (a->id) + { + case MVPP2_SEC_IF_ARG_DSA_PORT: + if (a->val_set) + port_id = vnet_dev_arg_get_uint32 (a); + break; + case MVPP2_SEC_IF_ARG_DSA_SWITCH: + switch_id = vnet_dev_arg_get_uint32 (a); + break; + default: + break; + } + } + + if (port_id == CLIB_U32_MAX) + { + log_err (port->dev, "missing dsa_port argument"); + return VNET_DEV_ERR_INVALID_ARG; + } + + log_debug (port->dev, "switch %u port %u", switch_id, port_id); + + mv_dsa_tag_t tag = { + .tag_type = MV_DSA_TAG_TYPE_FROM_CPU, + .src_port_or_lag = port_id, + .src_dev = switch_id, + }; + + index = switch_id << 5 | port_id; + + sif->user_data = tag.as_u32; + uword_bitmap_set_bits_at_index (mp->valid_dsa_src_bitmap, index, 1); + mp->dsa_to_sec_if[index] = sif->index; + return VNET_DEV_OK; +} + +vnet_dev_rv_t +mvpp2_port_del_sec_if (vlib_main_t *vm, vnet_dev_port_t *port, void *p) +{ + vnet_dev_port_interface_t *sif = p; + mvpp2_port_t *mp = vnet_dev_get_port_data (port); + mv_dsa_tag_t tag = { .as_u32 = sif->user_data }; + u32 index = tag.src_dev << 5 | tag.src_port_or_lag; + + log_debug (port->dev, "switch %u port %u", tag.src_dev, tag.src_port_or_lag); + + uword_bitmap_clear_bits_at_index (mp->valid_dsa_src_bitmap, index, 1); + return VNET_DEV_OK; +} + +vnet_dev_rv_t mvpp2_port_cfg_change_validate (vlib_main_t *vm, vnet_dev_port_t *port, vnet_dev_port_cfg_change_req_t *req) { @@ -211,6 +280,7 @@ mvpp2_port_cfg_change_validate (vlib_main_t *vm, vnet_dev_port_t *port, switch (req->type) { case VNET_DEV_PORT_CFG_PROMISC_MODE: + case VNET_DEV_PORT_CFG_CHANGE_PRIMARY_HW_ADDR: case VNET_DEV_PORT_CFG_ADD_SECONDARY_HW_ADDR: case VNET_DEV_PORT_CFG_REMOVE_SECONDARY_HW_ADDR: break; @@ -246,6 +316,19 @@ mvpp2_port_cfg_change (vlib_main_t *vm, vnet_dev_port_t *port, req->promisc); break; + case VNET_DEV_PORT_CFG_CHANGE_PRIMARY_HW_ADDR: + clib_memcpy (&addr, req->addr.eth_mac, sizeof (addr)); + mrv = pp2_ppio_set_mac_addr (mp->ppio, addr); + if (mrv) + { + log_err (port->dev, "pp2_ppio_set_mac_addr: failed, rv %d", mrv); + rv = VNET_DEV_ERR_INTERNAL; + } + else + log_debug (port->dev, "pp2_ppio_set_mac_addr: %U added", + format_ethernet_address, &addr); + break; + case VNET_DEV_PORT_CFG_ADD_SECONDARY_HW_ADDR: clib_memcpy (&addr, req->addr.eth_mac, sizeof (addr)); mrv = pp2_ppio_add_mac_addr (mp->ppio, addr); diff --git a/src/plugins/dev_armada/pp2/pp2.h b/src/plugins/dev_armada/pp2/pp2.h index 6b12dc737a7..160bfd20c5c 100644 --- a/src/plugins/dev_armada/pp2/pp2.h +++ b/src/plugins/dev_armada/pp2/pp2.h @@ -8,6 +8,7 @@ #include <vppinfra/clib.h> #include <vppinfra/error_bootstrap.h> #include <vppinfra/format.h> +#include <vppinfra/devicetree.h> #include <vnet/vnet.h> #include <vnet/dev/dev.h> @@ -29,6 +30,58 @@ #define MVPP2_NUM_BPOOLS 16 #define MVPP2_MAX_THREADS 4 #define MRVL_PP2_BUFF_BATCH_SZ 32 +#define MV_DSA_N_SRC 32 + +#define foreach_mv_dsa_tag_field \ + _ (12, vid) \ + _ (1, _zero13) \ + _ (3, pri) \ + _ (1, cfi_dei) \ + _ (1, _unused17) \ + _ (1, src_is_lag) \ + _ (5, src_port_or_lag) \ + _ (5, src_dev) \ + _ (1, src_tagged) \ + _ (2, tag_type) + +typedef enum +{ + MV_DSA_TAG_TYPE_TO_CPU = 0, + MV_DSA_TAG_TYPE_FROM_CPU = 1, + MV_DSA_TAG_TYPE_TO_SNIFFER = 2, + MV_DSA_TAG_TYPE_FORWARD = 3 +} mv_dsa_tag_type_t; + +typedef enum +{ + MVPP2_SEC_IF_ARG_DSA_SWITCH, + MVPP2_SEC_IF_ARG_DSA_PORT +} mvpp2_sec_if_args_t; + +typedef union +{ + struct + { +#define _(b, n) u32 (n) : (b); + foreach_mv_dsa_tag_field +#undef _ + }; + u32 as_u32; +} mv_dsa_tag_t; + +STATIC_ASSERT_SIZEOF (mv_dsa_tag_t, 4); + +static_always_inline mv_dsa_tag_t +mv_dsa_tag_read (void *p) +{ + return (mv_dsa_tag_t){ .as_u32 = clib_net_to_host_u32 (*(u32u *) p) }; +} + +static_always_inline void +mv_dsa_tag_write (void *p, mv_dsa_tag_t tag) +{ + ((mv_dsa_tag_t *) p)->as_u32 = clib_host_to_net_u32 (tag.as_u32); +} typedef struct { @@ -49,6 +102,13 @@ typedef struct struct pp2_ppio *ppio; u8 ppio_id; struct pp2_ppio_link_info last_link_info; + clib_dt_node_t *switch_node; + clib_dt_node_t *switch_port_node; + + struct pp2_ppio_desc descs[VLIB_FRAME_SIZE]; + struct pp2_ppio_desc *desc_ptrs[VLIB_FRAME_SIZE]; + uword valid_dsa_src_bitmap[1024 / uword_bits]; + u16 dsa_to_sec_if[1024]; } mvpp2_port_t; typedef struct @@ -65,21 +125,33 @@ typedef struct typedef struct { struct pp2_ppio_desc desc; - vnet_dev_rx_queue_t *rxq; + u32 sw_if_index; + u16 next_index; + mv_dsa_tag_t dsa_tag; } mvpp2_rx_trace_t; +/* counters.c */ +void mvpp2_port_add_counters (vlib_main_t *, vnet_dev_port_t *); +void mvpp2_port_clear_counters (vlib_main_t *, vnet_dev_port_t *); +void mvpp2_rxq_clear_counters (vlib_main_t *, vnet_dev_rx_queue_t *); +void mvpp2_txq_clear_counters (vlib_main_t *, vnet_dev_tx_queue_t *); +vnet_dev_rv_t mvpp2_port_get_stats (vlib_main_t *, vnet_dev_port_t *); + /* format.c */ format_function_t format_pp2_ppio_link_info; format_function_t format_mvpp2_port_status; format_function_t format_mvpp2_dev_info; format_function_t format_mvpp2_rx_trace; format_function_t format_mvpp2_rx_desc; +format_function_t format_mv_dsa_tag; /* port.c */ vnet_dev_port_op_t mvpp2_port_init; vnet_dev_port_op_no_rv_t mvpp2_port_deinit; vnet_dev_port_op_t mvpp2_port_start; vnet_dev_port_op_no_rv_t mvpp2_port_stop; +vnet_dev_port_op_with_ptr_t mvpp2_port_add_sec_if; +vnet_dev_port_op_with_ptr_t mvpp2_port_del_sec_if; vnet_dev_rv_t mvpp2_port_cfg_change (vlib_main_t *, vnet_dev_port_t *, vnet_dev_port_cfg_change_req_t *); vnet_dev_rv_t @@ -128,6 +200,7 @@ typedef enum "pp2_bpool_get_num_buffs error") \ _ (BPOOL_PUT_BUFFS, bpool_put_buffs, ERROR, "pp2_bpool_put_buffs error") \ _ (BUFFER_ALLOC, buffer_alloc, ERROR, "buffer alloc error") \ + _ (UNKNOWN_DSA_SRC, unknown_dsa_src, ERROR, "unknown DSA source") \ _ (MAC_CE, mac_ce, ERROR, "MAC error (CRC error)") \ _ (MAC_OR, mac_or, ERROR, "overrun error") \ _ (MAC_RSVD, mac_rsvd, ERROR, "unknown MAC error") \ diff --git a/src/plugins/dev_armada/pp2/rx.c b/src/plugins/dev_armada/pp2/rx.c index 81101ef9313..5b0e8d35000 100644 --- a/src/plugins/dev_armada/pp2/rx.c +++ b/src/plugins/dev_armada/pp2/rx.c @@ -5,104 +5,219 @@ #include <vlib/vlib.h> #include <vnet/dev/dev.h> #include <vnet/ethernet/ethernet.h> +#include <vppinfra/vector/mask_compare.h> +#include <vppinfra/vector/compress.h> #include <dev_armada/pp2/pp2.h> -static_always_inline void -mvpp2_rx_trace (vlib_main_t *vm, vlib_node_runtime_t *node, - vnet_dev_rx_queue_t *rxq, vlib_buffer_t *b0, uword *n_trace, - struct pp2_ppio_desc *d) +static_always_inline vlib_buffer_t * +desc_to_vlib_buffer (vlib_main_t *vm, struct pp2_ppio_desc *d) { - if (PREDICT_TRUE (vlib_trace_buffer (vm, node, rxq->next_index, b0, - /* follow_chain */ 0))) + return vlib_get_buffer (vm, pp2_ppio_inq_desc_get_cookie (d)); +} + +static_always_inline u64 +mrvl_pp2_rx_one_if (vlib_main_t *vm, vlib_node_runtime_t *node, + vnet_dev_rx_queue_t *rxq, + vnet_dev_rx_queue_if_rt_data_t *if_rt_data, + struct pp2_ppio_desc **desc_ptrs, u32 n_desc, + i32 current_data, i32 len_adj, mv_dsa_tag_t tag) +{ + vnet_main_t *vnm = vnet_get_main (); + u64 n_rx_bytes = 0; + vlib_buffer_t *b0, *b1; + u32 n_trace, n_left = n_desc; + u32 buffer_indices[VLIB_FRAME_SIZE], *bi = buffer_indices; + struct pp2_ppio_desc **dp = desc_ptrs; + u32 next_index = if_rt_data->next_index; + vlib_buffer_template_t bt = if_rt_data->buffer_template; + u32 sw_if_index = if_rt_data->sw_if_index; + + bt.current_data = current_data; + + for (; n_left >= 4; dp += 2, bi += 2, n_left -= 2) { - mvpp2_rx_trace_t *tr; - vlib_set_trace_count (vm, node, --(*n_trace)); - tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - tr->desc = *d; - tr->rxq = rxq; + clib_prefetch_store (desc_to_vlib_buffer (vm, dp[2])); + clib_prefetch_store (desc_to_vlib_buffer (vm, dp[3])); + b0 = desc_to_vlib_buffer (vm, dp[0]); + b1 = desc_to_vlib_buffer (vm, dp[1]); + bi[0] = pp2_ppio_inq_desc_get_cookie (dp[0]); + bi[1] = pp2_ppio_inq_desc_get_cookie (dp[1]); + b0->template = bt; + b1->template = bt; + + n_rx_bytes += b0->current_length = + pp2_ppio_inq_desc_get_pkt_len (dp[0]) + len_adj; + n_rx_bytes += b1->current_length = + pp2_ppio_inq_desc_get_pkt_len (dp[1]) + len_adj; + } + + for (; n_left; dp++, bi++, n_left--) + { + b0 = desc_to_vlib_buffer (vm, dp[0]); + bi[0] = pp2_ppio_inq_desc_get_cookie (dp[0]); + b0->template = bt; + + n_rx_bytes += b0->current_length = + pp2_ppio_inq_desc_get_pkt_len (dp[0]) + len_adj; } + + /* trace */ + n_trace = vlib_get_trace_count (vm, node); + if (PREDICT_FALSE (n_trace > 0)) + { + for (u32 i = 0; i < n_desc && n_trace > 0; i++) + { + vlib_buffer_t *b = desc_to_vlib_buffer (vm, desc_ptrs[i]); + if (PREDICT_TRUE (vlib_trace_buffer (vm, node, next_index, b, + /* follow_chain */ 0))) + { + mvpp2_rx_trace_t *tr; + tr = vlib_add_trace (vm, node, b, sizeof (*tr)); + tr->desc = *desc_ptrs[i]; + tr->next_index = next_index; + tr->sw_if_index = sw_if_index; + tr->dsa_tag = tag; + n_trace--; + } + } + vlib_set_trace_count (vm, node, n_trace); + } + vlib_buffer_enqueue_to_single_next (vm, node, buffer_indices, next_index, + n_desc); + + vlib_increment_combined_counter ( + vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, + vm->thread_index, sw_if_index, n_desc, n_rx_bytes); + + return n_rx_bytes; } static_always_inline uword mrvl_pp2_rx_inline (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, vnet_dev_rx_queue_t *rxq) { - vnet_main_t *vnm = vnet_get_main (); vnet_dev_port_t *port = rxq->port; - vnet_dev_t *dev = port->dev; - mvpp2_device_t *md = vnet_dev_get_data (dev); mvpp2_port_t *mp = vnet_dev_get_port_data (port); - vlib_buffer_template_t bt = rxq->buffer_template; - u32 thread_index = vm->thread_index; - uword n_trace = vlib_get_trace_count (vm, node); - u32 next_index = rxq->next_index; - u32 n_rx_packets = 0, n_rx_bytes = 0; - struct pp2_hif *hif = md->hif[thread_index]; - struct pp2_ppio_desc descs[VLIB_FRAME_SIZE], *d; - struct pp2_bpool *bpool = md->thread[thread_index].bpool; - struct buff_release_entry *bre = md->thread[thread_index].bre; + mv_dsa_tag_t dsa_tags[VLIB_FRAME_SIZE]; u16 n_desc = VLIB_FRAME_SIZE; - u32 buffers[VLIB_FRAME_SIZE]; - u32 n_bufs, *bi, i; - vlib_buffer_t *b0, *b1; + vlib_buffer_t *b; + u32 i; if (PREDICT_FALSE ( - pp2_ppio_recv (mp->ppio, 0, rxq->queue_id, descs, &n_desc))) + pp2_ppio_recv (mp->ppio, 0, rxq->queue_id, mp->descs, &n_desc))) { vlib_error_count (vm, node->node_index, MVPP2_RX_NODE_CTR_PPIO_RECV, 1); - n_desc = 0; + return 0; } - n_rx_packets = n_desc; + if (mp->is_dsa) + { + for (i = 0; i < n_desc; i++) + { + b = desc_to_vlib_buffer (vm, mp->descs + i); + u8 *start = b->data; + mv_dsa_tag_t tag = mv_dsa_tag_read (start + 14); + dsa_tags[i] = tag; + clib_memmove (start + 6, start + 2, 12); + } - for (i = 0; i < n_desc; i++) - buffers[i] = pp2_ppio_inq_desc_get_cookie (descs + i); + vlib_frame_bitmap_t avail_bmp = {}; + vlib_frame_bitmap_init (avail_bmp, n_desc); + u32 n_avail = n_desc; - bt.current_data = 2; + while (n_avail) + { + vlib_frame_bitmap_t selected_bmp = {}; + struct pp2_ppio_desc *sel_descs[VLIB_FRAME_SIZE]; + mv_dsa_tag_t tag; + u32 n_sel, index; - for (d = descs, bi = buffers; n_desc >= 4; d += 2, bi += 2, n_desc -= 2) - { - /* prefetch */ - b0 = vlib_get_buffer (vm, bi[0]); - b1 = vlib_get_buffer (vm, bi[1]); - b0->template = bt; - b1->template = bt; + tag = dsa_tags[vlib_frame_bitmap_find_first_set (avail_bmp)]; + index = tag.src_dev << 5 | tag.src_port_or_lag; - n_rx_bytes += b0->current_length = pp2_ppio_inq_desc_get_pkt_len (d); - n_rx_bytes += b1->current_length = pp2_ppio_inq_desc_get_pkt_len (d + 1); + clib_mask_compare_u32 (tag.as_u32, (u32 *) dsa_tags, selected_bmp, + n_desc); + n_sel = vlib_frame_bitmap_count_set_bits (selected_bmp); + n_avail -= n_sel; - if (PREDICT_FALSE (n_trace > 0)) - { - mvpp2_rx_trace (vm, node, rxq, b0, &n_trace, d); - if (n_trace > 0) - mvpp2_rx_trace (vm, node, rxq, b1, &n_trace, d + 1); - } - } + if (uword_bitmap_is_bit_set (mp->valid_dsa_src_bitmap, index)) + { + clib_compress_u64 ((uword *) sel_descs, (uword *) mp->desc_ptrs, + selected_bmp, n_desc); + mrvl_pp2_rx_one_if (vm, node, rxq, + vnet_dev_get_rx_queue_sec_if_rt_data ( + rxq, mp->dsa_to_sec_if[index]), + sel_descs, n_sel, 6, -4, tag); + } + else + { + u32 n_free = 0, buffer_indices[VLIB_FRAME_SIZE]; - for (; n_desc; d++, bi++, n_desc--) - { - b0 = vlib_get_buffer (vm, bi[0]); - b0->template = bt; + foreach_vlib_frame_bitmap_set_bit_index (i, selected_bmp) + buffer_indices[n_free++] = + pp2_ppio_inq_desc_get_cookie (mp->descs + i); - n_rx_bytes += b0->current_length = pp2_ppio_inq_desc_get_pkt_len (d); + u32 n_trace = vlib_get_trace_count (vm, node); + if (PREDICT_FALSE (n_trace > 0)) + { + foreach_vlib_frame_bitmap_set_bit_index (i, selected_bmp) + { + vlib_buffer_t *b = + desc_to_vlib_buffer (vm, mp->descs + i); - if (PREDICT_FALSE (n_trace > 0)) - mvpp2_rx_trace (vm, node, rxq, b0, &n_trace, d); + if (PREDICT_TRUE (vlib_trace_buffer ( + vm, node, VNET_DEV_ETH_RX_PORT_NEXT_DROP, b, + /* follow_chain */ 0))) + { + mvpp2_rx_trace_t *tr; + tr = vlib_add_trace (vm, node, b, sizeof (*tr)); + tr->desc = mp->descs[i]; + tr->next_index = VNET_DEV_ETH_RX_PORT_NEXT_DROP; + tr->sw_if_index = CLIB_U32_MAX; + tr->dsa_tag = dsa_tags[i]; + n_trace--; + } + if (n_trace == 0) + break; + } + vlib_set_trace_count (vm, node, n_trace); + } + + vlib_buffer_free (vm, buffer_indices, n_free); + vlib_error_count (vm, node->node_index, + MVPP2_RX_NODE_CTR_UNKNOWN_DSA_SRC, 1); + } + } + } + else + { + mrvl_pp2_rx_one_if (vm, node, rxq, + vnet_dev_get_rx_queue_if_rt_data (rxq), + mp->desc_ptrs, n_desc, 2, 0, (mv_dsa_tag_t){}); } - vlib_buffer_enqueue_to_single_next (vm, node, buffers, next_index, - n_rx_packets); + return n_desc; +} - vlib_increment_combined_counter ( - vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - thread_index, port->intf.sw_if_index, n_rx_packets, n_rx_bytes); +static_always_inline void +mrvl_pp2_rx_refill (vlib_main_t *vm, vlib_node_runtime_t *node, + vnet_dev_rx_queue_t *rxq) +{ + vnet_dev_port_t *port = rxq->port; + vnet_dev_t *dev = port->dev; + mvpp2_device_t *md = vnet_dev_get_data (dev); + u32 thread_index = vm->thread_index; + struct pp2_hif *hif = md->hif[thread_index]; + struct pp2_bpool *bpool = md->thread[thread_index].bpool; + struct buff_release_entry *bre = md->thread[thread_index].bre; + u32 n_bufs, *bi; if (PREDICT_FALSE (pp2_bpool_get_num_buffs (bpool, &n_bufs))) { vlib_error_count (vm, node->node_index, MVPP2_RX_NODE_CTR_BPOOL_GET_NUM_BUFFS, 1); - goto done; + return; } n_bufs = rxq->size - n_bufs; @@ -110,18 +225,18 @@ mrvl_pp2_rx_inline (vlib_main_t *vm, vlib_node_runtime_t *node, { u16 n_alloc, i; struct buff_release_entry *e = bre; + u32 buffer_indices[MRVL_PP2_BUFF_BATCH_SZ]; - n_alloc = vlib_buffer_alloc (vm, buffers, MRVL_PP2_BUFF_BATCH_SZ); - i = n_alloc; + n_alloc = vlib_buffer_alloc (vm, buffer_indices, MRVL_PP2_BUFF_BATCH_SZ); if (PREDICT_FALSE (n_alloc == 0)) { vlib_error_count (vm, node->node_index, MVPP2_RX_NODE_CTR_BUFFER_ALLOC, 1); - goto done; + return; } - for (bi = buffers; i--; e++, bi++) + for (i = n_alloc, bi = buffer_indices; i--; e++, bi++) { vlib_buffer_t *b = vlib_get_buffer (vm, bi[0]); @@ -129,23 +244,16 @@ mrvl_pp2_rx_inline (vlib_main_t *vm, vlib_node_runtime_t *node, e->buff.cookie = bi[0]; } - i = n_alloc; - if (PREDICT_FALSE (pp2_bpool_put_buffs (hif, bre, &i))) + if (PREDICT_FALSE (pp2_bpool_put_buffs (hif, bre, &n_alloc))) { vlib_error_count (vm, node->node_index, MVPP2_RX_NODE_CTR_BPOOL_PUT_BUFFS, 1); - vlib_buffer_free (vm, buffers, n_alloc); - goto done; + vlib_buffer_free (vm, buffer_indices, n_alloc); + return; } - if (PREDICT_FALSE (i != n_alloc)) - vlib_buffer_free (vm, buffers + i, n_alloc - i); - - n_bufs -= i; + n_bufs -= n_alloc; } - -done: - return n_rx_packets; } VNET_DEV_NODE_FN (mvpp2_rx_node) @@ -153,6 +261,9 @@ VNET_DEV_NODE_FN (mvpp2_rx_node) { u32 n_rx = 0; foreach_vnet_dev_rx_queue_runtime (rxq, node) - n_rx += mrvl_pp2_rx_inline (vm, node, frame, rxq); + { + n_rx += mrvl_pp2_rx_inline (vm, node, frame, rxq); + mrvl_pp2_rx_refill (vm, node, rxq); + } return n_rx; } diff --git a/src/plugins/dev_armada/pp2/tx.c b/src/plugins/dev_armada/pp2/tx.c index 1e6675c9746..583eec71d60 100644 --- a/src/plugins/dev_armada/pp2/tx.c +++ b/src/plugins/dev_armada/pp2/tx.c @@ -12,6 +12,7 @@ VNET_DEV_NODE_FN (mvpp2_tx_node) (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) { vnet_dev_tx_node_runtime_t *rt = vnet_dev_get_tx_node_runtime (node); + vnet_dev_instance_t *ins = vnet_dev_get_dev_instance (rt->dev_instance); vnet_dev_tx_queue_t *txq = rt->tx_queue; vnet_dev_port_t *port = txq->port; vnet_dev_t *dev = port->dev; @@ -27,6 +28,24 @@ VNET_DEV_NODE_FN (mvpp2_tx_node) struct pp2_ppio_desc descs[VLIB_FRAME_SIZE], *d = descs; u16 sz = txq->size; u16 mask = sz - 1; + i16 len_adj = 0; + + if (ins->is_primary_if == 0) + { + vnet_dev_port_interface_t *sif = + vnet_dev_port_get_sec_if_by_index (port, ins->sec_if_index); + + mv_dsa_tag_t tag = { .as_u32 = sif->user_data }; + + for (u32 i = 0; i < n_vectors; i++) + { + vlib_buffer_t *b = vlib_get_buffer (vm, buffers[i]); + u8 *start = vlib_buffer_get_current (b); + clib_memmove (start - 4, start, 12); + mv_dsa_tag_write (start + 8, tag); + } + len_adj = 4; + } if (mtq->n_enq) { @@ -51,9 +70,9 @@ VNET_DEV_NODE_FN (mvpp2_tx_node) u64 paddr = vlib_buffer_get_pa (vm, b0); pp2_ppio_outq_desc_reset (d); - pp2_ppio_outq_desc_set_phys_addr (d, paddr + b0->current_data); + pp2_ppio_outq_desc_set_phys_addr (d, paddr + b0->current_data - len_adj); pp2_ppio_outq_desc_set_pkt_offset (d, 0); - pp2_ppio_outq_desc_set_pkt_len (d, b0->current_length); + pp2_ppio_outq_desc_set_pkt_len (d, b0->current_length + len_adj); } buffers = vlib_frame_vector_args (frame); diff --git a/src/plugins/dev_ena/rx_node.c b/src/plugins/dev_ena/rx_node.c index 41fc5b8c943..51c6dbce84c 100644 --- a/src/plugins/dev_ena/rx_node.c +++ b/src/plugins/dev_ena/rx_node.c @@ -251,7 +251,6 @@ ena_device_input_inline (vlib_main_t *vm, vlib_node_runtime_t *node, vnet_dev_rx_queue_t *rxq) { ena_rxq_t *q = vnet_dev_get_rx_queue_data (rxq); - vnet_dev_port_t *port = rxq->port; vnet_main_t *vnm = vnet_get_main (); vlib_buffer_t *buffers[VLIB_FRAME_SIZE], **b; ena_rx_cdesc_status_t statuses[VLIB_FRAME_SIZE + 8]; @@ -260,13 +259,13 @@ ena_device_input_inline (vlib_main_t *vm, vlib_node_runtime_t *node, u16 *csi; uword n_rx_packets = 0, n_rx_bytes = 0; vlib_frame_bitmap_t head_bmp = {}; - u32 sw_if_index = port->intf.sw_if_index; - u32 hw_if_index = port->intf.hw_if_index; + u32 sw_if_index = vnet_dev_get_rx_queue_if_sw_if_index (rxq); + u32 hw_if_index = vnet_dev_get_rx_queue_if_hw_if_index (rxq); u32 n_trace, n_deq, n_left; u32 cq_next = q->cq_next; - u32 next_index = rxq->next_index; + u32 next_index = vnet_dev_get_rx_queue_if_next_index (rxq); vlib_frame_t *next_frame; - vlib_buffer_template_t bt = rxq->buffer_template; + vlib_buffer_template_t bt = vnet_dev_get_rx_queue_if_buffer_template (rxq); u32 *bi; int maybe_chained; diff --git a/src/plugins/dev_iavf/port.c b/src/plugins/dev_iavf/port.c index f1578fccb59..a0530822688 100644 --- a/src/plugins/dev_iavf/port.c +++ b/src/plugins/dev_iavf/port.c @@ -263,7 +263,7 @@ avf_msix_n_handler (vlib_main_t *vm, vnet_dev_t *dev, u16 line) iavf_reg_write (ad, IAVF_VFINT_DYN_CTLN (line), dyn_ctln_enabled.as_u32); vlib_node_set_interrupt_pending (vlib_get_main_by_index (line), - port->intf.rx_node_index); + vnet_dev_get_port_rx_node_index (port)); } vnet_dev_rv_t diff --git a/src/plugins/dev_iavf/rx_node.c b/src/plugins/dev_iavf/rx_node.c index ee6d7e8def0..bf650f9bfb9 100644 --- a/src/plugins/dev_iavf/rx_node.c +++ b/src/plugins/dev_iavf/rx_node.c @@ -249,14 +249,14 @@ iavf_device_input_inline (vlib_main_t *vm, vlib_node_runtime_t *node, u32 thr_idx = vlib_get_thread_index (); iavf_rt_data_t *rtd = vnet_dev_get_rt_temp_space (vm); iavf_rxq_t *arq = vnet_dev_get_rx_queue_data (rxq); - vlib_buffer_template_t bt = rxq->buffer_template; + vlib_buffer_template_t bt = vnet_dev_get_rx_queue_if_buffer_template (rxq); u32 n_trace, n_rx_packets = 0, n_rx_bytes = 0; u16 n_tail_desc = 0; u64 or_qw1 = 0; u32 *bi, *to_next, n_left_to_next; - u32 next_index = rxq->next_index; - u32 sw_if_index = port->intf.sw_if_index; - u32 hw_if_index = port->intf.hw_if_index; + u32 next_index = vnet_dev_get_rx_queue_if_next_index (rxq); + u32 sw_if_index = vnet_dev_get_rx_queue_if_sw_if_index (rxq); + u32 hw_if_index = vnet_dev_get_rx_queue_if_hw_if_index (rxq); u16 next = arq->next; u16 size = rxq->size; u16 mask = size - 1; diff --git a/src/plugins/dev_octeon/CMakeLists.txt b/src/plugins/dev_octeon/CMakeLists.txt index c6271ecdfba..6109de57a7d 100644 --- a/src/plugins/dev_octeon/CMakeLists.txt +++ b/src/plugins/dev_octeon/CMakeLists.txt @@ -36,6 +36,7 @@ add_vpp_plugin(dev_octeon tx_node.c flow.c counter.c + crypto.c MULTIARCH_SOURCES rx_node.c diff --git a/src/plugins/dev_octeon/crypto.c b/src/plugins/dev_octeon/crypto.c new file mode 100644 index 00000000000..7d3790f3ec9 --- /dev/null +++ b/src/plugins/dev_octeon/crypto.c @@ -0,0 +1,1782 @@ +/* + * Copyright (c) 2024 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#include <vnet/dev/dev.h> +#include <vnet/devices/devices.h> +#include <dev_octeon/octeon.h> +#include <dev_octeon/crypto.h> +#include <base/roc_api.h> +#include <common.h> + +oct_crypto_main_t oct_crypto_main; +oct_crypto_dev_t oct_crypto_dev; + +VLIB_REGISTER_LOG_CLASS (oct_log, static) = { + .class_name = "octeon", + .subclass_name = "crypto", +}; + +static_always_inline void +oct_map_keyindex_to_session (oct_crypto_sess_t *sess, u32 key_index, u8 type) +{ + oct_crypto_main_t *ocm = &oct_crypto_main; + oct_crypto_key_t *ckey; + + ckey = vec_elt_at_index (ocm->keys[type], key_index); + + ckey->sess = sess; + sess->key_index = key_index; +} + +static_always_inline oct_crypto_sess_t * +oct_crypto_session_alloc (vlib_main_t *vm, u8 type) +{ + extern oct_plt_init_param_t oct_plt_init_param; + oct_crypto_sess_t *addr = NULL; + oct_crypto_main_t *ocm; + oct_crypto_dev_t *ocd; + u32 size; + + ocm = &oct_crypto_main; + ocd = ocm->crypto_dev[type]; + + size = sizeof (oct_crypto_sess_t); + + addr = oct_plt_init_param.oct_plt_zmalloc (size, CLIB_CACHE_LINE_BYTES); + if (addr == NULL) + { + log_err (ocd->dev, "Failed to allocate crypto session memory"); + return NULL; + } + + return addr; +} + +static_always_inline i32 +oct_crypto_session_create (vlib_main_t *vm, vnet_crypto_key_index_t key_index, + int op_type) +{ + oct_crypto_main_t *ocm = &oct_crypto_main; + oct_crypto_sess_t *session; + vnet_crypto_key_t *key; + oct_crypto_key_t *ckey; + + key = vnet_crypto_get_key (key_index); + + if (key->type == VNET_CRYPTO_KEY_TYPE_LINK) + { + /* + * Read crypto or integ key session. And map link key index to same. + */ + if (key->index_crypto != UINT32_MAX) + { + ckey = vec_elt_at_index (ocm->keys[op_type], key->index_crypto); + session = ckey->sess; + } + else if (key->index_integ != UINT32_MAX) + { + ckey = vec_elt_at_index (ocm->keys[op_type], key->index_integ); + session = ckey->sess; + } + else + return -1; + } + else + { + session = oct_crypto_session_alloc (vm, op_type); + if (session == NULL) + return -1; + } + + oct_map_keyindex_to_session (session, key_index, op_type); + return 0; +} + +void +oct_crypto_key_del_handler (vlib_main_t *vm, vnet_crypto_key_index_t key_index) +{ + extern oct_plt_init_param_t oct_plt_init_param; + oct_crypto_main_t *ocm = &oct_crypto_main; + oct_crypto_key_t *ckey_linked; + oct_crypto_key_t *ckey; + + vec_validate (ocm->keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index); + + ckey = vec_elt_at_index (ocm->keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index); + if (ckey->sess) + { + /* + * If in case link algo is pointing to same sesison, reset the pointer. + */ + if (ckey->sess->key_index != key_index) + { + ckey_linked = vec_elt_at_index ( + ocm->keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], ckey->sess->key_index); + ckey_linked->sess = NULL; + } + oct_plt_init_param.oct_plt_free (ckey->sess); + ckey->sess = NULL; + } + + ckey = vec_elt_at_index (ocm->keys[VNET_CRYPTO_OP_TYPE_DECRYPT], key_index); + if (ckey->sess) + { + /* + * If in case link algo is pointing to same sesison, reset the pointer. + */ + if (ckey->sess->key_index != key_index) + { + ckey_linked = vec_elt_at_index ( + ocm->keys[VNET_CRYPTO_OP_TYPE_DECRYPT], ckey->sess->key_index); + ckey_linked->sess = NULL; + } + + oct_plt_init_param.oct_plt_free (ckey->sess); + ckey->sess = NULL; + } +} + +void +oct_crypto_key_add_handler (vlib_main_t *vm, vnet_crypto_key_index_t key_index) +{ + oct_crypto_main_t *ocm = &oct_crypto_main; + oct_crypto_key_t *ckey; + oct_crypto_dev_t *ocd = &oct_crypto_dev; + + vec_validate (ocm->keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index); + ckey = vec_elt_at_index (ocm->keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index); + if (ckey->sess == NULL) + { + if (oct_crypto_session_create (vm, key_index, + VNET_CRYPTO_OP_TYPE_ENCRYPT)) + { + log_err (ocd->dev, "Unable to create crypto session"); + return; + } + } + + vec_validate (ocm->keys[VNET_CRYPTO_OP_TYPE_DECRYPT], key_index); + ckey = vec_elt_at_index (ocm->keys[VNET_CRYPTO_OP_TYPE_DECRYPT], key_index); + if (ckey->sess == NULL) + { + if (oct_crypto_session_create (vm, key_index, + VNET_CRYPTO_OP_TYPE_DECRYPT)) + { + log_err (ocd->dev, "Unable to create crypto session"); + return; + } + } +} + +void +oct_crypto_key_handler (vlib_main_t *vm, vnet_crypto_key_op_t kop, + vnet_crypto_key_index_t idx) +{ + oct_crypto_main_t *ocm = &oct_crypto_main; + + if (kop == VNET_CRYPTO_KEY_OP_DEL) + { + oct_crypto_key_del_handler (vm, idx); + return; + } + oct_crypto_key_add_handler (vm, idx); + + ocm->started = 1; +} + +static_always_inline void +oct_crypto_session_free (vlib_main_t *vm, oct_crypto_sess_t *sess) +{ + extern oct_plt_init_param_t oct_plt_init_param; + + oct_plt_init_param.oct_plt_free (sess); + return; +} + +#ifdef PLATFORM_OCTEON9 +static inline void +oct_cpt_inst_submit (struct cpt_inst_s *inst, uint64_t lmtline, + uint64_t io_addr) +{ + uint64_t lmt_status; + + do + { + /* Copy CPT command to LMTLINE */ + roc_lmt_mov64 ((void *) lmtline, inst); + + /* + * Make sure compiler does not reorder memcpy and ldeor. + * LMTST transactions are always flushed from the write + * buffer immediately, a DMB is not required to push out + * LMTSTs. + */ + asm volatile ("dmb oshst" : : : "memory"); + lmt_status = roc_lmt_submit_ldeor (io_addr); + } + while (lmt_status == 0); +} +#endif + +static_always_inline void +oct_crypto_burst_submit (oct_crypto_dev_t *crypto_dev, struct cpt_inst_s *inst, + u32 n_left) +{ + u64 lmt_base; + u64 io_addr; + u32 count; + +#ifdef PLATFORM_OCTEON9 + lmt_base = crypto_dev->lf.lmt_base; + io_addr = crypto_dev->lf.io_addr; + + for (count = 0; count < n_left; count++) + oct_cpt_inst_submit (inst + count, lmt_base, io_addr); +#else + u64 *lmt_line[OCT_MAX_LMT_SZ]; + u64 lmt_arg, core_lmt_id; + + lmt_base = crypto_dev->lmtline.lmt_base; + io_addr = crypto_dev->lmtline.io_addr; + + ROC_LMT_CPT_BASE_ID_GET (lmt_base, core_lmt_id); + + for (count = 0; count < 16; count++) + { + lmt_line[count] = OCT_CPT_LMT_GET_LINE_ADDR (lmt_base, count); + } + + while (n_left > OCT_MAX_LMT_SZ) + { + + /* + * Add a memory barrier so that LMTLINEs from the previous iteration + * can be reused for a subsequent transfer. + */ + asm volatile ("dmb oshst" ::: "memory"); + + lmt_arg = ROC_CN10K_CPT_LMT_ARG | (u64) core_lmt_id; + + for (count = 0; count < 16; count++) + { + roc_lmt_mov_seg ((void *) lmt_line[count], inst + count, + CPT_LMT_SIZE_COPY); + } + + /* Set number of LMTSTs, excluding the first */ + lmt_arg |= (OCT_MAX_LMT_SZ - 1) << 12; + + roc_lmt_submit_steorl (lmt_arg, io_addr); + + inst += OCT_MAX_LMT_SZ; + n_left -= OCT_MAX_LMT_SZ; + } + + if (n_left > 0) + { + /* + * Add a memory barrier so that LMTLINEs from the previous iteration + * can be reused for a subsequent transfer. + */ + asm volatile ("dmb oshst" ::: "memory"); + + lmt_arg = ROC_CN10K_CPT_LMT_ARG | (u64) core_lmt_id; + + for (count = 0; count < n_left; count++) + { + roc_lmt_mov_seg ((void *) lmt_line[count], inst + count, + CPT_LMT_SIZE_COPY); + } + + /* Set number of LMTSTs, excluding the first */ + lmt_arg |= (n_left - 1) << 12; + + roc_lmt_submit_steorl (lmt_arg, io_addr); + } +#endif +} + +static_always_inline uint32_t +oct_crypto_fill_sg_comp_from_iov (struct roc_sglist_comp *list, uint32_t i, + struct roc_se_iov_ptr *from, + uint32_t from_offset, uint32_t *psize, + struct roc_se_buf_ptr *extra_buf, + uint32_t extra_offset) +{ + uint32_t extra_len = extra_buf ? extra_buf->size : 0; + uint32_t size = *psize; + int32_t j; + + for (j = 0; j < from->buf_cnt; j++) + { + struct roc_sglist_comp *to = &list[i >> 2]; + uint32_t buf_sz = from->bufs[j].size; + void *vaddr = from->bufs[j].vaddr; + uint64_t e_vaddr; + uint32_t e_len; + + if (PREDICT_FALSE (from_offset)) + { + if (from_offset >= buf_sz) + { + from_offset -= buf_sz; + continue; + } + e_vaddr = (uint64_t) vaddr + from_offset; + e_len = clib_min ((buf_sz - from_offset), size); + from_offset = 0; + } + else + { + e_vaddr = (uint64_t) vaddr; + e_len = clib_min (buf_sz, size); + } + + to->u.s.len[i % 4] = clib_host_to_net_u16 (e_len); + to->ptr[i % 4] = clib_host_to_net_u64 (e_vaddr); + + if (extra_len && (e_len >= extra_offset)) + { + /* Break the data at given offset */ + uint32_t next_len = e_len - extra_offset; + uint64_t next_vaddr = e_vaddr + extra_offset; + + if (!extra_offset) + { + i--; + } + else + { + e_len = extra_offset; + size -= e_len; + to->u.s.len[i % 4] = clib_host_to_net_u16 (e_len); + } + + extra_len = clib_min (extra_len, size); + /* Insert extra data ptr */ + if (extra_len) + { + i++; + to = &list[i >> 2]; + to->u.s.len[i % 4] = clib_host_to_net_u16 (extra_len); + to->ptr[i % 4] = + clib_host_to_net_u64 ((uint64_t) extra_buf->vaddr); + size -= extra_len; + } + + next_len = clib_min (next_len, size); + /* insert the rest of the data */ + if (next_len) + { + i++; + to = &list[i >> 2]; + to->u.s.len[i % 4] = clib_host_to_net_u16 (next_len); + to->ptr[i % 4] = clib_host_to_net_u64 (next_vaddr); + size -= next_len; + } + extra_len = 0; + } + else + { + size -= e_len; + } + if (extra_offset) + extra_offset -= size; + i++; + + if (PREDICT_FALSE (!size)) + break; + } + + *psize = size; + return (uint32_t) i; +} + +static_always_inline u32 +oct_crypto_fill_sg2_comp_from_iov (struct roc_sg2list_comp *list, u32 i, + struct roc_se_iov_ptr *from, + u32 from_offset, u32 *psize, + struct roc_se_buf_ptr *extra_buf, + u32 extra_offset) +{ + u32 extra_len = extra_buf ? extra_buf->size : 0; + u32 size = *psize, buf_sz, e_len, next_len; + struct roc_sg2list_comp *to; + u64 e_vaddr, next_vaddr; + void *vaddr; + i32 j; + + for (j = 0; j < from->buf_cnt; j++) + { + to = &list[i / 3]; + buf_sz = from->bufs[j].size; + vaddr = from->bufs[j].vaddr; + + if (PREDICT_FALSE (from_offset)) + { + if (from_offset >= buf_sz) + { + from_offset -= buf_sz; + continue; + } + e_vaddr = (u64) vaddr + from_offset; + e_len = clib_min ((buf_sz - from_offset), size); + from_offset = 0; + } + else + { + e_vaddr = (u64) vaddr; + e_len = clib_min (buf_sz, size); + } + + to->u.s.len[i % 3] = (e_len); + to->ptr[i % 3] = (e_vaddr); + to->u.s.valid_segs = (i % 3) + 1; + + if (extra_len && (e_len >= extra_offset)) + { + /* Break the data at given offset */ + next_len = e_len - extra_offset; + next_vaddr = e_vaddr + extra_offset; + + if (!extra_offset) + i--; + else + { + e_len = extra_offset; + size -= e_len; + to->u.s.len[i % 3] = (e_len); + } + + extra_len = clib_min (extra_len, size); + /* Insert extra data ptr */ + if (extra_len) + { + i++; + to = &list[i / 3]; + to->u.s.len[i % 3] = (extra_len); + to->ptr[i % 3] = ((u64) extra_buf->vaddr); + to->u.s.valid_segs = (i % 3) + 1; + size -= extra_len; + } + + next_len = clib_min (next_len, size); + /* insert the rest of the data */ + if (next_len) + { + i++; + to = &list[i / 3]; + to->u.s.len[i % 3] = (next_len); + to->ptr[i % 3] = (next_vaddr); + to->u.s.valid_segs = (i % 3) + 1; + size -= next_len; + } + extra_len = 0; + } + else + size -= e_len; + + if (extra_offset) + extra_offset -= size; + + i++; + + if (PREDICT_FALSE (!size)) + break; + } + + *psize = size; + return (u32) i; +} + +static_always_inline uint32_t +oct_crypto_fill_sg_comp_from_buf (struct roc_sglist_comp *list, uint32_t i, + struct roc_se_buf_ptr *from) +{ + struct roc_sglist_comp *to = &list[i >> 2]; + + to->u.s.len[i % 4] = clib_host_to_net_u16 (from->size); + to->ptr[i % 4] = clib_host_to_net_u64 ((uint64_t) from->vaddr); + return ++i; +} + +static_always_inline uint32_t +oct_crypto_fill_sg_comp (struct roc_sglist_comp *list, uint32_t i, + uint64_t dma_addr, uint32_t size) +{ + struct roc_sglist_comp *to = &list[i >> 2]; + + to->u.s.len[i % 4] = clib_host_to_net_u16 (size); + to->ptr[i % 4] = clib_host_to_net_u64 (dma_addr); + return ++i; +} + +static_always_inline u32 +oct_crypto_fill_sg2_comp (struct roc_sg2list_comp *list, u32 index, + u64 dma_addr, u32 size) +{ + struct roc_sg2list_comp *to = &list[index / 3]; + + to->u.s.len[index % 3] = (size); + to->ptr[index % 3] = (dma_addr); + to->u.s.valid_segs = (index % 3) + 1; + return ++index; +} + +static_always_inline u32 +oct_crypto_fill_sg2_comp_from_buf (struct roc_sg2list_comp *list, u32 index, + struct roc_se_buf_ptr *from) +{ + struct roc_sg2list_comp *to = &list[index / 3]; + + to->u.s.len[index % 3] = (from->size); + to->ptr[index % 3] = ((u64) from->vaddr); + to->u.s.valid_segs = (index % 3) + 1; + return ++index; +} + +static_always_inline int __attribute__ ((unused)) +oct_crypto_sg_inst_prep (struct roc_se_fc_params *params, + struct cpt_inst_s *inst, uint64_t offset_ctrl, + const uint8_t *iv_s, int iv_len, uint8_t pack_iv, + uint8_t pdcp_alg_type, int32_t inputlen, + int32_t outputlen, uint32_t passthrough_len, + uint32_t req_flags, int pdcp_flag, int decrypt) +{ + struct roc_sglist_comp *gather_comp, *scatter_comp; + void *m_vaddr = params->meta_buf.vaddr; + struct roc_se_buf_ptr *aad_buf = NULL; + uint32_t mac_len = 0, aad_len = 0; + struct roc_se_ctx *se_ctx; + uint32_t i, g_size_bytes; + uint64_t *offset_vaddr; + uint32_t s_size_bytes; + uint8_t *in_buffer; + uint32_t size; + uint8_t *iv_d; + int ret = 0; + + se_ctx = params->ctx; + mac_len = se_ctx->mac_len; + + if (PREDICT_FALSE (req_flags & ROC_SE_VALID_AAD_BUF)) + { + /* We don't support both AAD and auth data separately */ + aad_len = params->aad_buf.size; + aad_buf = ¶ms->aad_buf; + } + + /* save space for iv */ + offset_vaddr = m_vaddr; + + m_vaddr = + (uint8_t *) m_vaddr + ROC_SE_OFF_CTRL_LEN + PLT_ALIGN_CEIL (iv_len, 8); + + inst->w4.s.opcode_major |= (uint64_t) ROC_DMA_MODE_SG; + + /* iv offset is 0 */ + *offset_vaddr = offset_ctrl; + + iv_d = ((uint8_t *) offset_vaddr + ROC_SE_OFF_CTRL_LEN); + + if (PREDICT_TRUE (iv_len)) + memcpy (iv_d, iv_s, iv_len); + + /* DPTR has SG list */ + + /* TODO Add error check if space will be sufficient */ + gather_comp = (struct roc_sglist_comp *) ((uint8_t *) m_vaddr + 8); + + /* + * Input Gather List + */ + i = 0; + + /* Offset control word followed by iv */ + + i = oct_crypto_fill_sg_comp (gather_comp, i, (uint64_t) offset_vaddr, + ROC_SE_OFF_CTRL_LEN + iv_len); + + /* Add input data */ + if (decrypt && (req_flags & ROC_SE_VALID_MAC_BUF)) + { + size = inputlen - iv_len - mac_len; + + if (PREDICT_TRUE (size)) + { + uint32_t aad_offset = aad_len ? passthrough_len : 0; + i = oct_crypto_fill_sg_comp_from_iov ( + gather_comp, i, params->src_iov, 0, &size, aad_buf, aad_offset); + if (PREDICT_FALSE (size)) + { + clib_warning ("Cryptodev: Insufficient buffer" + " space, size %d needed", + size); + return -1; + } + } + + if (mac_len) + i = + oct_crypto_fill_sg_comp_from_buf (gather_comp, i, ¶ms->mac_buf); + } + else + { + /* input data */ + size = inputlen - iv_len; + if (size) + { + uint32_t aad_offset = aad_len ? passthrough_len : 0; + i = oct_crypto_fill_sg_comp_from_iov ( + gather_comp, i, params->src_iov, 0, &size, aad_buf, aad_offset); + if (PREDICT_FALSE (size)) + { + clib_warning ("Cryptodev: Insufficient buffer space," + " size %d needed", + size); + return -1; + } + } + } + + in_buffer = m_vaddr; + ((uint16_t *) in_buffer)[0] = 0; + ((uint16_t *) in_buffer)[1] = 0; + ((uint16_t *) in_buffer)[2] = clib_host_to_net_u16 (i); + + g_size_bytes = ((i + 3) / 4) * sizeof (struct roc_sglist_comp); + /* + * Output Scatter List + */ + + i = 0; + scatter_comp = + (struct roc_sglist_comp *) ((uint8_t *) gather_comp + g_size_bytes); + + i = oct_crypto_fill_sg_comp ( + scatter_comp, i, (uint64_t) offset_vaddr + ROC_SE_OFF_CTRL_LEN, iv_len); + + /* Add output data */ + if ((!decrypt) && (req_flags & ROC_SE_VALID_MAC_BUF)) + { + size = outputlen - iv_len - mac_len; + if (size) + { + + uint32_t aad_offset = aad_len ? passthrough_len : 0; + + i = oct_crypto_fill_sg_comp_from_iov ( + scatter_comp, i, params->dst_iov, 0, &size, aad_buf, aad_offset); + if (PREDICT_FALSE (size)) + { + clib_warning ("Cryptodev: Insufficient buffer space," + " size %d needed", + size); + return -1; + } + } + + /* mac data */ + if (mac_len) + i = + oct_crypto_fill_sg_comp_from_buf (scatter_comp, i, ¶ms->mac_buf); + } + else + { + /* Output including mac */ + size = outputlen - iv_len; + + if (size) + { + uint32_t aad_offset = aad_len ? passthrough_len : 0; + + i = oct_crypto_fill_sg_comp_from_iov ( + scatter_comp, i, params->dst_iov, 0, &size, aad_buf, aad_offset); + + if (PREDICT_FALSE (size)) + { + clib_warning ("Cryptodev: Insufficient buffer space," + " size %d needed", + size); + return -1; + } + } + } + ((uint16_t *) in_buffer)[3] = clib_host_to_net_u16 (i); + s_size_bytes = ((i + 3) / 4) * sizeof (struct roc_sglist_comp); + + size = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE; + + /* This is DPTR len in case of SG mode */ + inst->w4.s.dlen = size; + + if (PREDICT_FALSE (size > ROC_SG_MAX_DLEN_SIZE)) + { + clib_warning ( + "Cryptodev: Exceeds max supported components. Reduce segments"); + ret = -1; + } + + inst->dptr = (uint64_t) in_buffer; + return ret; +} + +static_always_inline int __attribute__ ((unused)) +oct_crypto_sg2_inst_prep (struct roc_se_fc_params *params, + struct cpt_inst_s *inst, u64 offset_ctrl, + const u8 *iv_s, int iv_len, u8 pack_iv, + u8 pdcp_alg_type, i32 inputlen, i32 outputlen, + u32 passthrough_len, u32 req_flags, int pdcp_flag, + int decrypt) +{ + u32 mac_len = 0, aad_len = 0, size, index, g_size_bytes; + struct roc_sg2list_comp *gather_comp, *scatter_comp; + void *m_vaddr = params->meta_buf.vaddr; + struct roc_se_buf_ptr *aad_buf = NULL; + union cpt_inst_w5 cpt_inst_w5; + union cpt_inst_w6 cpt_inst_w6; + u16 scatter_sz, gather_sz; + struct roc_se_ctx *se_ctx; + u64 *offset_vaddr; + int ret = 0; + u8 *iv_d; + + se_ctx = params->ctx; + mac_len = se_ctx->mac_len; + + if (PREDICT_FALSE (req_flags & ROC_SE_VALID_AAD_BUF)) + { + /* We don't support both AAD and auth data separately */ + aad_len = params->aad_buf.size; + aad_buf = ¶ms->aad_buf; + } + + /* save space for iv */ + offset_vaddr = m_vaddr; + + m_vaddr = (u8 *) m_vaddr + ROC_SE_OFF_CTRL_LEN + PLT_ALIGN_CEIL (iv_len, 8); + + inst->w4.s.opcode_major |= (u64) ROC_DMA_MODE_SG; + + /* This is DPTR len in case of SG mode */ + inst->w4.s.dlen = inputlen + ROC_SE_OFF_CTRL_LEN; + + /* iv offset is 0 */ + *offset_vaddr = offset_ctrl; + iv_d = ((u8 *) offset_vaddr + ROC_SE_OFF_CTRL_LEN); + + if (PREDICT_TRUE (iv_len)) + clib_memcpy (iv_d, iv_s, iv_len); + + /* DPTR has SG list */ + + gather_comp = (struct roc_sg2list_comp *) ((u8 *) m_vaddr); + + /* + * Input Gather List + */ + index = 0; + + /* Offset control word followed by iv */ + + index = oct_crypto_fill_sg2_comp (gather_comp, index, (u64) offset_vaddr, + ROC_SE_OFF_CTRL_LEN + iv_len); + + /* Add input data */ + if (decrypt && (req_flags & ROC_SE_VALID_MAC_BUF)) + { + size = inputlen - iv_len - mac_len; + if (size) + { + /* input data only */ + u32 aad_offset = aad_len ? passthrough_len : 0; + + index = oct_crypto_fill_sg2_comp_from_iov (gather_comp, index, + params->src_iov, 0, &size, + aad_buf, aad_offset); + + if (PREDICT_FALSE (size)) + { + clib_warning ("Cryptodev: Insufficient buffer" + " space, size %d needed", + size); + return -1; + } + } + + /* mac data */ + if (mac_len) + index = oct_crypto_fill_sg2_comp_from_buf (gather_comp, index, + ¶ms->mac_buf); + } + else + { + /* input data */ + size = inputlen - iv_len; + if (size) + { + u32 aad_offset = aad_len ? passthrough_len : 0; + + index = oct_crypto_fill_sg2_comp_from_iov (gather_comp, index, + params->src_iov, 0, &size, + aad_buf, aad_offset); + if (PREDICT_FALSE (size)) + { + clib_warning ("Cryptodev: Insufficient buffer space," + " size %d needed", + size); + return -1; + } + } + } + + gather_sz = (index + 2) / 3; + g_size_bytes = gather_sz * sizeof (struct roc_sg2list_comp); + + /* + * Output Scatter List + */ + + index = 0; + scatter_comp = + (struct roc_sg2list_comp *) ((u8 *) gather_comp + g_size_bytes); + + index = oct_crypto_fill_sg2_comp ( + scatter_comp, index, (u64) offset_vaddr + ROC_SE_OFF_CTRL_LEN, iv_len); + + /* Add output data */ + if ((!decrypt) && (req_flags & ROC_SE_VALID_MAC_BUF)) + { + size = outputlen - iv_len - mac_len; + if (size) + { + + u32 aad_offset = aad_len ? passthrough_len : 0; + + index = oct_crypto_fill_sg2_comp_from_iov (scatter_comp, index, + params->dst_iov, 0, &size, + aad_buf, aad_offset); + if (PREDICT_FALSE (size)) + { + clib_warning ("Cryptodev: Insufficient buffer space," + " size %d needed", + size); + return -1; + } + } + + /* mac data */ + if (mac_len) + index = oct_crypto_fill_sg2_comp_from_buf (scatter_comp, index, + ¶ms->mac_buf); + } + else + { + /* Output including mac */ + size = outputlen - iv_len; + if (size) + { + u32 aad_offset = aad_len ? passthrough_len : 0; + + index = oct_crypto_fill_sg2_comp_from_iov (scatter_comp, index, + params->dst_iov, 0, &size, + aad_buf, aad_offset); + + if (PREDICT_FALSE (size)) + { + clib_warning ("Cryptodev: Insufficient buffer space," + " size %d needed", + size); + return -1; + } + } + } + + scatter_sz = (index + 2) / 3; + + cpt_inst_w5.s.gather_sz = gather_sz; + cpt_inst_w6.s.scatter_sz = scatter_sz; + + cpt_inst_w5.s.dptr = (u64) gather_comp; + cpt_inst_w6.s.rptr = (u64) scatter_comp; + + inst->w5.u64 = cpt_inst_w5.u64; + inst->w6.u64 = cpt_inst_w6.u64; + + if (PREDICT_FALSE ((scatter_sz >> 4) || (gather_sz >> 4))) + { + clib_warning ( + "Cryptodev: Exceeds max supported components. Reduce segments"); + ret = -1; + } + + return ret; +} + +static_always_inline int +oct_crypto_cpt_hmac_prep (u32 flags, u64 d_offs, u64 d_lens, + struct roc_se_fc_params *fc_params, + struct cpt_inst_s *inst, u8 is_decrypt) +{ + u32 encr_data_len, auth_data_len, aad_len = 0; + i32 inputlen, outputlen, enc_dlen, auth_dlen; + u32 encr_offset, auth_offset, iv_offset = 0; + union cpt_inst_w4 cpt_inst_w4; + u32 cipher_type; + struct roc_se_ctx *se_ctx; + u32 passthrough_len = 0; + const u8 *src = NULL; + u64 offset_ctrl; + u8 iv_len = 16; + u8 op_minor; + u32 mac_len; + int ret; + + encr_offset = ROC_SE_ENCR_OFFSET (d_offs); + auth_offset = ROC_SE_AUTH_OFFSET (d_offs); + encr_data_len = ROC_SE_ENCR_DLEN (d_lens); + auth_data_len = ROC_SE_AUTH_DLEN (d_lens); + + if (PREDICT_FALSE (flags & ROC_SE_VALID_AAD_BUF)) + { + /* We don't support both AAD and auth data separately */ + auth_data_len = 0; + auth_offset = 0; + aad_len = fc_params->aad_buf.size; + } + + se_ctx = fc_params->ctx; + cipher_type = se_ctx->enc_cipher; + mac_len = se_ctx->mac_len; + cpt_inst_w4.u64 = se_ctx->template_w4.u64; + op_minor = cpt_inst_w4.s.opcode_minor; + + if (PREDICT_FALSE (flags & ROC_SE_VALID_AAD_BUF)) + { + /* + * When AAD is given, data above encr_offset is pass through + * Since AAD is given as separate pointer and not as offset, + * this is a special case as we need to fragment input data + * into passthrough + encr_data and then insert AAD in between. + */ + passthrough_len = encr_offset; + auth_offset = passthrough_len + iv_len; + encr_offset = passthrough_len + aad_len + iv_len; + auth_data_len = aad_len + encr_data_len; + } + else + { + encr_offset += iv_len; + auth_offset += iv_len; + } + + auth_dlen = auth_offset + auth_data_len; + enc_dlen = encr_data_len + encr_offset; + + cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_FC; + + if (is_decrypt) + { + cpt_inst_w4.s.opcode_minor |= ROC_SE_FC_MINOR_OP_DECRYPT; + + if (auth_dlen > enc_dlen) + { + inputlen = auth_dlen + mac_len; + outputlen = auth_dlen; + } + else + { + inputlen = enc_dlen + mac_len; + outputlen = enc_dlen; + } + } + else + { + cpt_inst_w4.s.opcode_minor |= ROC_SE_FC_MINOR_OP_ENCRYPT; + + /* Round up to 16 bytes alignment */ + if (PREDICT_FALSE (encr_data_len & 0xf)) + { + if (PREDICT_TRUE (cipher_type == ROC_SE_AES_CBC) || + (cipher_type == ROC_SE_DES3_CBC)) + enc_dlen = PLT_ALIGN_CEIL (encr_data_len, 8) + encr_offset; + } + + /* + * auth_dlen is larger than enc_dlen in Authentication cases + * like AES GMAC Authentication + */ + if (PREDICT_FALSE (auth_dlen > enc_dlen)) + { + inputlen = auth_dlen; + outputlen = auth_dlen + mac_len; + } + else + { + inputlen = enc_dlen; + outputlen = enc_dlen + mac_len; + } + } + + if (op_minor & ROC_SE_FC_MINOR_OP_HMAC_FIRST) + outputlen = enc_dlen; + + cpt_inst_w4.s.param1 = encr_data_len; + cpt_inst_w4.s.param2 = auth_data_len; + + if (PREDICT_FALSE ((encr_offset >> 16) || (iv_offset >> 8) || + (auth_offset >> 8))) + { + clib_warning ("Cryptodev: Offset not supported"); + clib_warning ( + "Cryptodev: enc_offset: %d, iv_offset : %d, auth_offset: %d", + encr_offset, iv_offset, auth_offset); + return -1; + } + + offset_ctrl = clib_host_to_net_u64 ( + ((u64) encr_offset << 16) | ((u64) iv_offset << 8) | ((u64) auth_offset)); + + src = fc_params->iv_buf; + + inst->w4.u64 = cpt_inst_w4.u64; + +#ifdef PLATFORM_OCTEON9 + ret = oct_crypto_sg_inst_prep (fc_params, inst, offset_ctrl, src, iv_len, 0, + 0, inputlen, outputlen, passthrough_len, + flags, 0, is_decrypt); +#else + ret = oct_crypto_sg2_inst_prep (fc_params, inst, offset_ctrl, src, iv_len, 0, + 0, inputlen, outputlen, passthrough_len, + flags, 0, is_decrypt); +#endif + + if (PREDICT_FALSE (ret)) + return -1; + + return 0; +} + +static_always_inline int +oct_crypto_fill_fc_params (oct_crypto_sess_t *sess, struct cpt_inst_s *inst, + const bool is_aead, u8 aad_length, u8 *payload, + vnet_crypto_async_frame_elt_t *elts, void *mdata, + u32 cipher_data_length, u32 cipher_data_offset, + u32 auth_data_length, u32 auth_data_offset, + vlib_buffer_t *b, u16 adj_len) +{ + struct roc_se_fc_params fc_params = { 0 }; + struct roc_se_ctx *ctx = &sess->cpt_ctx; + u64 d_offs = 0, d_lens = 0; + vlib_buffer_t *buffer = b; + u32 flags = 0, index = 0; + u8 op_minor = 0, cpt_op; + char src[SRC_IOV_SIZE]; + u32 *iv_buf; + + cpt_op = sess->cpt_op; + + if (is_aead) + { + flags |= ROC_SE_VALID_IV_BUF; + iv_buf = (u32 *) elts->iv; + iv_buf[3] = clib_host_to_net_u32 (0x1); + fc_params.iv_buf = elts->iv; + + d_offs = cipher_data_offset; + d_offs = d_offs << 16; + + d_lens = cipher_data_length; + d_lens = d_lens << 32; + + fc_params.aad_buf.vaddr = elts->aad; + fc_params.aad_buf.size = aad_length; + flags |= ROC_SE_VALID_AAD_BUF; + + if (sess->cpt_ctx.mac_len) + { + flags |= ROC_SE_VALID_MAC_BUF; + fc_params.mac_buf.size = sess->cpt_ctx.mac_len; + fc_params.mac_buf.vaddr = elts->tag; + } + } + else + { + op_minor = ctx->template_w4.s.opcode_minor; + + flags |= ROC_SE_VALID_IV_BUF; + + fc_params.iv_buf = elts->iv; + + d_offs = cipher_data_offset; + d_offs = (d_offs << 16) | auth_data_offset; + + d_lens = cipher_data_length; + d_lens = (d_lens << 32) | auth_data_length; + + if (PREDICT_TRUE (sess->cpt_ctx.mac_len)) + { + if (!(op_minor & ROC_SE_FC_MINOR_OP_HMAC_FIRST)) + { + flags |= ROC_SE_VALID_MAC_BUF; + fc_params.mac_buf.size = sess->cpt_ctx.mac_len; + fc_params.mac_buf.vaddr = elts->digest; + } + } + } + + fc_params.ctx = &sess->cpt_ctx; + + fc_params.src_iov = (void *) src; + + fc_params.src_iov->bufs[index].vaddr = payload; + fc_params.src_iov->bufs[index].size = b->current_length - adj_len; + index++; + + while (buffer->flags & VLIB_BUFFER_NEXT_PRESENT) + { + buffer = vlib_get_buffer (vlib_get_main (), buffer->next_buffer); + fc_params.src_iov->bufs[index].vaddr = + buffer->data + buffer->current_data; + fc_params.src_iov->bufs[index].size = buffer->current_length; + index++; + } + + fc_params.src_iov->buf_cnt = index; + + fc_params.dst_iov = (void *) src; + + fc_params.meta_buf.vaddr = mdata; + fc_params.meta_buf.size = OCT_SCATTER_GATHER_BUFFER_SIZE; + + return oct_crypto_cpt_hmac_prep (flags, d_offs, d_lens, &fc_params, inst, + cpt_op); +} + +static_always_inline u64 +oct_cpt_inst_w7_get (oct_crypto_sess_t *sess, struct roc_cpt *roc_cpt) +{ + union cpt_inst_w7 inst_w7; + + inst_w7.u64 = 0; + inst_w7.s.cptr = (u64) &sess->cpt_ctx.se_ctx.fctx; + /* Set the engine group */ + inst_w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_IE]; + + return inst_w7.u64; +} + +static_always_inline i32 +oct_crypto_link_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, + u32 key_index, u8 type) +{ + vnet_crypto_key_t *crypto_key, *auth_key; + roc_se_cipher_type enc_type = 0; + roc_se_auth_type auth_type = 0; + vnet_crypto_key_t *key; + u32 digest_len = ~0; + i32 rv = 0; + + key = vnet_crypto_get_key (key_index); + + switch (key->async_alg) + { + case VNET_CRYPTO_ALG_AES_128_CBC_SHA1_TAG12: + case VNET_CRYPTO_ALG_AES_192_CBC_SHA1_TAG12: + case VNET_CRYPTO_ALG_AES_256_CBC_SHA1_TAG12: + enc_type = ROC_SE_AES_CBC; + auth_type = ROC_SE_SHA1_TYPE; + digest_len = 12; + break; + case VNET_CRYPTO_ALG_AES_128_CBC_SHA224_TAG14: + case VNET_CRYPTO_ALG_AES_192_CBC_SHA224_TAG14: + case VNET_CRYPTO_ALG_AES_256_CBC_SHA224_TAG14: + enc_type = ROC_SE_AES_CBC; + auth_type = ROC_SE_SHA2_SHA224; + digest_len = 14; + break; + case VNET_CRYPTO_ALG_AES_128_CBC_SHA256_TAG16: + case VNET_CRYPTO_ALG_AES_192_CBC_SHA256_TAG16: + case VNET_CRYPTO_ALG_AES_256_CBC_SHA256_TAG16: + enc_type = ROC_SE_AES_CBC; + auth_type = ROC_SE_SHA2_SHA256; + digest_len = 16; + break; + case VNET_CRYPTO_ALG_AES_128_CBC_SHA384_TAG24: + case VNET_CRYPTO_ALG_AES_192_CBC_SHA384_TAG24: + case VNET_CRYPTO_ALG_AES_256_CBC_SHA384_TAG24: + enc_type = ROC_SE_AES_CBC; + auth_type = ROC_SE_SHA2_SHA384; + digest_len = 24; + break; + case VNET_CRYPTO_ALG_AES_128_CBC_SHA512_TAG32: + case VNET_CRYPTO_ALG_AES_192_CBC_SHA512_TAG32: + case VNET_CRYPTO_ALG_AES_256_CBC_SHA512_TAG32: + enc_type = ROC_SE_AES_CBC; + auth_type = ROC_SE_SHA2_SHA512; + digest_len = 32; + break; + case VNET_CRYPTO_ALG_AES_128_CBC_MD5_TAG12: + case VNET_CRYPTO_ALG_AES_192_CBC_MD5_TAG12: + case VNET_CRYPTO_ALG_AES_256_CBC_MD5_TAG12: + enc_type = ROC_SE_AES_CBC; + auth_type = ROC_SE_MD5_TYPE; + digest_len = 12; + break; + case VNET_CRYPTO_ALG_AES_128_CTR_SHA1_TAG12: + case VNET_CRYPTO_ALG_AES_192_CTR_SHA1_TAG12: + case VNET_CRYPTO_ALG_AES_256_CTR_SHA1_TAG12: + enc_type = ROC_SE_AES_CTR; + auth_type = ROC_SE_SHA1_TYPE; + digest_len = 12; + break; + case VNET_CRYPTO_ALG_3DES_CBC_MD5_TAG12: + enc_type = ROC_SE_DES3_CBC; + auth_type = ROC_SE_MD5_TYPE; + digest_len = 12; + break; + case VNET_CRYPTO_ALG_3DES_CBC_SHA1_TAG12: + enc_type = ROC_SE_DES3_CBC; + auth_type = ROC_SE_SHA1_TYPE; + digest_len = 12; + break; + case VNET_CRYPTO_ALG_3DES_CBC_SHA224_TAG14: + enc_type = ROC_SE_DES3_CBC; + auth_type = ROC_SE_SHA2_SHA224; + digest_len = 14; + break; + case VNET_CRYPTO_ALG_3DES_CBC_SHA256_TAG16: + enc_type = ROC_SE_DES3_CBC; + auth_type = ROC_SE_SHA2_SHA256; + digest_len = 16; + break; + case VNET_CRYPTO_ALG_3DES_CBC_SHA384_TAG24: + enc_type = ROC_SE_DES3_CBC; + auth_type = ROC_SE_SHA2_SHA384; + digest_len = 24; + break; + case VNET_CRYPTO_ALG_3DES_CBC_SHA512_TAG32: + enc_type = ROC_SE_DES3_CBC; + auth_type = ROC_SE_SHA2_SHA512; + digest_len = 32; + break; + default: + clib_warning ( + "Cryptodev: Undefined link algo %u specified. Key index %u", + key->async_alg, key_index); + return -1; + } + + if (type == VNET_CRYPTO_OP_TYPE_ENCRYPT) + sess->cpt_ctx.ciph_then_auth = true; + else + sess->cpt_ctx.auth_then_ciph = true; + + sess->iv_length = 16; + sess->cpt_op = type; + + crypto_key = vnet_crypto_get_key (key->index_crypto); + rv = roc_se_ciph_key_set (&sess->cpt_ctx, enc_type, crypto_key->data, + vec_len (crypto_key->data)); + if (rv) + { + clib_warning ("Cryptodev: Error in setting cipher key for enc type %u", + enc_type); + return -1; + } + + auth_key = vnet_crypto_get_key (key->index_integ); + + rv = roc_se_auth_key_set (&sess->cpt_ctx, auth_type, auth_key->data, + vec_len (auth_key->data), digest_len); + if (rv) + { + clib_warning ("Cryptodev: Error in setting auth key for auth type %u", + auth_type); + return -1; + } + + return 0; +} + +static_always_inline i32 +oct_crypto_aead_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, + u32 key_index, u8 type) +{ + vnet_crypto_key_t *key = vnet_crypto_get_key (key_index); + roc_se_cipher_type enc_type = 0; + roc_se_auth_type auth_type = 0; + u32 digest_len = ~0; + i32 rv = 0; + + switch (key->async_alg) + { + case VNET_CRYPTO_ALG_AES_128_GCM: + case VNET_CRYPTO_ALG_AES_192_GCM: + case VNET_CRYPTO_ALG_AES_256_GCM: + enc_type = ROC_SE_AES_GCM; + sess->aes_gcm = 1; + sess->iv_offset = 0; + sess->iv_length = 16; + sess->cpt_ctx.mac_len = 16; + sess->cpt_op = type; + digest_len = 16; + break; + case VNET_CRYPTO_ALG_CHACHA20_POLY1305: + enc_type = ROC_SE_CHACHA20; + auth_type = ROC_SE_POLY1305; + break; + default: + clib_warning ( + "Cryptodev: Undefined cipher algo %u specified. Key index %u", + key->async_alg, key_index); + return -1; + } + + rv = roc_se_ciph_key_set (&sess->cpt_ctx, enc_type, key->data, + vec_len (key->data)); + if (rv) + { + clib_warning ("Cryptodev: Error in setting cipher key for enc type %u", + enc_type); + return -1; + } + + rv = roc_se_auth_key_set (&sess->cpt_ctx, auth_type, NULL, 0, digest_len); + if (rv) + { + clib_warning ("Cryptodev: Error in setting auth key for auth type %u", + auth_type); + return -1; + } + + if (enc_type == ROC_SE_CHACHA20) + sess->cpt_ctx.template_w4.s.opcode_minor |= BIT (5); + + return 0; +} + +static_always_inline i32 +oct_crypto_session_init (vlib_main_t *vm, oct_crypto_sess_t *session, + vnet_crypto_key_index_t key_index, int op_type) +{ + oct_crypto_main_t *ocm = &oct_crypto_main; + vnet_crypto_key_t *key; + oct_crypto_dev_t *ocd; + i32 rv = 0; + + ocd = ocm->crypto_dev[op_type]; + + key = vnet_crypto_get_key (key_index); + + if (key->type == VNET_CRYPTO_KEY_TYPE_LINK) + rv = oct_crypto_link_session_update (vm, session, key_index, op_type); + else + rv = oct_crypto_aead_session_update (vm, session, key_index, op_type); + + if (rv) + { + oct_crypto_session_free (vm, session); + return -1; + } + + session->crypto_dev = ocd; + + session->cpt_inst_w7 = + oct_cpt_inst_w7_get (session, session->crypto_dev->roc_cpt); + + session->initialised = 1; + + return 0; +} + +static_always_inline void +oct_crypto_update_frame_error_status (vnet_crypto_async_frame_t *f, u32 index, + vnet_crypto_op_status_t s) +{ + u32 i; + + for (i = index; i < f->n_elts; i++) + f->elts[i].status = s; + + if (index == 0) + f->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED; +} + +static_always_inline int +oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, + const u8 is_aead, u8 aad_len, const u8 type) +{ + u32 i, enq_tail, enc_auth_len, buffer_index, nb_infl_allowed; + struct cpt_inst_s inst[VNET_CRYPTO_FRAME_SIZE]; + u32 crypto_start_offset, integ_start_offset; + oct_crypto_main_t *ocm = &oct_crypto_main; + vnet_crypto_async_frame_elt_t *elts; + oct_crypto_dev_t *crypto_dev = NULL; + oct_crypto_inflight_req_t *infl_req; + oct_crypto_pending_queue_t *pend_q; + u64 dptr_start_ptr, curr_ptr; + oct_crypto_sess_t *sess; + u32 crypto_total_length; + oct_crypto_key_t *key; + vlib_buffer_t *buffer; + u16 adj_len; + int ret; + + /* GCM packets having 8 bytes of aad and 8 bytes of iv */ + u8 aad_iv = 8 + 8; + + pend_q = &ocm->pend_q[vlib_get_thread_index ()]; + + enq_tail = pend_q->enq_tail; + + nb_infl_allowed = pend_q->n_desc - pend_q->n_crypto_inflight; + if (PREDICT_FALSE (nb_infl_allowed == 0)) + { + oct_crypto_update_frame_error_status ( + frame, 0, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); + return -1; + } + + infl_req = &pend_q->req_queue[enq_tail]; + infl_req->frame = frame; + + for (i = 0; i < frame->n_elts; i++) + { + elts = &frame->elts[i]; + buffer_index = frame->buffer_indices[i]; + key = vec_elt_at_index (ocm->keys[type], elts->key_index); + + if (PREDICT_FALSE (!key->sess)) + { + oct_crypto_update_frame_error_status ( + frame, i, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); + return -1; + } + + sess = key->sess; + + if (PREDICT_FALSE (!sess->initialised)) + oct_crypto_session_init (vm, sess, elts->key_index, type); + + crypto_dev = sess->crypto_dev; + + clib_memset (inst + i, 0, sizeof (struct cpt_inst_s)); + + buffer = vlib_get_buffer (vm, buffer_index); + + if (is_aead) + { + dptr_start_ptr = + (u64) (buffer->data + (elts->crypto_start_offset - aad_iv)); + curr_ptr = (u64) (buffer->data + buffer->current_data); + adj_len = (u16) (dptr_start_ptr - curr_ptr); + + crypto_total_length = elts->crypto_total_length; + crypto_start_offset = aad_iv; + integ_start_offset = 0; + + ret = oct_crypto_fill_fc_params ( + sess, inst + i, is_aead, aad_len, (u8 *) dptr_start_ptr, elts, + (oct_crypto_scatter_gather_t *) (infl_req->sg_data) + i, + crypto_total_length /* cipher_len */, + crypto_start_offset /* cipher_offset */, 0 /* auth_len */, + integ_start_offset /* auth_off */, buffer, adj_len); + if (PREDICT_FALSE (ret < 0)) + { + oct_crypto_update_frame_error_status ( + frame, i, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); + return -1; + } + } + else + { + dptr_start_ptr = (u64) (buffer->data + elts->integ_start_offset); + + enc_auth_len = elts->crypto_total_length + elts->integ_length_adj; + + curr_ptr = (u64) (buffer->data + buffer->current_data); + adj_len = (u16) (dptr_start_ptr - curr_ptr); + + crypto_total_length = elts->crypto_total_length; + crypto_start_offset = + elts->crypto_start_offset - elts->integ_start_offset; + integ_start_offset = 0; + + ret = oct_crypto_fill_fc_params ( + sess, inst + i, is_aead, aad_len, (u8 *) dptr_start_ptr, elts, + (oct_crypto_scatter_gather_t *) (infl_req->sg_data) + i, + crypto_total_length /* cipher_len */, + crypto_start_offset /* cipher_offset */, + enc_auth_len /* auth_len */, integ_start_offset /* auth_off */, + buffer, adj_len); + if (PREDICT_FALSE (ret < 0)) + { + oct_crypto_update_frame_error_status ( + frame, i, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); + return -1; + } + } + + inst[i].w7.u64 = sess->cpt_inst_w7; + inst[i].res_addr = (u64) &infl_req->res[i]; + } + + oct_crypto_burst_submit (crypto_dev, inst, frame->n_elts); + + infl_req->elts = frame->n_elts; + OCT_MOD_INC (pend_q->enq_tail, pend_q->n_desc); + pend_q->n_crypto_inflight++; + + return 0; +} + +int +oct_crypto_enqueue_linked_alg_enc (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame) +{ + return oct_crypto_enqueue_enc_dec ( + vm, frame, 0 /* is_aead */, 0 /* aad_len */, VNET_CRYPTO_OP_TYPE_ENCRYPT); +} + +int +oct_crypto_enqueue_linked_alg_dec (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame) +{ + return oct_crypto_enqueue_enc_dec ( + vm, frame, 0 /* is_aead */, 0 /* aad_len */, VNET_CRYPTO_OP_TYPE_DECRYPT); +} + +int +oct_crypto_enqueue_aead_aad_enc (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame, u8 aad_len) +{ + return oct_crypto_enqueue_enc_dec (vm, frame, 1 /* is_aead */, aad_len, + VNET_CRYPTO_OP_TYPE_ENCRYPT); +} + +static_always_inline int +oct_crypto_enqueue_aead_aad_dec (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame, u8 aad_len) +{ + return oct_crypto_enqueue_enc_dec (vm, frame, 1 /* is_aead */, aad_len, + VNET_CRYPTO_OP_TYPE_DECRYPT); +} + +int +oct_crypto_enqueue_aead_aad_8_enc (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame) +{ + return oct_crypto_enqueue_aead_aad_enc (vm, frame, 8); +} + +int +oct_crypto_enqueue_aead_aad_12_enc (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame) +{ + return oct_crypto_enqueue_aead_aad_enc (vm, frame, 12); +} + +int +oct_crypto_enqueue_aead_aad_0_enc (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame) +{ + return oct_crypto_enqueue_aead_aad_enc (vm, frame, 0); +} + +int +oct_crypto_enqueue_aead_aad_8_dec (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame) +{ + return oct_crypto_enqueue_aead_aad_dec (vm, frame, 8); +} + +int +oct_crypto_enqueue_aead_aad_12_dec (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame) +{ + return oct_crypto_enqueue_aead_aad_dec (vm, frame, 12); +} + +int +oct_crypto_enqueue_aead_aad_0_dec (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame) +{ + return oct_crypto_enqueue_aead_aad_dec (vm, frame, 0); +} + +vnet_crypto_async_frame_t * +oct_crypto_frame_dequeue (vlib_main_t *vm, u32 *nb_elts_processed, + u32 *enqueue_thread_idx) +{ + oct_crypto_main_t *ocm = &oct_crypto_main; + u32 deq_head, status = VNET_CRYPTO_OP_STATUS_COMPLETED; + vnet_crypto_async_frame_elt_t *fe = NULL; + oct_crypto_inflight_req_t *infl_req; + oct_crypto_pending_queue_t *pend_q; + vnet_crypto_async_frame_t *frame; + volatile union cpt_res_s *res; + int i; + + pend_q = &ocm->pend_q[vlib_get_thread_index ()]; + + if (!pend_q->n_crypto_inflight) + return NULL; + + deq_head = pend_q->deq_head; + infl_req = &pend_q->req_queue[deq_head]; + frame = infl_req->frame; + + fe = frame->elts; + + for (i = infl_req->deq_elts; i < infl_req->elts; ++i) + { + res = &infl_req->res[i]; + + if (PREDICT_FALSE (res->cn10k.compcode == CPT_COMP_NOT_DONE)) + return NULL; + + if (PREDICT_FALSE (res->cn10k.uc_compcode)) + { + if (res->cn10k.uc_compcode == ROC_SE_ERR_GC_ICV_MISCOMPARE) + status = fe[i].status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC; + else + status = fe[i].status = VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR; + } + + infl_req->deq_elts++; + } + + clib_memset ((void *) infl_req->res, 0, + sizeof (union cpt_res_s) * VNET_CRYPTO_FRAME_SIZE); + + OCT_MOD_INC (pend_q->deq_head, pend_q->n_desc); + pend_q->n_crypto_inflight--; + + frame->state = status == VNET_CRYPTO_OP_STATUS_COMPLETED ? + VNET_CRYPTO_FRAME_STATE_SUCCESS : + VNET_CRYPTO_FRAME_STATE_ELT_ERROR; + + *nb_elts_processed = frame->n_elts; + *enqueue_thread_idx = frame->enqueue_thread_index; + + infl_req->deq_elts = 0; + infl_req->elts = 0; + + return frame; +} + +int +oct_init_crypto_engine_handlers (vlib_main_t *vm, vnet_dev_t *dev) +{ + u32 engine_index; + + engine_index = vnet_crypto_register_engine (vm, "oct_cryptodev", 100, + "OCT Cryptodev Engine"); + +#define _(n, k, t, a) \ + vnet_crypto_register_enqueue_handler ( \ + vm, engine_index, VNET_CRYPTO_OP_##n##_TAG##t##_AAD##a##_ENC, \ + oct_crypto_enqueue_aead_aad_##a##_enc); \ + vnet_crypto_register_enqueue_handler ( \ + vm, engine_index, VNET_CRYPTO_OP_##n##_TAG##t##_AAD##a##_DEC, \ + oct_crypto_enqueue_aead_aad_##a##_dec); + foreach_oct_crypto_aead_async_alg +#undef _ + +#define _(c, h, k, d) \ + vnet_crypto_register_enqueue_handler ( \ + vm, engine_index, VNET_CRYPTO_OP_##c##_##h##_TAG##d##_ENC, \ + oct_crypto_enqueue_linked_alg_enc); \ + vnet_crypto_register_enqueue_handler ( \ + vm, engine_index, VNET_CRYPTO_OP_##c##_##h##_TAG##d##_DEC, \ + oct_crypto_enqueue_linked_alg_dec); + foreach_oct_crypto_link_async_alg; +#undef _ + + vnet_crypto_register_dequeue_handler (vm, engine_index, + oct_crypto_frame_dequeue); + + vnet_crypto_register_key_handler (vm, engine_index, oct_crypto_key_handler); + + return 0; +} + +int +oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev) +{ + oct_crypto_main_t *ocm = &oct_crypto_main; + vlib_thread_main_t *tm = vlib_get_thread_main (); + extern oct_plt_init_param_t oct_plt_init_param; + oct_crypto_inflight_req_t *infl_req_queue; + u32 n_inflight_req; + int i, j = 0; + + ocm->pend_q = oct_plt_init_param.oct_plt_zmalloc ( + tm->n_vlib_mains * sizeof (oct_crypto_pending_queue_t), + CLIB_CACHE_LINE_BYTES); + if (ocm->pend_q == NULL) + { + log_err (dev, "Failed to allocate memory for crypto pending queue"); + return -1; + } + + /* + * Each pending queue will get number of cpt desc / number of cores. + * And that desc count is shared across inflight entries. + */ + n_inflight_req = + (OCT_CPT_LF_MAX_NB_DESC / tm->n_vlib_mains) / VNET_CRYPTO_FRAME_SIZE; + + for (i = 0; i < tm->n_vlib_mains; ++i) + { + ocm->pend_q[i].n_desc = n_inflight_req; + + ocm->pend_q[i].req_queue = oct_plt_init_param.oct_plt_zmalloc ( + ocm->pend_q[i].n_desc * sizeof (oct_crypto_inflight_req_t), + CLIB_CACHE_LINE_BYTES); + if (ocm->pend_q[i].req_queue == NULL) + { + log_err (dev, + "Failed to allocate memory for crypto inflight request"); + goto free; + } + + for (j = 0; j <= ocm->pend_q[i].n_desc; ++j) + { + infl_req_queue = &ocm->pend_q[i].req_queue[j]; + + infl_req_queue->sg_data = oct_plt_init_param.oct_plt_zmalloc ( + OCT_SCATTER_GATHER_BUFFER_SIZE * VNET_CRYPTO_FRAME_SIZE, + CLIB_CACHE_LINE_BYTES); + if (infl_req_queue->sg_data == NULL) + { + log_err (dev, "Failed to allocate crypto scatter gather memory"); + goto free; + } + } + } + return 0; +free: + for (; i >= 0; i--) + { + if (ocm->pend_q[i].req_queue == NULL) + continue; + for (; j >= 0; j--) + { + infl_req_queue = &ocm->pend_q[i].req_queue[j]; + + if (infl_req_queue->sg_data == NULL) + continue; + + oct_plt_init_param.oct_plt_free (infl_req_queue->sg_data); + } + oct_plt_init_param.oct_plt_free (ocm->pend_q[i].req_queue); + } + oct_plt_init_param.oct_plt_free (ocm->pend_q); + + return -1; +} diff --git a/src/plugins/dev_octeon/crypto.h b/src/plugins/dev_octeon/crypto.h new file mode 100644 index 00000000000..27e1f600c68 --- /dev/null +++ b/src/plugins/dev_octeon/crypto.h @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2024 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#ifndef _CRYPTO_H_ +#define _CRYPTO_H_ +#include <vnet/crypto/crypto.h> +#include <vnet/ip/ip.h> + +#define OCT_MAX_N_CPT_DEV 2 + +#define OCT_CPT_LF_MAX_NB_DESC 128000 + +/* CRYPTO_ID, KEY_LENGTH_IN_BYTES, TAG_LEN, AAD_LEN */ +#define foreach_oct_crypto_aead_async_alg \ + _ (AES_128_GCM, 16, 16, 8) \ + _ (AES_128_GCM, 16, 16, 12) \ + _ (AES_192_GCM, 24, 16, 8) \ + _ (AES_192_GCM, 24, 16, 12) \ + _ (AES_256_GCM, 32, 16, 8) \ + _ (AES_256_GCM, 32, 16, 12) \ + _ (CHACHA20_POLY1305, 32, 16, 8) \ + _ (CHACHA20_POLY1305, 32, 16, 12) \ + _ (CHACHA20_POLY1305, 32, 16, 0) + +/* CRYPTO_ID, INTEG_ID, KEY_LENGTH_IN_BYTES, DIGEST_LEN */ +#define foreach_oct_crypto_link_async_alg \ + _ (AES_128_CBC, SHA1, 16, 12) \ + _ (AES_192_CBC, SHA1, 24, 12) \ + _ (AES_256_CBC, SHA1, 32, 12) \ + _ (AES_128_CBC, SHA256, 16, 16) \ + _ (AES_192_CBC, SHA256, 24, 16) \ + _ (AES_256_CBC, SHA256, 32, 16) \ + _ (AES_128_CBC, SHA384, 16, 24) \ + _ (AES_192_CBC, SHA384, 24, 24) \ + _ (AES_256_CBC, SHA384, 32, 24) \ + _ (AES_128_CBC, SHA512, 16, 32) \ + _ (AES_192_CBC, SHA512, 24, 32) \ + _ (AES_256_CBC, SHA512, 32, 32) \ + _ (AES_128_CBC, MD5, 16, 12) \ + _ (AES_192_CBC, MD5, 24, 12) \ + _ (AES_256_CBC, MD5, 32, 12) \ + _ (3DES_CBC, MD5, 24, 12) \ + _ (3DES_CBC, SHA1, 24, 12) \ + _ (3DES_CBC, SHA256, 24, 16) \ + _ (3DES_CBC, SHA384, 24, 24) \ + _ (3DES_CBC, SHA512, 24, 32) \ + _ (AES_128_CTR, SHA1, 16, 12) \ + _ (AES_192_CTR, SHA1, 24, 12) \ + _ (AES_256_CTR, SHA1, 32, 12) + +#define OCT_MOD_INC(i, l) ((i) == (l - 1) ? (i) = 0 : (i)++) + +#define OCT_SCATTER_GATHER_BUFFER_SIZE 1024 + +#define CPT_LMT_SIZE_COPY (sizeof (struct cpt_inst_s) / 16) +#define OCT_MAX_LMT_SZ 16 + +#define SRC_IOV_SIZE \ + (sizeof (struct roc_se_iov_ptr) + \ + (sizeof (struct roc_se_buf_ptr) * ROC_MAX_SG_CNT)) + +#define OCT_CPT_LMT_GET_LINE_ADDR(lmt_addr, lmt_num) \ + (void *) ((u64) (lmt_addr) + ((u64) (lmt_num) << ROC_LMT_LINE_SIZE_LOG2)) + +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + struct roc_cpt *roc_cpt; + struct roc_cpt_lmtline lmtline; + struct roc_cpt_lf lf; + vnet_dev_t *dev; +} oct_crypto_dev_t; + +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + /** CPT opcode */ + u16 cpt_op : 4; + /** Flag for AES GCM */ + u16 aes_gcm : 1; + /** IV length in bytes */ + u8 iv_length; + /** Auth IV length in bytes */ + u8 auth_iv_length; + /** IV offset in bytes */ + u16 iv_offset; + /** Auth IV offset in bytes */ + u16 auth_iv_offset; + /** CPT inst word 7 */ + u64 cpt_inst_w7; + /* initialise as part of first packet */ + u8 initialised; + /* store link key index in case of linked algo */ + vnet_crypto_key_index_t key_index; + oct_crypto_dev_t *crypto_dev; + struct roc_se_ctx cpt_ctx; +} oct_crypto_sess_t; + +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + oct_crypto_sess_t *sess; + oct_crypto_dev_t *crypto_dev; +} oct_crypto_key_t; + +typedef struct oct_crypto_scatter_gather +{ + u8 buf[OCT_SCATTER_GATHER_BUFFER_SIZE]; +} oct_crypto_scatter_gather_t; + +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + /** Result data of all entries in the frame */ + volatile union cpt_res_s res[VNET_CRYPTO_FRAME_SIZE]; + /** Scatter gather data */ + void *sg_data; + /** Frame pointer */ + vnet_crypto_async_frame_t *frame; + /** Number of async elements in frame */ + u16 elts; + /** Next read entry in frame, when dequeue */ + u16 deq_elts; +} oct_crypto_inflight_req_t; + +typedef struct +{ + /** Array of pending request */ + oct_crypto_inflight_req_t *req_queue; + /** Number of inflight operations in queue */ + u32 n_crypto_inflight; + /** Tail of queue to be used for enqueue */ + u16 enq_tail; + /** Head of queue to be used for dequeue */ + u16 deq_head; + /** Number of descriptors */ + u16 n_desc; +} oct_crypto_pending_queue_t; + +typedef struct +{ + oct_crypto_dev_t *crypto_dev[OCT_MAX_N_CPT_DEV]; + oct_crypto_key_t *keys[VNET_CRYPTO_ASYNC_OP_N_TYPES]; + oct_crypto_pending_queue_t *pend_q; + int n_cpt; + u8 started; +} oct_crypto_main_t; + +extern oct_crypto_main_t oct_crypto_main; + +void oct_crypto_key_del_handler (vlib_main_t *vm, + vnet_crypto_key_index_t key_index); + +void oct_crypto_key_add_handler (vlib_main_t *vm, + vnet_crypto_key_index_t key_index); + +void oct_crypto_key_handler (vlib_main_t *vm, vnet_crypto_key_op_t kop, + vnet_crypto_key_index_t idx); + +int oct_crypto_enqueue_linked_alg_enc (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame); +int oct_crypto_enqueue_linked_alg_dec (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame); +int oct_crypto_enqueue_aead_aad_8_enc (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame); +int oct_crypto_enqueue_aead_aad_12_enc (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame); +int oct_crypto_enqueue_aead_aad_0_enc (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame); +int oct_crypto_enqueue_aead_aad_8_dec (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame); +int oct_crypto_enqueue_aead_aad_12_dec (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame); +int oct_crypto_enqueue_aead_aad_0_dec (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame); +vnet_crypto_async_frame_t *oct_crypto_frame_dequeue (vlib_main_t *vm, + u32 *nb_elts_processed, + u32 *enqueue_thread_idx); +int oct_init_crypto_engine_handlers (vlib_main_t *vm, vnet_dev_t *dev); +int oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev); +#endif /* _CRYPTO_H_ */ diff --git a/src/plugins/dev_octeon/flow.c b/src/plugins/dev_octeon/flow.c index 5bef25f5369..e86425ec85d 100644 --- a/src/plugins/dev_octeon/flow.c +++ b/src/plugins/dev_octeon/flow.c @@ -131,6 +131,7 @@ oct_flow_validate_params (vlib_main_t *vm, vnet_dev_port_t *port, vnet_dev_port_cfg_type_t type, u32 flow_index, uword *priv_data) { + vnet_dev_port_interfaces_t *ifs = port->interfaces; vnet_flow_t *flow = vnet_get_flow (flow_index); u32 last_queue; u32 qid; @@ -151,11 +152,11 @@ oct_flow_validate_params (vlib_main_t *vm, vnet_dev_port_t *port, if (flow->actions & VNET_FLOW_ACTION_REDIRECT_TO_QUEUE) { qid = flow->redirect_queue; - if (qid > port->intf.num_rx_queues - 1 || qid < 0) + if (qid > ifs->num_rx_queues - 1 || qid < 0) { log_err (port->dev, "Given Q(%d) is invalid, supported range is %d-%d", qid, 0, - port->intf.num_rx_queues - 1); + ifs->num_rx_queues - 1); return VNET_DEV_ERR_NOT_SUPPORTED; } } @@ -163,12 +164,12 @@ oct_flow_validate_params (vlib_main_t *vm, vnet_dev_port_t *port, if (flow->actions & VNET_FLOW_ACTION_RSS) { last_queue = flow->queue_index + flow->queue_num; - if (last_queue > port->intf.num_rx_queues - 1) + if (last_queue > ifs->num_rx_queues - 1) { log_err (port->dev, "Given Q range(%d-%d) is invalid, supported range is %d-%d", flow->queue_index, flow->queue_index + flow->queue_num, 0, - port->intf.num_rx_queues - 1); + ifs->num_rx_queues - 1); return VNET_DEV_ERR_NOT_SUPPORTED; } } @@ -538,6 +539,7 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, struct roc_npc_item_info item_info[ROC_NPC_ITEM_TYPE_END] = {}; struct roc_npc_action actions[ROC_NPC_ITEM_TYPE_END] = {}; oct_port_t *oct_port = vnet_dev_get_port_data (port); + vnet_dev_port_interfaces_t *ifs = port->interfaces; ethernet_header_t eth_spec = {}, eth_mask = {}; sctp_header_t sctp_spec = {}, sctp_mask = {}; gtpu_header_t gtpu_spec = {}, gtpu_mask = {}; @@ -775,7 +777,7 @@ parse_flow_actions: log_err (port->dev, "RSS action has no queues"); return VNET_DEV_ERR_NOT_SUPPORTED; } - queues = clib_mem_alloc (sizeof (u16) * port->intf.num_rx_queues); + queues = clib_mem_alloc (sizeof (u16) * ifs->num_rx_queues); for (index = 0; index < flow->queue_num; index++) queues[index] = flow->queue_index++; diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 2f0c82c1c01..99cadddfc24 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -10,6 +10,7 @@ #include <vnet/plugin/plugin.h> #include <vpp/app/version.h> #include <dev_octeon/octeon.h> +#include <dev_octeon/crypto.h> #include <base/roc_api.h> #include <common.h> @@ -54,7 +55,9 @@ static struct _ (0xa064, RVU_VF, "Marvell Octeon Resource Virtualization Unit VF"), _ (0xa0f8, LBK_VF, "Marvell Octeon Loopback Unit VF"), _ (0xa0f7, SDP_VF, "Marvell Octeon System DPI Packet Interface Unit VF"), - _ (0xa0f3, CPT_VF, "Marvell Octeon Cryptographic Accelerator Unit VF"), + _ (0xa0f3, O10K_CPT_VF, + "Marvell Octeon-10 Cryptographic Accelerator Unit VF"), + _ (0xa0fe, O9K_CPT_VF, "Marvell Octeon-9 Cryptographic Accelerator Unit VF"), #undef _ }; @@ -191,17 +194,113 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) return vnet_dev_port_add (vm, dev, 0, &port_add_args); } +static int +oct_conf_cpt (vlib_main_t *vm, vnet_dev_t *dev, oct_crypto_dev_t *ocd, + int nb_lf) +{ + struct roc_cpt *roc_cpt = ocd->roc_cpt; + int rrv; + + if ((rrv = roc_cpt_eng_grp_add (roc_cpt, CPT_ENG_TYPE_SE)) < 0) + { + log_err (dev, "Could not add CPT SE engines"); + return cnx_return_roc_err (dev, rrv, "roc_cpt_eng_grp_add"); + } + if ((rrv = roc_cpt_eng_grp_add (roc_cpt, CPT_ENG_TYPE_IE)) < 0) + { + log_err (dev, "Could not add CPT IE engines"); + return cnx_return_roc_err (dev, rrv, "roc_cpt_eng_grp_add"); + } + if (roc_cpt->eng_grp[CPT_ENG_TYPE_IE] != ROC_CPT_DFLT_ENG_GRP_SE_IE) + { + log_err (dev, "Invalid CPT IE engine group configuration"); + return -1; + } + if (roc_cpt->eng_grp[CPT_ENG_TYPE_SE] != ROC_CPT_DFLT_ENG_GRP_SE) + { + log_err (dev, "Invalid CPT SE engine group configuration"); + return -1; + } + if ((rrv = roc_cpt_dev_configure (roc_cpt, nb_lf, false, 0)) < 0) + { + log_err (dev, "could not configure crypto device %U", + format_vlib_pci_addr, roc_cpt->pci_dev->addr); + return cnx_return_roc_err (dev, rrv, "roc_cpt_dev_configure"); + } + return 0; +} + +static vnet_dev_rv_t +oct_conf_cpt_queue (vlib_main_t *vm, vnet_dev_t *dev, oct_crypto_dev_t *ocd) +{ + struct roc_cpt *roc_cpt = ocd->roc_cpt; + struct roc_cpt_lmtline *cpt_lmtline; + struct roc_cpt_lf *cpt_lf; + int rrv; + + cpt_lf = &ocd->lf; + cpt_lmtline = &ocd->lmtline; + + cpt_lf->nb_desc = OCT_CPT_LF_MAX_NB_DESC; + cpt_lf->lf_id = 0; + if ((rrv = roc_cpt_lf_init (roc_cpt, cpt_lf)) < 0) + return cnx_return_roc_err (dev, rrv, "roc_cpt_lf_init"); + + roc_cpt_iq_enable (cpt_lf); + + if ((rrv = roc_cpt_lmtline_init (roc_cpt, cpt_lmtline, 0) < 0)) + return cnx_return_roc_err (dev, rrv, "roc_cpt_lmtline_init"); + + return 0; +} + static vnet_dev_rv_t oct_init_cpt (vlib_main_t *vm, vnet_dev_t *dev) { + oct_crypto_main_t *ocm = &oct_crypto_main; + extern oct_plt_init_param_t oct_plt_init_param; oct_device_t *cd = vnet_dev_get_data (dev); + oct_crypto_dev_t *ocd = NULL; int rrv; - struct roc_cpt cpt = { - .pci_dev = &cd->plt_pci_dev, - }; - if ((rrv = roc_cpt_dev_init (&cpt))) + if (ocm->n_cpt == OCT_MAX_N_CPT_DEV || ocm->started) + return VNET_DEV_ERR_NOT_SUPPORTED; + + ocd = oct_plt_init_param.oct_plt_zmalloc (sizeof (oct_crypto_dev_t), + CLIB_CACHE_LINE_BYTES); + + ocd->roc_cpt = oct_plt_init_param.oct_plt_zmalloc (sizeof (struct roc_cpt), + CLIB_CACHE_LINE_BYTES); + ocd->roc_cpt->pci_dev = &cd->plt_pci_dev; + + ocd->dev = dev; + + if ((rrv = roc_cpt_dev_init (ocd->roc_cpt))) return cnx_return_roc_err (dev, rrv, "roc_cpt_dev_init"); + + if ((rrv = oct_conf_cpt (vm, dev, ocd, 1))) + return rrv; + + if ((rrv = oct_conf_cpt_queue (vm, dev, ocd))) + return rrv; + + if (!ocm->n_cpt) + { + /* + * Initialize s/w queues, which are common across multiple + * crypto devices + */ + oct_conf_sw_queue (vm, dev); + + ocm->crypto_dev[0] = ocd; + } + + ocm->crypto_dev[1] = ocd; + + oct_init_crypto_engine_handlers (vm, dev); + + ocm->n_cpt++; + return VNET_DEV_OK; } @@ -256,7 +355,8 @@ oct_init (vlib_main_t *vm, vnet_dev_t *dev) case OCT_DEVICE_TYPE_SDP_VF: return oct_init_nix (vm, dev); - case OCT_DEVICE_TYPE_CPT_VF: + case OCT_DEVICE_TYPE_O10K_CPT_VF: + case OCT_DEVICE_TYPE_O9K_CPT_VF: return oct_init_cpt (vm, dev); default: diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index a87a5e3e1ed..ccf8f62880d 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -30,7 +30,8 @@ typedef enum OCT_DEVICE_TYPE_RVU_VF, OCT_DEVICE_TYPE_LBK_VF, OCT_DEVICE_TYPE_SDP_VF, - OCT_DEVICE_TYPE_CPT_VF, + OCT_DEVICE_TYPE_O10K_CPT_VF, + OCT_DEVICE_TYPE_O9K_CPT_VF, } __clib_packed oct_device_type_t; typedef struct @@ -41,7 +42,6 @@ typedef struct u8 full_duplex : 1; u32 speed; struct plt_pci_device plt_pci_dev; - struct roc_cpt cpt; struct roc_nix *nix; } oct_device_t; @@ -102,7 +102,6 @@ typedef struct u64 aura_handle; u64 io_addr; void *lmt_addr; - oct_npa_batch_alloc_cl128_t *ba_buffer; u8 ba_first_cl; u8 ba_num_cl; diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 528683fa3c7..f8a7d6ba7db 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -129,6 +129,7 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) vnet_dev_t *dev = port->dev; oct_device_t *cd = vnet_dev_get_data (dev); oct_port_t *cp = vnet_dev_get_port_data (port); + vnet_dev_port_interfaces_t *ifs = port->interfaces; u8 mac_addr[PLT_ETHER_ADDR_LEN]; struct roc_nix *nix = cd->nix; vnet_dev_rv_t rv; @@ -136,14 +137,14 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) log_debug (dev, "port init: port %u", port->port_id); - if ((rrv = roc_nix_lf_alloc (nix, port->intf.num_rx_queues, - port->intf.num_tx_queues, rxq_cfg))) + if ((rrv = roc_nix_lf_alloc (nix, ifs->num_rx_queues, ifs->num_tx_queues, + rxq_cfg))) { oct_port_deinit (vm, port); return oct_roc_err ( dev, rrv, "roc_nix_lf_alloc(nb_rxq = %u, nb_txq = %d, rxq_cfg=0x%lx) failed", - port->intf.num_rx_queues, port->intf.num_tx_queues, rxq_cfg); + ifs->num_rx_queues, ifs->num_tx_queues, rxq_cfg); } cp->lf_allocated = 1; @@ -428,6 +429,7 @@ oct_port_start (vlib_main_t *vm, vnet_dev_port_t *port) { vnet_dev_t *dev = port->dev; oct_device_t *cd = vnet_dev_get_data (dev); + oct_port_t *cp = vnet_dev_get_port_data (port); struct roc_nix *nix = cd->nix; struct roc_nix_eeprom_info eeprom_info = {}; vnet_dev_rv_t rv; @@ -451,6 +453,12 @@ oct_port_start (vlib_main_t *vm, vnet_dev_port_t *port) goto done; } + if ((rrv = roc_npc_mcam_enable_all_entries (&cp->npc, true))) + { + rv = oct_roc_err (dev, rrv, "roc_npc_mcam_enable_all_entries() failed"); + goto done; + } + vnet_dev_poll_port_add (vm, port, 0.5, oct_port_poll); if (roc_nix_eeprom_info_get (nix, &eeprom_info) == 0) @@ -469,6 +477,7 @@ oct_port_stop (vlib_main_t *vm, vnet_dev_port_t *port) { vnet_dev_t *dev = port->dev; oct_device_t *cd = vnet_dev_get_data (dev); + oct_port_t *cp = vnet_dev_get_port_data (port); struct roc_nix *nix = cd->nix; int rrv; @@ -476,6 +485,14 @@ oct_port_stop (vlib_main_t *vm, vnet_dev_port_t *port) vnet_dev_poll_port_remove (vm, port, oct_port_poll); + /* Disable all the NPC entries */ + rrv = roc_npc_mcam_enable_all_entries (&cp->npc, false); + if (rrv) + { + oct_roc_err (dev, rrv, "roc_npc_mcam_enable_all_entries() failed"); + return; + } + rrv = roc_nix_npc_rx_ena_dis (nix, false); if (rrv) { @@ -575,6 +592,10 @@ oct_port_add_del_eth_addr (vlib_main_t *vm, vnet_dev_port_t *port, rv = oct_roc_err (dev, rrv, "roc_nix_mac_addr_set() failed"); } } + + rrv = roc_nix_rss_default_setup (nix, default_rss_flowkey); + if (rrv) + rv = oct_roc_err (dev, rrv, "roc_nix_rss_default_setup() failed"); } } diff --git a/src/plugins/dev_octeon/roc_helper.c b/src/plugins/dev_octeon/roc_helper.c index 16e0a871a9d..c1166b654cf 100644 --- a/src/plugins/dev_octeon/roc_helper.c +++ b/src/plugins/dev_octeon/roc_helper.c @@ -75,13 +75,12 @@ oct_drv_physmem_alloc (vlib_main_t *vm, u32 size, u32 align) if (align) { - /* Force cache line alloc in case alignment is less than cache line */ - align = align < CLIB_CACHE_LINE_BYTES ? CLIB_CACHE_LINE_BYTES : align; + /* Force ROC align alloc in case alignment is less than ROC align */ + align = align < ROC_ALIGN ? ROC_ALIGN : align; mem = vlib_physmem_alloc_aligned_on_numa (vm, size, align, 0); } else - mem = - vlib_physmem_alloc_aligned_on_numa (vm, size, CLIB_CACHE_LINE_BYTES, 0); + mem = vlib_physmem_alloc_aligned_on_numa (vm, size, ROC_ALIGN, 0); if (!mem) return NULL; diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index b057c4d7047..833227eeea8 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -103,7 +103,7 @@ oct_rx_batch (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, vnet_dev_rx_queue_t *rxq, u32 n) { oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); - vlib_buffer_template_t bt = rxq->buffer_template; + vlib_buffer_template_t bt = vnet_dev_get_rx_queue_if_buffer_template (rxq); u32 b0_err_flags = 0, b1_err_flags = 0; u32 b2_err_flags = 0, b3_err_flags = 0; u32 n_left, err_flags = 0; @@ -347,9 +347,9 @@ oct_rx_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, oct_nix_rx_cqe_desc_t *descs = crq->cq.desc_base; oct_nix_lf_cq_op_status_t status; oct_rx_node_ctx_t _ctx = { - .next_index = rxq->next_index, - .sw_if_index = port->intf.sw_if_index, - .hw_if_index = port->intf.hw_if_index, + .next_index = vnet_dev_get_rx_queue_if_next_index(rxq), + .sw_if_index = vnet_dev_get_rx_queue_if_sw_if_index (rxq), + .hw_if_index = vnet_dev_get_rx_queue_if_hw_if_index (rxq), }, *ctx = &_ctx; /* get head and tail from NIX_LF_CQ_OP_STATUS */ diff --git a/src/plugins/dhcp/client.c b/src/plugins/dhcp/client.c index 8fa67c616b2..d81d2935577 100644 --- a/src/plugins/dhcp/client.c +++ b/src/plugins/dhcp/client.c @@ -1153,7 +1153,9 @@ dhcp_client_set_command_fn (vlib_main_t * vm, a->is_add = is_add; a->sw_if_index = sw_if_index; a->hostname = hostname; - a->client_identifier = format (0, "vpp 1.1%c", 0); + a->client_identifier = + format (0, "%U", format_ethernet_address, + vnet_sw_interface_get_hw_address (vnet_get_main (), sw_if_index)); a->set_broadcast_flag = set_broadcast_flag; /* diff --git a/src/plugins/dhcp/dhcp4_proxy_node.c b/src/plugins/dhcp/dhcp4_proxy_node.c index 2b49d49bb7f..740ae8043e0 100644 --- a/src/plugins/dhcp/dhcp4_proxy_node.c +++ b/src/plugins/dhcp/dhcp4_proxy_node.c @@ -321,7 +321,8 @@ dhcp_proxy_to_server_input (vlib_main_t * vm, o->length += id_len + 5; } - len = o->length + 3; + /* 2 bytes for option header 82+len */ + len = o->length + 2; b0->current_length += len; /* Fix IP header length and checksum */ old_l0 = ip0->length; diff --git a/src/plugins/dpdk/device/dpdk.h b/src/plugins/dpdk/device/dpdk.h index a069fbe3818..2440439989f 100644 --- a/src/plugins/dpdk/device/dpdk.h +++ b/src/plugins/dpdk/device/dpdk.h @@ -131,7 +131,6 @@ typedef struct u32 interface_number_from_port_id : 1; u32 use_intel_phdr_cksum : 1; u32 int_unmaskable : 1; - vlib_simple_counter_main_t *xstats_counters; } dpdk_driver_t; dpdk_driver_t *dpdk_driver_find (const char *name, const char **desc); @@ -211,6 +210,8 @@ typedef struct struct rte_eth_stats last_stats; struct rte_eth_xstat *xstats; f64 time_last_stats_update; + vlib_simple_counter_main_t xstats_counters; + u32 *xstats_symlinks; /* mac address */ u8 *default_mac_address; diff --git a/src/plugins/dpdk/device/dpdk_priv.h b/src/plugins/dpdk/device/dpdk_priv.h index e5b5a35df80..794953da55e 100644 --- a/src/plugins/dpdk/device/dpdk_priv.h +++ b/src/plugins/dpdk/device/dpdk_priv.h @@ -54,39 +54,28 @@ dpdk_get_xstats (dpdk_device_t *xd, u32 thread_index) { int ret; int i; - int len; if (!(xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP)) return; - if (xd->driver == 0) - return; - len = rte_eth_xstats_get (xd->port_id, NULL, 0); - if (len < 0) - return; - - vec_validate (xd->xstats, len - 1); - ret = rte_eth_xstats_get (xd->port_id, xd->xstats, len); - if (ret < 0 || ret > len) + ret = rte_eth_xstats_get (xd->port_id, xd->xstats, vec_len (xd->xstats)); + if (ret < 0) { - /* Failed, expand vector and try again on next time around the track. */ - vec_validate (xd->xstats, ret - 1); - vec_set_len (xd->xstats, 0); dpdk_log_warn ("rte_eth_xstats_get(%d) failed: %d", xd->port_id, ret); return; } - if (len == vec_len (xd->driver->xstats_counters)) + else if (ret != vec_len (xd->xstats)) { - vec_foreach_index (i, xd->xstats) - { - vlib_set_simple_counter (&xd->driver->xstats_counters[i], - thread_index, xd->sw_if_index, - xd->xstats[i].value); - } + dpdk_log_warn ( + "rte_eth_xstats_get(%d) returned %d/%d stats. Resetting counters.", + xd->port_id, ret, vec_len (xd->xstats)); + dpdk_counters_xstats_init (xd); + return; } - else + + vec_foreach_index (i, xd->xstats) { - dpdk_log_warn ("rte_eth_xstats_get vector size mismatch (%d/%d", len, - vec_len (xd->driver->xstats_counters)); + vlib_set_simple_counter (&xd->xstats_counters, thread_index, i, + xd->xstats[i].value); } } diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c index fa1b234874d..ec9e6045de7 100644 --- a/src/plugins/dpdk/device/init.c +++ b/src/plugins/dpdk/device/init.c @@ -227,71 +227,71 @@ dpdk_find_startup_config (struct rte_eth_dev_info *di) } /* - * Initialise or refresh the xstats counters for a device + * Initialise the xstats counters for a device */ void dpdk_counters_xstats_init (dpdk_device_t *xd) { int len, ret, i; struct rte_eth_xstat_name *xstats_names = 0; - char *name; - dpdk_driver_t *dr = xd->driver; - /* Only support xstats for supported drivers */ - if (!dr) - return; + if (vec_len (xd->xstats_symlinks) > 0) + { + /* xstats already initialized. Reset counters */ + vec_foreach_index (i, xd->xstats_symlinks) + { + vlib_stats_remove_entry (xd->xstats_symlinks[i]); + } + } + else + { + xd->xstats_counters.stat_segment_name = + (char *) format (0, "/if/xstats/%d%c", xd->sw_if_index, 0); + xd->xstats_counters.counters = 0; + } len = rte_eth_xstats_get_names (xd->port_id, 0, 0); if (len < 0) { - dpdk_log_err ("[%u] rte_eth_xstats_get_names failed: %d", xd->port_id, - len); - return; - } - /* Counters for this driver is already initialised */ - if (vec_len (dr->xstats_counters) == len) - { - vec_foreach_index (i, dr->xstats_counters) - { - vlib_validate_simple_counter (&dr->xstats_counters[i], - xd->sw_if_index); - vlib_zero_simple_counter (&dr->xstats_counters[i], xd->sw_if_index); - } + dpdk_log_err ("[%u] rte_eth_xstats_get_names failed: %d. DPDK xstats " + "not configured.", + xd->port_id, len); return; } - /* Same driver, different interface, different length of counter array. */ - ASSERT (vec_len (dr->xstats_counters) == 0); + vlib_validate_simple_counter (&xd->xstats_counters, len); + vlib_zero_simple_counter (&xd->xstats_counters, len); vec_validate (xstats_names, len - 1); + vec_validate (xd->xstats, len - 1); + vec_validate (xd->xstats_symlinks, len - 1); ret = rte_eth_xstats_get_names (xd->port_id, xstats_names, len); if (ret >= 0 && ret <= len) { - vec_validate (dr->xstats_counters, len - 1); vec_foreach_index (i, xstats_names) { - name = (char *) format (0, "/if/%s/%s%c", dr->drivers->name, - xstats_names[i].name, 0); - /* There is a bug in the ENA driver where the xstats names are not * unique. */ - if (vlib_stats_find_entry_index (name) != STAT_SEGMENT_INDEX_INVALID) + xd->xstats_symlinks[i] = vlib_stats_add_symlink ( + xd->xstats_counters.stats_entry_index, i, "/interfaces/%U/%s%c", + format_vnet_sw_if_index_name, vnet_get_main (), xd->sw_if_index, + xstats_names[i].name, 0); + if (xd->xstats_symlinks[i] == STAT_SEGMENT_INDEX_INVALID) { - vec_free (name); - name = (char *) format (0, "/if/%s/%s_%d%c", dr->drivers->name, - xstats_names[i].name, i, 0); + xd->xstats_symlinks[i] = vlib_stats_add_symlink ( + xd->xstats_counters.stats_entry_index, i, + "/interfaces/%U/%s_%d%c", format_vnet_sw_if_index_name, + vnet_get_main (), xd->sw_if_index, xstats_names[i].name, i, 0); } - - dr->xstats_counters[i].name = name; - dr->xstats_counters[i].stat_segment_name = name; - dr->xstats_counters[i].counters = 0; - vlib_validate_simple_counter (&dr->xstats_counters[i], - xd->sw_if_index); - vlib_zero_simple_counter (&dr->xstats_counters[i], xd->sw_if_index); - vec_free (name); } } + else + { + dpdk_log_err ("[%u] rte_eth_xstats_get_names failed: %d. DPDK xstats " + "not configured.", + xd->port_id, ret); + } vec_free (xstats_names); } @@ -503,6 +503,14 @@ dpdk_lib_init (dpdk_main_t * dm) else if (dr && dr->n_tx_desc) xd->conf.n_tx_desc = dr->n_tx_desc; + if (xd->conf.n_tx_desc > di.tx_desc_lim.nb_max) + { + dpdk_log_warn ("[%u] Configured number of TX descriptors (%u) is " + "bigger than maximum supported (%u)", + port_id, xd->conf.n_tx_desc, di.tx_desc_lim.nb_max); + xd->conf.n_tx_desc = di.tx_desc_lim.nb_max; + } + dpdk_log_debug ( "[%u] n_rx_queues: %u n_tx_queues: %u n_rx_desc: %u n_tx_desc: %u", port_id, xd->conf.n_rx_queues, xd->conf.n_tx_queues, diff --git a/src/plugins/hs_apps/CMakeLists.txt b/src/plugins/hs_apps/CMakeLists.txt index ba03e393f44..eae100949d4 100644 --- a/src/plugins/hs_apps/CMakeLists.txt +++ b/src/plugins/hs_apps/CMakeLists.txt @@ -21,7 +21,7 @@ add_vpp_plugin(hs_apps hs_apps.c http_cli.c http_client_cli.c - http_simple_post.c + http_client.c http_tps.c proxy.c test_builtins.c diff --git a/src/plugins/hs_apps/echo_client.c b/src/plugins/hs_apps/echo_client.c index 8dec5d86824..d5edffbd02e 100644 --- a/src/plugins/hs_apps/echo_client.c +++ b/src/plugins/hs_apps/echo_client.c @@ -946,15 +946,16 @@ ec_connect_rpc (void *args) a->api_context = ci; if (needs_crypto) { - session_endpoint_alloc_ext_cfg (&a->sep_ext, - TRANSPORT_ENDPT_EXT_CFG_CRYPTO); - a->sep_ext.ext_cfg->crypto.ckpair_index = ecm->ckpair_index; + transport_endpt_ext_cfg_t *ext_cfg = session_endpoint_add_ext_cfg ( + &a->sep_ext, TRANSPORT_ENDPT_EXT_CFG_CRYPTO, + sizeof (transport_endpt_crypto_cfg_t)); + ext_cfg->crypto.ckpair_index = ecm->ckpair_index; } rv = vnet_connect (a); if (needs_crypto) - clib_mem_free (a->sep_ext.ext_cfg); + session_endpoint_free_ext_cfgs (&a->sep_ext); if (rv) { diff --git a/src/plugins/hs_apps/echo_server.c b/src/plugins/hs_apps/echo_server.c index 756a1cc3451..b981e775b57 100644 --- a/src/plugins/hs_apps/echo_server.c +++ b/src/plugins/hs_apps/echo_server.c @@ -591,6 +591,7 @@ echo_server_listen () i32 rv; echo_server_main_t *esm = &echo_server_main; vnet_listen_args_t _args = {}, *args = &_args; + int needs_crypto; if ((rv = parse_uri (esm->server_uri, &args->sep_ext))) { @@ -598,11 +599,14 @@ echo_server_listen () } args->app_index = esm->app_index; args->sep_ext.port = hs_make_data_port (args->sep_ext.port); - if (echo_client_transport_needs_crypto (args->sep_ext.transport_proto)) + needs_crypto = + echo_client_transport_needs_crypto (args->sep_ext.transport_proto); + if (needs_crypto) { - session_endpoint_alloc_ext_cfg (&args->sep_ext, - TRANSPORT_ENDPT_EXT_CFG_CRYPTO); - args->sep_ext.ext_cfg->crypto.ckpair_index = esm->ckpair_index; + transport_endpt_ext_cfg_t *ext_cfg = session_endpoint_add_ext_cfg ( + &args->sep_ext, TRANSPORT_ENDPT_EXT_CFG_CRYPTO, + sizeof (transport_endpt_crypto_cfg_t)); + ext_cfg->crypto.ckpair_index = esm->ckpair_index; } if (args->sep_ext.transport_proto == TRANSPORT_PROTO_UDP) @@ -612,8 +616,8 @@ echo_server_listen () rv = vnet_listen (args); esm->listener_handle = args->handle; - if (args->sep_ext.ext_cfg) - clib_mem_free (args->sep_ext.ext_cfg); + if (needs_crypto) + session_endpoint_free_ext_cfgs (&args->sep_ext); return rv; } diff --git a/src/plugins/hs_apps/http_cli.c b/src/plugins/hs_apps/http_cli.c index 18b57f6c29d..dfa90f9eced 100644 --- a/src/plugins/hs_apps/http_cli.c +++ b/src/plugins/hs_apps/http_cli.c @@ -74,6 +74,10 @@ typedef struct /* pool of uri maps */ hcs_uri_map_t *uri_map_pool; + + /* for appns */ + u8 *appns_id; + u64 appns_secret; } hcs_main_t; static hcs_main_t hcs_main; @@ -402,7 +406,7 @@ hcs_ts_rx_callback (session_t *ts) } if (is_encoded) { - u8 *decoded = http_percent_decode (args.buf); + u8 *decoded = http_percent_decode (args.buf, vec_len (args.buf)); vec_free (args.buf); args.buf = decoded; } @@ -597,6 +601,11 @@ hcs_attach () hcm->fifo_size ? hcm->fifo_size : 32 << 10; a->options[APP_OPTIONS_FLAGS] = APP_OPTIONS_FLAGS_IS_BUILTIN; a->options[APP_OPTIONS_PREALLOC_FIFO_PAIRS] = hcm->prealloc_fifos; + if (hcm->appns_id) + { + a->namespace_id = hcm->appns_id; + a->options[APP_OPTIONS_NAMESPACE_SECRET] = hcm->appns_secret; + } if (vnet_application_attach (a)) { @@ -651,9 +660,10 @@ hcs_listen () if (need_crypto) { - session_endpoint_alloc_ext_cfg (&a->sep_ext, - TRANSPORT_ENDPT_EXT_CFG_CRYPTO); - a->sep_ext.ext_cfg->crypto.ckpair_index = hcm->ckpair_index; + transport_endpt_ext_cfg_t *ext_cfg = session_endpoint_add_ext_cfg ( + &a->sep_ext, TRANSPORT_ENDPT_EXT_CFG_CRYPTO, + sizeof (transport_endpt_crypto_cfg_t)); + ext_cfg->crypto.ckpair_index = hcm->ckpair_index; } rv = vnet_listen (a); @@ -667,15 +677,25 @@ hcs_listen () } if (need_crypto) - clib_mem_free (a->sep_ext.ext_cfg); + session_endpoint_free_ext_cfgs (&a->sep_ext); return rv; } +static void +hcs_detach () +{ + vnet_app_detach_args_t _a, *a = &_a; + hcs_main_t *hcm = &hcs_main; + a->app_index = hcm->app_index; + a->api_client_index = APP_INVALID_INDEX; + hcm->app_index = ~0; + vnet_application_detach (a); +} + static int hcs_unlisten () { - session_endpoint_cfg_t sep = SESSION_ENDPOINT_CFG_NULL; hcs_main_t *hcm = &hcs_main; vnet_unlisten_args_t _a, *a = &_a; char *uri; @@ -688,9 +708,6 @@ hcs_unlisten () uri = (char *) hcm->uri; ASSERT (uri); - if (parse_uri (uri, &sep)) - return -1; - value = hash_get_mem (hcm->index_by_uri, uri); if (value) { @@ -700,8 +717,11 @@ hcs_unlisten () rv = vnet_unlisten (a); if (rv == 0) { + hash_unset_mem (hcm->index_by_uri, uri); vec_free (map->uri); pool_put (hcm->uri_map_pool, map); + if (pool_elts (hcm->uri_map_pool) == 0) + hcs_detach (); } } else @@ -710,17 +730,6 @@ hcs_unlisten () return rv; } -static void -hcs_detach () -{ - vnet_app_detach_args_t _a, *a = &_a; - hcs_main_t *hcm = &hcs_main; - a->app_index = hcm->app_index; - a->api_client_index = APP_INVALID_INDEX; - hcm->app_index = ~0; - vnet_application_detach (a); -} - static int hcs_create (vlib_main_t *vm) { @@ -776,6 +785,10 @@ hcs_create_command_fn (vlib_main_t *vm, unformat_input_t *input, hcm->fifo_size <<= 10; else if (unformat (line_input, "uri %_%v%_", &hcm->uri)) ; + else if (unformat (line_input, "appns %_%v%_", &hcm->appns_id)) + ; + else if (unformat (line_input, "secret %lu", &hcm->appns_secret)) + ; else if (unformat (line_input, "listener")) { if (unformat (line_input, "add")) @@ -804,31 +817,30 @@ hcs_create_command_fn (vlib_main_t *vm, unformat_input_t *input, start_server: if (hcm->uri == 0) - hcm->uri = format (0, "tcp://0.0.0.0/80%c", 0); + hcm->uri = format (0, "tcp://0.0.0.0/80"); if (hcm->app_index != (u32) ~0) { + if (hcm->appns_id && (listener_add != ~0)) + { + error = clib_error_return ( + 0, "appns must not be specified for listener add/del"); + goto done; + } if (listener_add == 1) { if (hcs_listen ()) - { - error = clib_error_return (0, "failed to start listening %v", - hcm->uri); - goto done; - } - else - goto done; + error = + clib_error_return (0, "failed to start listening %v", hcm->uri); + goto done; } else if (listener_add == 0) { - if (hcs_unlisten () != 0) - { - error = - clib_error_return (0, "failed to stop listening %v", hcm->uri); - goto done; - } - else - goto done; + rv = hcs_unlisten (); + if (rv != 0) + error = clib_error_return ( + 0, "failed to stop listening %v, rv = %d", hcm->uri, rv); + goto done; } else { @@ -855,6 +867,7 @@ start_server: } done: + vec_free (hcm->appns_id); vec_free (hcm->uri); return error; } @@ -863,7 +876,7 @@ VLIB_CLI_COMMAND (hcs_create_command, static) = { .path = "http cli server", .short_help = "http cli server [uri <uri>] [fifo-size <nbytes>] " "[private-segment-size <nMG>] [prealloc-fifos <n>] " - "[listener <add|del>]", + "[listener <add|del>] [appns <app-ns> secret <appns-secret>]", .function = hcs_create_command_fn, }; diff --git a/src/plugins/hs_apps/http_client.c b/src/plugins/hs_apps/http_client.c new file mode 100644 index 00000000000..05a87ec7de8 --- /dev/null +++ b/src/plugins/hs_apps/http_client.c @@ -0,0 +1,743 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2024 Cisco Systems, Inc. + */ + +#include <vnet/session/application.h> +#include <vnet/session/application_interface.h> +#include <vnet/session/session.h> +#include <http/http.h> +#include <http/http_header_names.h> +#include <http/http_content_types.h> +#include <http/http_status_codes.h> +#include <vppinfra/unix.h> + +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + u32 session_index; + u32 thread_index; + u32 vpp_session_index; + u64 to_recv; + u8 is_closed; +} hc_session_t; + +typedef struct +{ + hc_session_t *sessions; + u32 thread_index; + vlib_main_t *vlib_main; +} hc_worker_t; + +typedef struct +{ + u32 app_index; + u32 cli_node_index; + u8 attached; + u8 *uri; + session_endpoint_cfg_t connect_sep; + u8 *target; + u8 *headers_buf; + u8 *data; + u64 data_offset; + hc_worker_t *wrk; + u8 *resp_headers; + u8 *http_response; + u8 *response_status; + http_header_ht_t *custom_header; + u8 is_file; + u8 use_ptr; + u8 *filename; + bool verbose; + f64 timeout; + http_req_method_t req_method; +} hc_main_t; + +typedef enum +{ + HC_CONNECT_FAILED = 1, + HC_TRANSPORT_CLOSED, + HC_REPLY_RECEIVED, +} hc_cli_signal_t; + +static hc_main_t hc_main; + +static inline hc_worker_t * +hc_worker_get (u32 thread_index) +{ + return &hc_main.wrk[thread_index]; +} + +static inline hc_session_t * +hc_session_get (u32 session_index, u32 thread_index) +{ + hc_worker_t *wrk = hc_worker_get (thread_index); + wrk->vlib_main = vlib_get_main_by_index (thread_index); + return pool_elt_at_index (wrk->sessions, session_index); +} + +static void +hc_ho_session_free (u32 hs_index) +{ + hc_worker_t *wrk = hc_worker_get (0); + pool_put_index (wrk->sessions, hs_index); +} + +static hc_session_t * +hc_session_alloc (hc_worker_t *wrk) +{ + hc_session_t *s; + + pool_get_zero (wrk->sessions, s); + s->session_index = s - wrk->sessions; + s->thread_index = wrk->thread_index; + + return s; +} + +static int +hc_session_connected_callback (u32 app_index, u32 hc_session_index, + session_t *s, session_error_t err) +{ + hc_main_t *hcm = &hc_main; + hc_session_t *hc_session, *new_hc_session; + hc_worker_t *wrk; + http_msg_t msg; + u64 to_send; + u32 n_enq; + u8 n_segs; + int rv; + http_header_ht_t *header; + http_header_t *req_headers = 0; + u32 new_hc_index; + + HTTP_DBG (1, "ho hc_index: %d", hc_session_index); + + if (err) + { + clib_warning ("hc_session_index[%d] connected error: %U", + hc_session_index, format_session_error, err); + vlib_process_signal_event_mt (hcm->wrk->vlib_main, hcm->cli_node_index, + HC_CONNECT_FAILED, 0); + return -1; + } + + hc_session = hc_session_get (hc_session_index, 0); + wrk = hc_worker_get (s->thread_index); + new_hc_session = hc_session_alloc (wrk); + new_hc_index = new_hc_session->session_index; + clib_memcpy_fast (new_hc_session, hc_session, sizeof (*hc_session)); + hc_session->vpp_session_index = s->session_index; + + new_hc_session->session_index = new_hc_index; + new_hc_session->thread_index = s->thread_index; + new_hc_session->vpp_session_index = s->session_index; + HTTP_DBG (1, "new hc_index: %d", new_hc_session->session_index); + s->opaque = new_hc_index; + + if (hcm->req_method == HTTP_REQ_POST) + { + if (hcm->is_file) + http_add_header ( + &req_headers, http_header_name_token (HTTP_HEADER_CONTENT_TYPE), + http_content_type_token (HTTP_CONTENT_APP_OCTET_STREAM)); + else + http_add_header ( + &req_headers, http_header_name_token (HTTP_HEADER_CONTENT_TYPE), + http_content_type_token (HTTP_CONTENT_APP_X_WWW_FORM_URLENCODED)); + } + + vec_foreach (header, hcm->custom_header) + http_add_header (&req_headers, (const char *) header->name, + vec_len (header->name), (const char *) header->value, + vec_len (header->value)); + + hcm->headers_buf = http_serialize_headers (req_headers); + vec_free (req_headers); + + msg.method_type = hcm->req_method; + if (hcm->req_method == HTTP_REQ_POST) + msg.data.body_len = vec_len (hcm->data); + else + msg.data.body_len = 0; + + msg.type = HTTP_MSG_REQUEST; + /* request target */ + msg.data.target_form = HTTP_TARGET_ORIGIN_FORM; + msg.data.target_path_len = vec_len (hcm->target); + /* custom headers */ + msg.data.headers_len = vec_len (hcm->headers_buf); + /* total length */ + msg.data.len = + msg.data.target_path_len + msg.data.headers_len + msg.data.body_len; + + if (hcm->use_ptr) + { + uword target = pointer_to_uword (hcm->target); + uword headers = pointer_to_uword (hcm->headers_buf); + uword body = pointer_to_uword (hcm->data); + msg.data.type = HTTP_MSG_DATA_PTR; + svm_fifo_seg_t segs[4] = { + { (u8 *) &msg, sizeof (msg) }, + { (u8 *) &target, sizeof (target) }, + { (u8 *) &headers, sizeof (headers) }, + { (u8 *) &body, sizeof (body) }, + }; + + n_segs = (hcm->req_method == HTTP_REQ_GET) ? 3 : 4; + rv = svm_fifo_enqueue_segments (s->tx_fifo, segs, n_segs, + 0 /* allow partial */); + if (hcm->req_method == HTTP_REQ_POST) + ASSERT (rv == (sizeof (msg) + sizeof (target) + sizeof (headers) + + sizeof (body))); + else + ASSERT (rv == (sizeof (msg) + sizeof (target) + sizeof (headers))); + goto done; + } + + msg.data.type = HTTP_MSG_DATA_INLINE; + msg.data.target_path_offset = 0; + msg.data.headers_offset = msg.data.target_path_len; + msg.data.body_offset = msg.data.headers_offset + msg.data.headers_len; + + rv = svm_fifo_enqueue (s->tx_fifo, sizeof (msg), (u8 *) &msg); + ASSERT (rv == sizeof (msg)); + + rv = svm_fifo_enqueue (s->tx_fifo, vec_len (hcm->target), hcm->target); + ASSERT (rv == vec_len (hcm->target)); + + rv = svm_fifo_enqueue (s->tx_fifo, vec_len (hcm->headers_buf), + hcm->headers_buf); + ASSERT (rv == msg.data.headers_len); + + if (hcm->req_method == HTTP_REQ_POST) + { + to_send = vec_len (hcm->data); + n_enq = clib_min (svm_fifo_size (s->tx_fifo), to_send); + + rv = svm_fifo_enqueue (s->tx_fifo, n_enq, hcm->data); + if (rv < to_send) + { + hcm->data_offset = (rv > 0) ? rv : 0; + svm_fifo_add_want_deq_ntf (s->tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); + } + } + +done: + if (svm_fifo_set_event (s->tx_fifo)) + session_program_tx_io_evt (s->handle, SESSION_IO_EVT_TX); + + return 0; +} + +static void +hc_session_disconnect_callback (session_t *s) +{ + hc_main_t *hcm = &hc_main; + vnet_disconnect_args_t _a = { 0 }, *a = &_a; + int rv; + + a->handle = session_handle (s); + a->app_index = hcm->app_index; + if ((rv = vnet_disconnect_session (a))) + clib_warning ("warning: disconnect returned: %U", format_session_error, + rv); +} + +static void +hc_session_transport_closed_callback (session_t *s) +{ + hc_main_t *hcm = &hc_main; + vlib_process_signal_event_mt (hcm->wrk->vlib_main, hcm->cli_node_index, + HC_TRANSPORT_CLOSED, 0); +} + +static void +hc_ho_cleanup_callback (session_t *ts) +{ + HTTP_DBG (1, "ho hc_index: %d:", ts->opaque); + hc_ho_session_free (ts->opaque); +} + +static void +hc_session_reset_callback (session_t *s) +{ + hc_main_t *hcm = &hc_main; + hc_session_t *hc_session; + vnet_disconnect_args_t _a = { 0 }, *a = &_a; + int rv; + + hc_session = hc_session_get (s->opaque, s->thread_index); + hc_session->is_closed = 1; + + a->handle = session_handle (s); + a->app_index = hcm->app_index; + if ((rv = vnet_disconnect_session (a))) + clib_warning ("warning: disconnect returned: %U", format_session_error, + rv); +} + +static int +hc_rx_callback (session_t *s) +{ + hc_main_t *hcm = &hc_main; + hc_session_t *hc_session; + http_msg_t msg; + int rv; + + hc_session = hc_session_get (s->opaque, s->thread_index); + + if (hc_session->is_closed) + { + clib_warning ("hc_session_index[%d] is closed", s->opaque); + return -1; + } + + if (hc_session->to_recv == 0) + { + rv = svm_fifo_dequeue (s->rx_fifo, sizeof (msg), (u8 *) &msg); + ASSERT (rv == sizeof (msg)); + + if (msg.type != HTTP_MSG_REPLY) + { + clib_warning ("unexpected msg type %d", msg.type); + return -1; + } + + if (msg.data.headers_len) + { + http_header_table_t *ht; + vec_validate (hcm->resp_headers, msg.data.headers_len - 1); + rv = svm_fifo_peek (s->rx_fifo, msg.data.headers_offset, + msg.data.headers_len, hcm->resp_headers); + + ASSERT (rv == msg.data.headers_len); + HTTP_DBG (1, (char *) hcm->resp_headers); + + if (http_parse_headers (hcm->resp_headers, &ht)) + { + clib_warning ("invalid headers received"); + return -1; + } + http_free_header_table (ht); + + hcm->response_status = + format (0, "%U", format_http_status_code, msg.code); + } + + if (msg.data.body_len == 0) + { + svm_fifo_dequeue_drop_all (s->rx_fifo); + goto done; + } + + /* drop everything up to body */ + svm_fifo_dequeue_drop (s->rx_fifo, msg.data.body_offset); + hc_session->to_recv = msg.data.body_len; + if (msg.code != HTTP_STATUS_OK && hc_session->to_recv == 0) + { + goto done; + } + vec_validate (hcm->http_response, msg.data.body_len - 1); + vec_reset_length (hcm->http_response); + } + + u32 max_deq = svm_fifo_max_dequeue (s->rx_fifo); + + u32 n_deq = clib_min (hc_session->to_recv, max_deq); + u32 curr = vec_len (hcm->http_response); + rv = svm_fifo_dequeue (s->rx_fifo, n_deq, hcm->http_response + curr); + if (rv < 0) + { + clib_warning ("app dequeue(n=%d) failed; rv = %d", n_deq, rv); + return -1; + } + + ASSERT (rv == n_deq); + vec_set_len (hcm->http_response, curr + n_deq); + ASSERT (hc_session->to_recv >= rv); + hc_session->to_recv -= rv; + +done: + if (hc_session->to_recv == 0) + { + hc_session_disconnect_callback (s); + vlib_process_signal_event_mt (hcm->wrk->vlib_main, hcm->cli_node_index, + HC_REPLY_RECEIVED, 0); + } + + return 0; +} + +static int +hc_tx_callback (session_t *s) +{ + hc_main_t *hcm = &hc_main; + u64 to_send; + int rv; + + to_send = vec_len (hcm->data) - hcm->data_offset; + rv = svm_fifo_enqueue (s->tx_fifo, to_send, hcm->data + hcm->data_offset); + + if (rv <= 0) + { + svm_fifo_add_want_deq_ntf (s->tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); + return 0; + } + + if (rv < to_send) + { + hcm->data_offset += rv; + svm_fifo_add_want_deq_ntf (s->tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); + } + + if (svm_fifo_set_event (s->tx_fifo)) + session_program_tx_io_evt (s->handle, SESSION_IO_EVT_TX); + + return 0; +} + +static session_cb_vft_t hc_session_cb_vft = { + .session_connected_callback = hc_session_connected_callback, + .session_disconnect_callback = hc_session_disconnect_callback, + .session_transport_closed_callback = hc_session_transport_closed_callback, + .session_reset_callback = hc_session_reset_callback, + .builtin_app_rx_callback = hc_rx_callback, + .builtin_app_tx_callback = hc_tx_callback, + .half_open_cleanup_callback = hc_ho_cleanup_callback, +}; + +static clib_error_t * +hc_attach () +{ + hc_main_t *hcm = &hc_main; + vnet_app_attach_args_t _a, *a = &_a; + u64 options[18]; + int rv; + + clib_memset (a, 0, sizeof (*a)); + clib_memset (options, 0, sizeof (options)); + + a->api_client_index = APP_INVALID_INDEX; + a->name = format (0, "http_client"); + a->session_cb_vft = &hc_session_cb_vft; + a->options = options; + a->options[APP_OPTIONS_FLAGS] = APP_OPTIONS_FLAGS_IS_BUILTIN; + + if ((rv = vnet_application_attach (a))) + return clib_error_return (0, "attach returned: %U", format_session_error, + rv); + + hcm->app_index = a->app_index; + vec_free (a->name); + hcm->attached = 1; + + return 0; +} + +static int +hc_connect_rpc (void *rpc_args) +{ + vnet_connect_args_t *a = rpc_args; + int rv; + + rv = vnet_connect (a); + if (rv > 0) + clib_warning (0, "connect returned: %U", format_session_error, rv); + + vec_free (a); + return rv; +} + +static void +hc_connect () +{ + hc_main_t *hcm = &hc_main; + vnet_connect_args_t *a = 0; + hc_worker_t *wrk; + hc_session_t *hc_session; + + vec_validate (a, 0); + clib_memset (a, 0, sizeof (a[0])); + + clib_memcpy (&a->sep_ext, &hcm->connect_sep, sizeof (hcm->connect_sep)); + a->app_index = hcm->app_index; + + /* allocate http session on main thread */ + wrk = hc_worker_get (0); + hc_session = hc_session_alloc (wrk); + a->api_context = hc_session->session_index; + + session_send_rpc_evt_to_thread_force (transport_cl_thread (), hc_connect_rpc, + a); +} + +static clib_error_t * +hc_run (vlib_main_t *vm) +{ + hc_main_t *hcm = &hc_main; + vlib_thread_main_t *vtm = vlib_get_thread_main (); + u32 num_threads; + hc_worker_t *wrk; + uword event_type, *event_data = 0; + clib_error_t *err; + FILE *file_ptr; + + num_threads = 1 /* main thread */ + vtm->n_threads; + vec_validate (hcm->wrk, num_threads - 1); + vec_foreach (wrk, hcm->wrk) + wrk->thread_index = wrk - hcm->wrk; + + if ((err = hc_attach ())) + return clib_error_return (0, "http client attach: %U", format_clib_error, + err); + + hc_connect (); + + vlib_process_wait_for_event_or_clock (vm, hcm->timeout); + event_type = vlib_process_get_events (vm, &event_data); + switch (event_type) + { + case ~0: + err = clib_error_return (0, "error: timeout"); + break; + case HC_CONNECT_FAILED: + err = clib_error_return (0, "error: failed to connect"); + break; + case HC_TRANSPORT_CLOSED: + err = clib_error_return (0, "error: transport closed"); + break; + case HC_REPLY_RECEIVED: + if (hcm->filename) + { + file_ptr = + fopen ((char *) format (0, "/tmp/%v", hcm->filename), "w"); + if (file_ptr == NULL) + { + vlib_cli_output (vm, "couldn't open file %v", hcm->filename); + } + else + { + fprintf (file_ptr, "< %s\n< %s\n< %s", hcm->response_status, + hcm->resp_headers, hcm->http_response); + fclose (file_ptr); + vlib_cli_output (vm, "file saved (/tmp/%v)", hcm->filename); + } + } + if (hcm->verbose) + vlib_cli_output (vm, "< %v\n< %v", hcm->response_status, + hcm->resp_headers); + vlib_cli_output (vm, "<\n%v", hcm->http_response); + + break; + default: + err = clib_error_return (0, "error: unexpected event %d", event_type); + break; + } + + vec_free (event_data); + return err; +} + +static int +hc_detach () +{ + hc_main_t *hcm = &hc_main; + vnet_app_detach_args_t _da, *da = &_da; + int rv; + + if (!hcm->attached) + return 0; + + da->app_index = hcm->app_index; + da->api_client_index = APP_INVALID_INDEX; + rv = vnet_application_detach (da); + hcm->attached = 0; + hcm->app_index = APP_INVALID_INDEX; + + return rv; +} + +static void +hcc_worker_cleanup (hc_worker_t *wrk) +{ + pool_free (wrk->sessions); +} + +static void +hc_cleanup () +{ + hc_main_t *hcm = &hc_main; + hc_worker_t *wrk; + http_header_ht_t *header; + + vec_foreach (wrk, hcm->wrk) + hcc_worker_cleanup (wrk); + + vec_free (hcm->uri); + vec_free (hcm->target); + vec_free (hcm->headers_buf); + vec_free (hcm->data); + vec_free (hcm->resp_headers); + vec_free (hcm->http_response); + vec_free (hcm->response_status); + vec_free (hcm->wrk); + vec_free (hcm->filename); + vec_foreach (header, hcm->custom_header) + { + vec_free (header->name); + vec_free (header->value); + } + vec_free (hcm->custom_header); +} + +static clib_error_t * +hc_command_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + hc_main_t *hcm = &hc_main; + clib_error_t *err = 0; + unformat_input_t _line_input, *line_input = &_line_input; + u8 *path = 0; + u8 *file_data; + http_header_ht_t new_header; + u8 *name; + u8 *value; + int rv; + hcm->timeout = 10; + + if (hcm->attached) + return clib_error_return (0, "failed: already running!"); + + hcm->use_ptr = 0; + + /* Get a line of input. */ + if (!unformat_user (input, unformat_line_input, line_input)) + return clib_error_return (0, "expected required arguments"); + + hcm->req_method = + (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) && + unformat (line_input, "post") ? + HTTP_REQ_POST : + HTTP_REQ_GET; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "uri %s", &hcm->uri)) + ; + else if (unformat (line_input, "data %v", &hcm->data)) + hcm->is_file = 0; + else if (unformat (line_input, "target %s", &hcm->target)) + ; + else if (unformat (line_input, "file %s", &path)) + hcm->is_file = 1; + else if (unformat (line_input, "use-ptr")) + hcm->use_ptr = 1; + else if (unformat (line_input, "save-to %s", &hcm->filename)) + { + if (strstr ((char *) hcm->filename, "..") || + strchr ((char *) hcm->filename, '/')) + { + err = clib_error_return ( + 0, "illegal characters in filename '%v'", hcm->filename); + goto done; + } + } + else if (unformat (line_input, "header %v:%v", &name, &value)) + { + new_header.name = name; + new_header.value = value; + vec_add1 (hcm->custom_header, new_header); + } + else if (unformat (line_input, "verbose")) + hcm->verbose = true; + else if (unformat (line_input, "timeout %f", &hcm->timeout)) + ; + else + { + err = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } + } + + if (!hcm->uri) + { + err = clib_error_return (0, "URI not defined"); + goto done; + } + if (!hcm->target) + { + err = clib_error_return (0, "target not defined"); + goto done; + } + if (!hcm->data && hcm->req_method == HTTP_REQ_POST) + { + if (path) + { + err = clib_file_contents ((char *) path, &file_data); + if (err) + goto done; + hcm->data = file_data; + } + else + { + err = clib_error_return (0, "data not defined"); + goto done; + } + } + + if ((rv = parse_uri ((char *) hcm->uri, &hcm->connect_sep))) + { + err = + clib_error_return (0, "URI parse error: %U", format_session_error, rv); + goto done; + } + + session_enable_disable_args_t args = { .is_en = 1, + .rt_engine_type = + RT_BACKEND_ENGINE_RULE_TABLE }; + vlib_worker_thread_barrier_sync (vm); + vnet_session_enable_disable (vm, &args); + vlib_worker_thread_barrier_release (vm); + + hcm->cli_node_index = vlib_get_current_process (vm)->node_runtime.node_index; + + err = hc_run (vm); + + if ((rv = hc_detach ())) + { + /* don't override last error */ + if (!err) + err = clib_error_return (0, "detach returned: %U", + format_session_error, rv); + else + clib_warning ("warning: detach returned: %U", format_session_error, + rv); + } + +done: + vec_free (path); + hc_cleanup (); + unformat_free (line_input); + return err; +} + +VLIB_CLI_COMMAND (hc_command, static) = { + .path = "http client", + .short_help = "[post] uri http://<ip-addr> target <origin-form> " + "[data <form-urlencoded> | file <file-path>] [use-ptr] " + "[save-to <filename>] [header <Key:Value>] [verbose] " + "[timeout <seconds> (default = 10)]", + .function = hc_command_fn, + .is_mp_safe = 1, +}; + +static clib_error_t * +hc_main_init () +{ + hc_main_t *hcm = &hc_main; + hcm->app_index = APP_INVALID_INDEX; + return 0; +} + +VLIB_INIT_FUNCTION (hc_main_init); diff --git a/src/plugins/hs_apps/http_client_cli.c b/src/plugins/hs_apps/http_client_cli.c index 4b8ef173bd9..861af7f03e2 100644 --- a/src/plugins/hs_apps/http_client_cli.c +++ b/src/plugins/hs_apps/http_client_cli.c @@ -98,6 +98,13 @@ hcc_session_get (u32 hs_index, u32 thread_index) return pool_elt_at_index (wrk->sessions, hs_index); } +static void +hcc_ho_session_free (u32 hs_index) +{ + hcc_worker_t *wrk = hcc_worker_get (0); + pool_put_index (wrk->sessions, hs_index); +} + static int hcc_ts_accept_callback (session_t *ts) { @@ -125,9 +132,10 @@ hcc_ts_connected_callback (u32 app_index, u32 hc_index, session_t *as, hcc_worker_t *wrk; http_msg_t msg; u8 *headers_buf; + u32 new_hs_index; int rv; - HCC_DBG ("hc_index: %d", hc_index); + HCC_DBG ("ho hc_index: %d", hc_index); if (err) { @@ -138,19 +146,22 @@ hcc_ts_connected_callback (u32 app_index, u32 hc_index, session_t *as, return -1; } - /* TODO delete half open session once the support is added in http layer */ hs = hcc_session_get (hc_index, 0); wrk = hcc_worker_get (as->thread_index); new_hs = hcc_session_alloc (wrk); + new_hs_index = new_hs->session_index; clib_memcpy_fast (new_hs, hs, sizeof (*hs)); + new_hs->session_index = new_hs_index; + new_hs->thread_index = as->thread_index; + new_hs->vpp_session_index = as->session_index; + HCC_DBG ("new hc_index: %d", new_hs->session_index); + as->opaque = new_hs_index; - hs->vpp_session_index = as->session_index; - - http_add_header (&hs->req_headers, + http_add_header (&new_hs->req_headers, http_header_name_token (HTTP_HEADER_ACCEPT), http_content_type_token (HTTP_CONTENT_TEXT_HTML)); - headers_buf = http_serialize_headers (hs->req_headers); - vec_free (hs->req_headers); + headers_buf = http_serialize_headers (new_hs->req_headers); + vec_free (new_hs->req_headers); msg.type = HTTP_MSG_REQUEST; msg.method_type = HTTP_REQ_GET; @@ -300,6 +311,13 @@ hcc_ts_transport_closed (session_t *s) HCC_TRANSPORT_CLOSED, 0); } +static void +hcc_ho_cleanup_callback (session_t *ts) +{ + HCC_DBG ("ho hc_index: %d:", ts->opaque); + hcc_ho_session_free (ts->opaque); +} + static session_cb_vft_t hcc_session_cb_vft = { .session_accept_callback = hcc_ts_accept_callback, .session_disconnect_callback = hcc_ts_disconnect_callback, @@ -308,6 +326,7 @@ static session_cb_vft_t hcc_session_cb_vft = { .builtin_app_tx_callback = hcc_ts_tx_callback, .session_reset_callback = hcc_ts_reset_callback, .session_transport_closed_callback = hcc_ts_transport_closed, + .half_open_cleanup_callback = hcc_ho_cleanup_callback, }; static clib_error_t * @@ -362,6 +381,7 @@ hcc_connect_rpc (void *rpc_args) if (rv) clib_warning (0, "connect returned: %U", format_session_error, rv); + session_endpoint_free_ext_cfgs (&a->sep_ext); vec_free (a); return rv; } @@ -380,6 +400,7 @@ hcc_connect () hcc_main_t *hcm = &hcc_main; hcc_worker_t *wrk; hcc_session_t *hs; + transport_endpt_ext_cfg_t *ext_cfg; vec_validate (a, 0); clib_memset (a, 0, sizeof (a[0])); @@ -387,6 +408,11 @@ hcc_connect () clib_memcpy (&a->sep_ext, &hcm->connect_sep, sizeof (hcm->connect_sep)); a->app_index = hcm->app_index; + /* set http (response) timeout to 10 seconds */ + ext_cfg = session_endpoint_add_ext_cfg ( + &a->sep_ext, TRANSPORT_ENDPT_EXT_CFG_HTTP, sizeof (ext_cfg->opaque)); + ext_cfg->opaque = 10; + /* allocate http session on main thread */ wrk = hcc_worker_get (0); hs = hcc_session_alloc (wrk); @@ -407,7 +433,7 @@ hcc_run (vlib_main_t *vm, int print_output) hcc_worker_t *wrk; num_threads = 1 /* main thread */ + vtm->n_threads; - vec_validate (hcm->wrk, num_threads); + vec_validate (hcm->wrk, num_threads - 1); vec_foreach (wrk, hcm->wrk) { wrk->thread_index = wrk - hcm->wrk; diff --git a/src/plugins/hs_apps/http_simple_post.c b/src/plugins/hs_apps/http_simple_post.c deleted file mode 100644 index 6212eac1c97..00000000000 --- a/src/plugins/hs_apps/http_simple_post.c +++ /dev/null @@ -1,581 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 - * Copyright(c) 2024 Cisco Systems, Inc. - */ - -#include <vnet/session/application.h> -#include <vnet/session/application_interface.h> -#include <vnet/session/session.h> -#include <http/http.h> -#include <http/http_header_names.h> -#include <http/http_content_types.h> -#include <vppinfra/unix.h> - -typedef struct -{ - CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); - u32 session_index; - u32 thread_index; - u32 vpp_session_index; - u8 is_closed; -} hsp_session_t; - -typedef struct -{ - hsp_session_t *sessions; - u32 thread_index; -} hsp_worker_t; - -typedef struct -{ - u32 app_index; - vlib_main_t *vlib_main; - u32 cli_node_index; - u8 attached; - u8 *uri; - session_endpoint_cfg_t connect_sep; - u8 *target; - u8 *headers_buf; - u8 *data; - u64 data_offset; - hsp_worker_t *wrk; - u8 *http_response; - u8 is_file; - u8 use_ptr; -} hsp_main_t; - -typedef enum -{ - HSP_CONNECT_FAILED = 1, - HSP_TRANSPORT_CLOSED, - HSP_REPLY_RECEIVED, -} hsp_cli_signal_t; - -static hsp_main_t hsp_main; - -static inline hsp_worker_t * -hsp_worker_get (u32 thread_index) -{ - return &hsp_main.wrk[thread_index]; -} - -static inline hsp_session_t * -hsp_session_get (u32 session_index, u32 thread_index) -{ - hsp_worker_t *wrk = hsp_worker_get (thread_index); - return pool_elt_at_index (wrk->sessions, session_index); -} - -static hsp_session_t * -hsp_session_alloc (hsp_worker_t *wrk) -{ - hsp_session_t *s; - - pool_get_zero (wrk->sessions, s); - s->session_index = s - wrk->sessions; - s->thread_index = wrk->thread_index; - - return s; -} - -static int -hsp_session_connected_callback (u32 app_index, u32 hsp_session_index, - session_t *s, session_error_t err) -{ - hsp_main_t *hspm = &hsp_main; - hsp_session_t *hsp_session, *new_hsp_session; - hsp_worker_t *wrk; - http_header_t *headers = 0; - http_msg_t msg; - u64 to_send; - u32 n_enq; - int rv; - - if (err) - { - clib_warning ("hsp_session_index[%d] connected error: %U", - hsp_session_index, format_session_error, err); - vlib_process_signal_event_mt (hspm->vlib_main, hspm->cli_node_index, - HSP_CONNECT_FAILED, 0); - return -1; - } - - hsp_session = hsp_session_get (hsp_session_index, 0); - wrk = hsp_worker_get (s->thread_index); - new_hsp_session = hsp_session_alloc (wrk); - clib_memcpy_fast (new_hsp_session, hsp_session, sizeof (*hsp_session)); - hsp_session->vpp_session_index = s->session_index; - - if (hspm->is_file) - { - http_add_header ( - &headers, http_header_name_token (HTTP_HEADER_CONTENT_TYPE), - http_content_type_token (HTTP_CONTENT_APP_OCTET_STREAM)); - } - else - { - http_add_header ( - &headers, http_header_name_token (HTTP_HEADER_CONTENT_TYPE), - http_content_type_token (HTTP_CONTENT_APP_X_WWW_FORM_URLENCODED)); - } - hspm->headers_buf = http_serialize_headers (headers); - vec_free (headers); - - msg.type = HTTP_MSG_REQUEST; - msg.method_type = HTTP_REQ_POST; - /* request target */ - msg.data.target_form = HTTP_TARGET_ORIGIN_FORM; - msg.data.target_path_len = vec_len (hspm->target); - /* custom headers */ - msg.data.headers_len = vec_len (hspm->headers_buf); - /* request body */ - msg.data.body_len = vec_len (hspm->data); - /* total length */ - msg.data.len = - msg.data.target_path_len + msg.data.headers_len + msg.data.body_len; - - if (hspm->use_ptr) - { - uword target = pointer_to_uword (hspm->target); - uword headers = pointer_to_uword (hspm->headers_buf); - uword body = pointer_to_uword (hspm->data); - msg.data.type = HTTP_MSG_DATA_PTR; - svm_fifo_seg_t segs[4] = { - { (u8 *) &msg, sizeof (msg) }, - { (u8 *) &target, sizeof (target) }, - { (u8 *) &headers, sizeof (headers) }, - { (u8 *) &body, sizeof (body) }, - }; - - rv = - svm_fifo_enqueue_segments (s->tx_fifo, segs, 4, 0 /* allow partial */); - ASSERT (rv == (sizeof (msg) + sizeof (target) + sizeof (headers) + - sizeof (body))); - goto done; - } - - msg.data.type = HTTP_MSG_DATA_INLINE; - msg.data.target_path_offset = 0; - msg.data.headers_offset = msg.data.target_path_len; - msg.data.body_offset = msg.data.headers_offset + msg.data.headers_len; - - rv = svm_fifo_enqueue (s->tx_fifo, sizeof (msg), (u8 *) &msg); - ASSERT (rv == sizeof (msg)); - - rv = svm_fifo_enqueue (s->tx_fifo, vec_len (hspm->target), hspm->target); - ASSERT (rv == vec_len (hspm->target)); - - rv = svm_fifo_enqueue (s->tx_fifo, vec_len (hspm->headers_buf), - hspm->headers_buf); - ASSERT (rv == msg.data.headers_len); - - to_send = vec_len (hspm->data); - n_enq = clib_min (svm_fifo_size (s->tx_fifo), to_send); - - rv = svm_fifo_enqueue (s->tx_fifo, n_enq, hspm->data); - - if (rv < to_send) - { - hspm->data_offset = (rv > 0) ? rv : 0; - svm_fifo_add_want_deq_ntf (s->tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); - } - -done: - if (svm_fifo_set_event (s->tx_fifo)) - session_program_tx_io_evt (s->handle, SESSION_IO_EVT_TX); - - return 0; -} - -static void -hsp_session_disconnect_callback (session_t *s) -{ - hsp_main_t *hspm = &hsp_main; - vnet_disconnect_args_t _a = { 0 }, *a = &_a; - int rv; - - a->handle = session_handle (s); - a->app_index = hspm->app_index; - if ((rv = vnet_disconnect_session (a))) - clib_warning ("warning: disconnect returned: %U", format_session_error, - rv); -} - -static void -hsp_session_transport_closed_callback (session_t *s) -{ - hsp_main_t *hspm = &hsp_main; - - vlib_process_signal_event_mt (hspm->vlib_main, hspm->cli_node_index, - HSP_TRANSPORT_CLOSED, 0); -} - -static void -hsp_session_reset_callback (session_t *s) -{ - hsp_main_t *hspm = &hsp_main; - hsp_session_t *hsp_session; - vnet_disconnect_args_t _a = { 0 }, *a = &_a; - int rv; - - hsp_session = hsp_session_get (s->opaque, s->thread_index); - hsp_session->is_closed = 1; - - a->handle = session_handle (s); - a->app_index = hspm->app_index; - if ((rv = vnet_disconnect_session (a))) - clib_warning ("warning: disconnect returned: %U", format_session_error, - rv); -} - -static int -hsp_rx_callback (session_t *s) -{ - hsp_main_t *hspm = &hsp_main; - hsp_session_t *hsp_session; - http_msg_t msg; - int rv; - - hsp_session = hsp_session_get (s->opaque, s->thread_index); - - if (hsp_session->is_closed) - { - clib_warning ("hsp_session_index[%d] is closed", s->opaque); - return -1; - } - - rv = svm_fifo_dequeue (s->rx_fifo, sizeof (msg), (u8 *) &msg); - ASSERT (rv == sizeof (msg)); - - if (msg.type != HTTP_MSG_REPLY) - { - clib_warning ("unexpected msg type %d", msg.type); - return -1; - } - - svm_fifo_dequeue_drop_all (s->rx_fifo); - - if (msg.code == HTTP_STATUS_OK) - hspm->http_response = format (0, "request success"); - else - hspm->http_response = format (0, "request failed"); - - hsp_session_disconnect_callback (s); - vlib_process_signal_event_mt (hspm->vlib_main, hspm->cli_node_index, - HSP_REPLY_RECEIVED, 0); - return 0; -} - -static int -hsp_tx_callback (session_t *s) -{ - hsp_main_t *hspm = &hsp_main; - u64 to_send; - u32 n_enq; - int rv; - - to_send = vec_len (hspm->data) - hspm->data_offset; - n_enq = clib_min (svm_fifo_size (s->tx_fifo), to_send); - - rv = svm_fifo_enqueue (s->tx_fifo, n_enq, hspm->data + hspm->data_offset); - - if (rv <= 0) - { - svm_fifo_add_want_deq_ntf (s->tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); - return 0; - } - - if (rv < to_send) - { - hspm->data_offset += rv; - svm_fifo_add_want_deq_ntf (s->tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); - } - - if (svm_fifo_set_event (s->tx_fifo)) - session_program_tx_io_evt (s->handle, SESSION_IO_EVT_TX); - - return 0; -} - -static session_cb_vft_t hsp_session_cb_vft = { - .session_connected_callback = hsp_session_connected_callback, - .session_disconnect_callback = hsp_session_disconnect_callback, - .session_transport_closed_callback = hsp_session_transport_closed_callback, - .session_reset_callback = hsp_session_reset_callback, - .builtin_app_rx_callback = hsp_rx_callback, - .builtin_app_tx_callback = hsp_tx_callback, -}; - -static clib_error_t * -hsp_attach () -{ - hsp_main_t *hspm = &hsp_main; - vnet_app_attach_args_t _a, *a = &_a; - u64 options[18]; - int rv; - - clib_memset (a, 0, sizeof (*a)); - clib_memset (options, 0, sizeof (options)); - - a->api_client_index = APP_INVALID_INDEX; - a->name = format (0, "http_simple_post"); - a->session_cb_vft = &hsp_session_cb_vft; - a->options = options; - a->options[APP_OPTIONS_FLAGS] = APP_OPTIONS_FLAGS_IS_BUILTIN; - - if ((rv = vnet_application_attach (a))) - return clib_error_return (0, "attach returned: %U", format_session_error, - rv); - - hspm->app_index = a->app_index; - vec_free (a->name); - hspm->attached = 1; - - return 0; -} - -static int -hsp_connect_rpc (void *rpc_args) -{ - vnet_connect_args_t *a = rpc_args; - int rv; - - rv = vnet_connect (a); - if (rv) - clib_warning (0, "connect returned: %U", format_session_error, rv); - - vec_free (a); - return rv; -} - -static void -hsp_connect () -{ - hsp_main_t *hspm = &hsp_main; - vnet_connect_args_t *a = 0; - hsp_worker_t *wrk; - hsp_session_t *hsp_session; - - vec_validate (a, 0); - clib_memset (a, 0, sizeof (a[0])); - - clib_memcpy (&a->sep_ext, &hspm->connect_sep, sizeof (hspm->connect_sep)); - a->app_index = hspm->app_index; - - /* allocate http session on main thread */ - wrk = hsp_worker_get (0); - hsp_session = hsp_session_alloc (wrk); - a->api_context = hsp_session->session_index; - - session_send_rpc_evt_to_thread_force (transport_cl_thread (), - hsp_connect_rpc, a); -} - -static clib_error_t * -hsp_run (vlib_main_t *vm) -{ - hsp_main_t *hspm = &hsp_main; - vlib_thread_main_t *vtm = vlib_get_thread_main (); - u32 num_threads; - hsp_worker_t *wrk; - uword event_type, *event_data = 0; - clib_error_t *err; - - num_threads = 1 /* main thread */ + vtm->n_threads; - vec_validate (hspm->wrk, num_threads); - vec_foreach (wrk, hspm->wrk) - wrk->thread_index = wrk - hspm->wrk; - - if ((err = hsp_attach ())) - return clib_error_return (0, "http simple post attach: %U", - format_clib_error, err); - - hsp_connect (); - - vlib_process_wait_for_event_or_clock (vm, 10); - event_type = vlib_process_get_events (vm, &event_data); - switch (event_type) - { - case ~0: - err = clib_error_return (0, "error: timeout"); - break; - case HSP_CONNECT_FAILED: - err = clib_error_return (0, "error: failed to connect"); - break; - case HSP_TRANSPORT_CLOSED: - err = clib_error_return (0, "error: transport closed"); - break; - case HSP_REPLY_RECEIVED: - vlib_cli_output (vm, "%v", hspm->http_response); - break; - default: - err = clib_error_return (0, "error: unexpected event %d", event_type); - break; - } - - vec_free (event_data); - return err; -} - -static int -hsp_detach () -{ - hsp_main_t *hspm = &hsp_main; - vnet_app_detach_args_t _da, *da = &_da; - int rv; - - if (!hspm->attached) - return 0; - - da->app_index = hspm->app_index; - da->api_client_index = APP_INVALID_INDEX; - rv = vnet_application_detach (da); - hspm->attached = 0; - hspm->app_index = APP_INVALID_INDEX; - - return rv; -} - -static void -hcc_worker_cleanup (hsp_worker_t *wrk) -{ - pool_free (wrk->sessions); -} - -static void -hsp_cleanup () -{ - hsp_main_t *hspm = &hsp_main; - hsp_worker_t *wrk; - - vec_foreach (wrk, hspm->wrk) - hcc_worker_cleanup (wrk); - - vec_free (hspm->uri); - vec_free (hspm->target); - vec_free (hspm->headers_buf); - vec_free (hspm->data); - vec_free (hspm->http_response); - vec_free (hspm->wrk); -} - -static clib_error_t * -hsp_command_fn (vlib_main_t *vm, unformat_input_t *input, - vlib_cli_command_t *cmd) -{ - hsp_main_t *hspm = &hsp_main; - clib_error_t *err = 0; - unformat_input_t _line_input, *line_input = &_line_input; - u8 *path = 0; - u8 *file_data; - int rv; - - if (hspm->attached) - return clib_error_return (0, "failed: already running!"); - - hspm->use_ptr = 0; - - /* Get a line of input. */ - if (!unformat_user (input, unformat_line_input, line_input)) - return clib_error_return (0, "expected required arguments"); - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (line_input, "uri %s", &hspm->uri)) - ; - else if (unformat (line_input, "data %v", &hspm->data)) - hspm->is_file = 0; - else if (unformat (line_input, "target %s", &hspm->target)) - ; - else if (unformat (line_input, "file %s", &path)) - hspm->is_file = 1; - else if (unformat (line_input, "use-ptr")) - hspm->use_ptr = 1; - else - { - err = clib_error_return (0, "unknown input `%U'", - format_unformat_error, line_input); - goto done; - } - } - - if (!hspm->uri) - { - err = clib_error_return (0, "URI not defined"); - goto done; - } - if (!hspm->target) - { - err = clib_error_return (0, "target not defined"); - goto done; - } - if (!hspm->data) - { - if (path) - { - err = clib_file_contents ((char *) path, &file_data); - if (err) - goto done; - hspm->data = file_data; - } - else - { - err = clib_error_return (0, "data not defined"); - goto done; - } - } - - if ((rv = parse_uri ((char *) hspm->uri, &hspm->connect_sep))) - { - err = - clib_error_return (0, "URI parse error: %U", format_session_error, rv); - goto done; - } - - session_enable_disable_args_t args = { .is_en = 1, - .rt_engine_type = - RT_BACKEND_ENGINE_RULE_TABLE }; - vlib_worker_thread_barrier_sync (vm); - vnet_session_enable_disable (vm, &args); - vlib_worker_thread_barrier_release (vm); - - hspm->cli_node_index = - vlib_get_current_process (vm)->node_runtime.node_index; - - err = hsp_run (vm); - - if ((rv = hsp_detach ())) - { - /* don't override last error */ - if (!err) - err = clib_error_return (0, "detach returned: %U", - format_session_error, rv); - else - clib_warning ("warning: detach returned: %U", format_session_error, - rv); - } - -done: - hsp_cleanup (); - unformat_free (line_input); - return err; -} - -VLIB_CLI_COMMAND (hsp_command, static) = { - .path = "http post", - .short_help = "uri http://<ip-addr> target <origin-form> " - "[data <form-urlencoded> | file <file-path>] [use-ptr]", - .function = hsp_command_fn, - .is_mp_safe = 1, -}; - -static clib_error_t * -hsp_main_init (vlib_main_t *vm) -{ - hsp_main_t *hspm = &hsp_main; - - hspm->app_index = APP_INVALID_INDEX; - hspm->vlib_main = vm; - return 0; -} - -VLIB_INIT_FUNCTION (hsp_main_init); diff --git a/src/plugins/hs_apps/http_tps.c b/src/plugins/hs_apps/http_tps.c index cdeafa5d54a..a40a31caf63 100644 --- a/src/plugins/hs_apps/http_tps.c +++ b/src/plugins/hs_apps/http_tps.c @@ -641,15 +641,16 @@ hts_start_listen (hts_main_t *htm, session_endpoint_cfg_t *sep, u8 *uri, if (need_crypto) { - session_endpoint_alloc_ext_cfg (&a->sep_ext, - TRANSPORT_ENDPT_EXT_CFG_CRYPTO); - a->sep_ext.ext_cfg->crypto.ckpair_index = htm->ckpair_index; + transport_endpt_ext_cfg_t *ext_cfg = session_endpoint_add_ext_cfg ( + &a->sep_ext, TRANSPORT_ENDPT_EXT_CFG_CRYPTO, + sizeof (transport_endpt_crypto_cfg_t)); + ext_cfg->crypto.ckpair_index = htm->ckpair_index; } rv = vnet_listen (a); if (need_crypto) - clib_mem_free (a->sep_ext.ext_cfg); + session_endpoint_free_ext_cfgs (&a->sep_ext); if (rv) return rv; diff --git a/src/plugins/hs_apps/proxy.c b/src/plugins/hs_apps/proxy.c index c7e7b2a653c..d7fe6fb54df 100644 --- a/src/plugins/hs_apps/proxy.c +++ b/src/plugins/hs_apps/proxy.c @@ -19,50 +19,145 @@ #include <vnet/session/application_interface.h> #include <hs_apps/proxy.h> #include <vnet/tcp/tcp.h> +#include <http/http.h> +#include <http/http_header_names.h> proxy_main_t proxy_main; #define TCP_MSS 1460 -typedef struct +static proxy_session_side_ctx_t * +proxy_session_side_ctx_alloc (proxy_worker_t *wrk) { - session_endpoint_cfg_t sep; - u32 app_index; - u32 api_context; -} proxy_connect_args_t; + proxy_session_side_ctx_t *ctx; + + pool_get_zero (wrk->ctx_pool, ctx); + ctx->sc_index = ctx - wrk->ctx_pool; + ctx->ps_index = ~0; + + return ctx; +} static void -proxy_cb_fn (void *data, u32 data_len) +proxy_session_side_ctx_free (proxy_worker_t *wrk, + proxy_session_side_ctx_t *ctx) { - proxy_connect_args_t *pa = (proxy_connect_args_t *) data; - vnet_connect_args_t a; + pool_put (wrk->ctx_pool, ctx); +} - clib_memset (&a, 0, sizeof (a)); - a.api_context = pa->api_context; - a.app_index = pa->app_index; - clib_memcpy (&a.sep_ext, &pa->sep, sizeof (pa->sep)); - vnet_connect (&a); - if (a.sep_ext.ext_cfg) - clib_mem_free (a.sep_ext.ext_cfg); +static proxy_session_side_ctx_t * +proxy_session_side_ctx_get (proxy_worker_t *wrk, u32 ctx_index) +{ + return pool_elt_at_index (wrk->ctx_pool, ctx_index); } static void -proxy_call_main_thread (vnet_connect_args_t * a) +proxy_send_http_resp (session_t *s, http_status_code_t sc, + http_header_t *resp_headers) { - if (vlib_get_thread_index () == 0) + http_msg_t msg; + int rv; + u8 *headers_buf = 0; + + if (vec_len (resp_headers)) { - vnet_connect (a); - if (a->sep_ext.ext_cfg) - clib_mem_free (a->sep_ext.ext_cfg); + headers_buf = http_serialize_headers (resp_headers); + msg.data.len = msg.data.headers_len = vec_len (headers_buf); } else + msg.data.len = msg.data.headers_len = 0; + + msg.type = HTTP_MSG_REPLY; + msg.code = sc; + msg.data.type = HTTP_MSG_DATA_INLINE; + msg.data.headers_offset = 0; + msg.data.body_len = 0; + msg.data.body_offset = 0; + rv = svm_fifo_enqueue (s->tx_fifo, sizeof (msg), (u8 *) &msg); + ASSERT (rv == sizeof (msg)); + if (msg.data.headers_len) { - proxy_connect_args_t args; - args.api_context = a->api_context; - args.app_index = a->app_index; - clib_memcpy (&args.sep, &a->sep_ext, sizeof (a->sep_ext)); - vl_api_rpc_call_main_thread (proxy_cb_fn, (u8 *) & args, sizeof (args)); + rv = svm_fifo_enqueue (s->tx_fifo, vec_len (headers_buf), headers_buf); + ASSERT (rv == vec_len (headers_buf)); + vec_free (headers_buf); } + + if (svm_fifo_set_event (s->tx_fifo)) + session_program_tx_io_evt (s->handle, SESSION_IO_EVT_TX); +} + +static void +proxy_do_connect (vnet_connect_args_t *a) +{ + ASSERT (session_vlib_thread_is_cl_thread ()); + vnet_connect (a); + session_endpoint_free_ext_cfgs (&a->sep_ext); +} + +static void +proxy_handle_connects_rpc (void *args) +{ + u32 thread_index = pointer_to_uword (args), n_connects = 0, n_pending; + proxy_worker_t *wrk; + u32 max_connects; + + wrk = proxy_worker_get (thread_index); + + clib_spinlock_lock (&wrk->pending_connects_lock); + + n_pending = clib_fifo_elts (wrk->pending_connects); + max_connects = clib_min (32, n_pending); + vec_validate (wrk->burst_connects, max_connects); + + while (n_connects < max_connects) + clib_fifo_sub1 (wrk->pending_connects, wrk->burst_connects[n_connects++]); + + clib_spinlock_unlock (&wrk->pending_connects_lock); + + /* Do connects without locking pending_connects */ + n_connects = 0; + while (n_connects < max_connects) + { + proxy_do_connect (&wrk->burst_connects[n_connects]); + n_connects += 1; + } + + /* More work to do, program rpc */ + if (max_connects < n_pending) + session_send_rpc_evt_to_thread_force ( + transport_cl_thread (), proxy_handle_connects_rpc, + uword_to_pointer ((uword) thread_index, void *)); +} + +static void +proxy_program_connect (vnet_connect_args_t *a) +{ + u32 connects_thread = transport_cl_thread (), thread_index, n_pending; + proxy_worker_t *wrk; + + thread_index = vlib_get_thread_index (); + + /* If already on first worker, handle request */ + if (thread_index == connects_thread) + { + proxy_do_connect (a); + return; + } + + /* If not on first worker, queue request */ + wrk = proxy_worker_get (thread_index); + + clib_spinlock_lock (&wrk->pending_connects_lock); + + clib_fifo_add1 (wrk->pending_connects, *a); + n_pending = clib_fifo_elts (wrk->pending_connects); + + clib_spinlock_unlock (&wrk->pending_connects_lock); + + if (n_pending == 1) + session_send_rpc_evt_to_thread_force ( + connects_thread, proxy_handle_connects_rpc, + uword_to_pointer ((uword) thread_index, void *)); } static proxy_session_t * @@ -85,16 +180,6 @@ proxy_session_get (u32 ps_index) return pool_elt_at_index (pm->sessions, ps_index); } -static inline proxy_session_t * -proxy_session_get_if_valid (u32 ps_index) -{ - proxy_main_t *pm = &proxy_main; - - if (pool_is_free_index (pm->sessions, ps_index)) - return 0; - return pool_elt_at_index (pm->sessions, ps_index); -} - static void proxy_session_free (proxy_session_t *ps) { @@ -115,7 +200,7 @@ proxy_session_postponed_free_rpc (void *arg) clib_spinlock_lock_if_init (&pm->sessions_lock); ps = proxy_session_get (ps_index); - segment_manager_dealloc_fifos (ps->server_rx_fifo, ps->server_tx_fifo); + segment_manager_dealloc_fifos (ps->po.rx_fifo, ps->po.tx_fifo); proxy_session_free (ps); clib_spinlock_unlock_if_init (&pm->sessions_lock); @@ -126,54 +211,79 @@ proxy_session_postponed_free_rpc (void *arg) static void proxy_session_postponed_free (proxy_session_t *ps) { - session_send_rpc_evt_to_thread (ps->po_thread_index, + /* Passive open session handle has been invalidated so we don't have thread + * index at this point */ + session_send_rpc_evt_to_thread (ps->po.rx_fifo->master_thread_index, proxy_session_postponed_free_rpc, uword_to_pointer (ps->ps_index, void *)); } static void +proxy_session_close_po (proxy_session_t *ps) +{ + vnet_disconnect_args_t _a = {}, *a = &_a; + proxy_main_t *pm = &proxy_main; + + ASSERT (!vlib_num_workers () || + CLIB_SPINLOCK_IS_LOCKED (&pm->sessions_lock)); + + a->handle = ps->po.session_handle; + a->app_index = pm->server_app_index; + vnet_disconnect_session (a); + + ps->po_disconnected = 1; +} + +static void +proxy_session_close_ao (proxy_session_t *ps) +{ + vnet_disconnect_args_t _a = {}, *a = &_a; + proxy_main_t *pm = &proxy_main; + + ASSERT (!vlib_num_workers () || + CLIB_SPINLOCK_IS_LOCKED (&pm->sessions_lock)); + + a->handle = ps->ao.session_handle; + a->app_index = pm->active_open_app_index; + vnet_disconnect_session (a); + + ps->ao_disconnected = 1; +} + +static void proxy_try_close_session (session_t * s, int is_active_open) { proxy_main_t *pm = &proxy_main; - proxy_session_t *ps = 0; - vnet_disconnect_args_t _a, *a = &_a; + proxy_session_side_ctx_t *sc; + proxy_session_t *ps; + proxy_worker_t *wrk; + + wrk = proxy_worker_get (s->thread_index); + sc = proxy_session_side_ctx_get (wrk, s->opaque); clib_spinlock_lock_if_init (&pm->sessions_lock); - ps = proxy_session_get (s->opaque); + ps = proxy_session_get (sc->ps_index); if (is_active_open) { - a->handle = ps->vpp_active_open_handle; - a->app_index = pm->active_open_app_index; - vnet_disconnect_session (a); - ps->ao_disconnected = 1; + proxy_session_close_ao (ps); if (!ps->po_disconnected) { - ASSERT (ps->vpp_server_handle != SESSION_INVALID_HANDLE); - a->handle = ps->vpp_server_handle; - a->app_index = pm->server_app_index; - vnet_disconnect_session (a); - ps->po_disconnected = 1; + ASSERT (ps->po.session_handle != SESSION_INVALID_HANDLE); + proxy_session_close_po (ps); } } else { - a->handle = ps->vpp_server_handle; - a->app_index = pm->server_app_index; - vnet_disconnect_session (a); - ps->po_disconnected = 1; + proxy_session_close_po (ps); if (!ps->ao_disconnected && !ps->active_open_establishing) { /* Proxy session closed before active open */ - if (ps->vpp_active_open_handle != SESSION_INVALID_HANDLE) - { - a->handle = ps->vpp_active_open_handle; - a->app_index = pm->active_open_app_index; - vnet_disconnect_session (a); - } + if (ps->ao.session_handle != SESSION_INVALID_HANDLE) + proxy_session_close_ao (ps); ps->ao_disconnected = 1; } } @@ -181,29 +291,63 @@ proxy_try_close_session (session_t * s, int is_active_open) } static void +proxy_try_side_ctx_cleanup (session_t *s) +{ + proxy_main_t *pm = &proxy_main; + proxy_session_t *ps; + proxy_session_side_ctx_t *sc; + proxy_worker_t *wrk; + + wrk = proxy_worker_get (s->thread_index); + sc = proxy_session_side_ctx_get (wrk, s->opaque); + if (sc->state == PROXY_SC_S_CREATED) + return; + + clib_spinlock_lock_if_init (&pm->sessions_lock); + + ps = proxy_session_get (sc->ps_index); + + if (!ps->po_disconnected) + proxy_session_close_po (ps); + + if (!ps->ao_disconnected) + proxy_session_close_ao (ps); + + clib_spinlock_unlock_if_init (&pm->sessions_lock); +} + +static void proxy_try_delete_session (session_t * s, u8 is_active_open) { proxy_main_t *pm = &proxy_main; proxy_session_t *ps = 0; + proxy_session_side_ctx_t *sc; + proxy_worker_t *wrk; + u32 ps_index; + + wrk = proxy_worker_get (s->thread_index); + sc = proxy_session_side_ctx_get (wrk, s->opaque); + ps_index = sc->ps_index; + + proxy_session_side_ctx_free (wrk, sc); clib_spinlock_lock_if_init (&pm->sessions_lock); - ps = proxy_session_get (s->opaque); + ps = proxy_session_get (ps_index); if (is_active_open) { - ps->vpp_active_open_handle = SESSION_INVALID_HANDLE; + ps->ao.session_handle = SESSION_INVALID_HANDLE; /* Revert master thread index change on connect notification */ - ps->server_rx_fifo->master_thread_index = ps->po_thread_index; + ps->po.rx_fifo->master_thread_index = + ps->po.tx_fifo->master_thread_index; /* Passive open already cleaned up */ - if (ps->vpp_server_handle == SESSION_INVALID_HANDLE) + if (ps->po.session_handle == SESSION_INVALID_HANDLE) { - ASSERT (s->rx_fifo->refcnt == 1); - /* The two sides of the proxy on different threads */ - if (ps->po_thread_index != s->thread_index) + if (ps->po.tx_fifo->master_thread_index != s->thread_index) { /* This is not the right thread to delete the fifos */ s->rx_fifo = 0; @@ -211,14 +355,17 @@ proxy_try_delete_session (session_t * s, u8 is_active_open) proxy_session_postponed_free (ps); } else - proxy_session_free (ps); + { + ASSERT (s->rx_fifo->refcnt == 1); + proxy_session_free (ps); + } } } else { - ps->vpp_server_handle = SESSION_INVALID_HANDLE; + ps->po.session_handle = SESSION_INVALID_HANDLE; - if (ps->vpp_active_open_handle == SESSION_INVALID_HANDLE) + if (ps->ao.session_handle == SESSION_INVALID_HANDLE) { if (!ps->active_open_establishing) proxy_session_free (ps); @@ -275,16 +422,26 @@ static int proxy_accept_callback (session_t * s) { proxy_main_t *pm = &proxy_main; + proxy_session_side_ctx_t *sc; proxy_session_t *ps; + proxy_worker_t *wrk; + transport_proto_t tp = session_get_transport_proto (s); + + wrk = proxy_worker_get (s->thread_index); + sc = proxy_session_side_ctx_alloc (wrk); + s->opaque = sc->sc_index; clib_spinlock_lock_if_init (&pm->sessions_lock); ps = proxy_session_alloc (); - ps->vpp_server_handle = session_handle (s); - ps->vpp_active_open_handle = SESSION_INVALID_HANDLE; - ps->po_thread_index = s->thread_index; - s->opaque = ps->ps_index; + ps->po.session_handle = session_handle (s); + ps->po.rx_fifo = s->rx_fifo; + ps->po.tx_fifo = s->tx_fifo; + + ps->ao.session_handle = SESSION_INVALID_HANDLE; + sc->ps_index = ps->ps_index; + sc->is_http = tp == TRANSPORT_PROTO_HTTP ? 1 : 0; clib_spinlock_unlock_if_init (&pm->sessions_lock); @@ -325,98 +482,166 @@ proxy_transport_needs_crypto (transport_proto_t proto) return proto == TRANSPORT_PROTO_TLS; } -static int -proxy_rx_callback (session_t * s) +static void +proxy_session_start_connect (proxy_session_side_ctx_t *sc, session_t *s) { + int actual_transfer __attribute__ ((unused)); + vnet_connect_args_t _a = {}, *a = &_a; proxy_main_t *pm = &proxy_main; - u32 thread_index = vlib_get_thread_index (); - svm_fifo_t *ao_tx_fifo; + u32 max_dequeue, ps_index; proxy_session_t *ps; - - ASSERT (s->thread_index == thread_index); + transport_proto_t tp = session_get_transport_proto (s); clib_spinlock_lock_if_init (&pm->sessions_lock); - ps = proxy_session_get (s->opaque); + ps = proxy_session_get (sc->ps_index); - if (PREDICT_TRUE (ps->vpp_active_open_handle != SESSION_INVALID_HANDLE)) + /* maybe we were already here */ + if (ps->active_open_establishing) { clib_spinlock_unlock_if_init (&pm->sessions_lock); + return; + } - ao_tx_fifo = s->rx_fifo; + ps->active_open_establishing = 1; + ps_index = ps->ps_index; - /* - * Send event for active open tx fifo - */ - if (svm_fifo_set_event (ao_tx_fifo)) + clib_spinlock_unlock_if_init (&pm->sessions_lock); + + if (tp == TRANSPORT_PROTO_HTTP) + { + http_msg_t msg; + u8 *target_buf = 0; + http_uri_t target_uri; + http_header_t *resp_headers = 0; + session_endpoint_cfg_t target_sep = SESSION_ENDPOINT_CFG_NULL; + int rv; + + rv = svm_fifo_dequeue (s->rx_fifo, sizeof (msg), (u8 *) &msg); + ASSERT (rv == sizeof (msg)); + + if (msg.type != HTTP_MSG_REQUEST) + { + proxy_send_http_resp (s, HTTP_STATUS_INTERNAL_ERROR, 0); + return; + } + if (msg.method_type != HTTP_REQ_CONNECT) + { + http_add_header (&resp_headers, + http_header_name_token (HTTP_HEADER_ALLOW), + http_token_lit ("CONNECT")); + proxy_send_http_resp (s, HTTP_STATUS_METHOD_NOT_ALLOWED, + resp_headers); + vec_free (resp_headers); + return; + } + + if (msg.data.target_form != HTTP_TARGET_AUTHORITY_FORM || + msg.data.target_path_len == 0) { - u32 ao_thread_index = ao_tx_fifo->master_thread_index; - u32 ao_session_index = ao_tx_fifo->shr->master_session_index; - if (session_send_io_evt_to_thread_custom (&ao_session_index, - ao_thread_index, - SESSION_IO_EVT_TX)) - clib_warning ("failed to enqueue tx evt"); + proxy_send_http_resp (s, HTTP_STATUS_BAD_REQUEST, 0); + return; } - if (svm_fifo_max_enqueue (ao_tx_fifo) <= TCP_MSS) - svm_fifo_add_want_deq_ntf (ao_tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); + /* read target uri */ + target_buf = vec_new (u8, msg.data.target_path_len); + rv = svm_fifo_peek (s->rx_fifo, msg.data.target_path_offset, + msg.data.target_path_len, target_buf); + ASSERT (rv == msg.data.target_path_len); + svm_fifo_dequeue_drop (s->rx_fifo, msg.data.len); + rv = http_parse_authority_form_target (target_buf, &target_uri); + vec_free (target_buf); + if (rv) + { + proxy_send_http_resp (s, HTTP_STATUS_BAD_REQUEST, 0); + return; + } + target_sep.is_ip4 = target_uri.is_ip4; + target_sep.ip = target_uri.ip; + target_sep.port = target_uri.port; + target_sep.transport_proto = TRANSPORT_PROTO_TCP; + clib_memcpy (&a->sep_ext, &target_sep, sizeof (target_sep)); } else { - vnet_connect_args_t _a, *a = &_a; - svm_fifo_t *tx_fifo, *rx_fifo; - u32 max_dequeue, ps_index; - int actual_transfer __attribute__ ((unused)); + max_dequeue = svm_fifo_max_dequeue_cons (s->rx_fifo); + if (PREDICT_FALSE (max_dequeue == 0)) + return; - /* maybe we were already here */ - if (ps->active_open_establishing) - { - clib_spinlock_unlock_if_init (&pm->sessions_lock); - return 0; - } + max_dequeue = clib_min (pm->rcv_buffer_size, max_dequeue); + actual_transfer = + svm_fifo_peek (s->rx_fifo, 0 /* relative_offset */, max_dequeue, + pm->rx_buf[s->thread_index]); - rx_fifo = s->rx_fifo; - tx_fifo = s->tx_fifo; + /* Expectation is that here actual data just received is parsed and based + * on its contents, the destination and parameters of the connect to the + * upstream are decided + */ - ASSERT (rx_fifo->master_thread_index == thread_index); - ASSERT (tx_fifo->master_thread_index == thread_index); + clib_memcpy (&a->sep_ext, &pm->client_sep, sizeof (pm->client_sep)); + } - max_dequeue = svm_fifo_max_dequeue_cons (s->rx_fifo); + a->api_context = ps_index; + a->app_index = pm->active_open_app_index; - if (PREDICT_FALSE (max_dequeue == 0)) + if (proxy_transport_needs_crypto (a->sep.transport_proto)) + { + transport_endpt_ext_cfg_t *ext_cfg = session_endpoint_add_ext_cfg ( + &a->sep_ext, TRANSPORT_ENDPT_EXT_CFG_CRYPTO, + sizeof (transport_endpt_crypto_cfg_t)); + ext_cfg->crypto.ckpair_index = pm->ckpair_index; + } + + proxy_program_connect (a); +} + +static int +proxy_rx_callback (session_t *s) +{ + proxy_session_side_ctx_t *sc; + svm_fifo_t *ao_tx_fifo; + proxy_session_t *ps; + proxy_worker_t *wrk; + + ASSERT (s->thread_index == vlib_get_thread_index ()); + + wrk = proxy_worker_get (s->thread_index); + sc = proxy_session_side_ctx_get (wrk, s->opaque); + + if (PREDICT_FALSE (sc->state < PROXY_SC_S_ESTABLISHED)) + { + proxy_main_t *pm = &proxy_main; + + if (sc->state == PROXY_SC_S_CREATED) { - clib_spinlock_unlock_if_init (&pm->sessions_lock); + proxy_session_start_connect (sc, s); + sc->state = PROXY_SC_S_CONNECTING; return 0; } - max_dequeue = clib_min (pm->rcv_buffer_size, max_dequeue); - actual_transfer = svm_fifo_peek (rx_fifo, 0 /* relative_offset */ , - max_dequeue, pm->rx_buf[thread_index]); + clib_spinlock_lock_if_init (&pm->sessions_lock); - /* $$$ your message in this space: parse url, etc. */ + ps = proxy_session_get (sc->ps_index); + sc->pair = ps->ao; - clib_memset (a, 0, sizeof (*a)); + clib_spinlock_unlock_if_init (&pm->sessions_lock); - ps->server_rx_fifo = rx_fifo; - ps->server_tx_fifo = tx_fifo; - ps->active_open_establishing = 1; - ps_index = ps->ps_index; + if (sc->pair.session_handle == SESSION_INVALID_HANDLE) + return 0; - clib_spinlock_unlock_if_init (&pm->sessions_lock); + sc->state = PROXY_SC_S_ESTABLISHED; + } - clib_memcpy (&a->sep_ext, &pm->client_sep, sizeof (pm->client_sep)); - a->api_context = ps_index; - a->app_index = pm->active_open_app_index; + ao_tx_fifo = s->rx_fifo; - if (proxy_transport_needs_crypto (a->sep.transport_proto)) - { - session_endpoint_alloc_ext_cfg (&a->sep_ext, - TRANSPORT_ENDPT_EXT_CFG_CRYPTO); - a->sep_ext.ext_cfg->crypto.ckpair_index = pm->ckpair_index; - } + /* + * Send event for active open tx fifo + */ + if (svm_fifo_set_event (ao_tx_fifo)) + session_program_tx_io_evt (sc->pair.session_handle, SESSION_IO_EVT_TX); - proxy_call_main_thread (a); - } + if (svm_fifo_max_enqueue (ao_tx_fifo) <= TCP_MSS) + svm_fifo_add_want_deq_ntf (ao_tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); return 0; } @@ -437,8 +662,8 @@ proxy_force_ack (void *handlep) static int proxy_tx_callback (session_t * proxy_s) { - proxy_main_t *pm = &proxy_main; - proxy_session_t *ps; + proxy_session_side_ctx_t *sc; + proxy_worker_t *wrk; u32 min_free; min_free = clib_min (svm_fifo_size (proxy_s->tx_fifo) >> 3, 128 << 10); @@ -448,21 +673,17 @@ proxy_tx_callback (session_t * proxy_s) return 0; } - clib_spinlock_lock_if_init (&pm->sessions_lock); - - ps = proxy_session_get (proxy_s->opaque); - - if (ps->vpp_active_open_handle == SESSION_INVALID_HANDLE) - goto unlock; + wrk = proxy_worker_get (proxy_s->thread_index); + sc = proxy_session_side_ctx_get (wrk, proxy_s->opaque); + if (sc->state < PROXY_SC_S_ESTABLISHED) + return 0; /* Force ack on active open side to update rcv wnd. Make sure it's done on * the right thread */ - void *arg = uword_to_pointer (ps->vpp_active_open_handle, void *); - session_send_rpc_evt_to_thread (ps->server_rx_fifo->master_thread_index, - proxy_force_ack, arg); - -unlock: - clib_spinlock_unlock_if_init (&pm->sessions_lock); + void *arg = uword_to_pointer (sc->pair.session_handle, void *); + session_send_rpc_evt_to_thread ( + session_thread_from_handle (sc->pair.session_handle), proxy_force_ack, + arg); return 0; } @@ -471,7 +692,10 @@ static void proxy_cleanup_callback (session_t * s, session_cleanup_ntf_t ntf) { if (ntf == SESSION_CLEANUP_TRANSPORT) - return; + { + proxy_try_side_ctx_cleanup (s); + return; + } proxy_try_delete_session (s, 0 /* is_active_open */ ); } @@ -497,10 +721,17 @@ active_open_alloc_session_fifos (session_t *s) clib_spinlock_lock_if_init (&pm->sessions_lock); + /* Active open opaque is pointing at proxy session */ ps = proxy_session_get (s->opaque); - txf = ps->server_rx_fifo; - rxf = ps->server_tx_fifo; + if (ps->po_disconnected) + { + clib_spinlock_unlock_if_init (&pm->sessions_lock); + return SESSION_E_ALLOC; + } + + txf = ps->po.rx_fifo; + rxf = ps->po.tx_fifo; /* * Reset the active-open tx-fifo master indices so the active-open session @@ -531,31 +762,43 @@ active_open_connected_callback (u32 app_index, u32 opaque, { proxy_main_t *pm = &proxy_main; proxy_session_t *ps; - u8 thread_index = vlib_get_thread_index (); - - /* - * Setup proxy session handle. - */ - clib_spinlock_lock_if_init (&pm->sessions_lock); - - ps = proxy_session_get (opaque); + proxy_worker_t *wrk; + proxy_session_side_ctx_t *sc; + session_t *po_s; + transport_proto_t tp; /* Connection failed */ if (err) { - vnet_disconnect_args_t _a, *a = &_a; + clib_spinlock_lock_if_init (&pm->sessions_lock); - a->handle = ps->vpp_server_handle; - a->app_index = pm->server_app_index; - vnet_disconnect_session (a); - ps->po_disconnected = 1; - } - else - { - ps->vpp_active_open_handle = session_handle (s); - ps->active_open_establishing = 0; + ps = proxy_session_get (opaque); + po_s = session_get_from_handle (ps->po.session_handle); + tp = session_get_transport_proto (po_s); + if (tp == TRANSPORT_PROTO_HTTP) + { + proxy_send_http_resp (po_s, HTTP_STATUS_BAD_GATEWAY, 0); + } + ps->ao_disconnected = 1; + proxy_session_close_po (ps); + + clib_spinlock_unlock_if_init (&pm->sessions_lock); + + return 0; } + wrk = proxy_worker_get (s->thread_index); + + clib_spinlock_lock_if_init (&pm->sessions_lock); + + ps = proxy_session_get (opaque); + + ps->ao.rx_fifo = s->rx_fifo; + ps->ao.tx_fifo = s->tx_fifo; + ps->ao.session_handle = session_handle (s); + + ps->active_open_establishing = 0; + /* Passive open session was already closed! */ if (ps->po_disconnected) { @@ -565,21 +808,136 @@ active_open_connected_callback (u32 app_index, u32 opaque, return -1; } - s->opaque = opaque; + po_s = session_get_from_handle (ps->po.session_handle); + tp = session_get_transport_proto (po_s); + + sc = proxy_session_side_ctx_alloc (wrk); + sc->pair = ps->po; + sc->ps_index = ps->ps_index; clib_spinlock_unlock_if_init (&pm->sessions_lock); - /* - * Send event for active open tx fifo - */ - ASSERT (s->thread_index == thread_index); - if (svm_fifo_set_event (s->tx_fifo)) - session_send_io_evt_to_thread (s->tx_fifo, SESSION_IO_EVT_TX); + sc->state = PROXY_SC_S_ESTABLISHED; + s->opaque = sc->sc_index; + sc->is_http = tp == TRANSPORT_PROTO_HTTP ? 1 : 0; + + if (tp == TRANSPORT_PROTO_HTTP) + { + proxy_send_http_resp (po_s, HTTP_STATUS_OK, 0); + } + else + { + /* + * Send event for active open tx fifo + */ + ASSERT (s->thread_index == vlib_get_thread_index ()); + if (svm_fifo_set_event (s->tx_fifo)) + session_program_tx_io_evt (session_handle (s), SESSION_IO_EVT_TX); + } return 0; } static void +active_open_migrate_po_fixup_rpc (void *arg) +{ + u32 ps_index = pointer_to_uword (arg); + proxy_session_side_ctx_t *po_sc; + proxy_main_t *pm = &proxy_main; + session_handle_t po_sh; + proxy_worker_t *wrk; + proxy_session_t *ps; + session_t *po_s; + + wrk = proxy_worker_get (vlib_get_thread_index ()); + + clib_spinlock_lock_if_init (&pm->sessions_lock); + + ps = proxy_session_get (ps_index); + + po_s = session_get_from_handle (ps->po.session_handle); + po_s->rx_fifo = ps->po.rx_fifo; + po_s->tx_fifo = ps->po.tx_fifo; + + po_sc = proxy_session_side_ctx_get (wrk, po_s->opaque); + po_sc->pair = ps->ao; + po_sh = ps->po.session_handle; + + clib_spinlock_unlock_if_init (&pm->sessions_lock); + + session_program_tx_io_evt (po_sh, SESSION_IO_EVT_TX); +} + +static void +active_open_migrate_rpc (void *arg) +{ + u32 ps_index = pointer_to_uword (arg); + proxy_main_t *pm = &proxy_main; + proxy_session_side_ctx_t *sc; + proxy_worker_t *wrk; + proxy_session_t *ps; + session_t *s; + + wrk = proxy_worker_get (vlib_get_thread_index ()); + sc = proxy_session_side_ctx_alloc (wrk); + + clib_spinlock_lock_if_init (&pm->sessions_lock); + + ps = proxy_session_get (ps_index); + sc->ps_index = ps->ps_index; + + s = session_get_from_handle (ps->ao.session_handle); + s->opaque = sc->sc_index; + s->flags &= ~SESSION_F_IS_MIGRATING; + + /* Fixup passive open session because of migration and zc */ + ps->ao.rx_fifo = ps->po.tx_fifo = s->rx_fifo; + ps->ao.tx_fifo = ps->po.rx_fifo = s->tx_fifo; + + ps->po.tx_fifo->shr->master_session_index = + session_index_from_handle (ps->po.session_handle); + ps->po.tx_fifo->master_thread_index = + session_thread_from_handle (ps->po.session_handle); + + sc->pair = ps->po; + + clib_spinlock_unlock_if_init (&pm->sessions_lock); + + session_send_rpc_evt_to_thread ( + session_thread_from_handle (sc->pair.session_handle), + active_open_migrate_po_fixup_rpc, uword_to_pointer (sc->ps_index, void *)); +} + +static void +active_open_migrate_callback (session_t *s, session_handle_t new_sh) +{ + proxy_main_t *pm = &proxy_main; + proxy_session_side_ctx_t *sc; + proxy_session_t *ps; + proxy_worker_t *wrk; + + wrk = proxy_worker_get (s->thread_index); + sc = proxy_session_side_ctx_get (wrk, s->opaque); + + /* NOTE: this is just an example. ZC makes this migration rather + * tedious. Probably better approaches could be found */ + clib_spinlock_lock_if_init (&pm->sessions_lock); + + ps = proxy_session_get (sc->ps_index); + ps->ao.session_handle = new_sh; + ps->ao.rx_fifo = 0; + ps->ao.tx_fifo = 0; + + clib_spinlock_unlock_if_init (&pm->sessions_lock); + + session_send_rpc_evt_to_thread (session_thread_from_handle (new_sh), + active_open_migrate_rpc, + uword_to_pointer (sc->ps_index, void *)); + + proxy_session_side_ctx_free (wrk, sc); +} + +static void active_open_reset_callback (session_t * s) { proxy_try_close_session (s, 1 /* is_active_open */ ); @@ -625,8 +983,8 @@ active_open_rx_callback (session_t * s) static int active_open_tx_callback (session_t * ao_s) { - proxy_main_t *pm = &proxy_main; - proxy_session_t *ps; + proxy_session_side_ctx_t *sc; + proxy_worker_t *wrk; u32 min_free; min_free = clib_min (svm_fifo_size (ao_s->tx_fifo) >> 3, 128 << 10); @@ -636,22 +994,27 @@ active_open_tx_callback (session_t * ao_s) return 0; } - clib_spinlock_lock_if_init (&pm->sessions_lock); - - ps = proxy_session_get_if_valid (ao_s->opaque); - if (!ps) - goto unlock; + wrk = proxy_worker_get (ao_s->thread_index); + sc = proxy_session_side_ctx_get (wrk, ao_s->opaque); - if (ps->vpp_server_handle == SESSION_INVALID_HANDLE) - goto unlock; - - /* Force ack on proxy side to update rcv wnd */ - void *arg = uword_to_pointer (ps->vpp_server_handle, void *); - session_send_rpc_evt_to_thread ( - session_thread_from_handle (ps->vpp_server_handle), proxy_force_ack, arg); + if (sc->state < PROXY_SC_S_ESTABLISHED) + return 0; -unlock: - clib_spinlock_unlock_if_init (&pm->sessions_lock); + if (sc->is_http) + { + /* notify HTTP transport */ + session_t *po = session_get_from_handle (sc->pair.session_handle); + session_send_io_evt_to_thread_custom ( + &po->session_index, po->thread_index, SESSION_IO_EVT_RX); + } + else + { + /* Force ack on proxy side to update rcv wnd */ + void *arg = uword_to_pointer (sc->pair.session_handle, void *); + session_send_rpc_evt_to_thread ( + session_thread_from_handle (sc->pair.session_handle), proxy_force_ack, + arg); + } return 0; } @@ -668,6 +1031,7 @@ active_open_cleanup_callback (session_t * s, session_cleanup_ntf_t ntf) static session_cb_vft_t active_open_clients = { .session_reset_callback = active_open_reset_callback, .session_connected_callback = active_open_connected_callback, + .session_migrate_callback = active_open_migrate_callback, .session_accept_callback = active_open_create_callback, .session_disconnect_callback = active_open_disconnect_callback, .session_cleanup_callback = active_open_cleanup_callback, @@ -760,22 +1124,33 @@ proxy_server_listen () { proxy_main_t *pm = &proxy_main; vnet_listen_args_t _a, *a = &_a; - int rv; + int rv, need_crypto; clib_memset (a, 0, sizeof (*a)); a->app_index = pm->server_app_index; clib_memcpy (&a->sep_ext, &pm->server_sep, sizeof (pm->server_sep)); - if (proxy_transport_needs_crypto (a->sep.transport_proto)) + /* Make sure listener is marked connected for transports like udp */ + a->sep_ext.transport_flags = TRANSPORT_CFG_F_CONNECTED; + need_crypto = proxy_transport_needs_crypto (a->sep.transport_proto); + if (need_crypto) { - session_endpoint_alloc_ext_cfg (&a->sep_ext, - TRANSPORT_ENDPT_EXT_CFG_CRYPTO); - a->sep_ext.ext_cfg->crypto.ckpair_index = pm->ckpair_index; + transport_endpt_ext_cfg_t *ext_cfg = session_endpoint_add_ext_cfg ( + &a->sep_ext, TRANSPORT_ENDPT_EXT_CFG_CRYPTO, + sizeof (transport_endpt_crypto_cfg_t)); + ext_cfg->crypto.ckpair_index = pm->ckpair_index; + } + /* set http timeout for connect-proxy */ + if (pm->server_sep.transport_proto == TRANSPORT_PROTO_HTTP) + { + transport_endpt_ext_cfg_t *ext_cfg = session_endpoint_add_ext_cfg ( + &a->sep_ext, TRANSPORT_ENDPT_EXT_CFG_HTTP, sizeof (ext_cfg->opaque)); + ext_cfg->opaque = pm->idle_timeout; } rv = vnet_listen (a); - if (a->sep_ext.ext_cfg) - clib_mem_free (a->sep_ext.ext_cfg); + if (need_crypto) + session_endpoint_free_ext_cfgs (&a->sep_ext); return rv; } @@ -801,15 +1176,25 @@ proxy_server_create (vlib_main_t * vm) { vlib_thread_main_t *vtm = vlib_get_thread_main (); proxy_main_t *pm = &proxy_main; + proxy_worker_t *wrk; u32 num_threads; int i; + if (vlib_num_workers ()) + clib_spinlock_init (&pm->sessions_lock); + num_threads = 1 /* main thread */ + vtm->n_threads; vec_validate (pm->rx_buf, num_threads - 1); for (i = 0; i < num_threads; i++) vec_validate (pm->rx_buf[i], pm->rcv_buffer_size); + vec_validate (pm->workers, vlib_num_workers ()); + vec_foreach (wrk, pm->workers) + { + clib_spinlock_init (&wrk->pending_connects_lock); + } + proxy_server_add_ckpair (); if (proxy_server_attach ()) @@ -817,11 +1202,6 @@ proxy_server_create (vlib_main_t * vm) clib_warning ("failed to attach server app"); return -1; } - if (proxy_server_listen ()) - { - clib_warning ("failed to start listening"); - return -1; - } if (active_open_attach ()) { clib_warning ("failed to attach active open app"); @@ -853,9 +1233,6 @@ proxy_server_create_command_fn (vlib_main_t * vm, unformat_input_t * input, pm->private_segment_count = 0; pm->segment_size = 512 << 20; - if (vlib_num_workers ()) - clib_spinlock_init (&pm->sessions_lock); - if (!unformat_user (input, unformat_line_input, line_input)) return 0; @@ -887,6 +1264,8 @@ proxy_server_create_command_fn (vlib_main_t * vm, unformat_input_t * input, vec_add1 (server_uri, 0); else if (unformat (line_input, "client-uri %s", &client_uri)) vec_add1 (client_uri, 0); + else if (unformat (line_input, "idle-timeout %d", &pm->idle_timeout)) + ; else { error = clib_error_return (0, "unknown input `%U'", @@ -901,38 +1280,45 @@ proxy_server_create_command_fn (vlib_main_t * vm, unformat_input_t * input, default_server_uri); server_uri = format (0, "%s%c", default_server_uri, 0); } - if (!client_uri) - { - clib_warning ("No client-uri provided, Using default: %s", - default_client_uri); - client_uri = format (0, "%s%c", default_client_uri, 0); - } - if (parse_uri ((char *) server_uri, &pm->server_sep)) { error = clib_error_return (0, "Invalid server uri %v", server_uri); goto done; } - if (parse_uri ((char *) client_uri, &pm->client_sep)) + + /* http proxy get target within request */ + if (pm->server_sep.transport_proto != TRANSPORT_PROTO_HTTP) { - error = clib_error_return (0, "Invalid client uri %v", client_uri); - goto done; + if (!client_uri) + { + clib_warning ("No client-uri provided, Using default: %s", + default_client_uri); + client_uri = format (0, "%s%c", default_client_uri, 0); + } + if (parse_uri ((char *) client_uri, &pm->client_sep)) + { + error = clib_error_return (0, "Invalid client uri %v", client_uri); + goto done; + } } - session_enable_disable_args_t args = { .is_en = 1, - .rt_engine_type = - RT_BACKEND_ENGINE_RULE_TABLE }; - vnet_session_enable_disable (vm, &args); - - rv = proxy_server_create (vm); - switch (rv) + if (pm->server_app_index == APP_INVALID_INDEX) { - case 0: - break; - default: - error = clib_error_return (0, "server_create returned %d", rv); + session_enable_disable_args_t args = { .is_en = 1, + .rt_engine_type = + RT_BACKEND_ENGINE_RULE_TABLE }; + vnet_session_enable_disable (vm, &args); + rv = proxy_server_create (vm); + if (rv) + { + error = clib_error_return (0, "server_create returned %d", rv); + goto done; + } } + if (proxy_server_listen ()) + error = clib_error_return (0, "failed to start listening"); + done: unformat_free (line_input); vec_free (client_uri); @@ -940,14 +1326,14 @@ done: return error; } -VLIB_CLI_COMMAND (proxy_create_command, static) = -{ +VLIB_CLI_COMMAND (proxy_create_command, static) = { .path = "test proxy server", - .short_help = "test proxy server [server-uri <tcp://ip/port>]" - "[client-uri <tcp://ip/port>][fifo-size <nn>[k|m]]" - "[max-fifo-size <nn>[k|m]][high-watermark <nn>]" - "[low-watermark <nn>][rcv-buf-size <nn>][prealloc-fifos <nn>]" - "[private-segment-size <mem>][private-segment-count <nn>]", + .short_help = "test proxy server [server-uri <proto://ip/port>]" + "[client-uri <tcp://ip/port>][fifo-size <nn>[k|m]]" + "[max-fifo-size <nn>[k|m]][high-watermark <nn>]" + "[low-watermark <nn>][rcv-buf-size <nn>][prealloc-fifos <nn>]" + "[private-segment-size <mem>][private-segment-count <nn>]" + "[idle-timeout <nn>]", .function = proxy_server_create_command_fn, }; @@ -957,6 +1343,8 @@ proxy_main_init (vlib_main_t * vm) proxy_main_t *pm = &proxy_main; pm->server_client_index = ~0; pm->active_open_client_index = ~0; + pm->server_app_index = APP_INVALID_INDEX; + pm->idle_timeout = 600; /* connect-proxy default idle timeout 10 minutes */ return 0; } diff --git a/src/plugins/hs_apps/proxy.h b/src/plugins/hs_apps/proxy.h index 26f4de2f729..75567e4c1ba 100644 --- a/src/plugins/hs_apps/proxy.h +++ b/src/plugins/hs_apps/proxy.h @@ -26,23 +26,57 @@ #include <vnet/session/session.h> #include <vnet/session/application_interface.h> +#define foreach_proxy_session_side_state \ + _ (CREATED, "created") \ + _ (CONNECTING, "connecting") \ + _ (ESTABLISHED, "establiehed") \ + _ (CLOSED, "closed") + +typedef enum proxy_session_side_state_ +{ +#define _(sym, str) PROXY_SC_S_##sym, + foreach_proxy_session_side_state +#undef _ +} proxy_session_side_state_t; +typedef struct proxy_session_side_ +{ + session_handle_t session_handle; + svm_fifo_t *rx_fifo; + svm_fifo_t *tx_fifo; +} proxy_session_side_t; + +typedef struct proxy_session_side_ctx_ +{ + proxy_session_side_t pair; + proxy_session_side_state_t state; + u32 sc_index; + u32 ps_index; + u8 is_http; +} proxy_session_side_ctx_t; + typedef struct { - svm_fifo_t *server_rx_fifo; - svm_fifo_t *server_tx_fifo; + proxy_session_side_t po; /**< passive open side */ + proxy_session_side_t ao; /**< active open side */ - session_handle_t vpp_server_handle; - session_handle_t vpp_active_open_handle; volatile int active_open_establishing; volatile int po_disconnected; volatile int ao_disconnected; u32 ps_index; - u32 po_thread_index; } proxy_session_t; +typedef struct proxy_worker_ +{ + proxy_session_side_ctx_t *ctx_pool; + clib_spinlock_t pending_connects_lock; + vnet_connect_args_t *pending_connects; + vnet_connect_args_t *burst_connects; +} proxy_worker_t; + typedef struct { + proxy_worker_t *workers; /**< per-thread data */ proxy_session_t *sessions; /**< session pool, shared */ clib_spinlock_t sessions_lock; /**< lock for session pool */ u8 **rx_buf; /**< intermediate rx buffers */ @@ -63,6 +97,7 @@ typedef struct u32 private_segment_count; /**< Number of private fifo segs */ u64 segment_size; /**< size of fifo segs */ u8 prealloc_fifos; /**< Request fifo preallocation */ + u32 idle_timeout; /**< connect-proxy timeout for idle connections */ int rcv_buffer_size; session_endpoint_cfg_t server_sep; session_endpoint_cfg_t client_sep; @@ -75,6 +110,13 @@ typedef struct extern proxy_main_t proxy_main; +static inline proxy_worker_t * +proxy_worker_get (u32 thread_index) +{ + proxy_main_t *pm = &proxy_main; + return vec_elt_at_index (pm->workers, thread_index); +} + #endif /* __included_proxy_h__ */ /* diff --git a/src/plugins/hs_apps/sapi/vpp_echo_common.c b/src/plugins/hs_apps/sapi/vpp_echo_common.c index 5ce04d1b75b..09ba583cf78 100644 --- a/src/plugins/hs_apps/sapi/vpp_echo_common.c +++ b/src/plugins/hs_apps/sapi/vpp_echo_common.c @@ -330,8 +330,8 @@ format_transport_proto (u8 * s, va_list * args) case TRANSPORT_PROTO_UDP: s = format (s, "UDP"); break; - case TRANSPORT_PROTO_NONE: - s = format (s, "NONE"); + case TRANSPORT_PROTO_CT: + s = format (s, "CT"); break; case TRANSPORT_PROTO_TLS: s = format (s, "TLS"); diff --git a/src/plugins/hs_apps/test_builtins.c b/src/plugins/hs_apps/test_builtins.c index 631c1f1a8a2..c314e71b5df 100644 --- a/src/plugins/hs_apps/test_builtins.c +++ b/src/plugins/hs_apps/test_builtins.c @@ -16,6 +16,7 @@ typedef struct tb_main_ tw_timer_elt_t *delayed_resps; tw_timer_wheel_2t_1w_2048sl_t tw; hss_session_send_fn send_data; + u8 *test_data; } tb_main_t; static tb_main_t tb_main; @@ -51,7 +52,7 @@ VLIB_REGISTER_NODE (test_builtins_timer_process_node) = { }; static void -send_data_to_hss (hss_session_handle_t sh, u8 *data) +send_data_to_hss (hss_session_handle_t sh, u8 *data, u8 free_vec_data) { tb_main_t *tbm = &tb_main; hss_url_handler_args_t args = {}; @@ -61,7 +62,7 @@ send_data_to_hss (hss_session_handle_t sh, u8 *data) args.data_len = vec_len (data); args.ct = HTTP_CONTENT_TEXT_PLAIN; args.sc = HTTP_STATUS_OK; - args.free_vec_data = 1; + args.free_vec_data = free_vec_data; tbm->send_data (&args); } @@ -73,7 +74,7 @@ handle_get_test1 (hss_url_handler_args_t *args) clib_warning ("get request on test1"); data = format (0, "hello"); - send_data_to_hss (args->sh, data); + send_data_to_hss (args->sh, data, 1); return HSS_URL_HANDLER_ASYNC; } @@ -85,7 +86,7 @@ handle_get_test2 (hss_url_handler_args_t *args) clib_warning ("get request on test2"); data = format (0, "some data"); - send_data_to_hss (args->sh, data); + send_data_to_hss (args->sh, data, 1); return HSS_URL_HANDLER_ASYNC; } @@ -105,7 +106,7 @@ delayed_resp_cb (u32 *expired_timers) e = pool_elt_at_index (tbm->delayed_resps, pool_index); clib_warning ("sending delayed data"); data = format (0, "delayed data"); - send_data_to_hss (e->sh, data); + send_data_to_hss (e->sh, data, 1); pool_put (tbm->delayed_resps, e); } } @@ -128,7 +129,15 @@ handle_get_test_delayed (hss_url_handler_args_t *args) static hss_url_handler_rc_t handle_post_test3 (hss_url_handler_args_t *args) { - send_data_to_hss (args->sh, 0); + send_data_to_hss (args->sh, 0, 0); + return HSS_URL_HANDLER_ASYNC; +} + +static hss_url_handler_rc_t +handle_get_64bytes (hss_url_handler_args_t *args) +{ + tb_main_t *tbm = &tb_main; + send_data_to_hss (args->sh, tbm->test_data, 0); return HSS_URL_HANDLER_ASYNC; } @@ -148,10 +157,14 @@ test_builtins_init (vlib_main_t *vm) return; } + tbm->test_data = format ( + 0, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"); + (*fp) (handle_get_test1, "test1", HTTP_REQ_GET); (*fp) (handle_get_test2, "test2", HTTP_REQ_GET); (*fp) (handle_get_test_delayed, "test_delayed", HTTP_REQ_GET); (*fp) (handle_post_test3, "test3", HTTP_REQ_POST); + (*fp) (handle_get_64bytes, "64B", HTTP_REQ_GET); tbm->send_data = vlib_get_plugin_symbol ("http_static_plugin.so", "hss_session_send_data"); diff --git a/src/plugins/hs_apps/vcl/vcl_test_server.c b/src/plugins/hs_apps/vcl/vcl_test_server.c index 5de53173784..008539f2585 100644 --- a/src/plugins/hs_apps/vcl/vcl_test_server.c +++ b/src/plugins/hs_apps/vcl/vcl_test_server.c @@ -416,36 +416,41 @@ static void vcl_test_init_endpoint_addr (vcl_test_server_main_t * vsm) { struct sockaddr_storage *servaddr = &vsm->servaddr; - memset (servaddr, 0, sizeof (*servaddr)); if (vsm->server_cfg.address_ip6) { struct sockaddr_in6 *server_addr = (struct sockaddr_in6 *) servaddr; - server_addr->sin6_family = AF_INET6; - server_addr->sin6_addr = in6addr_any; - server_addr->sin6_port = htons (vsm->server_cfg.port); + vsm->server_cfg.endpt.is_ip4 = 0; + vsm->server_cfg.endpt.ip = (uint8_t *) &server_addr->sin6_addr; + vsm->server_cfg.endpt.port = htons (vsm->server_cfg.port); } else { struct sockaddr_in *server_addr = (struct sockaddr_in *) servaddr; - server_addr->sin_family = AF_INET; - server_addr->sin_addr.s_addr = htonl (INADDR_ANY); - server_addr->sin_port = htons (vsm->server_cfg.port); + vsm->server_cfg.endpt.is_ip4 = 1; + vsm->server_cfg.endpt.ip = (uint8_t *) &server_addr->sin_addr; + vsm->server_cfg.endpt.port = htons (vsm->server_cfg.port); } +} + +static void +vcl_test_clear_endpoint_addr (vcl_test_server_main_t *vsm) +{ + struct sockaddr_storage *servaddr = &vsm->servaddr; + + memset (&vsm->servaddr, 0, sizeof (vsm->servaddr)); if (vsm->server_cfg.address_ip6) { struct sockaddr_in6 *server_addr = (struct sockaddr_in6 *) servaddr; - vsm->server_cfg.endpt.is_ip4 = 0; - vsm->server_cfg.endpt.ip = (uint8_t *) &server_addr->sin6_addr; - vsm->server_cfg.endpt.port = (uint16_t) server_addr->sin6_port; + server_addr->sin6_family = AF_INET6; + server_addr->sin6_addr = in6addr_any; } else { struct sockaddr_in *server_addr = (struct sockaddr_in *) servaddr; - vsm->server_cfg.endpt.is_ip4 = 1; - vsm->server_cfg.endpt.ip = (uint8_t *) &server_addr->sin_addr; - vsm->server_cfg.endpt.port = (uint16_t) server_addr->sin_port; + server_addr->sin_family = AF_INET; + server_addr->sin_addr.s_addr = htonl (INADDR_ANY); } } @@ -456,9 +461,10 @@ vcl_test_server_process_opts (vcl_test_server_main_t * vsm, int argc, int v, c; vsm->server_cfg.proto = VPPCOM_PROTO_TCP; + vcl_test_clear_endpoint_addr (vsm); opterr = 0; - while ((c = getopt (argc, argv, "6DLsw:hp:S")) != -1) + while ((c = getopt (argc, argv, "6DLsw:hp:SB:")) != -1) switch (c) { case '6': @@ -469,7 +475,22 @@ vcl_test_server_process_opts (vcl_test_server_main_t * vsm, int argc, if (vppcom_unformat_proto (&vsm->server_cfg.proto, optarg)) vtwrn ("Invalid vppcom protocol %s, defaulting to TCP", optarg); break; - + case 'B': + if (vsm->server_cfg.address_ip6) + { + if (inet_pton ( + AF_INET6, optarg, + &((struct sockaddr_in6 *) &vsm->servaddr)->sin6_addr) != 1) + vtwrn ("couldn't parse ipv6 addr %s", optarg); + } + else + { + if (inet_pton ( + AF_INET, optarg, + &((struct sockaddr_in *) &vsm->servaddr)->sin_addr) != 1) + vtwrn ("couldn't parse ipv4 addr %s", optarg); + } + break; case 'D': vsm->server_cfg.proto = VPPCOM_PROTO_UDP; break; diff --git a/src/plugins/http/CMakeLists.txt b/src/plugins/http/CMakeLists.txt index c51a7dce36d..075b8d6817b 100644 --- a/src/plugins/http/CMakeLists.txt +++ b/src/plugins/http/CMakeLists.txt @@ -16,5 +16,9 @@ add_vpp_plugin(http http.c http_buffer.c http_timer.c - http_test.c +) + +add_vpp_plugin(http_unittest + SOURCES + test/http_test.c ) diff --git a/src/plugins/http/http.c b/src/plugins/http/http.c index 4f741c2e6b4..1ea5a08fbf6 100644 --- a/src/plugins/http/http.c +++ b/src/plugins/http/http.c @@ -36,57 +36,52 @@ const http_buffer_type_t msg_to_buf_type[] = { }; static u8 * -format_http_state (u8 *s, va_list *va) +format_http_req_state (u8 *s, va_list *va) { - http_state_t state = va_arg (*va, http_state_t); + http_req_state_t state = va_arg (*va, http_req_state_t); + u8 *t = 0; switch (state) { - case HTTP_STATE_IDLE: - return format (s, "idle"); - case HTTP_STATE_WAIT_APP_METHOD: - return format (s, "wait app method"); - case HTTP_STATE_WAIT_SERVER_REPLY: - return format (s, "wait server reply"); - case HTTP_STATE_CLIENT_IO_MORE_DATA: - return format (s, "client io more data"); - case HTTP_STATE_WAIT_CLIENT_METHOD: - return format (s, "wait client method"); - case HTTP_STATE_WAIT_APP_REPLY: - return format (s, "wait app reply"); - case HTTP_STATE_APP_IO_MORE_DATA: - return format (s, "app io more data"); - default: - break; - } - return format (s, "unknown"); -} - -#define http_state_change(_hc, _state) \ +#define _(n, s, str) \ + case HTTP_REQ_STATE_##s: \ + t = (u8 *) str; \ + break; + foreach_http_req_state +#undef _ + default : return format (s, "unknown"); + } + return format (s, "%s", t); +} + +#define http_req_state_change(_hc, _state) \ do \ { \ - HTTP_DBG (1, "changing http state %U -> %U", format_http_state, \ - (_hc)->http_state, format_http_state, _state); \ - (_hc)->http_state = _state; \ + HTTP_DBG (1, "changing http req state: %U -> %U", \ + format_http_req_state, (_hc)->req_state, \ + format_http_req_state, _state); \ + ASSERT ((_hc)->req_state != HTTP_REQ_STATE_TUNNEL); \ + (_hc)->req_state = _state; \ } \ while (0) -static inline int -http_state_is_tx_valid (http_conn_t *hc) +static u8 * +format_http_conn_state (u8 *s, va_list *args) { - http_state_t state = hc->http_state; - return (state == HTTP_STATE_APP_IO_MORE_DATA || - state == HTTP_STATE_WAIT_APP_REPLY || - state == HTTP_STATE_WAIT_APP_METHOD); -} + http_conn_t *hc = va_arg (*args, http_conn_t *); + u8 *t = 0; -static inline int -http_state_is_rx_valid (http_conn_t *hc) -{ - http_state_t state = hc->http_state; - return (state == HTTP_STATE_WAIT_SERVER_REPLY || - state == HTTP_STATE_CLIENT_IO_MORE_DATA || - state == HTTP_STATE_WAIT_CLIENT_METHOD); + switch (hc->state) + { +#define _(s, str) \ + case HTTP_CONN_STATE_##s: \ + t = (u8 *) str; \ + break; + foreach_http_conn_state +#undef _ + default : return format (s, "unknown"); + } + return format (s, "%s", t); } static inline http_worker_t * @@ -117,6 +112,15 @@ http_conn_get_w_thread (u32 hc_index, u32 thread_index) return pool_elt_at_index (wrk->conn_pool, hc_index); } +static inline http_conn_t * +http_conn_get_w_thread_if_valid (u32 hc_index, u32 thread_index) +{ + http_worker_t *wrk = http_worker_get (thread_index); + if (pool_is_free_index (wrk->conn_pool, hc_index)) + return 0; + return pool_elt_at_index (wrk->conn_pool, hc_index); +} + void http_conn_free (http_conn_t *hc) { @@ -124,6 +128,35 @@ http_conn_free (http_conn_t *hc) pool_put (wrk->conn_pool, hc); } +static inline http_conn_t * +http_ho_conn_get (u32 ho_hc_index) +{ + http_main_t *hm = &http_main; + return pool_elt_at_index (hm->ho_conn_pool, ho_hc_index); +} + +void +http_ho_conn_free (http_conn_t *ho_hc) +{ + http_main_t *hm = &http_main; + pool_put (hm->ho_conn_pool, ho_hc); +} + +static inline u32 +http_ho_conn_alloc (void) +{ + http_main_t *hm = &http_main; + http_conn_t *hc; + + pool_get_aligned_safe (hm->ho_conn_pool, hc, CLIB_CACHE_LINE_BYTES); + clib_memset (hc, 0, sizeof (*hc)); + hc->h_hc_index = hc - hm->ho_conn_pool; + hc->h_pa_session_handle = SESSION_INVALID_HANDLE; + hc->h_tc_session_handle = SESSION_INVALID_HANDLE; + hc->timeout = HTTP_CONN_TIMEOUT; + return hc->h_hc_index; +} + static u32 http_listener_alloc (void) { @@ -132,6 +165,7 @@ http_listener_alloc (void) pool_get_zero (hm->listener_pool, lhc); lhc->c_c_index = lhc - hm->listener_pool; + lhc->timeout = HTTP_CONN_TIMEOUT; return lhc->c_c_index; } @@ -167,20 +201,47 @@ http_disconnect_transport (http_conn_t *hc) } static void +http_conn_invalidate_timer_cb (u32 hs_handle) +{ + http_conn_t *hc; + + hc = + http_conn_get_w_thread_if_valid (hs_handle & 0x00FFFFFF, hs_handle >> 24); + + HTTP_DBG (1, "hc [%u]%x", hs_handle >> 24, hs_handle & 0x00FFFFFF); + if (!hc) + { + HTTP_DBG (1, "already deleted"); + return; + } + + hc->timer_handle = HTTP_TIMER_HANDLE_INVALID; + hc->pending_timer = 1; +} + +static void http_conn_timeout_cb (void *hc_handlep) { http_conn_t *hc; uword hs_handle; hs_handle = pointer_to_uword (hc_handlep); - hc = http_conn_get_w_thread (hs_handle & 0x00FFFFFF, hs_handle >> 24); + hc = + http_conn_get_w_thread_if_valid (hs_handle & 0x00FFFFFF, hs_handle >> 24); - HTTP_DBG (1, "terminate thread %d index %d hs %llx", hs_handle >> 24, - hs_handle & 0x00FFFFFF, hc); + HTTP_DBG (1, "hc [%u]%x", hs_handle >> 24, hs_handle & 0x00FFFFFF); if (!hc) - return; + { + HTTP_DBG (1, "already deleted"); + return; + } + + if (!hc->pending_timer) + { + HTTP_DBG (1, "timer not pending"); + return; + } - hc->timer_handle = ~0; session_transport_closing_notify (&hc->connection); http_disconnect_transport (hc); } @@ -200,6 +261,7 @@ http_ts_accept_callback (session_t *ts) hc_index = http_conn_alloc_w_thread (ts->thread_index); hc = http_conn_get_w_thread (hc_index, ts->thread_index); clib_memcpy_fast (hc, lhc, sizeof (*lhc)); + hc->timer_handle = HTTP_TIMER_HANDLE_INVALID; hc->c_thread_index = ts->thread_index; hc->h_hc_index = hc_index; @@ -207,7 +269,7 @@ http_ts_accept_callback (session_t *ts) hc->c_flags |= TRANSPORT_CONNECTION_F_NO_LOOKUP; hc->state = HTTP_CONN_STATE_ESTABLISHED; - http_state_change (hc, HTTP_STATE_WAIT_CLIENT_METHOD); + http_req_state_change (hc, HTTP_REQ_STATE_WAIT_TRANSPORT_METHOD); ts->session_state = SESSION_STATE_READY; ts->opaque = hc_index; @@ -232,6 +294,7 @@ http_ts_accept_callback (session_t *ts) if ((rv = app_worker_init_accepted (as))) { HTTP_DBG (1, "failed to allocate fifos"); + hc->h_pa_session_handle = SESSION_INVALID_HANDLE; session_free (as); return rv; } @@ -273,7 +336,7 @@ http_ts_connected_callback (u32 http_app_index, u32 ho_hc_index, session_t *ts, app_worker_t *app_wrk; int rv; - ho_hc = http_conn_get_w_thread (ho_hc_index, 0); + ho_hc = http_ho_conn_get (ho_hc_index); ASSERT (ho_hc->state == HTTP_CONN_STATE_CONNECTING); if (err) @@ -291,12 +354,13 @@ http_ts_connected_callback (u32 http_app_index, u32 ho_hc_index, session_t *ts, clib_memcpy_fast (hc, ho_hc, sizeof (*hc)); + hc->timer_handle = HTTP_TIMER_HANDLE_INVALID; hc->c_thread_index = ts->thread_index; hc->h_tc_session_handle = session_handle (ts); hc->c_c_index = new_hc_index; hc->c_flags |= TRANSPORT_CONNECTION_F_NO_LOOKUP; hc->state = HTTP_CONN_STATE_ESTABLISHED; - http_state_change (hc, HTTP_STATE_WAIT_APP_METHOD); + http_req_state_change (hc, HTTP_REQ_STATE_WAIT_APP_METHOD); ts->session_state = SESSION_STATE_READY; ts->opaque = new_hc_index; @@ -312,8 +376,8 @@ http_ts_connected_callback (u32 http_app_index, u32 ho_hc_index, session_t *ts, as->session_type = session_type_from_proto_and_ip ( TRANSPORT_PROTO_HTTP, session_type_is_ip4 (ts->session_type)); - HTTP_DBG (1, "half-open hc index %d, hc index %d", ho_hc_index, - new_hc_index); + HTTP_DBG (1, "half-open hc index %x, hc [%u]%x", ho_hc_index, + ts->thread_index, new_hc_index); app_wrk = app_worker_get (hc->h_pa_wrk_index); if (!app_wrk) @@ -359,7 +423,7 @@ http_ts_reset_callback (session_t *ts) hc->state = HTTP_CONN_STATE_CLOSED; http_buffer_free (&hc->tx_buf); - http_state_change (hc, HTTP_STATE_WAIT_CLIENT_METHOD); + http_req_state_change (hc, HTTP_REQ_STATE_WAIT_TRANSPORT_METHOD); session_transport_reset_notify (&hc->connection); http_disconnect_transport (hc); @@ -378,9 +442,9 @@ static const char *http_error_template = "HTTP/1.1 %s\r\n" */ static const char *http_response_template = "HTTP/1.1 %s\r\n" "Date: %U GMT\r\n" - "Server: %v\r\n" - "Content-Length: %llu\r\n" - "%s"; + "Server: %v\r\n"; + +static const char *content_len_template = "Content-Length: %llu\r\n"; /** * http request boilerplate @@ -433,7 +497,7 @@ http_send_error (http_conn_t *hc, http_status_code_t ec) now = clib_timebase_now (&hm->timebase); data = format (0, http_error_template, http_status_code_str[ec], format_clib_timebase_time, now); - HTTP_DBG (1, "%v", data); + HTTP_DBG (3, "%v", data); http_send_data (hc, data, vec_len (data)); vec_free (data); } @@ -604,7 +668,7 @@ http_parse_request_line (http_conn_t *hc, http_status_code_t *ec) *ec = HTTP_STATUS_BAD_REQUEST; return -1; } - HTTP_DBG (0, "request line length: %d", i); + HTTP_DBG (2, "request line length: %d", i); hc->control_data_len = i + 2; next_line_offset = hc->control_data_len; @@ -636,18 +700,25 @@ http_parse_request_line (http_conn_t *hc, http_status_code_t *ec) hc->method = HTTP_REQ_POST; hc->target_path_offset = method_offset + 5; } + else if (!memcmp (hc->rx_buf + method_offset, "CONNECT ", 8)) + { + HTTP_DBG (0, "CONNECT method"); + hc->method = HTTP_REQ_CONNECT; + hc->target_path_offset = method_offset + 8; + hc->is_tunnel = 1; + } else { - if (hc->rx_buf[method_offset] - 'A' <= 'Z' - hc->rx_buf[method_offset]) + if (hc->rx_buf[method_offset] - 'A' <= 'Z' - 'A') { - clib_warning ("not method name: %8v", hc->rx_buf); - *ec = HTTP_STATUS_BAD_REQUEST; + clib_warning ("method not implemented: %8v", hc->rx_buf); + *ec = HTTP_STATUS_NOT_IMPLEMENTED; return -1; } else { - clib_warning ("method not implemented: %8v", hc->rx_buf); - *ec = HTTP_STATUS_NOT_IMPLEMENTED; + clib_warning ("not method name: %8v", hc->rx_buf); + *ec = HTTP_STATUS_BAD_REQUEST; return -1; } } @@ -679,9 +750,9 @@ http_parse_request_line (http_conn_t *hc, http_status_code_t *ec) } /* parse request-target */ - HTTP_DBG (0, "http at %d", i); + HTTP_DBG (2, "http at %d", i); target_len = i - hc->target_path_offset; - HTTP_DBG (0, "target_len %d", target_len); + HTTP_DBG (2, "target_len %d", target_len); if (target_len < 1) { clib_warning ("request-target not present"); @@ -697,10 +768,10 @@ http_parse_request_line (http_conn_t *hc, http_status_code_t *ec) *ec = HTTP_STATUS_BAD_REQUEST; return -1; } - HTTP_DBG (0, "request-target path length: %u", hc->target_path_len); - HTTP_DBG (0, "request-target path offset: %u", hc->target_path_offset); - HTTP_DBG (0, "request-target query length: %u", hc->target_query_len); - HTTP_DBG (0, "request-target query offset: %u", hc->target_query_offset); + HTTP_DBG (2, "request-target path length: %u", hc->target_path_len); + HTTP_DBG (2, "request-target path offset: %u", hc->target_path_offset); + HTTP_DBG (2, "request-target query length: %u", hc->target_query_len); + HTTP_DBG (2, "request-target query offset: %u", hc->target_query_offset); /* set buffer offset to nex line start */ hc->rx_buf_offset = next_line_offset; @@ -742,7 +813,7 @@ http_parse_status_line (http_conn_t *hc) clib_warning ("status line incomplete"); return -1; } - HTTP_DBG (0, "status line length: %d", i); + HTTP_DBG (2, "status line length: %d", i); if (i < 12) { clib_warning ("status line too short (%d)", i); @@ -824,7 +895,7 @@ http_identify_headers (http_conn_t *hc, http_status_code_t *ec) (hc->rx_buf[hc->rx_buf_offset + 1] == '\n')) { /* just another CRLF -> no headers */ - HTTP_DBG (0, "no headers"); + HTTP_DBG (2, "no headers"); hc->headers_len = 0; hc->control_data_len += 2; return 0; @@ -841,8 +912,8 @@ http_identify_headers (http_conn_t *hc, http_status_code_t *ec) hc->headers_offset = hc->rx_buf_offset; hc->headers_len = i - hc->rx_buf_offset + 2; hc->control_data_len += (hc->headers_len + 2); - HTTP_DBG (0, "headers length: %u", hc->headers_len); - HTTP_DBG (0, "headers offset: %u", hc->headers_offset); + HTTP_DBG (2, "headers length: %u", hc->headers_len); + HTTP_DBG (2, "headers offset: %u", hc->headers_offset); return 0; } @@ -850,16 +921,20 @@ http_identify_headers (http_conn_t *hc, http_status_code_t *ec) static int http_identify_message_body (http_conn_t *hc, http_status_code_t *ec) { - unformat_input_t input; - int i, len; - u8 *line; - u64 body_len; + int i, value_len; + u8 *end, *p, *value_start; + u64 body_len = 0, digit; hc->body_len = 0; if (hc->headers_len == 0) { - HTTP_DBG (0, "no header, no message-body"); + HTTP_DBG (2, "no header, no message-body"); + return 0; + } + if (hc->is_tunnel) + { + HTTP_DBG (2, "tunnel, no message-body"); return 0; } @@ -870,7 +945,7 @@ http_identify_message_body (http_conn_t *hc, http_status_code_t *ec) "Content-Length:"); if (i < 0) { - HTTP_DBG (0, "Content-Length header not present, no message-body"); + HTTP_DBG (2, "Content-Length header not present, no message-body"); return 0; } hc->rx_buf_offset = i + 15; @@ -882,37 +957,83 @@ http_identify_message_body (http_conn_t *hc, http_status_code_t *ec) *ec = HTTP_STATUS_BAD_REQUEST; return -1; } - len = i - hc->rx_buf_offset; - if (len < 1) + value_len = i - hc->rx_buf_offset; + if (value_len < 1) { clib_warning ("invalid header, content length value missing"); *ec = HTTP_STATUS_BAD_REQUEST; return -1; } - line = vec_new (u8, len); - clib_memcpy (line, hc->rx_buf + hc->rx_buf_offset, len); - HTTP_DBG (0, "%v", line); + end = hc->rx_buf + hc->rx_buf_offset + value_len; + p = hc->rx_buf + hc->rx_buf_offset; + /* skip leading whitespace */ + while (1) + { + if (p == end) + { + clib_warning ("value not found"); + *ec = HTTP_STATUS_BAD_REQUEST; + return -1; + } + else if (*p != ' ' && *p != '\t') + { + break; + } + p++; + value_len--; + } + value_start = p; + /* skip trailing whitespace */ + p = value_start + value_len - 1; + while (*p == ' ' || *p == '\t') + { + p--; + value_len--; + } - unformat_init_vector (&input, line); - if (!unformat (&input, "%llu", &body_len)) + if (value_len < 1) { - clib_warning ("failed to unformat content length value"); + clib_warning ("value not found"); *ec = HTTP_STATUS_BAD_REQUEST; return -1; } - unformat_free (&input); + + p = value_start; + for (i = 0; i < value_len; i++) + { + /* check for digit */ + if (!isdigit (*p)) + { + clib_warning ("expected digit"); + *ec = HTTP_STATUS_BAD_REQUEST; + return -1; + } + digit = *p - '0'; + u64 new_body_len = body_len * 10 + digit; + /* check for overflow */ + if (new_body_len < body_len) + { + clib_warning ("too big number, overflow"); + *ec = HTTP_STATUS_BAD_REQUEST; + return -1; + } + body_len = new_body_len; + p++; + } + hc->body_len = body_len; hc->body_offset = hc->headers_offset + hc->headers_len + 2; - HTTP_DBG (0, "body length: %llu", hc->body_len); - HTTP_DBG (0, "body offset: %u", hc->body_offset); + HTTP_DBG (2, "body length: %llu", hc->body_len); + HTTP_DBG (2, "body offset: %u", hc->body_offset); return 0; } static http_sm_result_t -http_state_wait_server_reply (http_conn_t *hc, transport_send_params_t *sp) +http_req_state_wait_transport_reply (http_conn_t *hc, + transport_send_params_t *sp) { int rv; http_msg_t msg = {}; @@ -931,7 +1052,7 @@ http_state_wait_server_reply (http_conn_t *hc, transport_send_params_t *sp) return HTTP_SM_STOP; } - HTTP_DBG (0, "%v", hc->rx_buf); + HTTP_DBG (3, "%v", hc->rx_buf); if (vec_len (hc->rx_buf) < 8) { @@ -955,6 +1076,7 @@ http_state_wait_server_reply (http_conn_t *hc, transport_send_params_t *sp) * if there is some space send also portion of body */ as = session_get_from_handle (hc->h_pa_session_handle); max_enq = svm_fifo_max_enqueue (as->rx_fifo); + max_enq -= sizeof (msg); if (max_enq < hc->control_data_len) { clib_warning ("not enough room for control data in app's rx fifo"); @@ -984,12 +1106,12 @@ http_state_wait_server_reply (http_conn_t *hc, transport_send_params_t *sp) if (hc->to_recv == 0) { /* all sent, we are done */ - http_state_change (hc, HTTP_STATE_WAIT_APP_METHOD); + http_req_state_change (hc, HTTP_REQ_STATE_WAIT_APP_METHOD); } else { /* stream rest of the response body */ - http_state_change (hc, HTTP_STATE_CLIENT_IO_MORE_DATA); + http_req_state_change (hc, HTTP_REQ_STATE_TRANSPORT_IO_MORE_DATA); } app_wrk = app_worker_get_if_valid (as->app_wrk_index); @@ -1006,7 +1128,8 @@ error: } static http_sm_result_t -http_state_wait_client_method (http_conn_t *hc, transport_send_params_t *sp) +http_req_state_wait_transport_method (http_conn_t *hc, + transport_send_params_t *sp) { http_status_code_t ec; app_worker_t *app_wrk; @@ -1022,7 +1145,7 @@ http_state_wait_client_method (http_conn_t *hc, transport_send_params_t *sp) if (rv) return HTTP_SM_STOP; - HTTP_DBG (0, "%v", hc->rx_buf); + HTTP_DBG (3, "%v", hc->rx_buf); if (vec_len (hc->rx_buf) < 8) { @@ -1084,13 +1207,13 @@ http_state_wait_client_method (http_conn_t *hc, transport_send_params_t *sp) /* drop everything, we do not support pipelining */ http_read_message_drop_all (hc); /* all sent, we are done */ - http_state_change (hc, HTTP_STATE_WAIT_APP_REPLY); + http_req_state_change (hc, HTTP_REQ_STATE_WAIT_APP_REPLY); } else { http_read_message_drop (hc, len); /* stream rest of the response body */ - http_state_change (hc, HTTP_STATE_CLIENT_IO_MORE_DATA); + http_req_state_change (hc, HTTP_REQ_STATE_TRANSPORT_IO_MORE_DATA); } app_wrk = app_worker_get_if_valid (as->app_wrk_index); @@ -1109,7 +1232,7 @@ error: } static http_sm_result_t -http_state_wait_app_reply (http_conn_t *hc, transport_send_params_t *sp) +http_req_state_wait_app_reply (http_conn_t *hc, transport_send_params_t *sp) { http_main_t *hm = &http_main; u8 *response; @@ -1120,6 +1243,7 @@ http_state_wait_app_reply (http_conn_t *hc, transport_send_params_t *sp) http_msg_t msg; int rv; http_sm_result_t sm_result = HTTP_SM_ERROR; + http_req_state_t next_state = HTTP_REQ_STATE_WAIT_TRANSPORT_METHOD; as = session_get_from_handle (hc->h_pa_session_handle); @@ -1157,11 +1281,21 @@ http_state_wait_app_reply (http_conn_t *hc, transport_send_params_t *sp) /* Date */ format_clib_timebase_time, now, /* Server */ - hc->app_name, - /* Length */ - msg.data.body_len, - /* Any headers from app? */ - msg.data.headers_len ? "" : "\r\n"); + hc->app_name); + + /* RFC9110 9.3.6: A server MUST NOT send Content-Length header field in a + * 2xx (Successful) response to CONNECT. */ + if (hc->is_tunnel && http_status_code_str[msg.code][0] == '2') + { + ASSERT (msg.data.body_len == 0); + next_state = HTTP_REQ_STATE_TUNNEL; + /* cleanup some stuff we don't need anymore in tunnel mode */ + http_conn_timer_stop (hc); + vec_free (hc->rx_buf); + http_buffer_free (&hc->tx_buf); + } + else + response = format (response, content_len_template, msg.data.body_len); /* Add headers from app (if any) */ if (msg.data.headers_len) @@ -1184,7 +1318,12 @@ http_state_wait_app_reply (http_conn_t *hc, transport_send_params_t *sp) ASSERT (rv == msg.data.headers_len); } } - HTTP_DBG (0, "%v", response); + else + { + /* No headers from app */ + response = format (response, "\r\n"); + } + HTTP_DBG (3, "%v", response); sent = http_send_data (hc, response, vec_len (response)); if (sent != vec_len (response)) @@ -1201,30 +1340,30 @@ http_state_wait_app_reply (http_conn_t *hc, transport_send_params_t *sp) /* Start sending the actual data */ http_buffer_init (&hc->tx_buf, msg_to_buf_type[msg.data.type], as->tx_fifo, msg.data.body_len); - http_state_change (hc, HTTP_STATE_APP_IO_MORE_DATA); + next_state = HTTP_REQ_STATE_APP_IO_MORE_DATA; sm_result = HTTP_SM_CONTINUE; } else { /* No response body, we are done */ - http_state_change (hc, HTTP_STATE_WAIT_CLIENT_METHOD); sm_result = HTTP_SM_STOP; } + http_req_state_change (hc, next_state); + ASSERT (sp->max_burst_size >= sent); sp->max_burst_size -= sent; return sm_result; error: http_send_error (hc, sc); - http_state_change (hc, HTTP_STATE_WAIT_CLIENT_METHOD); session_transport_closing_notify (&hc->connection); http_disconnect_transport (hc); return HTTP_SM_STOP; } static http_sm_result_t -http_state_wait_app_method (http_conn_t *hc, transport_send_params_t *sp) +http_req_state_wait_app_method (http_conn_t *hc, transport_send_params_t *sp) { http_msg_t msg; session_t *as; @@ -1232,7 +1371,7 @@ http_state_wait_app_method (http_conn_t *hc, transport_send_params_t *sp) u32 sent; int rv; http_sm_result_t sm_result = HTTP_SM_ERROR; - http_state_t next_state; + http_req_state_t next_state; as = session_get_from_handle (hc->h_pa_session_handle); @@ -1292,7 +1431,7 @@ http_state_wait_app_method (http_conn_t *hc, transport_send_params_t *sp) /* Any headers from app? */ msg.data.headers_len ? "" : "\r\n"); - next_state = HTTP_STATE_WAIT_SERVER_REPLY; + next_state = HTTP_REQ_STATE_WAIT_TRANSPORT_REPLY; sm_result = HTTP_SM_STOP; } else if (msg.method_type == HTTP_REQ_POST) @@ -1323,7 +1462,7 @@ http_state_wait_app_method (http_conn_t *hc, transport_send_params_t *sp) http_buffer_init (&hc->tx_buf, msg_to_buf_type[msg.data.type], as->tx_fifo, msg.data.body_len); - next_state = HTTP_STATE_APP_IO_MORE_DATA; + next_state = HTTP_REQ_STATE_APP_IO_MORE_DATA; sm_result = HTTP_SM_CONTINUE; } else @@ -1353,7 +1492,7 @@ http_state_wait_app_method (http_conn_t *hc, transport_send_params_t *sp) ASSERT (rv == msg.data.headers_len); } } - HTTP_DBG (0, "%v", request); + HTTP_DBG (3, "%v", request); sent = http_send_data (hc, request, vec_len (request)); if (sent != vec_len (request)) @@ -1363,7 +1502,7 @@ http_state_wait_app_method (http_conn_t *hc, transport_send_params_t *sp) goto error; } - http_state_change (hc, next_state); + http_req_state_change (hc, next_state); goto done; error: @@ -1379,7 +1518,8 @@ done: } static http_sm_result_t -http_state_client_io_more_data (http_conn_t *hc, transport_send_params_t *sp) +http_req_state_transport_io_more_data (http_conn_t *hc, + transport_send_params_t *sp) { session_t *as, *ts; app_worker_t *app_wrk; @@ -1426,18 +1566,18 @@ http_state_client_io_more_data (http_conn_t *hc, transport_send_params_t *sp) clib_warning ("http protocol error: received more data than expected"); session_transport_closing_notify (&hc->connection); http_disconnect_transport (hc); - http_state_change (hc, HTTP_STATE_WAIT_APP_METHOD); + http_req_state_change (hc, HTTP_REQ_STATE_WAIT_APP_METHOD); return HTTP_SM_ERROR; } hc->to_recv -= rv; HTTP_DBG (1, "drained %d from ts; remains %lu", rv, hc->to_recv); /* Finished transaction: - * server back to HTTP_STATE_WAIT_APP_REPLY - * client to HTTP_STATE_WAIT_APP_METHOD */ + * server back to HTTP_REQ_STATE_WAIT_APP_REPLY + * client to HTTP_REQ_STATE_WAIT_APP_METHOD */ if (hc->to_recv == 0) - http_state_change (hc, hc->is_server ? HTTP_STATE_WAIT_APP_REPLY : - HTTP_STATE_WAIT_APP_METHOD); + http_req_state_change (hc, hc->is_server ? HTTP_REQ_STATE_WAIT_APP_REPLY : + HTTP_REQ_STATE_WAIT_APP_METHOD); app_wrk = app_worker_get_if_valid (as->app_wrk_index); if (app_wrk) @@ -1450,7 +1590,7 @@ http_state_client_io_more_data (http_conn_t *hc, transport_send_params_t *sp) } static http_sm_result_t -http_state_app_io_more_data (http_conn_t *hc, transport_send_params_t *sp) +http_req_state_app_io_more_data (http_conn_t *hc, transport_send_params_t *sp) { u32 max_send = 64 << 10, n_segs; http_buffer_t *hb = &hc->tx_buf; @@ -1492,37 +1632,157 @@ http_state_app_io_more_data (http_conn_t *hc, transport_send_params_t *sp) session_program_tx_io_evt (ts->handle, SESSION_IO_EVT_TX_FLUSH); /* Finished transaction: - * server back to HTTP_STATE_WAIT_METHOD - * client to HTTP_STATE_WAIT_SERVER_REPLY */ - http_state_change (hc, hc->is_server ? HTTP_STATE_WAIT_CLIENT_METHOD : - HTTP_STATE_WAIT_SERVER_REPLY); + * server back to HTTP_REQ_STATE_WAIT_TRANSPORT_METHOD + * client to HTTP_REQ_STATE_WAIT_TRANSPORT_REPLY */ + http_req_state_change (hc, hc->is_server ? + HTTP_REQ_STATE_WAIT_TRANSPORT_METHOD : + HTTP_REQ_STATE_WAIT_TRANSPORT_REPLY); http_buffer_free (&hc->tx_buf); } return HTTP_SM_STOP; } +static http_sm_result_t +http_req_state_tunnel_rx (http_conn_t *hc, transport_send_params_t *sp) +{ + u32 max_deq, max_enq, max_read, n_segs = 2; + svm_fifo_seg_t segs[n_segs]; + int n_written = 0; + session_t *as, *ts; + app_worker_t *app_wrk; + + HTTP_DBG (1, "tunnel received data from client"); + + as = session_get_from_handle (hc->h_pa_session_handle); + ts = session_get_from_handle (hc->h_tc_session_handle); + + max_deq = svm_fifo_max_dequeue (ts->rx_fifo); + if (PREDICT_FALSE (max_deq == 0)) + { + HTTP_DBG (1, "max_deq == 0"); + return HTTP_SM_STOP; + } + max_enq = svm_fifo_max_enqueue (as->rx_fifo); + if (max_enq == 0) + { + HTTP_DBG (1, "app's rx fifo full"); + svm_fifo_add_want_deq_ntf (as->rx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); + return HTTP_SM_STOP; + } + max_read = clib_min (max_enq, max_deq); + svm_fifo_segments (ts->rx_fifo, 0, segs, &n_segs, max_read); + n_written = svm_fifo_enqueue_segments (as->rx_fifo, segs, n_segs, 0); + ASSERT (n_written > 0); + HTTP_DBG (1, "transfered %u bytes", n_written); + svm_fifo_dequeue_drop (ts->rx_fifo, n_written); + app_wrk = app_worker_get_if_valid (as->app_wrk_index); + if (app_wrk) + app_worker_rx_notify (app_wrk, as); + if (svm_fifo_max_dequeue_cons (ts->rx_fifo)) + session_program_rx_io_evt (session_handle (ts)); + + return HTTP_SM_STOP; +} + +static http_sm_result_t +http_req_state_tunnel_tx (http_conn_t *hc, transport_send_params_t *sp) +{ + u32 max_deq, max_enq, max_read, n_segs = 2; + svm_fifo_seg_t segs[n_segs]; + session_t *as, *ts; + int n_written = 0; + + HTTP_DBG (1, "tunnel received data from target"); + + as = session_get_from_handle (hc->h_pa_session_handle); + ts = session_get_from_handle (hc->h_tc_session_handle); + + max_deq = svm_fifo_max_dequeue_cons (as->tx_fifo); + if (PREDICT_FALSE (max_deq == 0)) + { + HTTP_DBG (1, "max_deq == 0"); + goto check_fifo; + } + max_enq = svm_fifo_max_enqueue_prod (ts->tx_fifo); + if (max_enq == 0) + { + HTTP_DBG (1, "ts tx fifo full"); + goto check_fifo; + } + max_read = clib_min (max_enq, max_deq); + max_read = clib_min (max_read, sp->max_burst_size); + svm_fifo_segments (as->tx_fifo, 0, segs, &n_segs, max_read); + n_written = svm_fifo_enqueue_segments (ts->tx_fifo, segs, n_segs, 0); + ASSERT (n_written > 0); + HTTP_DBG (1, "transfered %u bytes", n_written); + sp->bytes_dequeued += n_written; + sp->max_burst_size -= n_written; + svm_fifo_dequeue_drop (as->tx_fifo, n_written); + if (svm_fifo_set_event (ts->tx_fifo)) + session_program_tx_io_evt (ts->handle, SESSION_IO_EVT_TX); + +check_fifo: + /* Deschedule and wait for deq notification if ts fifo is almost full */ + if (svm_fifo_max_enqueue (ts->tx_fifo) < HTTP_FIFO_THRESH) + { + svm_fifo_add_want_deq_ntf (ts->tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); + transport_connection_deschedule (&hc->connection); + sp->flags |= TRANSPORT_SND_F_DESCHED; + } + + return HTTP_SM_STOP; +} + typedef http_sm_result_t (*http_sm_handler) (http_conn_t *, transport_send_params_t *sp); -static http_sm_handler state_funcs[HTTP_N_STATES] = { - 0, /* idle state */ - http_state_wait_app_method, - http_state_wait_client_method, - http_state_wait_server_reply, - http_state_wait_app_reply, - http_state_client_io_more_data, - http_state_app_io_more_data, +static http_sm_handler tx_state_funcs[HTTP_REQ_N_STATES] = { + 0, /* idle */ + http_req_state_wait_app_method, + 0, /* wait transport reply */ + 0, /* transport io more data */ + 0, /* wait transport method */ + http_req_state_wait_app_reply, + http_req_state_app_io_more_data, + http_req_state_tunnel_tx, }; -static void -http_req_run_state_machine (http_conn_t *hc, transport_send_params_t *sp) +static_always_inline int +http_req_state_is_tx_valid (http_conn_t *hc) +{ + return tx_state_funcs[hc->req_state] ? 1 : 0; +} + +static http_sm_handler rx_state_funcs[HTTP_REQ_N_STATES] = { + 0, /* idle */ + 0, /* wait app method */ + http_req_state_wait_transport_reply, + http_req_state_transport_io_more_data, + http_req_state_wait_transport_method, + 0, /* wait app reply */ + 0, /* app io more data */ + http_req_state_tunnel_rx, +}; + +static_always_inline int +http_req_state_is_rx_valid (http_conn_t *hc) +{ + return rx_state_funcs[hc->req_state] ? 1 : 0; +} + +static_always_inline void +http_req_run_state_machine (http_conn_t *hc, transport_send_params_t *sp, + u8 is_tx) { http_sm_result_t res; do { - res = state_funcs[hc->http_state](hc, sp); + if (is_tx) + res = tx_state_funcs[hc->req_state](hc, sp); + else + res = rx_state_funcs[hc->req_state](hc, sp); if (res == HTTP_SM_ERROR) { HTTP_DBG (1, "error in state machine %d", res); @@ -1540,24 +1800,29 @@ http_ts_rx_callback (session_t *ts) { http_conn_t *hc; + HTTP_DBG (1, "hc [%u]%x", ts->thread_index, ts->opaque); + hc = http_conn_get_w_thread (ts->opaque, ts->thread_index); - if (!hc) + + if (hc->state == HTTP_CONN_STATE_CLOSED) { - clib_warning ("http connection not found (ts %d)", ts->opaque); - return -1; + HTTP_DBG (1, "conn closed"); + svm_fifo_dequeue_drop_all (ts->rx_fifo); + return 0; } - if (!http_state_is_rx_valid (hc)) + if (!http_req_state_is_rx_valid (hc)) { - if (hc->state != HTTP_CONN_STATE_CLOSED) - clib_warning ("app data req state '%U' session state %u", - format_http_state, hc->http_state, hc->state); - svm_fifo_dequeue_drop_all (ts->tx_fifo); + clib_warning ("hc [%u]%x invalid rx state: http req state " + "'%U', session state '%U'", + ts->thread_index, ts->opaque, format_http_req_state, + hc->req_state, format_http_conn_state, hc); + svm_fifo_dequeue_drop_all (ts->rx_fifo); return 0; } HTTP_DBG (1, "run state machine"); - http_req_run_state_machine (hc, 0); + http_req_run_state_machine (hc, 0, 0); if (hc->state == HTTP_CONN_STATE_TRANSPORT_CLOSED) { @@ -1573,6 +1838,7 @@ http_ts_builtin_tx_callback (session_t *ts) http_conn_t *hc; hc = http_conn_get_w_thread (ts->opaque, ts->thread_index); + HTTP_DBG (1, "transport connection reschedule"); transport_connection_reschedule (&hc->connection); return 0; @@ -1587,17 +1853,15 @@ http_ts_cleanup_callback (session_t *ts, session_cleanup_ntf_t ntf) return; hc = http_conn_get_w_thread (ts->opaque, ts->thread_index); - if (!hc) - { - clib_warning ("no http connection for %u", ts->session_index); - return; - } - HTTP_DBG (1, "going to free session %x", ts->opaque); + + HTTP_DBG (1, "going to free hc [%u]%x", ts->thread_index, ts->opaque); vec_free (hc->rx_buf); http_buffer_free (&hc->tx_buf); - http_conn_timer_stop (hc); + + if (hc->pending_timer == 0) + http_conn_timer_stop (hc); session_transport_delete_notify (&hc->connection); @@ -1609,6 +1873,16 @@ http_ts_cleanup_callback (session_t *ts, session_cleanup_ntf_t ntf) http_conn_free (hc); } +static void +http_ts_ho_cleanup_callback (session_t *ts) +{ + http_conn_t *ho_hc; + HTTP_DBG (1, "half open: %x", ts->opaque); + ho_hc = http_ho_conn_get (ts->opaque); + session_half_open_delete_notify (&ho_hc->connection); + http_ho_conn_free (ho_hc); +} + int http_add_segment_callback (u32 client_index, u64 segment_handle) { @@ -1628,6 +1902,7 @@ static session_cb_vft_t http_app_cb_vft = { .session_connected_callback = http_ts_connected_callback, .session_reset_callback = http_ts_reset_callback, .session_cleanup_callback = http_ts_cleanup_callback, + .half_open_cleanup_callback = http_ts_ho_cleanup_callback, .add_segment_callback = http_add_segment_callback, .del_segment_callback = http_del_segment_callback, .builtin_app_rx_callback = http_ts_rx_callback, @@ -1679,7 +1954,7 @@ http_transport_enable (vlib_main_t *vm, u8 is_en) clib_timebase_init (&hm->timebase, 0 /* GMT */, CLIB_TIMEBASE_DAYLIGHT_NONE, &vm->clib_time /* share the system clock */); - http_timers_init (vm, http_conn_timeout_cb); + http_timers_init (vm, http_conn_timeout_cb, http_conn_invalidate_timer_cb); hm->is_init = 1; return 0; @@ -1695,6 +1970,8 @@ http_transport_connect (transport_endpoint_cfg_t *tep) http_conn_t *hc; int error; u32 hc_index; + session_t *ho; + transport_endpt_ext_cfg_t *ext_cfg; app_worker_t *app_wrk = app_worker_get (sep->app_wrk_index); clib_memset (cargs, 0, sizeof (*cargs)); @@ -1704,13 +1981,20 @@ http_transport_connect (transport_endpoint_cfg_t *tep) app = application_get (app_wrk->app_index); cargs->sep_ext.ns_index = app->ns_index; - hc_index = http_conn_alloc_w_thread (0 /* ts->thread_index */); - hc = http_conn_get_w_thread (hc_index, 0); + hc_index = http_ho_conn_alloc (); + hc = http_ho_conn_get (hc_index); hc->h_pa_wrk_index = sep->app_wrk_index; hc->h_pa_app_api_ctx = sep->opaque; hc->state = HTTP_CONN_STATE_CONNECTING; cargs->api_context = hc_index; + ext_cfg = session_endpoint_get_ext_cfg (sep, TRANSPORT_ENDPT_EXT_CFG_HTTP); + if (ext_cfg) + { + HTTP_DBG (1, "app set timeout %u", ext_cfg->opaque); + hc->timeout = ext_cfg->opaque; + } + hc->is_server = 0; if (vec_len (app->name)) @@ -1730,6 +2014,15 @@ http_transport_connect (transport_endpoint_cfg_t *tep) if ((error = vnet_connect (cargs))) return error; + ho = session_alloc_for_half_open (&hc->connection); + ho->app_wrk_index = app_wrk->wrk_index; + ho->ho_index = app_worker_add_half_open (app_wrk, session_handle (ho)); + ho->opaque = sep->opaque; + ho->session_type = + session_type_from_proto_and_ip (TRANSPORT_PROTO_HTTP, sep->is_ip4); + hc->h_tc_session_handle = cargs->sh; + hc->c_s_index = ho->session_index; + return 0; } @@ -1741,11 +2034,12 @@ http_start_listen (u32 app_listener_index, transport_endpoint_cfg_t *tep) http_main_t *hm = &http_main; session_endpoint_cfg_t *sep; app_worker_t *app_wrk; - transport_proto_t tp; + transport_proto_t tp = TRANSPORT_PROTO_TCP; app_listener_t *al; application_t *app; http_conn_t *lhc; u32 lhc_index; + transport_endpt_ext_cfg_t *ext_cfg; sep = (session_endpoint_cfg_t *) tep; @@ -1755,7 +2049,13 @@ http_start_listen (u32 app_listener_index, transport_endpoint_cfg_t *tep) args->app_index = hm->app_index; args->sep_ext = *sep; args->sep_ext.ns_index = app->ns_index; - tp = sep->ext_cfg ? TRANSPORT_PROTO_TLS : TRANSPORT_PROTO_TCP; + + ext_cfg = session_endpoint_get_ext_cfg (sep, TRANSPORT_ENDPT_EXT_CFG_CRYPTO); + if (ext_cfg) + { + HTTP_DBG (1, "app set tls"); + tp = TRANSPORT_PROTO_TLS; + } args->sep_ext.transport_proto = tp; if (vnet_listen (args)) @@ -1764,6 +2064,13 @@ http_start_listen (u32 app_listener_index, transport_endpoint_cfg_t *tep) lhc_index = http_listener_alloc (); lhc = http_listener_get (lhc_index); + ext_cfg = session_endpoint_get_ext_cfg (sep, TRANSPORT_ENDPT_EXT_CFG_HTTP); + if (ext_cfg && ext_cfg->opaque) + { + HTTP_DBG (1, "app set timeout %u", ext_cfg->opaque); + lhc->timeout = ext_cfg->opaque; + } + /* Grab transport connection listener and link to http listener */ lhc->h_tc_session_handle = args->handle; al = app_listener_get_w_handle (lhc->h_tc_session_handle); @@ -1815,7 +2122,7 @@ http_transport_close (u32 hc_index, u32 thread_index) session_t *as; http_conn_t *hc; - HTTP_DBG (1, "App disconnecting %x", hc_index); + HTTP_DBG (1, "App disconnecting [%u]%x", thread_index, hc_index); hc = http_conn_get_w_thread (hc_index, thread_index); if (hc->state == HTTP_CONN_STATE_CONNECTING) @@ -1865,23 +2172,26 @@ http_app_tx_callback (void *session, transport_send_params_t *sp) u32 max_burst_sz, sent; http_conn_t *hc; - HTTP_DBG (1, "app session conn index %x", as->connection_index); + HTTP_DBG (1, "hc [%u]%x", as->thread_index, as->connection_index); hc = http_conn_get_w_thread (as->connection_index, as->thread_index); - if (!http_state_is_tx_valid (hc)) + + max_burst_sz = sp->max_burst_size * TRANSPORT_PACER_MIN_MSS; + sp->max_burst_size = max_burst_sz; + + if (!http_req_state_is_tx_valid (hc)) { - if (hc->state != HTTP_CONN_STATE_CLOSED) - clib_warning ("app data req state '%U' session state %u", - format_http_state, hc->http_state, hc->state); + clib_warning ("hc [%u]%x invalid tx state: http req state " + "'%U', session state '%U'", + as->thread_index, as->connection_index, + format_http_req_state, hc->req_state, + format_http_conn_state, hc); svm_fifo_dequeue_drop_all (as->tx_fifo); return 0; } - max_burst_sz = sp->max_burst_size * TRANSPORT_PACER_MIN_MSS; - sp->max_burst_size = max_burst_sz; - HTTP_DBG (1, "run state machine"); - http_req_run_state_machine (hc, sp); + http_req_run_state_machine (hc, sp, 1); if (hc->state == HTTP_CONN_STATE_APP_CLOSED) { @@ -1894,6 +2204,18 @@ http_app_tx_callback (void *session, transport_send_params_t *sp) return sent > 0 ? clib_max (sent / TRANSPORT_PACER_MIN_MSS, 1) : 0; } +static int +http_app_rx_evt_cb (transport_connection_t *tc) +{ + http_conn_t *hc = (http_conn_t *) tc; + HTTP_DBG (1, "hc [%u]%x", vlib_get_thread_index (), hc->h_hc_index); + + if (hc->req_state == HTTP_REQ_STATE_TUNNEL) + http_req_state_tunnel_rx (hc, 0); + + return 0; +} + static void http_transport_get_endpoint (u32 hc_index, u32 thread_index, transport_endpoint_t *tep, u8 is_lcl) @@ -1936,36 +2258,6 @@ format_http_listener (u8 *s, va_list *args) } static u8 * -format_http_conn_state (u8 *s, va_list *args) -{ - http_conn_t *hc = va_arg (*args, http_conn_t *); - - switch (hc->state) - { - case HTTP_CONN_STATE_LISTEN: - s = format (s, "LISTEN"); - break; - case HTTP_CONN_STATE_CONNECTING: - s = format (s, "CONNECTING"); - break; - case HTTP_CONN_STATE_ESTABLISHED: - s = format (s, "ESTABLISHED"); - break; - case HTTP_CONN_STATE_TRANSPORT_CLOSED: - s = format (s, "TRANSPORT_CLOSED"); - break; - case HTTP_CONN_STATE_APP_CLOSED: - s = format (s, "APP_CLOSED"); - break; - case HTTP_CONN_STATE_CLOSED: - s = format (s, "CLOSED"); - break; - } - - return s; -} - -static u8 * format_http_transport_connection (u8 *s, va_list *args) { u32 tc_index = va_arg (*args, u32); @@ -2002,18 +2294,61 @@ format_http_transport_listener (u8 *s, va_list *args) return s; } +static u8 * +format_http_transport_half_open (u8 *s, va_list *args) +{ + u32 ho_index = va_arg (*args, u32); + u32 __clib_unused thread_index = va_arg (*args, u32); + u32 __clib_unused verbose = va_arg (*args, u32); + http_conn_t *ho_hc; + session_t *tcp_ho; + + ho_hc = http_ho_conn_get (ho_index); + tcp_ho = session_get_from_handle (ho_hc->h_tc_session_handle); + + s = format (s, "[%d:%d][H] half-open app_wrk %u ts %d:%d", + ho_hc->c_thread_index, ho_hc->c_s_index, ho_hc->h_pa_wrk_index, + tcp_ho->thread_index, tcp_ho->session_index); + return s; +} + +static transport_connection_t * +http_transport_get_ho (u32 ho_hc_index) +{ + http_conn_t *ho_hc; + + HTTP_DBG (1, "half open: %x", ho_hc_index); + ho_hc = http_ho_conn_get (ho_hc_index); + return &ho_hc->connection; +} + +static void +http_transport_cleanup_ho (u32 ho_hc_index) +{ + http_conn_t *ho_hc; + + HTTP_DBG (1, "half open: %x", ho_hc_index); + ho_hc = http_ho_conn_get (ho_hc_index); + session_cleanup_half_open (ho_hc->h_tc_session_handle); + http_ho_conn_free (ho_hc); +} + static const transport_proto_vft_t http_proto = { .enable = http_transport_enable, .connect = http_transport_connect, .start_listen = http_start_listen, .stop_listen = http_stop_listen, .close = http_transport_close, + .cleanup_ho = http_transport_cleanup_ho, .custom_tx = http_app_tx_callback, + .app_rx_evt = http_app_rx_evt_cb, .get_connection = http_transport_get_connection, .get_listener = http_transport_get_listener, + .get_half_open = http_transport_get_ho, .get_transport_endpoint = http_transport_get_endpoint, .format_connection = format_http_transport_connection, .format_listener = format_http_transport_listener, + .format_half_open = format_http_transport_half_open, .transport_options = { .name = "http", .short_name = "H", diff --git a/src/plugins/http/http.h b/src/plugins/http/http.h index 5f74edb5e47..b293c125465 100644 --- a/src/plugins/http/http.h +++ b/src/plugins/http/http.h @@ -59,32 +59,44 @@ typedef struct #define http_token_lit(s) (s), sizeof (s) - 1 +#define foreach_http_conn_state \ + _ (LISTEN, "listen") \ + _ (CONNECTING, "connecting") \ + _ (ESTABLISHED, "established") \ + _ (TRANSPORT_CLOSED, "transport-closed") \ + _ (APP_CLOSED, "app-closed") \ + _ (CLOSED, "closed") + typedef enum http_conn_state_ { - HTTP_CONN_STATE_LISTEN, - HTTP_CONN_STATE_CONNECTING, - HTTP_CONN_STATE_ESTABLISHED, - HTTP_CONN_STATE_TRANSPORT_CLOSED, - HTTP_CONN_STATE_APP_CLOSED, - HTTP_CONN_STATE_CLOSED +#define _(s, str) HTTP_CONN_STATE_##s, + foreach_http_conn_state +#undef _ } http_conn_state_t; -typedef enum http_state_ +#define foreach_http_req_state \ + _ (0, IDLE, "idle") \ + _ (1, WAIT_APP_METHOD, "wait app method") \ + _ (2, WAIT_TRANSPORT_REPLY, "wait transport reply") \ + _ (3, TRANSPORT_IO_MORE_DATA, "transport io more data") \ + _ (4, WAIT_TRANSPORT_METHOD, "wait transport method") \ + _ (5, WAIT_APP_REPLY, "wait app reply") \ + _ (6, APP_IO_MORE_DATA, "app io more data") \ + _ (7, TUNNEL, "tunnel") + +typedef enum http_req_state_ { - HTTP_STATE_IDLE = 0, - HTTP_STATE_WAIT_APP_METHOD, - HTTP_STATE_WAIT_CLIENT_METHOD, - HTTP_STATE_WAIT_SERVER_REPLY, - HTTP_STATE_WAIT_APP_REPLY, - HTTP_STATE_CLIENT_IO_MORE_DATA, - HTTP_STATE_APP_IO_MORE_DATA, - HTTP_N_STATES, -} http_state_t; +#define _(n, s, str) HTTP_REQ_STATE_##s = n, + foreach_http_req_state +#undef _ + HTTP_REQ_N_STATES +} http_req_state_t; typedef enum http_req_method_ { HTTP_REQ_GET = 0, HTTP_REQ_POST, + HTTP_REQ_CONNECT, } http_req_method_t; typedef enum http_msg_type_ @@ -388,6 +400,8 @@ typedef struct http_tc_ http_conn_state_t state; u32 timer_handle; + u32 timeout; + u8 pending_timer; u8 *app_name; u8 *host; u8 is_server; @@ -395,7 +409,7 @@ typedef struct http_tc_ /* * Current request */ - http_state_t http_state; + http_req_state_t req_state; http_req_method_t method; u8 *rx_buf; u32 rx_buf_offset; @@ -413,6 +427,7 @@ typedef struct http_tc_ u32 body_offset; u64 body_len; u16 status_code; + u8 is_tunnel; } http_conn_t; typedef struct http_worker_ @@ -424,6 +439,7 @@ typedef struct http_main_ { http_worker_t *wrk; http_conn_t *listener_pool; + http_conn_t *ho_conn_pool; u32 app_index; clib_timebase_t timebase; @@ -444,9 +460,10 @@ typedef struct http_main_ } http_main_t; always_inline int -_validate_target_syntax (u8 *target, int is_query, int *is_encoded) +_validate_target_syntax (u8 *target, u32 len, int is_query, int *is_encoded) { - int i, encoded = 0; + int encoded = 0; + u32 i; static uword valid_chars[4] = { /* !$&'()*+,-./0123456789:;= */ @@ -457,7 +474,7 @@ _validate_target_syntax (u8 *target, int is_query, int *is_encoded) 0x0000000000000000, }; - for (i = 0; i < vec_len (target); i++) + for (i = 0; i < len; i++) { if (clib_bitmap_get_no_check (valid_chars, target[i])) continue; @@ -468,7 +485,7 @@ _validate_target_syntax (u8 *target, int is_query, int *is_encoded) /* pct-encoded = "%" HEXDIG HEXDIG */ if (target[i] == '%') { - if ((i + 2) > vec_len (target)) + if ((i + 2) >= len) return -1; if (!isxdigit (target[i + 1]) || !isxdigit (target[i + 2])) return -1; @@ -487,7 +504,7 @@ _validate_target_syntax (u8 *target, int is_query, int *is_encoded) /** * An "absolute-path" rule validation (RFC9110 section 4.1). * - * @param path Target path to validate. + * @param path Vector of target path to validate. * @param is_encoded Return flag that indicates if percent-encoded (optional). * * @return @c 0 on success. @@ -495,13 +512,13 @@ _validate_target_syntax (u8 *target, int is_query, int *is_encoded) always_inline int http_validate_abs_path_syntax (u8 *path, int *is_encoded) { - return _validate_target_syntax (path, 0, is_encoded); + return _validate_target_syntax (path, vec_len (path), 0, is_encoded); } /** * A "query" rule validation (RFC3986 section 2.1). * - * @param query Target query to validate. + * @param query Vector of target query to validate. * @param is_encoded Return flag that indicates if percent-encoded (optional). * * @return @c 0 on success. @@ -509,7 +526,7 @@ http_validate_abs_path_syntax (u8 *path, int *is_encoded) always_inline int http_validate_query_syntax (u8 *query, int *is_encoded) { - return _validate_target_syntax (query, 1, is_encoded); + return _validate_target_syntax (query, vec_len (query), 1, is_encoded); } #define htoi(x) (isdigit (x) ? (x - '0') : (tolower (x) - 'a' + 10)) @@ -518,18 +535,19 @@ http_validate_query_syntax (u8 *query, int *is_encoded) * Decode percent-encoded data. * * @param src Data to decode. + * @param len Length of data to decode. * * @return New vector with decoded data. * * The caller is always responsible to free the returned vector. */ always_inline u8 * -http_percent_decode (u8 *src) +http_percent_decode (u8 *src, u32 len) { - int i; + u32 i; u8 *decoded_uri = 0; - for (i = 0; i < vec_len (src); i++) + for (i = 0; i < len; i++) { if (src[i] == '%') { @@ -974,6 +992,233 @@ http_serialize_authority_form_target (http_uri_t *authority) return s; } +typedef enum http_url_scheme_ +{ + HTTP_URL_SCHEME_HTTP, + HTTP_URL_SCHEME_HTTPS, +} http_url_scheme_t; + +typedef struct +{ + http_url_scheme_t scheme; + u16 port; + u32 host_offset; + u32 host_len; + u32 path_offset; + u32 path_len; + u8 host_is_ip6; +} http_url_t; + +always_inline int +_parse_port (u8 **pos, u8 *end, u16 *port) +{ + u32 value = 0; + u8 *p = *pos; + + if (!isdigit (*p)) + return -1; + value = *p - '0'; + p++; + + while (p != end) + { + if (!isdigit (*p)) + break; + value = value * 10 + *p - '0'; + if (value > CLIB_U16_MAX) + return -1; + p++; + } + *pos = p; + *port = clib_host_to_net_u16 ((u16) value); + return 0; +} + +/** + * An "absolute-form" URL parsing. + * + * @param url Vector of target URL to validate. + * @param parsed Parsed URL metadata in case of success. + * + * @return @c 0 on success. + */ +always_inline int +http_parse_absolute_form (u8 *url, http_url_t *parsed) +{ + u8 *token_start, *token_end, *end; + int is_encoded = 0; + + static uword valid_chars[4] = { + /* -.0123456789 */ + 0x03ff600000000000, + /* ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz */ + 0x07fffffe07fffffe, + 0x0000000000000000, + 0x0000000000000000, + }; + + if (vec_len (url) < 9) + { + clib_warning ("uri too short"); + return -1; + } + + clib_memset (parsed, 0, sizeof (*parsed)); + + end = url + vec_len (url); + + /* parse scheme */ + if (!memcmp (url, "http:// ", 7)) + { + parsed->scheme = HTTP_URL_SCHEME_HTTP; + parsed->port = clib_host_to_net_u16 (80); + parsed->host_offset = 7; + } + else if (!memcmp (url, "https:// ", 8)) + { + parsed->scheme = HTTP_URL_SCHEME_HTTPS; + parsed->port = clib_host_to_net_u16 (443); + parsed->host_offset = 8; + } + else + { + clib_warning ("invalid scheme"); + return -1; + } + token_start = url + parsed->host_offset; + + /* parse host */ + if (*token_start == '[') + /* IPv6 address */ + { + parsed->host_is_ip6 = 1; + parsed->host_offset++; + token_end = ++token_start; + while (1) + { + if (token_end == end) + { + clib_warning ("invalid host, IPv6 addr not terminated with ']'"); + return -1; + } + else if (*token_end == ']') + { + parsed->host_len = token_end - token_start; + token_start = token_end + 1; + break; + } + else if (*token_end != ':' && *token_end != '.' && + !isxdigit (*token_end)) + { + clib_warning ("invalid character '%u'", *token_end); + return -1; + } + token_end++; + } + } + else + { + token_end = token_start; + while (token_end != end && *token_end != ':' && *token_end != '/') + { + if (!clib_bitmap_get_no_check (valid_chars, *token_end)) + { + clib_warning ("invalid character '%u'", *token_end); + return -1; + } + token_end++; + } + parsed->host_len = token_end - token_start; + token_start = token_end; + } + + if (!parsed->host_len) + { + clib_warning ("zero length host"); + return -1; + } + + /* parse port, if any */ + if (token_start != end && *token_start == ':') + { + token_end = ++token_start; + if (_parse_port (&token_end, end, &parsed->port)) + { + clib_warning ("invalid port"); + return -1; + } + token_start = token_end; + } + + if (token_start == end) + return 0; + + token_start++; /* drop leading slash */ + parsed->path_offset = token_start - url; + parsed->path_len = end - token_start; + + if (parsed->path_len) + return _validate_target_syntax (token_start, parsed->path_len, 0, + &is_encoded); + + return 0; +} + +/** + * Parse target host and port of UDP tunnel over HTTP. + * + * @param path Path in format "{target_host}/{target_port}/". + * @param path_len Length of given path. + * @param parsed Parsed target in case of success.. + * + * @return @c 0 on success. + * + * @note Only IPv4 literals and IPv6 literals supported. + */ +always_inline int +http_parse_masque_host_port (u8 *path, u32 path_len, http_uri_t *parsed) +{ + u8 *p, *end, *decoded_host; + u32 host_len; + unformat_input_t input; + + p = path; + end = path + path_len; + clib_memset (parsed, 0, sizeof (*parsed)); + + while (p != end && *p != '/') + p++; + + host_len = p - path; + if (!host_len || (host_len == path_len) || (host_len + 1 == path_len)) + return -1; + decoded_host = http_percent_decode (path, host_len); + unformat_init_vector (&input, decoded_host); + if (unformat (&input, "%U", unformat_ip4_address, &parsed->ip.ip4)) + parsed->is_ip4 = 1; + else if (unformat (&input, "%U", unformat_ip6_address, &parsed->ip.ip6)) + parsed->is_ip4 = 0; + else + { + unformat_free (&input); + clib_warning ("unsupported target_host format"); + return -1; + } + unformat_free (&input); + + p++; + if (_parse_port (&p, end, &parsed->port)) + { + clib_warning ("invalid port"); + return -1; + } + + if (p == end || *p != '/') + return -1; + + return 0; +} + #endif /* SRC_PLUGINS_HTTP_HTTP_H_ */ /* diff --git a/src/plugins/http/http_plugin.rst b/src/plugins/http/http_plugin.rst index 56da3a810b9..f86c796bd83 100644 --- a/src/plugins/http/http_plugin.rst +++ b/src/plugins/http/http_plugin.rst @@ -16,7 +16,8 @@ Usage The plugin exposes following inline functions: ``http_validate_abs_path_syntax``, ``http_validate_query_syntax``, ``http_percent_decode``, ``http_path_remove_dot_segments``, ``http_parse_headers``, ``http_get_header``, -``http_free_header_table``, ``http_add_header``, ``http_serialize_headers``. +``http_free_header_table``, ``http_add_header``, ``http_serialize_headers``, ``http_parse_authority_form_target``, +``http_serialize_authority_form_target``, ``http_parse_absolute_form``, ``http_parse_masque_host_port``. It relies on the hoststack constructs and uses ``http_msg_data_t`` data structure for passing metadata to/from applications. @@ -82,7 +83,7 @@ Example bellow validates "absolute-path" rule, as described in RFC9110 section 4 } if (is_encoded) { - u8 *decoded = http_percent_decode (target_path); + u8 *decoded = http_percent_decode (target_path, vec_len (target_path)); vec_free (target_path); target_path = decoded; } @@ -505,3 +506,32 @@ Now we can start reading body content, following block of code could be executed /* close the session if you don't want to send another request */ /* and update state machine... */ } + +HTTP timeout +^^^^^^^^^^^^ + +HTTP plugin sets session inactivity timeout by default to 60 seconds. +Client and server applications can pass custom timeout value (in seconds) using extended configuration when doing connect or start listening respectively. +You just need to add extended configuration to session endpoint configuration which is part of ``vnet_connect_args_t`` and ``vnet_listen_args_t``. +HTTP plugin use ``opaque`` member of ``transport_endpt_ext_cfg_t``, unsigned 32bit integer seems to be sufficient (allowing the timeout to be set up to 136 years). + +The example below sets HTTP session timeout to 30 seconds (server application): + +.. code-block:: C + + vnet_listen_args_t _a, *a = &_a; + session_endpoint_cfg_t sep = SESSION_ENDPOINT_CFG_NULL; + transport_endpt_ext_cfg_t *ext_cfg; + int rv; + clib_memset (a, 0, sizeof (*a)); + clib_memcpy (&a->sep_ext, &sep, sizeof (sep)); + /* add new extended config entry */ + ext_cfg = session_endpoint_add_ext_cfg ( + &a->sep_ext, TRANSPORT_ENDPT_EXT_CFG_HTTP, sizeof (ext_cfg->opaque)); + /* your custom timeout value in seconds */ + ext_cfg->opaque = 30; + /* rest of the settings omitted for brevity */ + rv = vnet_listen (a); + /* don't forget to free extended config */ + session_endpoint_free_ext_cfgs (&a->sep_ext); + /* ... */ diff --git a/src/plugins/http/http_test.c b/src/plugins/http/http_test.c deleted file mode 100644 index 1f2f21dd19a..00000000000 --- a/src/plugins/http/http_test.c +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 - * Copyright(c) 2024 Cisco Systems, Inc. - */ - -#include <http/http.h> - -static clib_error_t * -test_http_authority_command_fn (vlib_main_t *vm, unformat_input_t *input, - vlib_cli_command_t *cmd) -{ - u8 *target = 0; - http_uri_t authority; - int rv; - - if (!unformat (input, "%v", &target)) - return clib_error_return (0, "error: no input provided"); - - rv = http_parse_authority_form_target (target, &authority); - vec_free (target); - if (rv) - return clib_error_return (0, "error: parsing failed"); - - target = http_serialize_authority_form_target (&authority); - vlib_cli_output (vm, "%v", target); - vec_free (target); - - return 0; -} - -VLIB_CLI_COMMAND (test_http_authority_command) = { - .path = "test http authority-form", - .short_help = "test dns authority-form", - .function = test_http_authority_command_fn, -}; diff --git a/src/plugins/http/http_timer.c b/src/plugins/http/http_timer.c index 5ee8efc8551..580f31657a9 100644 --- a/src/plugins/http/http_timer.c +++ b/src/plugins/http/http_timer.c @@ -29,7 +29,15 @@ http_timer_process_expired_cb (u32 *expired_timers) { /* Get session handle. The first bit is the timer id */ hs_handle = expired_timers[i] & 0x7FFFFFFF; - session_send_rpc_evt_to_thread (hs_handle >> 24, twc->cb_fn, + twc->invalidate_cb (hs_handle); + } + for (i = 0; i < vec_len (expired_timers); i++) + { + /* Get session handle. The first bit is the timer id */ + hs_handle = expired_timers[i] & 0x7FFFFFFF; + HTTP_DBG (1, "rpc to hc [%u]%x", hs_handle >> 24, + hs_handle & 0x00FFFFFF); + session_send_rpc_evt_to_thread (hs_handle >> 24, twc->rpc_cb, uword_to_pointer (hs_handle, void *)); } } @@ -66,7 +74,8 @@ VLIB_REGISTER_NODE (http_timer_process_node) = { }; void -http_timers_init (vlib_main_t *vm, http_conn_timeout_fn *cb_fn) +http_timers_init (vlib_main_t *vm, http_conn_timeout_fn *rpc_cb, + http_conn_invalidate_timer_fn *invalidate_cb) { http_tw_ctx_t *twc = &http_tw_ctx; vlib_node_t *n; @@ -76,7 +85,8 @@ http_timers_init (vlib_main_t *vm, http_conn_timeout_fn *cb_fn) tw_timer_wheel_init_2t_1w_2048sl (&twc->tw, http_timer_process_expired_cb, 1.0 /* timer interval */, ~0); clib_spinlock_init (&twc->tw_lock); - twc->cb_fn = cb_fn; + twc->rpc_cb = rpc_cb; + twc->invalidate_cb = invalidate_cb; vlib_node_set_state (vm, http_timer_process_node.index, VLIB_NODE_STATE_POLLING); diff --git a/src/plugins/http/http_timer.h b/src/plugins/http/http_timer.h index eec5a4595fe..43d20d004d8 100644 --- a/src/plugins/http/http_timer.h +++ b/src/plugins/http/http_timer.h @@ -19,34 +19,37 @@ #include <http/http.h> #include <vppinfra/tw_timer_2t_1w_2048sl.h> -#define HTTP_CONN_TIMEOUT 60 +#define HTTP_CONN_TIMEOUT 60 +#define HTTP_TIMER_HANDLE_INVALID ((u32) ~0) typedef void (http_conn_timeout_fn) (void *); +typedef void (http_conn_invalidate_timer_fn) (u32 hs_handle); typedef struct http_tw_ctx_ { tw_timer_wheel_2t_1w_2048sl_t tw; clib_spinlock_t tw_lock; - http_conn_timeout_fn *cb_fn; + http_conn_timeout_fn *rpc_cb; + http_conn_invalidate_timer_fn *invalidate_cb; } http_tw_ctx_t; extern http_tw_ctx_t http_tw_ctx; -void http_timers_init (vlib_main_t *vm, http_conn_timeout_fn *cb_fn); +void http_timers_init (vlib_main_t *vm, http_conn_timeout_fn *rpc_cb, + http_conn_invalidate_timer_fn *invalidate_cb); static inline void http_conn_timer_start (http_conn_t *hc) { http_tw_ctx_t *twc = &http_tw_ctx; u32 hs_handle; - u64 timeout; - timeout = HTTP_CONN_TIMEOUT; + ASSERT (hc->timer_handle == HTTP_TIMER_HANDLE_INVALID); hs_handle = hc->c_thread_index << 24 | hc->c_c_index; clib_spinlock_lock (&twc->tw_lock); hc->timer_handle = - tw_timer_start_2t_1w_2048sl (&twc->tw, hs_handle, 0, timeout); + tw_timer_start_2t_1w_2048sl (&twc->tw, hs_handle, 0, hc->timeout); clib_spinlock_unlock (&twc->tw_lock); } @@ -55,12 +58,13 @@ http_conn_timer_stop (http_conn_t *hc) { http_tw_ctx_t *twc = &http_tw_ctx; - if (hc->timer_handle == ~0) + hc->pending_timer = 0; + if (hc->timer_handle == HTTP_TIMER_HANDLE_INVALID) return; clib_spinlock_lock (&twc->tw_lock); tw_timer_stop_2t_1w_2048sl (&twc->tw, hc->timer_handle); - hc->timer_handle = ~0; + hc->timer_handle = HTTP_TIMER_HANDLE_INVALID; clib_spinlock_unlock (&twc->tw_lock); } @@ -68,15 +72,17 @@ static inline void http_conn_timer_update (http_conn_t *hc) { http_tw_ctx_t *twc = &http_tw_ctx; - u64 timeout; - - if (hc->timer_handle == ~0) - return; - - timeout = HTTP_CONN_TIMEOUT; + u32 hs_handle; clib_spinlock_lock (&twc->tw_lock); - tw_timer_update_2t_1w_2048sl (&twc->tw, hc->timer_handle, timeout); + if (hc->timer_handle != HTTP_TIMER_HANDLE_INVALID) + tw_timer_update_2t_1w_2048sl (&twc->tw, hc->timer_handle, hc->timeout); + else + { + hs_handle = hc->c_thread_index << 24 | hc->c_c_index; + hc->timer_handle = + tw_timer_start_2t_1w_2048sl (&twc->tw, hs_handle, 0, hc->timeout); + } clib_spinlock_unlock (&twc->tw_lock); } diff --git a/src/plugins/http/test/http_test.c b/src/plugins/http/test/http_test.c new file mode 100644 index 00000000000..d4ac8f46f29 --- /dev/null +++ b/src/plugins/http/test/http_test.c @@ -0,0 +1,360 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2024 Cisco Systems, Inc. + */ + +#include <vnet/plugin/plugin.h> +#include <vpp/app/version.h> +#include <http/http.h> + +#define HTTP_TEST_I(_cond, _comment, _args...) \ + ({ \ + int _evald = (_cond); \ + if (!(_evald)) \ + { \ + vlib_cli_output (vm, "FAIL:%d: " _comment "\n", __LINE__, ##_args); \ + } \ + else \ + { \ + vlib_cli_output (vm, "PASS:%d: " _comment "\n", __LINE__, ##_args); \ + } \ + _evald; \ + }) + +#define HTTP_TEST(_cond, _comment, _args...) \ + { \ + if (!HTTP_TEST_I (_cond, _comment, ##_args)) \ + { \ + return 1; \ + } \ + } + +static int +http_test_authority_form (vlib_main_t *vm) +{ + u8 *target = 0, *formated_target = 0; + http_uri_t authority; + int rv; + + target = format (0, "10.10.2.45:20"); + rv = http_parse_authority_form_target (target, &authority); + HTTP_TEST ((rv == 0), "'%v' should be valid", target); + formated_target = http_serialize_authority_form_target (&authority); + rv = vec_cmp (target, formated_target); + HTTP_TEST ((rv == 0), "'%v' should match '%v'", target, formated_target); + vec_free (target); + vec_free (formated_target); + + target = format (0, "[dead:beef::1234]:443"); + rv = http_parse_authority_form_target (target, &authority); + HTTP_TEST ((rv == 0), "'%v' should be valid", target); + formated_target = http_serialize_authority_form_target (&authority); + rv = vec_cmp (target, formated_target); + HTTP_TEST ((rv == 0), "'%v' should match '%v'", target, formated_target); + vec_free (target); + vec_free (formated_target); + + target = format (0, "example.com:80"); + rv = http_parse_authority_form_target (target, &authority); + HTTP_TEST ((rv != 0), "'%v' reg-name not supported", target); + vec_free (target); + + target = format (0, "10.10.2.45"); + rv = http_parse_authority_form_target (target, &authority); + HTTP_TEST ((rv != 0), "'%v' should be invalid", target); + vec_free (target); + + target = format (0, "1000.10.2.45:20"); + rv = http_parse_authority_form_target (target, &authority); + HTTP_TEST ((rv != 0), "'%v' should be invalid", target); + vec_free (target); + + target = format (0, "[xyz0::1234]:443"); + rv = http_parse_authority_form_target (target, &authority); + HTTP_TEST ((rv != 0), "'%v' should be invalid", target); + vec_free (target); + + return 0; +} + +static int +http_test_absolute_form (vlib_main_t *vm) +{ + u8 *url = 0; + http_url_t parsed_url; + int rv; + + url = format (0, "https://example.org/.well-known/masque/udp/1.2.3.4/123/"); + rv = http_parse_absolute_form (url, &parsed_url); + HTTP_TEST ((rv == 0), "'%v' should be valid", url); + HTTP_TEST ((parsed_url.scheme == HTTP_URL_SCHEME_HTTPS), + "scheme should be https"); + HTTP_TEST ((parsed_url.host_is_ip6 == 0), "host_is_ip6=%u should be 0", + parsed_url.host_is_ip6); + HTTP_TEST ((parsed_url.host_offset == strlen ("https://")), + "host_offset=%u should be %u", parsed_url.host_offset, + strlen ("https://")); + HTTP_TEST ((parsed_url.host_len == strlen ("example.org")), + "host_len=%u should be %u", parsed_url.host_len, + strlen ("example.org")); + HTTP_TEST ((clib_net_to_host_u16 (parsed_url.port) == 443), + "port=%u should be 443", clib_net_to_host_u16 (parsed_url.port)); + HTTP_TEST ((parsed_url.path_offset == strlen ("https://example.org/")), + "path_offset=%u should be %u", parsed_url.path_offset, + strlen ("https://example.org/")); + HTTP_TEST ( + (parsed_url.path_len == strlen (".well-known/masque/udp/1.2.3.4/123/")), + "path_len=%u should be %u", parsed_url.path_len, + strlen (".well-known/masque/udp/1.2.3.4/123/")); + vec_free (url); + + url = format (0, "http://vpp-example.org"); + rv = http_parse_absolute_form (url, &parsed_url); + HTTP_TEST ((rv == 0), "'%v' should be valid", url); + HTTP_TEST ((parsed_url.scheme == HTTP_URL_SCHEME_HTTP), + "scheme should be http"); + HTTP_TEST ((parsed_url.host_is_ip6 == 0), "host_is_ip6=%u should be 0", + parsed_url.host_is_ip6); + HTTP_TEST ((parsed_url.host_offset == strlen ("http://")), + "host_offset=%u should be %u", parsed_url.host_offset, + strlen ("http://")); + HTTP_TEST ((parsed_url.host_len == strlen ("vpp-example.org")), + "host_len=%u should be %u", parsed_url.host_len, + strlen ("vpp-example.org")); + HTTP_TEST ((clib_net_to_host_u16 (parsed_url.port) == 80), + "port=%u should be 80", clib_net_to_host_u16 (parsed_url.port)); + HTTP_TEST ((parsed_url.path_len == 0), "path_len=%u should be 0", + parsed_url.path_len); + vec_free (url); + + url = format (0, "http://1.2.3.4:8080/abcd"); + rv = http_parse_absolute_form (url, &parsed_url); + HTTP_TEST ((rv == 0), "'%v' should be valid", url); + HTTP_TEST ((parsed_url.scheme == HTTP_URL_SCHEME_HTTP), + "scheme should be http"); + HTTP_TEST ((parsed_url.host_is_ip6 == 0), "host_is_ip6=%u should be 0", + parsed_url.host_is_ip6); + HTTP_TEST ((parsed_url.host_offset == strlen ("http://")), + "host_offset=%u should be %u", parsed_url.host_offset, + strlen ("http://")); + HTTP_TEST ((parsed_url.host_len == strlen ("1.2.3.4")), + "host_len=%u should be %u", parsed_url.host_len, + strlen ("1.2.3.4")); + HTTP_TEST ((clib_net_to_host_u16 (parsed_url.port) == 8080), + "port=%u should be 8080", clib_net_to_host_u16 (parsed_url.port)); + HTTP_TEST ((parsed_url.path_offset == strlen ("http://1.2.3.4:8080/")), + "path_offset=%u should be %u", parsed_url.path_offset, + strlen ("http://1.2.3.4:8080/")); + HTTP_TEST ((parsed_url.path_len == strlen ("abcd")), + "path_len=%u should be %u", parsed_url.path_len, strlen ("abcd")); + vec_free (url); + + url = format (0, "https://[dead:beef::1234]/abcd"); + rv = http_parse_absolute_form (url, &parsed_url); + HTTP_TEST ((rv == 0), "'%v' should be valid", url); + HTTP_TEST ((parsed_url.scheme == HTTP_URL_SCHEME_HTTPS), + "scheme should be https"); + HTTP_TEST ((parsed_url.host_is_ip6 == 1), "host_is_ip6=%u should be 1", + parsed_url.host_is_ip6); + HTTP_TEST ((parsed_url.host_offset == strlen ("https://[")), + "host_offset=%u should be %u", parsed_url.host_offset, + strlen ("https://[")); + HTTP_TEST ((parsed_url.host_len == strlen ("dead:beef::1234")), + "host_len=%u should be %u", parsed_url.host_len, + strlen ("dead:beef::1234")); + HTTP_TEST ((clib_net_to_host_u16 (parsed_url.port) == 443), + "port=%u should be 443", clib_net_to_host_u16 (parsed_url.port)); + HTTP_TEST ((parsed_url.path_offset == strlen ("https://[dead:beef::1234]/")), + "path_offset=%u should be %u", parsed_url.path_offset, + strlen ("https://[dead:beef::1234]/")); + HTTP_TEST ((parsed_url.path_len == strlen ("abcd")), + "path_len=%u should be %u", parsed_url.path_len, strlen ("abcd")); + vec_free (url); + + url = format (0, "http://[::ffff:192.0.2.128]:8080/"); + rv = http_parse_absolute_form (url, &parsed_url); + HTTP_TEST ((rv == 0), "'%v' should be valid", url); + HTTP_TEST ((parsed_url.scheme == HTTP_URL_SCHEME_HTTP), + "scheme should be http"); + HTTP_TEST ((parsed_url.host_is_ip6 == 1), "host_is_ip6=%u should be 1", + parsed_url.host_is_ip6); + HTTP_TEST ((parsed_url.host_offset == strlen ("http://[")), + "host_offset=%u should be %u", parsed_url.host_offset, + strlen ("http://[")); + HTTP_TEST ((parsed_url.host_len == strlen ("::ffff:192.0.2.128")), + "host_len=%u should be %u", parsed_url.host_len, + strlen ("::ffff:192.0.2.128")); + HTTP_TEST ((clib_net_to_host_u16 (parsed_url.port) == 8080), + "port=%u should be 8080", clib_net_to_host_u16 (parsed_url.port)); + HTTP_TEST ((parsed_url.path_len == 0), "path_len=%u should be 0", + parsed_url.path_len); + vec_free (url); + + url = format (0, "http://[dead:beef::1234/abc"); + rv = http_parse_absolute_form (url, &parsed_url); + HTTP_TEST ((rv != 0), "'%v' should be invalid", url); + vec_free (url); + + url = format (0, "http://[dead|beef::1234]/abc"); + rv = http_parse_absolute_form (url, &parsed_url); + HTTP_TEST ((rv != 0), "'%v' should be invalid", url); + vec_free (url); + + url = format (0, "http:example.org:8080/abcd"); + rv = http_parse_absolute_form (url, &parsed_url); + HTTP_TEST ((rv != 0), "'%v' should be invalid", url); + vec_free (url); + + url = format (0, "htt://example.org:8080/abcd"); + rv = http_parse_absolute_form (url, &parsed_url); + HTTP_TEST ((rv != 0), "'%v' should be invalid", url); + vec_free (url); + + url = format (0, "http://"); + rv = http_parse_absolute_form (url, &parsed_url); + HTTP_TEST ((rv != 0), "'%v' should be invalid", url); + vec_free (url); + + url = format (0, "http:///abcd"); + rv = http_parse_absolute_form (url, &parsed_url); + HTTP_TEST ((rv != 0), "'%v' should be invalid", url); + vec_free (url); + + url = format (0, "http://example.org:808080/abcd"); + rv = http_parse_absolute_form (url, &parsed_url); + HTTP_TEST ((rv != 0), "'%v' should be invalid", url); + vec_free (url); + + url = format (0, "http://example.org/a%%3Xbcd"); + rv = http_parse_absolute_form (url, &parsed_url); + HTTP_TEST ((rv != 0), "'%v' should be invalid", url); + vec_free (url); + + url = format (0, "http://example.org/a%%3"); + rv = http_parse_absolute_form (url, &parsed_url); + HTTP_TEST ((rv != 0), "'%v' should be invalid", url); + vec_free (url); + + url = format (0, "http://example.org/a[b]cd"); + rv = http_parse_absolute_form (url, &parsed_url); + HTTP_TEST ((rv != 0), "'%v' should be invalid", url); + vec_free (url); + + url = format (0, "http://exa[m]ple.org/abcd"); + rv = http_parse_absolute_form (url, &parsed_url); + HTTP_TEST ((rv != 0), "'%v' should be invalid", url); + vec_free (url); + + return 0; +} + +static int +http_test_parse_masque_host_port (vlib_main_t *vm) +{ + u8 *path = 0; + http_uri_t target; + int rv; + + path = format (0, "10.10.2.45/443/"); + rv = http_parse_masque_host_port (path, vec_len (path), &target); + HTTP_TEST ((rv == 0), "'%v' should be valid", path); + HTTP_TEST ((target.is_ip4 == 1), "is_ip4=%d should be 1", target.is_ip4); + HTTP_TEST ((clib_net_to_host_u16 (target.port) == 443), + "port=%u should be 443", clib_net_to_host_u16 (target.port)); + HTTP_TEST ((target.ip.ip4.data[0] == 10 && target.ip.ip4.data[1] == 10 && + target.ip.ip4.data[2] == 2 && target.ip.ip4.data[3] == 45), + "target.ip=%U should be 10.10.2.45", format_ip4_address, + &target.ip.ip4); + vec_free (path); + + path = format (0, "dead%%3Abeef%%3A%%3A1234/80/"); + rv = http_parse_masque_host_port (path, vec_len (path), &target); + HTTP_TEST ((rv == 0), "'%v' should be valid", path); + HTTP_TEST ((target.is_ip4 == 0), "is_ip4=%d should be 0", target.is_ip4); + HTTP_TEST ((clib_net_to_host_u16 (target.port) == 80), + "port=%u should be 80", clib_net_to_host_u16 (target.port)); + HTTP_TEST ((clib_net_to_host_u16 (target.ip.ip6.as_u16[0]) == 0xdead && + clib_net_to_host_u16 (target.ip.ip6.as_u16[1]) == 0xbeef && + target.ip.ip6.as_u16[2] == 0 && target.ip.ip6.as_u16[3] == 0 && + target.ip.ip6.as_u16[4] == 0 && target.ip.ip6.as_u16[5] == 0 && + target.ip.ip6.as_u16[6] == 0 && + clib_net_to_host_u16 (target.ip.ip6.as_u16[7]) == 0x1234), + "target.ip=%U should be dead:beef::1234", format_ip6_address, + &target.ip.ip6); + vec_free (path); + + path = format (0, "example.com/443/"); + rv = http_parse_masque_host_port (path, vec_len (path), &target); + HTTP_TEST ((rv != 0), "'%v' reg-name not supported", path); + vec_free (path); + + path = format (0, "10.10.2.45/443443/"); + rv = http_parse_masque_host_port (path, vec_len (path), &target); + HTTP_TEST ((rv != 0), "'%v' should be invalid", path); + vec_free (path); + + path = format (0, "/443/"); + rv = http_parse_masque_host_port (path, vec_len (path), &target); + HTTP_TEST ((rv != 0), "'%v' should be invalid", path); + vec_free (path); + + path = format (0, "10.10.2.45/"); + rv = http_parse_masque_host_port (path, vec_len (path), &target); + HTTP_TEST ((rv != 0), "'%v' should be invalid", path); + vec_free (path); + + path = format (0, "10.10.2.45"); + rv = http_parse_masque_host_port (path, vec_len (path), &target); + HTTP_TEST ((rv != 0), "'%v' should be invalid", path); + vec_free (path); + + path = format (0, "10.10.2.45/443"); + rv = http_parse_masque_host_port (path, vec_len (path), &target); + HTTP_TEST ((rv != 0), "'%v' should be invalid", path); + vec_free (path); + + return 0; +} + +static clib_error_t * +test_http_command_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + int res = 0; + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "authority-form")) + res = http_test_authority_form (vm); + else if (unformat (input, "absolute-form")) + res = http_test_absolute_form (vm); + else if (unformat (input, "parse-masque-host-port")) + res = http_test_parse_masque_host_port (vm); + else if (unformat (input, "all")) + { + if ((res = http_test_authority_form (vm))) + goto done; + if ((res = http_test_absolute_form (vm))) + goto done; + if ((res = http_test_parse_masque_host_port (vm))) + goto done; + } + else + break; + } + +done: + if (res) + return clib_error_return (0, "HTTP unit test failed"); + return 0; +} + +VLIB_CLI_COMMAND (test_http_command) = { + .path = "test http", + .short_help = "http unit tests", + .function = test_http_command_fn, +}; + +VLIB_PLUGIN_REGISTER () = { + .version = VPP_BUILD_VER, + .description = "HTTP - Unit Test", + .default_disabled = 1, +}; diff --git a/src/plugins/http_static/http_static.api b/src/plugins/http_static/http_static.api index dd4f513a420..60c0369848d 100644 --- a/src/plugins/http_static/http_static.api +++ b/src/plugins/http_static/http_static.api @@ -3,20 +3,21 @@ This file defines static http server control-plane API messages */ -option version = "2.2.0"; +option version = "2.3.0"; /** \brief Configure and enable the static http server @param client_index - opaque cookie to identify the sender @param context - sender context, to match reply w/ request @param fifo_size - size (in bytes) of the session FIFOs @param cache_size_limit - size (in bytes) of the in-memory file data cache + @param max_age - how long a response is considered fresh (in seconds) @param prealloc_fifos - number of preallocated fifos (usually 0) @param private_segment_size - fifo segment size (usually 0) @param www_root - html root path @param uri - bind URI, defaults to "tcp://0.0.0.0/80" */ -autoreply define http_static_enable { +autoreply define http_static_enable_v2 { option deprecated; /* Client identifier, set from api_main.my_client_index */ @@ -27,6 +28,7 @@ autoreply define http_static_enable { /* Typical options */ u32 fifo_size; u32 cache_size_limit; + u32 max_age [default=600]; /* Unusual options */ u32 prealloc_fifos; u32 private_segment_size; @@ -43,13 +45,16 @@ autoreply define http_static_enable { @param fifo_size - size (in bytes) of the session FIFOs @param cache_size_limit - size (in bytes) of the in-memory file data cache @param max_age - how long a response is considered fresh (in seconds) + @param keepalive_timeout - timeout during which client connection will stay open (in seconds) @param prealloc_fifos - number of preallocated fifos (usually 0) @param private_segment_size - fifo segment size (usually 0) @param www_root - html root path @param uri - bind URI, defaults to "tcp://0.0.0.0/80" */ -autoreply define http_static_enable_v2 { +autoreply define http_static_enable_v3 { + option deprecated; + /* Client identifier, set from api_main.my_client_index */ u32 client_index; @@ -59,6 +64,7 @@ autoreply define http_static_enable_v2 { u32 fifo_size; u32 cache_size_limit; u32 max_age [default=600]; + u32 keepalive_timeout [default=60]; /* Unusual options */ u32 prealloc_fifos; u32 private_segment_size; diff --git a/src/plugins/http_static/http_static.c b/src/plugins/http_static/http_static.c index 967b8474af8..464fd27e90d 100644 --- a/src/plugins/http_static/http_static.c +++ b/src/plugins/http_static/http_static.c @@ -66,7 +66,8 @@ hss_register_url_handler (hss_url_handler_fn fp, const char *url, */ static int hss_enable_api (u32 fifo_size, u32 cache_limit, u32 prealloc_fifos, - u32 private_segment_size, u8 *www_root, u8 *uri, u32 max_age) + u32 private_segment_size, u8 *www_root, u8 *uri, u32 max_age, + u32 keepalive_timeout) { hss_main_t *hsm = &hss_main; int rv; @@ -78,6 +79,7 @@ hss_enable_api (u32 fifo_size, u32 cache_limit, u32 prealloc_fifos, hsm->www_root = format (0, "%s%c", www_root, 0); hsm->uri = format (0, "%s%c", uri, 0); hsm->max_age = max_age; + hsm->keepalive_timeout = keepalive_timeout; if (vec_len (hsm->www_root) < 2) return VNET_API_ERROR_INVALID_VALUE; @@ -104,10 +106,10 @@ hss_enable_api (u32 fifo_size, u32 cache_limit, u32 prealloc_fifos, } /* API message handler */ -static void vl_api_http_static_enable_t_handler - (vl_api_http_static_enable_t * mp) +static void +vl_api_http_static_enable_v2_t_handler (vl_api_http_static_enable_v2_t *mp) { - vl_api_http_static_enable_reply_t *rmp; + vl_api_http_static_enable_v2_reply_t *rmp; hss_main_t *hsm = &hss_main; int rv; @@ -117,16 +119,16 @@ static void vl_api_http_static_enable_t_handler rv = hss_enable_api (ntohl (mp->fifo_size), ntohl (mp->cache_size_limit), ntohl (mp->prealloc_fifos), ntohl (mp->private_segment_size), mp->www_root, mp->uri, - HSS_DEFAULT_MAX_AGE); + ntohl (mp->max_age), HSS_DEFAULT_KEEPALIVE_TIMEOUT); - REPLY_MACRO (VL_API_HTTP_STATIC_ENABLE_REPLY); + REPLY_MACRO (VL_API_HTTP_STATIC_ENABLE_V2_REPLY); } /* API message handler */ static void -vl_api_http_static_enable_v2_t_handler (vl_api_http_static_enable_v2_t *mp) +vl_api_http_static_enable_v3_t_handler (vl_api_http_static_enable_v3_t *mp) { - vl_api_http_static_enable_v2_reply_t *rmp; + vl_api_http_static_enable_v3_reply_t *rmp; hss_main_t *hsm = &hss_main; int rv; @@ -136,9 +138,9 @@ vl_api_http_static_enable_v2_t_handler (vl_api_http_static_enable_v2_t *mp) rv = hss_enable_api (ntohl (mp->fifo_size), ntohl (mp->cache_size_limit), ntohl (mp->prealloc_fifos), ntohl (mp->private_segment_size), mp->www_root, mp->uri, - ntohl (mp->max_age)); + ntohl (mp->max_age), ntohl (mp->keepalive_timeout)); - REPLY_MACRO (VL_API_HTTP_STATIC_ENABLE_V2_REPLY); + REPLY_MACRO (VL_API_HTTP_STATIC_ENABLE_V3_REPLY); } #include <http_static/http_static.api.c> diff --git a/src/plugins/http_static/http_static.h b/src/plugins/http_static/http_static.h index bee79090d2b..5e0654fae24 100644 --- a/src/plugins/http_static/http_static.h +++ b/src/plugins/http_static/http_static.h @@ -24,6 +24,7 @@ #include <http_static/http_cache.h> #define HSS_DEFAULT_MAX_AGE 600 +#define HSS_DEFAULT_KEEPALIVE_TIMEOUT 60 /** @file http_static.h * Static http server definitions @@ -162,6 +163,8 @@ typedef struct u32 max_age; /** Formatted max_age: "max-age=xyz" */ u8 *max_age_formatted; + /** Timeout during which client connection will stay open */ + u32 keepalive_timeout; /** hash table of file extensions to mime types string indices */ uword *mime_type_indices_by_file_extensions; diff --git a/src/plugins/http_static/http_static_test.c b/src/plugins/http_static/http_static_test.c index f701c8b9ee7..edb016f9e05 100644 --- a/src/plugins/http_static/http_static_test.c +++ b/src/plugins/http_static/http_static_test.c @@ -39,10 +39,10 @@ http_static_test_main_t http_static_test_main; #include <vlibapi/vat_helper_macros.h> static int -api_http_static_enable (vat_main_t * vam) +api_http_static_enable_v2 (vat_main_t *vam) { unformat_input_t *line_input = vam->input; - vl_api_http_static_enable_t *mp; + vl_api_http_static_enable_v2_t *mp; u64 tmp; u8 *www_root = 0; u8 *uri = 0; @@ -50,6 +50,7 @@ api_http_static_enable (vat_main_t * vam) u32 private_segment_size = 0; u32 fifo_size = 8 << 10; u32 cache_size_limit = 1 << 20; + u32 max_age = HSS_DEFAULT_MAX_AGE; int ret; /* Parse args required to build the message */ @@ -89,7 +90,8 @@ api_http_static_enable (vat_main_t * vam) } cache_size_limit = (u32) tmp; } - + else if (unformat (line_input, "max-age %d", &max_age)) + ; else if (unformat (line_input, "uri %s", &uri)) ; else @@ -108,16 +110,15 @@ api_http_static_enable (vat_main_t * vam) if (uri == 0) uri = format (0, "tcp://0.0.0.0/80%c", 0); - - /* Construct the API message */ - M (HTTP_STATIC_ENABLE, mp); + M (HTTP_STATIC_ENABLE_V2, mp); strncpy_s ((char *) mp->www_root, 256, (const char *) www_root, 256); strncpy_s ((char *) mp->uri, 256, (const char *) uri, 256); mp->fifo_size = ntohl (fifo_size); mp->cache_size_limit = ntohl (cache_size_limit); mp->prealloc_fifos = ntohl (prealloc_fifos); mp->private_segment_size = ntohl (private_segment_size); + mp->max_age = ntohl (max_age); /* send it... */ S (mp); @@ -128,10 +129,10 @@ api_http_static_enable (vat_main_t * vam) } static int -api_http_static_enable_v2 (vat_main_t *vam) +api_http_static_enable_v3 (vat_main_t *vam) { unformat_input_t *line_input = vam->input; - vl_api_http_static_enable_v2_t *mp; + vl_api_http_static_enable_v3_t *mp; u64 tmp; u8 *www_root = 0; u8 *uri = 0; @@ -140,6 +141,7 @@ api_http_static_enable_v2 (vat_main_t *vam) u32 fifo_size = 8 << 10; u32 cache_size_limit = 1 << 20; u32 max_age = HSS_DEFAULT_MAX_AGE; + u32 keepalive_timeout = HSS_DEFAULT_KEEPALIVE_TIMEOUT; int ret; /* Parse args required to build the message */ @@ -181,6 +183,9 @@ api_http_static_enable_v2 (vat_main_t *vam) } else if (unformat (line_input, "max-age %d", &max_age)) ; + else if (unformat (line_input, "keepalive-timeout %d", + &keepalive_timeout)) + ; else if (unformat (line_input, "uri %s", &uri)) ; else @@ -200,7 +205,7 @@ api_http_static_enable_v2 (vat_main_t *vam) uri = format (0, "tcp://0.0.0.0/80%c", 0); /* Construct the API message */ - M (HTTP_STATIC_ENABLE_V2, mp); + M (HTTP_STATIC_ENABLE_V3, mp); strncpy_s ((char *) mp->www_root, 256, (const char *) www_root, 256); strncpy_s ((char *) mp->uri, 256, (const char *) uri, 256); mp->fifo_size = ntohl (fifo_size); @@ -208,6 +213,7 @@ api_http_static_enable_v2 (vat_main_t *vam) mp->prealloc_fifos = ntohl (prealloc_fifos); mp->private_segment_size = ntohl (private_segment_size); mp->max_age = ntohl (max_age); + mp->keepalive_timeout = ntohl (keepalive_timeout); /* send it... */ S (mp); diff --git a/src/plugins/http_static/static_server.c b/src/plugins/http_static/static_server.c index 48e71f51629..9cc3f5dd658 100644 --- a/src/plugins/http_static/static_server.c +++ b/src/plugins/http_static/static_server.c @@ -126,6 +126,9 @@ start_send_data (hss_session_t *hs, http_status_code_t status) ASSERT (rv == sizeof (headers)); } + if (!msg.data.body_len) + goto done; + uword data = pointer_to_uword (hs->data); rv = svm_fifo_enqueue (ts->tx_fifo, sizeof (data), (u8 *) &data); ASSERT (rv == sizeof (data)); @@ -747,7 +750,7 @@ hss_attach () hss_main_t *hsm = &hss_main; u64 options[APP_OPTIONS_N_OPTIONS]; vnet_app_attach_args_t _a, *a = &_a; - u32 segment_size = 128 << 20; + u64 segment_size = 128 << 20; clib_memset (a, 0, sizeof (*a)); clib_memset (options, 0, sizeof (options)); @@ -804,6 +807,7 @@ hss_listen (void) vnet_listen_args_t _a, *a = &_a; char *uri = "tcp://0.0.0.0/80"; u8 need_crypto; + transport_endpt_ext_cfg_t *ext_cfg; int rv; clib_memset (a, 0, sizeof (*a)); @@ -820,17 +824,21 @@ hss_listen (void) sep.transport_proto = TRANSPORT_PROTO_HTTP; clib_memcpy (&a->sep_ext, &sep, sizeof (sep)); + ext_cfg = session_endpoint_add_ext_cfg ( + &a->sep_ext, TRANSPORT_ENDPT_EXT_CFG_HTTP, sizeof (ext_cfg->opaque)); + ext_cfg->opaque = hsm->keepalive_timeout; + if (need_crypto) { - session_endpoint_alloc_ext_cfg (&a->sep_ext, - TRANSPORT_ENDPT_EXT_CFG_CRYPTO); - a->sep_ext.ext_cfg->crypto.ckpair_index = hsm->ckpair_index; + ext_cfg = session_endpoint_add_ext_cfg ( + &a->sep_ext, TRANSPORT_ENDPT_EXT_CFG_CRYPTO, + sizeof (transport_endpt_crypto_cfg_t)); + ext_cfg->crypto.ckpair_index = hsm->ckpair_index; } rv = vnet_listen (a); - if (need_crypto) - clib_mem_free (a->sep_ext.ext_cfg); + session_endpoint_free_ext_cfgs (&a->sep_ext); return rv; } @@ -897,6 +905,7 @@ hss_create_command_fn (vlib_main_t *vm, unformat_input_t *input, hsm->fifo_size = 0; hsm->cache_size = 10 << 20; hsm->max_age = HSS_DEFAULT_MAX_AGE; + hsm->keepalive_timeout = HSS_DEFAULT_KEEPALIVE_TIMEOUT; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -921,6 +930,9 @@ hss_create_command_fn (vlib_main_t *vm, unformat_input_t *input, ; else if (unformat (line_input, "debug %d", &hsm->debug_level)) ; + else if (unformat (line_input, "keepalive-timeout %d", + &hsm->keepalive_timeout)) + ; else if (unformat (line_input, "debug")) hsm->debug_level = 1; else if (unformat (line_input, "ptr-thresh %U", unformat_memory_size, @@ -984,14 +996,16 @@ done: * http static server www-root /tmp/www uri tcp://0.0.0.0/80 cache-size 2m * @cliend * @cliexcmd{http static server www-root <path> [prealloc-fios <nn>] - * [private-segment-size <nnMG>] [fifo-size <nbytes>] [uri <uri>]} + * [private-segment-size <nnMG>] [fifo-size <nbytes>] [uri <uri>] + * [keepalive-timeout <nn>]} ?*/ VLIB_CLI_COMMAND (hss_create_command, static) = { .path = "http static server", .short_help = "http static server www-root <path> [prealloc-fifos <nn>]\n" "[private-segment-size <nnMG>] [fifo-size <nbytes>] [max-age <nseconds>]\n" - "[uri <uri>] [ptr-thresh <nn>] [url-handlers] [debug [nn]]\n", + "[uri <uri>] [ptr-thresh <nn>] [url-handlers] [debug [nn]]\n" + "[keepalive-timeout <nn>]\n", .function = hss_create_command_fn, }; diff --git a/src/plugins/map/ip6_map_t.c b/src/plugins/map/ip6_map_t.c index 51853d619e6..f8d894a013a 100644 --- a/src/plugins/map/ip6_map_t.c +++ b/src/plugins/map/ip6_map_t.c @@ -151,9 +151,8 @@ ip6_map_t_icmp (vlib_main_t * vm, vnet_buffer (p0)->map_t.map_domain_index); ctx0.d = d0; ctx0.sender_port = 0; - if (!ip6_get_port - (vm, p0, ip60, p0->current_length, NULL, &ctx0.sender_port, - NULL, NULL, NULL, NULL)) + if (!ip6_get_port (vm, p0, ip60, p0->current_length, NULL, + &ctx0.sender_port, NULL, NULL, NULL, NULL, NULL)) { // In case of 1:1 mapping, we don't care about the port if (!(d0->ea_bits_len == 0 && d0->rules)) diff --git a/src/plugins/mss_clamp/mss_clamp.c b/src/plugins/mss_clamp/mss_clamp.c index cdac5456641..f1c435a347b 100644 --- a/src/plugins/mss_clamp/mss_clamp.c +++ b/src/plugins/mss_clamp/mss_clamp.c @@ -46,7 +46,7 @@ mssc_enable_disable_feat (u32 sw_if_index, u8 dir4, u8 dir6, int enable) sw_if_index, enable, 0, 0); } -int +__clib_export int mssc_enable_disable (u32 sw_if_index, u8 dir4, u8 dir6, u16 mss4, u16 mss6) { mssc_main_t *cm = &mssc_main; @@ -81,7 +81,7 @@ mssc_enable_disable (u32 sw_if_index, u8 dir4, u8 dir6, u16 mss4, u16 mss6) return rv; } -int +__clib_export int mssc_get_mss (u32 sw_if_index, u8 *dir4, u8 *dir6, u16 *mss4, u16 *mss6) { mssc_main_t *cm = &mssc_main; diff --git a/src/plugins/quic/quic.c b/src/plugins/quic/quic.c index 3f7a3426069..3797cd2b4ea 100644 --- a/src/plugins/quic/quic.c +++ b/src/plugins/quic/quic.c @@ -1332,14 +1332,16 @@ quic_connect_connection (session_endpoint_cfg_t * sep) quic_ctx_t *ctx; app_worker_t *app_wrk; application_t *app; + transport_endpt_ext_cfg_t *ext_cfg; int error; - if (!sep->ext_cfg) + ext_cfg = session_endpoint_get_ext_cfg (sep, TRANSPORT_ENDPT_EXT_CFG_CRYPTO); + if (!ext_cfg) return SESSION_E_NOEXTCFG; /* Use pool on thread 1 if we have workers because of UDP */ thread_index = transport_cl_thread (); - ccfg = &sep->ext_cfg->crypto; + ccfg = &ext_cfg->crypto; clib_memset (cargs, 0, sizeof (*cargs)); ctx_index = quic_ctx_alloc (thread_index); @@ -1475,13 +1477,15 @@ quic_start_listen (u32 quic_listen_session_index, quic_ctx_t *lctx; u32 lctx_index; app_listener_t *app_listener; + transport_endpt_ext_cfg_t *ext_cfg; int rv; sep = (session_endpoint_cfg_t *) tep; - if (!sep->ext_cfg) + ext_cfg = session_endpoint_get_ext_cfg (sep, TRANSPORT_ENDPT_EXT_CFG_CRYPTO); + if (!ext_cfg) return SESSION_E_NOEXTCFG; - ccfg = &sep->ext_cfg->crypto; + ccfg = &ext_cfg->crypto; app_wrk = app_worker_get (sep->app_wrk_index); app = application_get (app_wrk->app_index); QUIC_DBG (2, "Called quic_start_listen for app %d", app_wrk->app_index); diff --git a/src/plugins/snort/daq_vpp.c b/src/plugins/snort/daq_vpp.c index 386092a0382..6fc0bf5506a 100644 --- a/src/plugins/snort/daq_vpp.c +++ b/src/plugins/snort/daq_vpp.c @@ -10,6 +10,7 @@ #include <sys/socket.h> #include <sys/un.h> #include <sys/mman.h> +#include <sys/time.h> #include <errno.h> #include <sys/epoll.h> @@ -521,6 +522,7 @@ vpp_daq_msg_receive_one (VPP_Context_t *vc, VPPQueuePair *qp, { uint32_t n_recv, n_left; uint32_t head, next, mask = qp->queue_size - 1; + struct timeval tv; if (max_recv == 0) return 0; @@ -535,11 +537,14 @@ vpp_daq_msg_receive_one (VPP_Context_t *vc, VPPQueuePair *qp, n_left = n_recv = max_recv; } + gettimeofday (&tv, NULL); while (n_left--) { uint32_t desc_index = qp->enq_ring[next & mask]; daq_vpp_desc_t *d = qp->descs + desc_index; VPPDescData *dd = qp->desc_data + desc_index; + dd->pkthdr.ts.tv_sec = tv.tv_sec; + dd->pkthdr.ts.tv_usec = tv.tv_usec; dd->pkthdr.pktlen = d->length; dd->pkthdr.address_space_id = d->address_space_id; dd->msg.data = vc->bpools[d->buffer_pool].base + d->offset; diff --git a/src/plugins/snort/snort_api.c b/src/plugins/snort/snort_api.c index 334a84b4341..adad0d8763f 100644 --- a/src/plugins/snort/snort_api.c +++ b/src/plugins/snort/snort_api.c @@ -80,10 +80,16 @@ vl_api_snort_interface_attach_t_handler (vl_api_snort_interface_attach_t *mp) u8 snort_dir = mp->snort_dir; int rv = VNET_API_ERROR_NO_SUCH_ENTRY; - instance = snort_get_instance_by_index (instance_index); - if (instance) - rv = snort_interface_enable_disable ( - vm, (char *) instance->name, sw_if_index, 1 /* is_enable */, snort_dir); + if (sw_if_index == INDEX_INVALID) + rv = VNET_API_ERROR_NO_MATCHING_INTERFACE; + else + { + instance = snort_get_instance_by_index (instance_index); + if (instance) + rv = snort_interface_enable_disable (vm, (char *) instance->name, + sw_if_index, 1 /* is_enable */, + snort_dir); + } REPLY_MACRO (VL_API_SNORT_INTERFACE_ATTACH_REPLY); } @@ -346,10 +352,11 @@ vl_api_snort_interface_detach_t_handler (vl_api_snort_interface_detach_t *mp) vlib_main_t *vm = vlib_get_main (); vl_api_snort_interface_detach_reply_t *rmp; u32 sw_if_index = clib_net_to_host_u32 (mp->sw_if_index); - int rv; + int rv = VNET_API_ERROR_NO_MATCHING_INTERFACE; - rv = snort_interface_enable_disable (vm, NULL, sw_if_index, - 0 /* is_enable */, 0); + if (sw_if_index != INDEX_INVALID) + rv = snort_interface_enable_disable (vm, NULL, sw_if_index, + 0 /* is_enable */, SNORT_INOUT); REPLY_MACRO (VL_API_SNORT_INTERFACE_DETACH_REPLY); } diff --git a/src/plugins/srtp/srtp.c b/src/plugins/srtp/srtp.c index bb54e672918..6862301d2d2 100644 --- a/src/plugins/srtp/srtp.c +++ b/src/plugins/srtp/srtp.c @@ -641,10 +641,12 @@ srtp_connect (transport_endpoint_cfg_t *tep) application_t *app; srtp_tc_t *ctx; u32 ctx_index; + transport_endpt_ext_cfg_t *ext_cfg; int rv; sep = (session_endpoint_cfg_t *) tep; - if (!sep->ext_cfg) + ext_cfg = session_endpoint_get_ext_cfg (sep, TRANSPORT_ENDPT_EXT_CFG_NONE); + if (!ext_cfg) return SESSION_E_NOEXTCFG; app_wrk = app_worker_get (sep->app_wrk_index); @@ -658,7 +660,7 @@ srtp_connect (transport_endpoint_cfg_t *tep) ctx->srtp_ctx_handle = ctx_index; ctx->c_flags |= TRANSPORT_CONNECTION_F_NO_LOOKUP; - srtp_init_policy (ctx, (transport_endpt_cfg_srtp_t *) sep->ext_cfg->data); + srtp_init_policy (ctx, (transport_endpt_cfg_srtp_t *) ext_cfg->data); clib_memcpy_fast (&cargs->sep, sep, sizeof (session_endpoint_t)); cargs->sep.transport_proto = TRANSPORT_PROTO_UDP; @@ -723,9 +725,11 @@ srtp_start_listen (u32 app_listener_index, transport_endpoint_cfg_t *tep) app_listener_t *al; srtp_tc_t *lctx; u32 lctx_index; + transport_endpt_ext_cfg_t *ext_cfg; sep = (session_endpoint_cfg_t *) tep; - if (!sep->ext_cfg) + ext_cfg = session_endpoint_get_ext_cfg (sep, TRANSPORT_ENDPT_EXT_CFG_NONE); + if (!ext_cfg) return SESSION_E_NOEXTCFG; app_wrk = app_worker_get (sep->app_wrk_index); @@ -756,7 +760,7 @@ srtp_start_listen (u32 app_listener_index, transport_endpoint_cfg_t *tep) lctx->c_s_index = app_listener_index; lctx->c_flags |= TRANSPORT_CONNECTION_F_NO_LOOKUP; - srtp_init_policy (lctx, (transport_endpt_cfg_srtp_t *) sep->ext_cfg->data); + srtp_init_policy (lctx, (transport_endpt_cfg_srtp_t *) ext_cfg->data); SRTP_DBG (1, "Started listening %d", lctx_index); return lctx_index; diff --git a/src/plugins/tracenode/node.c b/src/plugins/tracenode/node.c index 444d93f1708..c56df589826 100644 --- a/src/plugins/tracenode/node.c +++ b/src/plugins/tracenode/node.c @@ -55,23 +55,19 @@ tracenode_inline (vlib_main_t *vm, vlib_node_runtime_t *node, /* enqueue b0 to the current next frame */ vnet_feature_next_u16 (next, b[0]); - /* buffer already traced */ - if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED)) - goto skip; - if (is_pcap && vnet_is_packet_pcaped (pp, b[0], ~0)) { pcap_add_buffer (&pp->pcap_main, vm, from0[0], pp->max_bytes_per_pkt); } - else if (!is_pcap && vlib_trace_buffer (vm, node, next[0], b[0], - 1 /* follow_chain */)) + else if (!is_pcap && !(b[0]->flags & VLIB_BUFFER_IS_TRACED) && + vlib_trace_buffer (vm, node, next[0], b[0], + 1 /* follow_chain */)) { tracenode_trace_t *tr = vlib_add_trace (vm, node, b[0], sizeof *tr); tr->sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_RX]; } - skip: b++; from0++; next++; diff --git a/src/plugins/unittest/session_test.c b/src/plugins/unittest/session_test.c index f01e661157c..7702e817070 100644 --- a/src/plugins/unittest/session_test.c +++ b/src/plugins/unittest/session_test.c @@ -825,6 +825,8 @@ session_test_rule_table (vlib_main_t * vm, unformat_input_t * input) session_test_enable_rule_table_engine (vm); session_table_init (st, FIB_PROTOCOL_MAX); + vec_add1 (st->appns_index, + app_namespace_index (app_namespace_get_default ())); session_rules_table_init (st, FIB_PROTOCOL_MAX); ip4_address_t lcl_ip = { @@ -2238,6 +2240,8 @@ session_test_sdl (vlib_main_t *vm, unformat_input_t *input) session_test_enable_sdl_engine (vm); session_table_init (st, FIB_PROTOCOL_MAX); + vec_add1 (st->appns_index, + app_namespace_index (app_namespace_get_default ())); session_rules_table_init (st, FIB_PROTOCOL_MAX); /* Add 1.2.0.0/16 */ @@ -2389,6 +2393,50 @@ session_test_sdl (vlib_main_t *vm, unformat_input_t *input) return 0; } +static int +session_test_ext_cfg (vlib_main_t *vm, unformat_input_t *input) +{ + session_endpoint_cfg_t sep = SESSION_ENDPOINT_CFG_NULL; + transport_endpt_ext_cfg_t *ext_cfg; + + ext_cfg = session_endpoint_add_ext_cfg (&sep, TRANSPORT_ENDPT_EXT_CFG_HTTP, + sizeof (ext_cfg->opaque)); + ext_cfg->opaque = 60; + + ext_cfg = + session_endpoint_add_ext_cfg (&sep, TRANSPORT_ENDPT_EXT_CFG_CRYPTO, + sizeof (transport_endpt_crypto_cfg_t)); + ext_cfg->crypto.ckpair_index = 1; + + ext_cfg = session_endpoint_add_ext_cfg (&sep, TRANSPORT_ENDPT_EXT_CFG_NONE, + sizeof (ext_cfg->opaque)); + ext_cfg->opaque = 345; + + ext_cfg = session_endpoint_get_ext_cfg (&sep, TRANSPORT_ENDPT_EXT_CFG_HTTP); + SESSION_TEST ((ext_cfg != 0), + "TRANSPORT_ENDPT_EXT_CFG_HTTP should be present"); + SESSION_TEST ((ext_cfg->opaque == 60), + "TRANSPORT_ENDPT_EXT_CFG_HTTP opaque value should be 60: %u", + ext_cfg->opaque); + ext_cfg = + session_endpoint_get_ext_cfg (&sep, TRANSPORT_ENDPT_EXT_CFG_CRYPTO); + SESSION_TEST ((ext_cfg != 0), + "TRANSPORT_ENDPT_EXT_CFG_CRYPTO should be present"); + SESSION_TEST ( + (ext_cfg->crypto.ckpair_index == 1), + "TRANSPORT_ENDPT_EXT_CFG_HTTP ckpair_index value should be 1: %u", + ext_cfg->crypto.ckpair_index); + ext_cfg = session_endpoint_get_ext_cfg (&sep, TRANSPORT_ENDPT_EXT_CFG_NONE); + SESSION_TEST ((ext_cfg != 0), + "TRANSPORT_ENDPT_EXT_CFG_NONE should be present"); + SESSION_TEST ((ext_cfg->opaque == 345), + "TRANSPORT_ENDPT_EXT_CFG_HTTP opaque value should be 345: %u", + ext_cfg->opaque); + session_endpoint_free_ext_cfgs (&sep); + + return 0; +} + static clib_error_t * session_test (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd_arg) @@ -2419,6 +2467,8 @@ session_test (vlib_main_t * vm, res = session_test_enable_disable (vm, input); else if (unformat (input, "sdl")) res = session_test_sdl (vm, input); + else if (unformat (input, "ext-cfg")) + res = session_test_ext_cfg (vm, input); else if (unformat (input, "all")) { if ((res = session_test_basic (vm, input))) @@ -2439,6 +2489,8 @@ session_test (vlib_main_t * vm, goto done; if ((res = session_test_sdl (vm, input))) goto done; + if ((res = session_test_ext_cfg (vm, input))) + goto done; if ((res = session_test_enable_disable (vm, input))) goto done; } |