diff options
| -rw-r--r-- | src/vnet/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | src/vnet/udp/udp.c | 19 | ||||
| -rw-r--r-- | src/vnet/udp/udp.h | 4 | ||||
| -rw-r--r-- | src/vnet/udp/udp_error.def | 2 | ||||
| -rw-r--r-- | src/vnet/udp/udp_output.c | 269 |
5 files changed, 283 insertions, 12 deletions
diff --git a/src/vnet/CMakeLists.txt b/src/vnet/CMakeLists.txt index ef187dc2f43..f4394ac0a78 100644 --- a/src/vnet/CMakeLists.txt +++ b/src/vnet/CMakeLists.txt @@ -670,6 +670,7 @@ list(APPEND VNET_SOURCES udp/udp_encap.c udp/udp_decap.c udp/udp_api.c + udp/udp_output.c ) list(APPEND VNET_MULTIARCH_SOURCES diff --git a/src/vnet/udp/udp.c b/src/vnet/udp/udp.c index 9b2ed886d0f..c919c958254 100644 --- a/src/vnet/udp/udp.c +++ b/src/vnet/udp/udp.c @@ -226,16 +226,9 @@ always_inline u32 udp_push_one_header (vlib_main_t *vm, udp_connection_t *uc, vlib_buffer_t *b) { vlib_buffer_push_udp (b, uc->c_lcl_port, uc->c_rmt_port, 1); - if (uc->c_is_ip4) - vlib_buffer_push_ip4_custom (vm, b, &uc->c_lcl_ip4, &uc->c_rmt_ip4, - IP_PROTOCOL_UDP, 1 /* csum offload */, - 0 /* is_df */, uc->c_dscp); - else - vlib_buffer_push_ip6 (vm, b, &uc->c_lcl_ip6, &uc->c_rmt_ip6, - IP_PROTOCOL_UDP); - vnet_buffer (b)->sw_if_index[VLIB_RX] = uc->sw_if_index; - vnet_buffer (b)->sw_if_index[VLIB_TX] = uc->c_fib_index; b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED; + /* reuse tcp medatada for now */ + vnet_buffer (b)->tcp.connection_index = uc->c_c_index; return 0; } @@ -412,6 +405,8 @@ conn_alloc: clib_spinlock_init (&uc->rx_lock); uc->c_flags |= TRANSPORT_CONNECTION_F_CLESS; } + uc->next_node_index = rmt->next_node_index; + uc->next_node_opaque = rmt->next_node_opaque; return uc->c_c_index; } @@ -505,11 +500,11 @@ udp_init (vlib_main_t * vm) pi->format_header = format_udp_header; pi->unformat_pg_edit = unformat_pg_udp_header; - /* Register as transport with URI */ + /* Register as transport with session layer */ transport_register_protocol (TRANSPORT_PROTO_UDP, &udp_proto, - FIB_PROTOCOL_IP4, ip4_lookup_node.index); + FIB_PROTOCOL_IP4, udp4_output_node.index); transport_register_protocol (TRANSPORT_PROTO_UDP, &udp_proto, - FIB_PROTOCOL_IP6, ip6_lookup_node.index); + FIB_PROTOCOL_IP6, udp6_output_node.index); /* * Initialize data structures diff --git a/src/vnet/udp/udp.h b/src/vnet/udp/udp.h index d51805099ce..96316a43fc8 100644 --- a/src/vnet/udp/udp.h +++ b/src/vnet/udp/udp.h @@ -64,6 +64,8 @@ typedef struct u8 flags; /**< connection flags */ u16 mss; /**< connection mss */ u32 sw_if_index; /**< connection sw_if_index */ + u32 next_node_index; /**< Can be used to control next node in output */ + u32 next_node_opaque; /**< Opaque to pass to next node */ } udp_connection_t; typedef struct @@ -129,6 +131,8 @@ extern vlib_node_registration_t udp4_input_node; extern vlib_node_registration_t udp6_input_node; extern vlib_node_registration_t udp4_local_node; extern vlib_node_registration_t udp6_local_node; +extern vlib_node_registration_t udp4_output_node; +extern vlib_node_registration_t udp6_output_node; void udp_add_dst_port (udp_main_t * um, udp_dst_port_t dst_port, char *dst_port_name, u8 is_ip4); diff --git a/src/vnet/udp/udp_error.def b/src/vnet/udp/udp_error.def index 178d5c96b2c..717285d9b76 100644 --- a/src/vnet/udp/udp_error.def +++ b/src/vnet/udp/udp_error.def @@ -25,3 +25,5 @@ udp_error (NOT_READY, not_ready, ERROR, "Connection not ready") udp_error (ACCEPT, accept, INFO, "Accepted session") udp_error (CREATE_SESSION, create_session, ERROR, "Failed to create session") udp_error (MQ_FULL, mq_full, ERROR, "Application msg queue full") +udp_error (INVALID_CONNECTION, invalid_connection, ERROR, "Invalid connection") +udp_error (PKTS_SENT, pkts_sent, INFO, "Packets sent") diff --git a/src/vnet/udp/udp_output.c b/src/vnet/udp/udp_output.c new file mode 100644 index 00000000000..1252bfb3bad --- /dev/null +++ b/src/vnet/udp/udp_output.c @@ -0,0 +1,269 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2022 Cisco Systems, Inc. + */ + +#include <vnet/udp/udp.h> +#include <vnet/ip/ip4_inlines.h> +#include <vnet/ip/ip6_inlines.h> + +#define udp_node_index(node_id, is_ip4) \ + ((is_ip4) ? udp4_##node_id##_node.index : udp6_##node_id##_node.index) + +typedef enum udp_output_next_ +{ + UDP_OUTPUT_NEXT_DROP, + UDP_OUTPUT_NEXT_IP_LOOKUP, + UDP_OUTPUT_N_NEXT +} udp_output_next_t; + +#define foreach_udp4_output_next \ + _ (DROP, "error-drop") \ + _ (IP_LOOKUP, "ip4-lookup") + +#define foreach_udp6_output_next \ + _ (DROP, "error-drop") \ + _ (IP_LOOKUP, "ip6-lookup") + +static vlib_error_desc_t udp_output_error_counters[] = { +#define udp_error(f, n, s, d) { #n, d, VL_COUNTER_SEVERITY_##s }, +#include <vnet/udp/udp_error.def> +#undef udp_error +}; + +typedef struct udp_tx_trace_ +{ + udp_header_t udp_header; + udp_connection_t udp_connection; +} udp_tx_trace_t; + +static u8 * +format_udp_tx_trace (u8 *s, va_list *args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + udp_tx_trace_t *t = va_arg (*args, udp_tx_trace_t *); + udp_connection_t *uc = &t->udp_connection; + u32 indent = format_get_indent (s); + + s = format (s, "%U\n%U%U", format_udp_connection, uc, format_white_space, + indent, format_udp_header, &t->udp_header, 128); + + return s; +} + +static void +udp46_output_trace_frame (vlib_main_t *vm, vlib_node_runtime_t *node, + u32 *to_next, u32 n_bufs) +{ + udp_connection_t *uc; + udp_tx_trace_t *t; + vlib_buffer_t *b; + udp_header_t *uh; + int i; + + for (i = 0; i < n_bufs; i++) + { + b = vlib_get_buffer (vm, to_next[i]); + if (!(b->flags & VLIB_BUFFER_IS_TRACED)) + continue; + uh = vlib_buffer_get_current (b); + uc = udp_connection_get (vnet_buffer (b)->tcp.connection_index, + vm->thread_index); + t = vlib_add_trace (vm, node, b, sizeof (*t)); + clib_memcpy_fast (&t->udp_header, uh, sizeof (t->udp_header)); + clib_memcpy_fast (&t->udp_connection, uc, sizeof (t->udp_connection)); + } +} + +always_inline void +udp_output_push_ip (vlib_main_t *vm, vlib_buffer_t *b, udp_connection_t *uc, + u8 is_ip4) +{ + if (uc->c_is_ip4) + vlib_buffer_push_ip4_custom (vm, b, &uc->c_lcl_ip4, &uc->c_rmt_ip4, + IP_PROTOCOL_UDP, 1 /* csum offload */, + 0 /* is_df */, uc->c_dscp); + else + vlib_buffer_push_ip6 (vm, b, &uc->c_lcl_ip6, &uc->c_rmt_ip6, + IP_PROTOCOL_UDP); + vnet_buffer (b)->sw_if_index[VLIB_RX] = uc->sw_if_index; + vnet_buffer (b)->sw_if_index[VLIB_TX] = uc->c_fib_index; +} + +always_inline void +udp_output_handle_packet (udp_connection_t *uc0, vlib_buffer_t *b0, + vlib_node_runtime_t *error_node, u16 *next0, + u8 is_ip4) +{ + /* If next_index is not drop use it */ + if (uc0->next_node_index) + { + *next0 = uc0->next_node_index; + vnet_buffer (b0)->tcp.next_node_opaque = uc0->next_node_opaque; + } + else + { + *next0 = UDP_OUTPUT_NEXT_IP_LOOKUP; + } + + vnet_buffer (b0)->sw_if_index[VLIB_TX] = uc0->c_fib_index; + vnet_buffer (b0)->sw_if_index[VLIB_RX] = uc0->sw_if_index; +} + +always_inline uword +udp46_output_inline (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_frame_t *frame, int is_ip4) +{ + u32 n_left_from, *from, thread_index = vm->thread_index; + vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b; + u16 nexts[VLIB_FRAME_SIZE], *next; + + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + + if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE)) + udp46_output_trace_frame (vm, node, from, n_left_from); + + vlib_get_buffers (vm, from, bufs, n_left_from); + b = bufs; + next = nexts; + + while (n_left_from >= 4) + { + udp_connection_t *uc0, *uc1; + + vlib_prefetch_buffer_header (b[2], STORE); + CLIB_PREFETCH (b[2]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE); + + vlib_prefetch_buffer_header (b[3], STORE); + CLIB_PREFETCH (b[3]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE); + + uc0 = udp_connection_get (vnet_buffer (b[0])->tcp.connection_index, + thread_index); + uc1 = udp_connection_get (vnet_buffer (b[1])->tcp.connection_index, + thread_index); + + if (PREDICT_TRUE (!uc0 + !uc1 == 0)) + { + udp_output_push_ip (vm, b[0], uc0, is_ip4); + udp_output_push_ip (vm, b[1], uc1, is_ip4); + + udp_output_handle_packet (uc0, b[0], node, &next[0], is_ip4); + udp_output_handle_packet (uc1, b[1], node, &next[1], is_ip4); + } + else + { + if (uc0 != 0) + { + udp_output_push_ip (vm, b[0], uc0, is_ip4); + udp_output_handle_packet (uc0, b[0], node, &next[0], is_ip4); + } + else + { + b[0]->error = node->errors[UDP_ERROR_INVALID_CONNECTION]; + next[0] = UDP_OUTPUT_NEXT_DROP; + } + if (uc1 != 0) + { + udp_output_push_ip (vm, b[1], uc1, is_ip4); + udp_output_handle_packet (uc1, b[1], node, &next[1], is_ip4); + } + else + { + b[1]->error = node->errors[UDP_ERROR_INVALID_CONNECTION]; + next[1] = UDP_OUTPUT_NEXT_DROP; + } + } + + b += 2; + next += 2; + n_left_from -= 2; + } + while (n_left_from > 0) + { + udp_connection_t *uc0; + + if (n_left_from > 1) + { + vlib_prefetch_buffer_header (b[1], STORE); + CLIB_PREFETCH (b[1]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE); + } + + uc0 = udp_connection_get (vnet_buffer (b[0])->tcp.connection_index, + thread_index); + + if (PREDICT_TRUE (uc0 != 0)) + { + udp_output_push_ip (vm, b[0], uc0, is_ip4); + udp_output_handle_packet (uc0, b[0], node, &next[0], is_ip4); + } + else + { + b[0]->error = node->errors[UDP_ERROR_INVALID_CONNECTION]; + next[0] = UDP_OUTPUT_NEXT_DROP; + } + + b += 1; + next += 1; + n_left_from -= 1; + } + + vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors); + vlib_node_increment_counter (vm, udp_node_index (output, is_ip4), + UDP_ERROR_PKTS_SENT, frame->n_vectors); + return frame->n_vectors; +} + +VLIB_NODE_FN (udp4_output_node) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame) +{ + return udp46_output_inline (vm, node, from_frame, 1 /* is_ip4 */); +} + +VLIB_NODE_FN (udp6_output_node) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame) +{ + return udp46_output_inline (vm, node, from_frame, 0 /* is_ip4 */); +} + +VLIB_REGISTER_NODE (udp4_output_node) = +{ + .name = "udp4-output", + .vector_size = sizeof (u32), + .n_errors = UDP_N_ERROR, + .protocol_hint = VLIB_NODE_PROTO_HINT_UDP, + .error_counters = udp_output_error_counters, + .n_next_nodes = UDP_OUTPUT_N_NEXT, + .next_nodes = { +#define _(s, n) [UDP_OUTPUT_NEXT_##s] = n, + foreach_udp4_output_next +#undef _ + }, + .format_buffer = format_udp_header, + .format_trace = format_udp_tx_trace, +}; + +VLIB_REGISTER_NODE (udp6_output_node) = +{ + .name = "udp6-output", + .vector_size = sizeof (u32), + .n_errors = UDP_N_ERROR, + .protocol_hint = VLIB_NODE_PROTO_HINT_UDP, + .error_counters = udp_output_error_counters, + .n_next_nodes = UDP_OUTPUT_N_NEXT, + .next_nodes = { +#define _(s, n) [UDP_OUTPUT_NEXT_##s] = n, + foreach_udp6_output_next +#undef _ + }, + .format_buffer = format_udp_header, + .format_trace = format_udp_tx_trace, +}; + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ |