udp: explicit udp output node

This allows for custom next node selection on output.

Type: feature

Signed-off-by: Florin Coras <fcoras@cisco.com>
Change-Id: Ib0fee71a01847184e95c18097bbbfdadfbd9d030

5 files changed, 283 insertions, 12 deletions

diff --git a/src/vnet/CMakeLists.txt b/src/vnet/CMakeLists.txt
index ef187dc2f43..f4394ac0a78 100644
--- a/src/vnet/CMakeLists.txt
+++ b/src/vnet/CMakeLists.txt
@@ -670,6 +670,7 @@ list(APPEND VNET_SOURCES
   udp/udp_encap.c
   udp/udp_decap.c
   udp/udp_api.c
+  udp/udp_output.c
 )
 
 list(APPEND VNET_MULTIARCH_SOURCES
diff --git a/src/vnet/udp/udp.c b/src/vnet/udp/udp.c
index 9b2ed886d0f..c919c958254 100644
--- a/src/vnet/udp/udp.c
+++ b/src/vnet/udp/udp.c
@@ -226,16 +226,9 @@ always_inline u32
 udp_push_one_header (vlib_main_t *vm, udp_connection_t *uc, vlib_buffer_t *b)
 {
   vlib_buffer_push_udp (b, uc->c_lcl_port, uc->c_rmt_port, 1);
-  if (uc->c_is_ip4)
-    vlib_buffer_push_ip4_custom (vm, b, &uc->c_lcl_ip4, &uc->c_rmt_ip4,
-				 IP_PROTOCOL_UDP, 1 /* csum offload */,
-				 0 /* is_df */, uc->c_dscp);
-  else
-    vlib_buffer_push_ip6 (vm, b, &uc->c_lcl_ip6, &uc->c_rmt_ip6,
-			  IP_PROTOCOL_UDP);
-  vnet_buffer (b)->sw_if_index[VLIB_RX] = uc->sw_if_index;
-  vnet_buffer (b)->sw_if_index[VLIB_TX] = uc->c_fib_index;
   b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
+  /* reuse tcp medatada for now */
+  vnet_buffer (b)->tcp.connection_index = uc->c_c_index;
 
   return 0;
 }
@@ -412,6 +405,8 @@ conn_alloc:
       clib_spinlock_init (&uc->rx_lock);
       uc->c_flags |= TRANSPORT_CONNECTION_F_CLESS;
     }
+  uc->next_node_index = rmt->next_node_index;
+  uc->next_node_opaque = rmt->next_node_opaque;
 
   return uc->c_c_index;
 }
@@ -505,11 +500,11 @@ udp_init (vlib_main_t * vm)
   pi->format_header = format_udp_header;
   pi->unformat_pg_edit = unformat_pg_udp_header;
 
-  /* Register as transport with URI */
+  /* Register as transport with session layer */
   transport_register_protocol (TRANSPORT_PROTO_UDP, &udp_proto,
-			       FIB_PROTOCOL_IP4, ip4_lookup_node.index);
+			       FIB_PROTOCOL_IP4, udp4_output_node.index);
   transport_register_protocol (TRANSPORT_PROTO_UDP, &udp_proto,
-			       FIB_PROTOCOL_IP6, ip6_lookup_node.index);
+			       FIB_PROTOCOL_IP6, udp6_output_node.index);
 
   /*
    * Initialize data structures
diff --git a/src/vnet/udp/udp.h b/src/vnet/udp/udp.h
index d51805099ce..96316a43fc8 100644
--- a/src/vnet/udp/udp.h
+++ b/src/vnet/udp/udp.h
@@ -64,6 +64,8 @@ typedef struct
   u8 flags;				/**< connection flags */
   u16 mss;				/**< connection mss */
   u32 sw_if_index;			/**< connection sw_if_index */
+  u32 next_node_index;	/**< Can be used to control next node in output */
+  u32 next_node_opaque; /**< Opaque to pass to next node */
 } udp_connection_t;
 
 typedef struct
@@ -129,6 +131,8 @@ extern vlib_node_registration_t udp4_input_node;
 extern vlib_node_registration_t udp6_input_node;
 extern vlib_node_registration_t udp4_local_node;
 extern vlib_node_registration_t udp6_local_node;
+extern vlib_node_registration_t udp4_output_node;
+extern vlib_node_registration_t udp6_output_node;
 
 void udp_add_dst_port (udp_main_t * um, udp_dst_port_t dst_port,
 		       char *dst_port_name, u8 is_ip4);
diff --git a/src/vnet/udp/udp_error.def b/src/vnet/udp/udp_error.def
index 178d5c96b2c..717285d9b76 100644
--- a/src/vnet/udp/udp_error.def
+++ b/src/vnet/udp/udp_error.def
@@ -25,3 +25,5 @@ udp_error (NOT_READY, not_ready, ERROR, "Connection not ready")
 udp_error (ACCEPT, accept, INFO, "Accepted session")
 udp_error (CREATE_SESSION, create_session, ERROR, "Failed to create session")
 udp_error (MQ_FULL, mq_full, ERROR, "Application msg queue full")
+udp_error (INVALID_CONNECTION, invalid_connection, ERROR, "Invalid connection")
+udp_error (PKTS_SENT, pkts_sent, INFO, "Packets sent")
diff --git a/src/vnet/udp/udp_output.c b/src/vnet/udp/udp_output.c
new file mode 100644
index 00000000000..1252bfb3bad
--- /dev/null
+++ b/src/vnet/udp/udp_output.c
@@ -0,0 +1,269 @@
+/* SPDX-License-Identifier: Apache-2.0
+ * Copyright(c) 2022 Cisco Systems, Inc.
+ */
+
+#include <vnet/udp/udp.h>
+#include <vnet/ip/ip4_inlines.h>
+#include <vnet/ip/ip6_inlines.h>
+
+#define udp_node_index(node_id, is_ip4)                                       \
+  ((is_ip4) ? udp4_##node_id##_node.index : udp6_##node_id##_node.index)
+
+typedef enum udp_output_next_
+{
+  UDP_OUTPUT_NEXT_DROP,
+  UDP_OUTPUT_NEXT_IP_LOOKUP,
+  UDP_OUTPUT_N_NEXT
+} udp_output_next_t;
+
+#define foreach_udp4_output_next                                              \
+  _ (DROP, "error-drop")                                                      \
+  _ (IP_LOOKUP, "ip4-lookup")
+
+#define foreach_udp6_output_next                                              \
+  _ (DROP, "error-drop")                                                      \
+  _ (IP_LOOKUP, "ip6-lookup")
+
+static vlib_error_desc_t udp_output_error_counters[] = {
+#define udp_error(f, n, s, d) { #n, d, VL_COUNTER_SEVERITY_##s },
+#include <vnet/udp/udp_error.def>
+#undef udp_error
+};
+
+typedef struct udp_tx_trace_
+{
+  udp_header_t udp_header;
+  udp_connection_t udp_connection;
+} udp_tx_trace_t;
+
+static u8 *
+format_udp_tx_trace (u8 *s, va_list *args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  udp_tx_trace_t *t = va_arg (*args, udp_tx_trace_t *);
+  udp_connection_t *uc = &t->udp_connection;
+  u32 indent = format_get_indent (s);
+
+  s = format (s, "%U\n%U%U", format_udp_connection, uc, format_white_space,
+	      indent, format_udp_header, &t->udp_header, 128);
+
+  return s;
+}
+
+static void
+udp46_output_trace_frame (vlib_main_t *vm, vlib_node_runtime_t *node,
+			  u32 *to_next, u32 n_bufs)
+{
+  udp_connection_t *uc;
+  udp_tx_trace_t *t;
+  vlib_buffer_t *b;
+  udp_header_t *uh;
+  int i;
+
+  for (i = 0; i < n_bufs; i++)
+    {
+      b = vlib_get_buffer (vm, to_next[i]);
+      if (!(b->flags & VLIB_BUFFER_IS_TRACED))
+	continue;
+      uh = vlib_buffer_get_current (b);
+      uc = udp_connection_get (vnet_buffer (b)->tcp.connection_index,
+			       vm->thread_index);
+      t = vlib_add_trace (vm, node, b, sizeof (*t));
+      clib_memcpy_fast (&t->udp_header, uh, sizeof (t->udp_header));
+      clib_memcpy_fast (&t->udp_connection, uc, sizeof (t->udp_connection));
+    }
+}
+
+always_inline void
+udp_output_push_ip (vlib_main_t *vm, vlib_buffer_t *b, udp_connection_t *uc,
+		    u8 is_ip4)
+{
+  if (uc->c_is_ip4)
+    vlib_buffer_push_ip4_custom (vm, b, &uc->c_lcl_ip4, &uc->c_rmt_ip4,
+				 IP_PROTOCOL_UDP, 1 /* csum offload */,
+				 0 /* is_df */, uc->c_dscp);
+  else
+    vlib_buffer_push_ip6 (vm, b, &uc->c_lcl_ip6, &uc->c_rmt_ip6,
+			  IP_PROTOCOL_UDP);
+  vnet_buffer (b)->sw_if_index[VLIB_RX] = uc->sw_if_index;
+  vnet_buffer (b)->sw_if_index[VLIB_TX] = uc->c_fib_index;
+}
+
+always_inline void
+udp_output_handle_packet (udp_connection_t *uc0, vlib_buffer_t *b0,
+			  vlib_node_runtime_t *error_node, u16 *next0,
+			  u8 is_ip4)
+{
+  /* If next_index is not drop use it */
+  if (uc0->next_node_index)
+    {
+      *next0 = uc0->next_node_index;
+      vnet_buffer (b0)->tcp.next_node_opaque = uc0->next_node_opaque;
+    }
+  else
+    {
+      *next0 = UDP_OUTPUT_NEXT_IP_LOOKUP;
+    }
+
+  vnet_buffer (b0)->sw_if_index[VLIB_TX] = uc0->c_fib_index;
+  vnet_buffer (b0)->sw_if_index[VLIB_RX] = uc0->sw_if_index;
+}
+
+always_inline uword
+udp46_output_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
+		     vlib_frame_t *frame, int is_ip4)
+{
+  u32 n_left_from, *from, thread_index = vm->thread_index;
+  vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
+  u16 nexts[VLIB_FRAME_SIZE], *next;
+
+  from = vlib_frame_vector_args (frame);
+  n_left_from = frame->n_vectors;
+
+  if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
+    udp46_output_trace_frame (vm, node, from, n_left_from);
+
+  vlib_get_buffers (vm, from, bufs, n_left_from);
+  b = bufs;
+  next = nexts;
+
+  while (n_left_from >= 4)
+    {
+      udp_connection_t *uc0, *uc1;
+
+      vlib_prefetch_buffer_header (b[2], STORE);
+      CLIB_PREFETCH (b[2]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
+
+      vlib_prefetch_buffer_header (b[3], STORE);
+      CLIB_PREFETCH (b[3]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
+
+      uc0 = udp_connection_get (vnet_buffer (b[0])->tcp.connection_index,
+				thread_index);
+      uc1 = udp_connection_get (vnet_buffer (b[1])->tcp.connection_index,
+				thread_index);
+
+      if (PREDICT_TRUE (!uc0 + !uc1 == 0))
+	{
+	  udp_output_push_ip (vm, b[0], uc0, is_ip4);
+	  udp_output_push_ip (vm, b[1], uc1, is_ip4);
+
+	  udp_output_handle_packet (uc0, b[0], node, &next[0], is_ip4);
+	  udp_output_handle_packet (uc1, b[1], node, &next[1], is_ip4);
+	}
+      else
+	{
+	  if (uc0 != 0)
+	    {
+	      udp_output_push_ip (vm, b[0], uc0, is_ip4);
+	      udp_output_handle_packet (uc0, b[0], node, &next[0], is_ip4);
+	    }
+	  else
+	    {
+	      b[0]->error = node->errors[UDP_ERROR_INVALID_CONNECTION];
+	      next[0] = UDP_OUTPUT_NEXT_DROP;
+	    }
+	  if (uc1 != 0)
+	    {
+	      udp_output_push_ip (vm, b[1], uc1, is_ip4);
+	      udp_output_handle_packet (uc1, b[1], node, &next[1], is_ip4);
+	    }
+	  else
+	    {
+	      b[1]->error = node->errors[UDP_ERROR_INVALID_CONNECTION];
+	      next[1] = UDP_OUTPUT_NEXT_DROP;
+	    }
+	}
+
+      b += 2;
+      next += 2;
+      n_left_from -= 2;
+    }
+  while (n_left_from > 0)
+    {
+      udp_connection_t *uc0;
+
+      if (n_left_from > 1)
+	{
+	  vlib_prefetch_buffer_header (b[1], STORE);
+	  CLIB_PREFETCH (b[1]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
+	}
+
+      uc0 = udp_connection_get (vnet_buffer (b[0])->tcp.connection_index,
+				thread_index);
+
+      if (PREDICT_TRUE (uc0 != 0))
+	{
+	  udp_output_push_ip (vm, b[0], uc0, is_ip4);
+	  udp_output_handle_packet (uc0, b[0], node, &next[0], is_ip4);
+	}
+      else
+	{
+	  b[0]->error = node->errors[UDP_ERROR_INVALID_CONNECTION];
+	  next[0] = UDP_OUTPUT_NEXT_DROP;
+	}
+
+      b += 1;
+      next += 1;
+      n_left_from -= 1;
+    }
+
+  vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
+  vlib_node_increment_counter (vm, udp_node_index (output, is_ip4),
+			       UDP_ERROR_PKTS_SENT, frame->n_vectors);
+  return frame->n_vectors;
+}
+
+VLIB_NODE_FN (udp4_output_node)
+(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
+{
+  return udp46_output_inline (vm, node, from_frame, 1 /* is_ip4 */);
+}
+
+VLIB_NODE_FN (udp6_output_node)
+(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
+{
+  return udp46_output_inline (vm, node, from_frame, 0 /* is_ip4 */);
+}
+
+VLIB_REGISTER_NODE (udp4_output_node) =
+{
+  .name = "udp4-output",
+  .vector_size = sizeof (u32),
+  .n_errors = UDP_N_ERROR,
+  .protocol_hint = VLIB_NODE_PROTO_HINT_UDP,
+  .error_counters = udp_output_error_counters,
+  .n_next_nodes = UDP_OUTPUT_N_NEXT,
+  .next_nodes = {
+#define _(s, n) [UDP_OUTPUT_NEXT_##s] = n,
+    foreach_udp4_output_next
+#undef _
+  },
+  .format_buffer = format_udp_header,
+  .format_trace = format_udp_tx_trace,
+};
+
+VLIB_REGISTER_NODE (udp6_output_node) =
+{
+  .name = "udp6-output",
+  .vector_size = sizeof (u32),
+  .n_errors = UDP_N_ERROR,
+  .protocol_hint = VLIB_NODE_PROTO_HINT_UDP,
+  .error_counters = udp_output_error_counters,
+  .n_next_nodes = UDP_OUTPUT_N_NEXT,
+  .next_nodes = {
+#define _(s, n) [UDP_OUTPUT_NEXT_##s] = n,
+    foreach_udp6_output_next
+#undef _
+  },
+  .format_buffer = format_udp_header,
+  .format_trace = format_udp_tx_trace,
+};
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */