aboutsummaryrefslogtreecommitdiffstats
path: root/src/vlib
diff options
context:
space:
mode:
Diffstat (limited to 'src/vlib')
-rw-r--r--src/vlib/CMakeLists.txt4
-rw-r--r--src/vlib/buffer.h11
-rw-r--r--src/vlib/punt.c629
-rw-r--r--src/vlib/punt.h93
-rw-r--r--src/vlib/punt_node.c327
-rw-r--r--src/vlib/vlib.h1
6 files changed, 1063 insertions, 2 deletions
diff --git a/src/vlib/CMakeLists.txt b/src/vlib/CMakeLists.txt
index ff64baea980..acd66359f9b 100644
--- a/src/vlib/CMakeLists.txt
+++ b/src/vlib/CMakeLists.txt
@@ -63,6 +63,8 @@ add_vpp_library(vlib
node_format.c
pci/pci.c
physmem.c
+ punt.c
+ punt_node.c
threads.c
threads_cli.c
trace.c
@@ -77,6 +79,7 @@ add_vpp_library(vlib
MULTIARCH_SOURCES
drop.c
+ punt_node.c
INSTALL_HEADERS
buffer_funcs.h
@@ -102,6 +105,7 @@ add_vpp_library(vlib
pci/pci.h
physmem_funcs.h
physmem.h
+ punt.h
threads.h
trace_funcs.h
trace.h
diff --git a/src/vlib/buffer.h b/src/vlib/buffer.h
index 514e73bd7ca..0fd57f5275b 100644
--- a/src/vlib/buffer.h
+++ b/src/vlib/buffer.h
@@ -139,8 +139,15 @@ typedef union
* VLIB_BUFFER_NEXT_PRESENT flag is set. */
u32 next_buffer;
- /** Used by feature subgraph arcs to visit enabled feature nodes */
- u32 current_config_index;
+ /** The following fields can be in a union because once a packet enters
+ * the punt path, it is no longer on a feature arc */
+ union
+ {
+ /** Used by feature subgraph arcs to visit enabled feature nodes */
+ u32 current_config_index;
+ /* the reason the packet once punted */
+ u32 punt_reason;
+ };
/** Opaque data used by sub-graphs for their own purposes. */
u32 opaque[10];
diff --git a/src/vlib/punt.c b/src/vlib/punt.c
new file mode 100644
index 00000000000..09f30f4f810
--- /dev/null
+++ b/src/vlib/punt.c
@@ -0,0 +1,629 @@
+/*
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vlib/punt.h>
+
+/**
+ * The last allocated punt reason
+ */
+static vlib_punt_reason_t punt_reason_last;
+
+/**
+ * Counters per punt-reason
+ */
+vlib_combined_counter_main_t punt_counters = {
+ .name = "punt",
+ .stat_segment_name = "/net/punt",
+};
+
+/**
+ * A punt reason
+ */
+typedef struct punt_reason_data_t_
+{
+ /**
+ * The reason name
+ */
+ u8 *pd_name;
+
+ /**
+ * The allocated reason value
+ */
+ vlib_punt_reason_t pd_reason;
+
+ /**
+ * Clients/owners that have registered this reason
+ */
+ u32 *pd_owners;
+} punt_reason_data_t;
+
+/**
+ * data for each punt reason
+ */
+static punt_reason_data_t *punt_reason_data;
+
+typedef enum punt_format_flags_t_
+{
+ PUNT_FORMAT_FLAG_NONE = 0,
+ PUNT_FORMAT_FLAG_DETAIL = (1 << 0),
+} punt_format_flags_t;
+
+/**
+ * A registration, by a client, to direct punted traffic to a given node
+ */
+typedef struct punt_reg_t_
+{
+ /**
+ * Reason the packets were punted
+ */
+ vlib_punt_reason_t pr_reason;
+
+ /**
+ * number of clients that have made this registration
+ */
+ u16 pr_locks;
+
+ /**
+ * The edge from the punt dispatch node to the requested node
+ */
+ u16 pr_edge;
+
+ /**
+ * node-index to send punted packets to
+ */
+ u32 pr_node_index;
+} punt_reg_t;
+
+/**
+ * Pool of registrations
+ */
+static punt_reg_t *punt_reg_pool;
+
+/**
+ * A DB of all the register nodes against punt reason and node index
+ */
+static uword *punt_reg_db;
+
+/**
+ * A DB used in the DP per-reason to dispatch packets to the requested nodes.
+ * this is a vector of edges per-reason
+ */
+u16 **punt_dp_db;
+
+/**
+ * A client using the punt serivce and its registrations
+ */
+typedef struct punt_client_t_
+{
+ /**
+ * The name of the client
+ */
+ u8 *pc_name;
+
+ /**
+ * The registrations is has made
+ */
+ u32 *pc_regs;
+} punt_client_t;
+
+/**
+ * Pool of clients
+ */
+static punt_client_t *punt_client_pool;
+
+/**
+ * DB of clients key'd by their name
+ */
+static uword *punt_client_db;
+
+u8 *
+format_vlib_punt_reason (u8 * s, va_list * args)
+{
+ vlib_punt_reason_t pr = va_arg (*args, int);
+
+ return (format (s, "[%d] %v", pr, punt_reason_data[pr].pd_name));
+}
+
+vlib_punt_hdl_t
+vlib_punt_client_register (const char *who)
+{
+ u8 *pc_name;
+ uword *p;
+ u32 pci;
+
+ pc_name = format (NULL, "%s", who);
+ p = hash_get_mem (punt_client_db, pc_name);
+
+ if (NULL == p)
+ {
+ punt_client_t *pc;
+
+ pool_get (punt_client_pool, pc);
+ pci = pc - punt_client_pool;
+
+ pc->pc_name = pc_name;
+
+ hash_set_mem (punt_client_db, pc->pc_name, pci);
+ }
+ else
+ {
+ pci = p[0];
+ vec_free (pc_name);
+ }
+
+ return (pci);
+}
+
+static int
+punt_validate_client (vlib_punt_hdl_t client)
+{
+ return (!pool_is_free_index (punt_client_pool, client));
+}
+
+static u64
+punt_reg_mk_key (vlib_punt_reason_t reason, u32 node_index)
+{
+ return (((u64) node_index) << 32 | reason);
+}
+
+static u32
+punt_reg_find (vlib_punt_reason_t reason, u32 node_index)
+{
+ uword *p;
+
+ p = hash_get (punt_reg_db, punt_reg_mk_key (reason, node_index));
+
+ if (p)
+ return p[0];
+
+ return ~0;
+}
+
+static void
+punt_reg_add (const punt_reg_t * pr)
+{
+ hash_set (punt_reg_db, punt_reg_mk_key (pr->pr_reason,
+ pr->pr_node_index),
+ pr - punt_reg_pool);
+}
+
+static void
+punt_reg_remove (const punt_reg_t * pr)
+{
+ hash_unset (punt_reg_db, punt_reg_mk_key (pr->pr_reason,
+ pr->pr_node_index));
+}
+
+/**
+ * reconstruct the DP per-reason DB
+ */
+static void
+punt_reg_mk_dp (vlib_punt_reason_t reason)
+{
+ u32 pri, *prip, *pris;
+ const punt_reg_t *pr;
+ u16 *edges, *old;
+ u64 key;
+
+ pris = NULL;
+ edges = NULL;
+ vec_validate (punt_dp_db, reason);
+
+ old = punt_dp_db[reason];
+
+ /* *INDENT-OFF* */
+ hash_foreach (key, pri, punt_reg_db,
+ ({
+ vec_add1(pris, pri);
+ }));
+ /* *INDENT-ON* */
+
+ /*
+ * A check for an empty vector is done in the DP, so the a zero
+ * length vector here is ok
+ */
+ vec_foreach (prip, pris)
+ {
+ pr = pool_elt_at_index (punt_reg_pool, *prip);
+
+ if (pr->pr_reason == reason)
+ vec_add1 (edges, pr->pr_edge);
+ }
+
+ /* atomic update of the DP */
+ punt_dp_db[reason] = edges;
+
+ vec_free (old);
+}
+
+int
+vlib_punt_register (vlib_punt_hdl_t client, vlib_punt_reason_t reason,
+ const char *node_name)
+{
+ vlib_node_t *punt_to, *punt_from;
+ punt_client_t *pc;
+ vlib_main_t *vm;
+ punt_reg_t *pr;
+ u32 pri;
+
+ if (reason >= punt_reason_last)
+ return -1;
+ if (!punt_validate_client (client))
+ return -2;
+
+ vm = vlib_get_main ();
+ pc = pool_elt_at_index (punt_client_pool, client);
+ punt_to = vlib_get_node_by_name (vm, (u8 *) node_name);
+ punt_from = vlib_get_node_by_name (vm, (u8 *) "punt-dispatch");
+
+ /*
+ * find a global matching registration
+ */
+ pri = punt_reg_find (reason, punt_to->index);
+
+ if (~0 != pri)
+ {
+ u32 pos;
+
+ pos = vec_search (pc->pc_regs, pri);
+
+ if (~0 != pos)
+ {
+ /* duplicate registration for this client */
+ return -1;
+ }
+
+ pr = pool_elt_at_index (punt_reg_pool, pri);
+ }
+ else
+ {
+ pool_get (punt_reg_pool, pr);
+
+ pr->pr_reason = reason;
+ pr->pr_node_index = punt_to->index;
+ pr->pr_edge = vlib_node_add_next (vm,
+ punt_from->index, pr->pr_node_index);
+
+ pri = pr - punt_reg_pool;
+
+ punt_reg_add (pr);
+ }
+
+ /*
+ * add this reg to the list the client has made
+ */
+ pr->pr_locks++;
+ vec_add1 (pc->pc_regs, pri);
+
+ punt_reg_mk_dp (reason);
+
+ return 0;
+}
+
+int
+vlib_punt_unregister (vlib_punt_hdl_t client,
+ vlib_punt_reason_t reason, const char *node_name)
+{
+ vlib_node_t *punt_to;
+ punt_client_t *pc;
+ vlib_main_t *vm;
+ punt_reg_t *pr;
+ u32 pri;
+
+ if (reason >= punt_reason_last)
+ return -1;
+
+ vm = vlib_get_main ();
+ pc = pool_elt_at_index (punt_client_pool, client);
+ punt_to = vlib_get_node_by_name (vm, (u8 *) node_name);
+
+ /*
+ * construct a registration and check if it's one this client already has
+ */
+ pri = punt_reg_find (reason, punt_to->index);
+
+ if (~0 != pri)
+ {
+ u32 pos;
+
+ pos = vec_search (pc->pc_regs, pri);
+
+ if (~0 == pos)
+ {
+ /* not a registration for this client */
+ return -1;
+ }
+ vec_del1 (pc->pc_regs, pos);
+
+ pr = pool_elt_at_index (punt_reg_pool, pri);
+
+ pr->pr_locks--;
+
+ if (0 == pr->pr_locks)
+ {
+ punt_reg_remove (pr);
+ pool_put (punt_reg_pool, pr);
+ }
+ }
+
+ /*
+ * rebuild the DP data-base
+ */
+ punt_reg_mk_dp (reason);
+
+ return (0);
+}
+
+int
+vlib_punt_reason_alloc (vlib_punt_hdl_t client,
+ const char *reason_name, vlib_punt_reason_t * reason)
+{
+ vlib_punt_reason_t new;
+
+ if (!punt_validate_client (client))
+ return -2;
+
+ new = punt_reason_last++;
+ vec_validate (punt_reason_data, new);
+ punt_reason_data[new].pd_name = format (NULL, "%s", reason_name);
+ punt_reason_data[new].pd_reason = new;
+ vec_add1 (punt_reason_data[new].pd_owners, client);
+
+ vlib_validate_combined_counter (&punt_counters, new);
+ vlib_zero_combined_counter (&punt_counters, new);
+
+ *reason = new;
+
+ /* build the DP data-base */
+ punt_reg_mk_dp (*reason);
+
+ return (0);
+}
+
+/* Parse node name -> node index. */
+uword
+unformat_punt_client (unformat_input_t * input, va_list * args)
+{
+ u32 *result = va_arg (*args, u32 *);
+
+ return unformat_user (input, unformat_hash_vec_string,
+ punt_client_db, result);
+}
+
+u8 *
+format_punt_reg (u8 * s, va_list * args)
+{
+ u32 pri = va_arg (*args, u32);
+ punt_reg_t *pr;
+
+ pr = pool_elt_at_index (punt_reg_pool, pri);
+
+ s = format (s, "%U -> %U",
+ format_vlib_punt_reason, pr->pr_reason,
+ format_vlib_node_name, vlib_get_main (), pr->pr_node_index);
+
+ return (s);
+}
+
+u8 *
+format_punt_reason_data (u8 * s, va_list * args)
+{
+ punt_reason_data_t *pd = va_arg (*args, punt_reason_data_t *);
+ punt_client_t *pc;
+ u32 *pci;
+
+ s = format (s, "[%d] %v from:[", pd->pd_reason, pd->pd_name);
+ vec_foreach (pci, pd->pd_owners)
+ {
+ pc = pool_elt_at_index (punt_client_pool, *pci);
+ s = format (s, "%v ", pc->pc_name);
+ }
+ s = format (s, "]");
+
+ return (s);
+}
+
+u8 *
+format_punt_client (u8 * s, va_list * args)
+{
+ u32 pci = va_arg (*args, u32);
+ punt_format_flags_t flags = va_arg (*args, punt_format_flags_t);
+ punt_client_t *pc;
+
+ pc = pool_elt_at_index (punt_client_pool, pci);
+
+ s = format (s, "%v", pc->pc_name);
+
+ if (flags & PUNT_FORMAT_FLAG_DETAIL)
+ {
+ punt_reason_data_t *pd;
+ u32 *pri;
+
+ s = format (s, "\n registrations:");
+ vec_foreach (pri, pc->pc_regs)
+ {
+ s = format (s, "\n [%U]", format_punt_reg, *pri);
+ }
+
+ s = format (s, "\n reasons:");
+
+ vec_foreach (pd, punt_reason_data)
+ {
+ u32 *tmp;
+
+ vec_foreach (tmp, pd->pd_owners)
+ {
+ if (*tmp == pci)
+ s = format (s, "\n %U", format_punt_reason_data, pd);
+ }
+ }
+ }
+ return (s);
+}
+
+static clib_error_t *
+punt_client_show (vlib_main_t * vm,
+ unformat_input_t * input, vlib_cli_command_t * cmd)
+{
+ u32 pci = ~0;
+
+ while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+ {
+ if (unformat (input, "%U", unformat_punt_client, &pci))
+ ;
+ else
+ break;
+ }
+
+ if (~0 != pci)
+ {
+ vlib_cli_output (vm, "%U", format_punt_client, pci,
+ PUNT_FORMAT_FLAG_DETAIL);
+ }
+ else
+ {
+ u8 *name;
+
+ /* *INDENT-OFF* */
+ hash_foreach(name, pci, punt_client_db,
+ ({
+ vlib_cli_output (vm, "%U", format_punt_client, pci,
+ PUNT_FORMAT_FLAG_NONE);
+ }));
+ /* *INDENT-ON* */
+ }
+
+ return (NULL);
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (punt_client_show_command, static) =
+{
+ .path = "show punt client",
+ .short_help = "show client[s] registered with the punt infra",
+ .function = punt_client_show,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+punt_reason_show (vlib_main_t * vm,
+ unformat_input_t * input, vlib_cli_command_t * cmd)
+{
+ const punt_reason_data_t *pd;
+
+ vec_foreach (pd, punt_reason_data)
+ {
+ vlib_cli_output (vm, "%U", format_punt_reason_data, pd);
+ }
+
+ return (NULL);
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (punt_reason_show_command, static) =
+{
+ .path = "show punt reasons",
+ .short_help = "show all punt reasons",
+ .function = punt_reason_show,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+punt_db_show (vlib_main_t * vm,
+ unformat_input_t * input, vlib_cli_command_t * cmd)
+{
+ u32 pri, ii, jj;
+ u64 key;
+
+ /* *INDENT-OFF* */
+ hash_foreach (key, pri, punt_reg_db,
+ ({
+ vlib_cli_output (vm, " %U", format_punt_reg, pri);
+ }));
+ /* *INDENT-ON* */
+
+ vlib_cli_output (vm, "\nDerived data-plane data-base:");
+ vlib_cli_output (vm,
+ " (for each punt-reason the edge[s] from punt-dispatch)");
+
+ vec_foreach_index (ii, punt_dp_db)
+ {
+ u8 *s = NULL;
+ vlib_cli_output (vm, " %U", format_vlib_punt_reason, ii);
+
+ vec_foreach_index (jj, punt_dp_db[ii])
+ {
+ s = format (s, "%d ", punt_dp_db[ii][jj]);
+ }
+ vlib_cli_output (vm, " [%v]", s);
+ vec_free (s);
+ }
+
+ return (NULL);
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (punt_db_show_command, static) =
+{
+ .path = "show punt db",
+ .short_help = "show the punt DB",
+ .function = punt_db_show,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+punt_stats_show (vlib_main_t * vm,
+ unformat_input_t * input, vlib_cli_command_t * cmd)
+{
+ vlib_combined_counter_main_t *cm = &punt_counters;
+ vlib_counter_t c;
+ u32 ii;
+
+ for (ii = 0; ii < vlib_combined_counter_n_counters (cm); ii++)
+ {
+ vlib_get_combined_counter (cm, ii, &c);
+ vlib_cli_output (vm, "%U packets:%lld bytes:%lld",
+ format_vlib_punt_reason, ii, c.packets, c.bytes);
+ }
+
+ return (NULL);
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (punt_stats_show_command, static) =
+{
+ .path = "show punt stats",
+ .short_help = "show the punt stats",
+ .function = punt_stats_show,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+punt_init (vlib_main_t * vm)
+{
+ punt_client_db = hash_create_vec (0, sizeof (u8), sizeof (u32));
+
+ return (NULL);
+}
+
+VLIB_INIT_FUNCTION (punt_init);
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vlib/punt.h b/src/vlib/punt.h
new file mode 100644
index 00000000000..adb25f91a5e
--- /dev/null
+++ b/src/vlib/punt.h
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __PUNT_H__
+#define __PUNT_H__
+
+#include <vlib/vlib.h>
+
+/**
+ * The 'syatem' defined punt reasons.
+ * Only add to this list reasons defined and used within the vlib subsystem.
+ * To define new reasons in e.g. plgins, use punt_reason_alloc()
+ */
+typedef enum vlib_punt_reason_t_
+{
+ PUNT_N_REASONS,
+} vlib_punt_reason_t;
+
+
+/**
+ * @brief Format a punt reason
+ */
+extern u8 *format_vlib_punt_reason (u8 * s, va_list * args);
+
+/**
+ * Typedef for a client handle
+ */
+typedef int vlib_punt_hdl_t;
+
+/**
+ * @brief Register a new clinet
+ *
+ * @param who - The name of the client
+ *
+ * @retrun the handle the punt infra allocated for this client that must
+ * be used when the client wishes to use the infra
+ */
+vlib_punt_hdl_t vlib_punt_client_register (const char *who);
+
+/**
+ * Allocate a new punt reason
+ */
+extern int vlib_punt_reason_alloc (vlib_punt_hdl_t client,
+ const char *reason_name,
+ vlib_punt_reason_t * reason);
+
+/**
+ * @brief Register a node to receive particular punted buffers
+ *
+ * @paran client - The registered client registering for the packets
+ * @param reason - The reason the packet was punted
+ * @param node - The node to which the punted packets will be sent
+ */
+extern int vlib_punt_register (vlib_punt_hdl_t client,
+ vlib_punt_reason_t reason, const char *node);
+extern int vlib_punt_unregister (vlib_punt_hdl_t client,
+ vlib_punt_reason_t pr, const char *node);
+
+/**
+ * FOR USE IN THE DP ONLY
+ *
+ * Arc[s] to follow for each reason
+ */
+extern u16 **punt_dp_db;
+
+/**
+ * FOR USE IN THE DP ONLY
+ *
+ * Per-reason counters
+ */
+extern vlib_combined_counter_main_t punt_counters;
+
+#endif
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vlib/punt_node.c b/src/vlib/punt_node.c
new file mode 100644
index 00000000000..c87d2b6fe36
--- /dev/null
+++ b/src/vlib/punt_node.c
@@ -0,0 +1,327 @@
+/*
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vlib/punt.h>
+
+#define foreach_punt_error \
+ _(DISPATCHED, "dispatched") \
+ _(NO_REASON, "No such punt reason") \
+ _(NO_REG, "No registrations") \
+ _(REP_FAIL, "Replication Faliure")
+
+typedef enum punt_error_t_
+{
+#define _(v,s) PUNT_ERROR_##v,
+ foreach_punt_error
+#undef _
+ PUNT_N_ERRORS,
+} punt_error_t;
+
+static char *punt_error_strings[] = {
+#define _(v,s) [PUNT_ERROR_##v] = s,
+ foreach_punt_error
+#undef _
+};
+
+typedef enum punt_next_t_
+{
+ PUNT_NEXT_DROP,
+ PUNT_N_NEXT,
+} punt_next_t;
+
+typedef struct punt_trace_t_
+{
+ vlib_punt_reason_t pt_reason;
+} punt_trace_t;
+
+/**
+ * Per-thread clone vectors
+ */
+#ifndef CLIB_MARCH_VARIANT
+u32 **punt_clones;
+#else
+extern u32 **punt_clones;
+#endif
+
+static u8 *
+format_punt_trace (u8 * s, va_list * args)
+{
+ CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+ CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+ punt_trace_t *t = va_arg (*args, punt_trace_t *);
+
+ s = format (s, "reason: %U", format_vlib_punt_reason, t->pt_reason);
+
+ return s;
+}
+
+always_inline u32
+punt_replicate (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ u32 thread_index,
+ vlib_buffer_t * b0,
+ u32 bi0,
+ vlib_punt_reason_t pr0,
+ u32 * next_index,
+ u32 * n_left_to_next, u32 ** to_next, u32 * pkts_rep_fail)
+{
+ /* multiple clients => replicate a copy to each */
+ u16 n_clones0, n_cloned0, clone0;
+ u32 ci0, next0;
+
+ n_clones0 = vec_len (punt_dp_db[pr0]);
+ vec_validate (punt_clones[thread_index], n_clones0);
+
+ n_cloned0 = vlib_buffer_clone (vm, bi0,
+ punt_clones[thread_index],
+ n_clones0, 2 * CLIB_CACHE_LINE_BYTES);
+
+ if (PREDICT_FALSE (n_cloned0 != n_clones0))
+ {
+ b0->error = node->errors[PUNT_ERROR_REP_FAIL];
+ *pkts_rep_fail += 1;
+ }
+
+ for (clone0 = 1; clone0 < n_cloned0; clone0++)
+ {
+ ci0 = punt_clones[thread_index][clone0];
+
+ *to_next[0] = ci0;
+ *to_next += 1;
+ *n_left_to_next -= 1;
+
+ next0 = punt_dp_db[pr0][clone0];
+
+ if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ vlib_buffer_t *c0;
+ punt_trace_t *t;
+
+ c0 = vlib_get_buffer (vm, ci0);
+
+ if (c0 != b0)
+ vlib_buffer_copy_trace_flag (vm, b0, ci0);
+
+ t = vlib_add_trace (vm, node, c0, sizeof (*t));
+ t->pt_reason = pr0;
+ }
+
+ vlib_validate_buffer_enqueue_x1 (vm, node, *next_index,
+ *to_next, *n_left_to_next, ci0, next0);
+
+ /* replications here always go to different next-nodes
+ * so there's no need to check if the to_next frame
+ * is full */
+ }
+
+ /* The original buffer is the first clone */
+ next0 = punt_dp_db[pr0][0];
+ *to_next[0] = bi0;
+ return next0;
+}
+
+always_inline u32
+punt_dispatch_one (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ vlib_combined_counter_main_t * cm,
+ u32 thread_index,
+ u32 bi0,
+ u32 * next_index,
+ u32 * n_left_to_next, u32 ** to_next, u32 * pkts_errors)
+{
+ vlib_punt_reason_t pr0;
+ vlib_buffer_t *b0;
+ u32 next0;
+
+ b0 = vlib_get_buffer (vm, bi0);
+ pr0 = b0->punt_reason;
+
+ if (PREDICT_FALSE (pr0 >= vec_len (punt_dp_db)))
+ {
+ b0->error = node->errors[PUNT_ERROR_NO_REASON];
+ next0 = PUNT_NEXT_DROP;
+ pkts_errors[PUNT_ERROR_NO_REASON] += 1;
+ }
+ else
+ {
+ vlib_increment_combined_counter
+ (cm, thread_index, pr0, 1, vlib_buffer_length_in_chain (vm, b0));
+
+ if (PREDICT_TRUE (1 == vec_len (punt_dp_db[pr0])))
+ {
+ /*
+ * one registered client => give it the packet
+ * This is the most likely outcome.
+ */
+ next0 = punt_dp_db[pr0][0];
+ pkts_errors[PUNT_ERROR_DISPATCHED] += 1;
+ }
+ else if (0 == vec_len (punt_dp_db[pr0]))
+ {
+ /* no registered clients => drop */
+ next0 = PUNT_NEXT_DROP;
+ pkts_errors[PUNT_ERROR_NO_REG] += 1;
+ }
+ else
+ {
+ /*
+ * multiple registered clients => replicate
+ */
+ pkts_errors[PUNT_ERROR_DISPATCHED] += 1;
+
+ next0 = punt_replicate (vm, node, thread_index, b0, bi0, pr0,
+ next_index, n_left_to_next, to_next,
+ &pkts_errors[PUNT_ERROR_REP_FAIL]);
+ }
+ }
+
+ if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ punt_trace_t *t;
+
+ t = vlib_add_trace (vm, node, b0, sizeof (*t));
+ t->pt_reason = pr0;
+ }
+
+ return (next0);
+}
+
+VLIB_NODE_FN (punt_dispatch_node) (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ vlib_frame_t * frame)
+{
+ u32 n_left_from, *from, *to_next, next_index, thread_index;
+ vlib_combined_counter_main_t *cm;
+ u32 pkt_errors[PUNT_N_ERRORS] = { 0 };
+
+ cm = &punt_counters;
+ from = vlib_frame_vector_args (frame);
+ n_left_from = frame->n_vectors;
+ next_index = node->cached_next_index;
+ thread_index = vlib_get_thread_index ();
+
+ while (n_left_from > 0)
+ {
+ u32 n_left_to_next;
+
+ vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+ while (n_left_from > 4 && n_left_to_next > 2)
+ {
+ punt_next_t next0, next1;
+ u32 bi0, bi1;
+
+ {
+ vlib_buffer_t *b2, *b3;
+
+ b2 = vlib_get_buffer (vm, from[2]);
+ b3 = vlib_get_buffer (vm, from[3]);
+
+ vlib_prefetch_buffer_header (b2, LOAD);
+ vlib_prefetch_buffer_header (b3, LOAD);
+ }
+
+ bi0 = to_next[0] = from[0];
+ bi1 = to_next[1] = from[1];
+ from += 2;
+ n_left_from -= 2;
+
+ next0 = punt_dispatch_one (vm, node, cm, thread_index, bi0,
+ &next_index, &n_left_to_next,
+ &to_next, pkt_errors);
+ next1 = punt_dispatch_one (vm, node, cm, thread_index, bi1,
+ &next_index, &n_left_to_next,
+ &to_next, pkt_errors);
+
+ to_next += 2;
+ n_left_to_next -= 2;
+
+ vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
+ to_next, n_left_to_next,
+ bi0, bi1, next0, next1);
+ }
+ while (n_left_from > 0 && n_left_to_next > 0)
+ {
+ punt_next_t next0;
+ u32 bi0;
+
+ bi0 = to_next[0] = from[0];
+ from += 1;
+ n_left_from -= 1;
+
+ next0 = punt_dispatch_one (vm, node, cm, thread_index, bi0,
+ &next_index, &n_left_to_next,
+ &to_next, pkt_errors);
+
+ to_next += 1;
+ n_left_to_next -= 1;
+
+ vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+ to_next, n_left_to_next,
+ bi0, next0);
+ }
+ vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+ }
+
+ vlib_node_increment_counter (vm, node->node_index,
+ PUNT_ERROR_DISPATCHED,
+ pkt_errors[PUNT_ERROR_DISPATCHED]);
+ vlib_node_increment_counter (vm, node->node_index,
+ PUNT_ERROR_NO_REASON,
+ pkt_errors[PUNT_ERROR_NO_REASON]);
+ vlib_node_increment_counter (vm, node->node_index,
+ PUNT_ERROR_NO_REG,
+ pkt_errors[PUNT_ERROR_NO_REG]);
+ vlib_node_increment_counter (vm, node->node_index,
+ PUNT_ERROR_REP_FAIL,
+ pkt_errors[PUNT_ERROR_REP_FAIL]);
+
+ return frame->n_vectors;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (punt_dispatch_node) = {
+ .name = "punt-dispatch",
+ .vector_size = sizeof (u32),
+ .format_trace = format_punt_trace,
+ .n_errors = PUNT_N_ERRORS,
+ .error_strings = punt_error_strings,
+ .n_next_nodes = PUNT_N_NEXT,
+ .next_nodes = {
+ [PUNT_NEXT_DROP] = "drop",
+ },
+};
+
+/* *INDENT-ON* */
+
+#ifndef CLIB_MARCH_VARIANT
+clib_error_t *
+punt_node_init (vlib_main_t * vm)
+{
+ vec_validate (punt_clones, vlib_num_workers ());
+
+ return NULL;
+}
+
+VLIB_INIT_FUNCTION (punt_node_init);
+#endif
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vlib/vlib.h b/src/vlib/vlib.h
index 2149c4763ab..b5fe47b3729 100644
--- a/src/vlib/vlib.h
+++ b/src/vlib/vlib.h
@@ -58,6 +58,7 @@ typedef u32 vlib_log_class_t;
#include <vlib/error.h>
#include <vlib/init.h>
#include <vlib/node.h>
+#include <vlib/punt.h>
#include <vlib/trace.h>
#include <vlib/log.h>