summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorFilip Varga <fivarga@cisco.com>2020-10-22 11:13:00 +0200
committerMatthew Smith <mgsmith@netgate.com>2020-12-03 22:55:02 +0000
commitc05e2852f6755f38065e633a3c1bcf7a502e1bf3 (patch)
tree6177daf99d28112367dd75f619971ee43d8aae94
parent090096bff365b4ad69c061303c0852bf214b4f03 (diff)
nat: fix forwarding handoff workaround
A special case when out2in packet needs to be handoffed to other worker thread. We are not able to determine which thread they belong to in the first processing of nat handoff node. These packets needs to go through out2in slowpath before we are able to tell where to handoff them. Type: fix Ticket: VPP-1941 Change-Id: I7173bda970ce6a91d81f48fc72aa2457586a076f Signed-off-by: Filip Varga <fivarga@cisco.com>
-rw-r--r--src/plugins/nat/nat.c1
-rw-r--r--src/plugins/nat/nat.h14
-rw-r--r--src/plugins/nat/nat44_handoff.c14
-rw-r--r--src/plugins/nat/nat_ha.c2
-rw-r--r--src/plugins/nat/out2in_ed.c165
-rw-r--r--src/vnet/buffer.h5
6 files changed, 170 insertions, 31 deletions
diff --git a/src/plugins/nat/nat.c b/src/plugins/nat/nat.c
index 0ca2042191d..91f14d682f4 100644
--- a/src/plugins/nat/nat.c
+++ b/src/plugins/nat/nat.c
@@ -4752,6 +4752,7 @@ VLIB_REGISTER_NODE (nat_default_node) = {
[NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
[NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
[NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
+ [NAT_NEXT_OUT2IN_ED_HANDOFF] = "nat44-ed-out2in-handoff",
[NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
[NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
},
diff --git a/src/plugins/nat/nat.h b/src/plugins/nat/nat.h
index a713f7b51cf..daebe451f4f 100644
--- a/src/plugins/nat/nat.h
+++ b/src/plugins/nat/nat.h
@@ -88,6 +88,7 @@ typedef enum
NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH,
NAT_NEXT_OUT2IN_ED_FAST_PATH,
NAT_NEXT_OUT2IN_ED_SLOW_PATH,
+ NAT_NEXT_OUT2IN_ED_HANDOFF,
NAT_NEXT_IN2OUT_CLASSIFY,
NAT_NEXT_OUT2IN_CLASSIFY,
NAT_N_NEXT,
@@ -162,6 +163,19 @@ typedef enum
NAT_IN2OUT_ED_N_ERROR,
} nat_in2out_ed_error_t;
+#define foreach_nat44_handoff_error \
+_(CONGESTION_DROP, "congestion drop") \
+_(SAME_WORKER, "same worker") \
+_(DO_HANDOFF, "do handoff")
+
+typedef enum
+{
+#define _(sym,str) NAT44_HANDOFF_ERROR_##sym,
+ foreach_nat44_handoff_error
+#undef _
+ NAT44_HANDOFF_N_ERROR,
+} nat44_handoff_error_t;
+
#define foreach_nat_out2in_ed_error \
_(UNSUPPORTED_PROTOCOL, "unsupported protocol") \
_(OUT_OF_PORTS, "out of ports") \
diff --git a/src/plugins/nat/nat44_handoff.c b/src/plugins/nat/nat44_handoff.c
index 69cdc42870f..651c8d1f4ac 100644
--- a/src/plugins/nat/nat44_handoff.c
+++ b/src/plugins/nat/nat44_handoff.c
@@ -33,26 +33,12 @@ typedef struct
u8 output;
} nat44_handoff_trace_t;
-#define foreach_nat44_handoff_error \
-_(CONGESTION_DROP, "congestion drop") \
-_(SAME_WORKER, "same worker") \
-_(DO_HANDOFF, "do handoff")
-
-typedef enum
-{
-#define _(sym,str) NAT44_HANDOFF_ERROR_##sym,
- foreach_nat44_handoff_error
-#undef _
- NAT44_HANDOFF_N_ERROR,
-} nat44_handoff_error_t;
-
static char *nat44_handoff_error_strings[] = {
#define _(sym,string) string,
foreach_nat44_handoff_error
#undef _
};
-
static u8 *
format_nat44_handoff_trace (u8 * s, va_list * args)
{
diff --git a/src/plugins/nat/nat_ha.c b/src/plugins/nat/nat_ha.c
index 3ae572ba930..08b065dec1d 100644
--- a/src/plugins/nat/nat_ha.c
+++ b/src/plugins/nat/nat_ha.c
@@ -1057,7 +1057,7 @@ typedef enum
#define _(sym,str) NAT_HA_HANDOFF_ERROR_##sym,
foreach_nat_ha_handoff_error
#undef _
- NAT44_HANDOFF_N_ERROR,
+ NAT_HA_HANDOFF_N_ERROR,
} nat_ha_handoff_error_t;
static char *nat_ha_handoff_error_strings[] = {
diff --git a/src/plugins/nat/out2in_ed.c b/src/plugins/nat/out2in_ed.c
index b0e8892f77a..95e2ea972af 100644
--- a/src/plugins/nat/out2in_ed.c
+++ b/src/plugins/nat/out2in_ed.c
@@ -46,6 +46,11 @@ typedef struct
u32 is_slow_path;
} nat44_ed_out2in_trace_t;
+typedef struct
+{
+ u16 thread_next;
+} nat44_ed_out2in_handoff_trace_t;
+
static u8 *
format_nat44_ed_out2in_trace (u8 * s, va_list * args)
{
@@ -513,16 +518,24 @@ create_bypass_for_fwd (snat_main_t * sm, vlib_buffer_t * b, ip4_header_t * ip,
nat44_session_update_lru (sm, s, thread_index);
}
-static inline void
-create_bypass_for_fwd_worker (snat_main_t * sm, vlib_buffer_t * b,
- ip4_header_t * ip, u32 rx_fib_index)
+static_always_inline int
+create_bypass_for_fwd_worker (snat_main_t * sm,
+ vlib_buffer_t * b, ip4_header_t * ip,
+ u32 rx_fib_index, u32 thread_index)
{
- ip4_header_t ip_wkr = {
+ ip4_header_t tmp = {
.src_address = ip->dst_address,
};
- u32 thread_index = sm->worker_in2out_cb (&ip_wkr, rx_fib_index, 0);
+ u32 index = sm->worker_in2out_cb (&tmp, rx_fib_index, 0);
+
+ if (index != thread_index)
+ {
+ vnet_buffer2 (b)->nat.thread_next = index;
+ return 1;
+ }
create_bypass_for_fwd (sm, b, ip, rx_fib_index, thread_index);
+ return 0;
}
#ifndef CLIB_MARCH_VARIANT
@@ -590,10 +603,19 @@ icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
else
{
if (sm->num_workers > 1)
- create_bypass_for_fwd_worker (sm, b, ip, rx_fib_index);
+ {
+ if (create_bypass_for_fwd_worker (sm, b, ip,
+ rx_fib_index,
+ thread_index))
+ {
+ next = NAT_NEXT_OUT2IN_ED_HANDOFF;
+ }
+ }
else
- create_bypass_for_fwd (sm, b, ip, rx_fib_index,
- thread_index);
+ {
+ create_bypass_for_fwd (sm, b, ip, rx_fib_index,
+ thread_index);
+ }
}
}
goto out;
@@ -1208,12 +1230,18 @@ nat44_ed_out2in_slow_path_node_fn_inline (vlib_main_t * vm,
}
else
{
- if (sm->num_workers > 1)
- create_bypass_for_fwd_worker (sm, b0, ip0,
- rx_fib_index0);
+ if ((sm->num_workers > 1)
+ && create_bypass_for_fwd_worker (sm, b0, ip0,
+ rx_fib_index0,
+ thread_index))
+ {
+ next[0] = NAT_NEXT_OUT2IN_ED_HANDOFF;
+ }
else
- create_bypass_for_fwd (sm, b0, ip0, rx_fib_index0,
- thread_index);
+ {
+ create_bypass_for_fwd (sm, b0, ip0, rx_fib_index0,
+ thread_index);
+ }
}
}
goto trace0;
@@ -1383,6 +1411,88 @@ nat44_ed_out2in_slow_path_node_fn_inline (vlib_main_t * vm,
return frame->n_vectors;
}
+static inline uword
+nat_handoff_node_fn_inline (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ vlib_frame_t * frame, u32 fq_index)
+{
+ u32 n_enq, n_left_from, *from;
+
+ u16 thread_indices[VLIB_FRAME_SIZE], *ti = thread_indices;
+ vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
+
+ from = vlib_frame_vector_args (frame);
+ n_left_from = frame->n_vectors;
+
+ vlib_get_buffers (vm, from, b, n_left_from);
+
+ while (n_left_from >= 4)
+ {
+ if (PREDICT_TRUE (n_left_from >= 8))
+ {
+ vlib_prefetch_buffer_header (b[4], LOAD);
+ vlib_prefetch_buffer_header (b[5], LOAD);
+ vlib_prefetch_buffer_header (b[6], LOAD);
+ vlib_prefetch_buffer_header (b[7], LOAD);
+ CLIB_PREFETCH (&b[4]->data, CLIB_CACHE_LINE_BYTES, LOAD);
+ CLIB_PREFETCH (&b[5]->data, CLIB_CACHE_LINE_BYTES, LOAD);
+ CLIB_PREFETCH (&b[6]->data, CLIB_CACHE_LINE_BYTES, LOAD);
+ CLIB_PREFETCH (&b[7]->data, CLIB_CACHE_LINE_BYTES, LOAD);
+ }
+
+ ti[0] = vnet_buffer2 (b[0])->nat.thread_next;
+ ti[1] = vnet_buffer2 (b[1])->nat.thread_next;
+ ti[2] = vnet_buffer2 (b[2])->nat.thread_next;
+ ti[3] = vnet_buffer2 (b[3])->nat.thread_next;
+
+ b += 4;
+ ti += 4;
+ n_left_from -= 4;
+ }
+
+ while (n_left_from > 0)
+ {
+ ti[0] = vnet_buffer2 (b[0])->nat.thread_next;
+
+ b += 1;
+ ti += 1;
+ n_left_from -= 1;
+ }
+
+ if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+ {
+ u32 i;
+ b = bufs;
+ ti = thread_indices;
+
+ for (i = 0; i < frame->n_vectors; i++)
+ {
+ if (b[0]->flags & VLIB_BUFFER_IS_TRACED)
+ {
+ nat44_ed_out2in_handoff_trace_t *t =
+ vlib_add_trace (vm, node, b[0], sizeof (*t));
+ t->thread_next = ti[0];
+ b += 1;
+ ti += 1;
+ }
+ else
+ break;
+ }
+ }
+
+ n_enq = vlib_buffer_enqueue_to_thread (vm, fq_index, from, thread_indices,
+ frame->n_vectors, 1);
+
+ if (n_enq < frame->n_vectors)
+ {
+ vlib_node_increment_counter (vm, node->node_index,
+ NAT44_HANDOFF_ERROR_CONGESTION_DROP,
+ frame->n_vectors - n_enq);
+ }
+
+ return frame->n_vectors;
+}
+
VLIB_NODE_FN (nat44_ed_out2in_node) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
@@ -1431,6 +1541,35 @@ VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
/* *INDENT-ON* */
static u8 *
+format_nat44_ed_out2in_handoff_trace (u8 * s, va_list * args)
+{
+ CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+ CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+ nat44_ed_out2in_handoff_trace_t *t =
+ va_arg (*args, nat44_ed_out2in_handoff_trace_t *);
+ return format (s, "out2in ed handoff thread_next index %d", t->thread_next);
+}
+
+VLIB_NODE_FN (nat44_ed_out2in_handoff_node) (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ vlib_frame_t * frame)
+{
+ return nat_handoff_node_fn_inline (vm, node, frame,
+ snat_main.ed_out2in_node_index);
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (nat44_ed_out2in_handoff_node) = {
+ .name = "nat44-ed-out2in-handoff",
+ .vector_size = sizeof (u32),
+ .sibling_of = "nat-default",
+ .format_trace = format_nat44_ed_out2in_handoff_trace,
+ .type = VLIB_NODE_TYPE_INTERNAL,
+ .n_errors = 0,
+};
+/* *INDENT-ON* */
+
+static u8 *
format_nat_pre_trace (u8 * s, va_list * args)
{
CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
diff --git a/src/vnet/buffer.h b/src/vnet/buffer.h
index 50515c435e2..9e997b81c52 100644
--- a/src/vnet/buffer.h
+++ b/src/vnet/buffer.h
@@ -452,11 +452,10 @@ typedef struct
/* size of L4 prototol header */
u16 gso_l4_hdr_sz;
- /* The union below has a u64 alignment, so this space is unused */
- u32 __unused2[1];
-
struct
{
+ u16 unused;
+ u16 thread_next;
u32 arc_next;
u32 ed_out2in_nat_session_index;
} nat;