aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/plugins/nat/nat.c1
-rw-r--r--src/plugins/nat/nat.h14
-rw-r--r--src/plugins/nat/nat44_handoff.c14
-rw-r--r--src/plugins/nat/nat_ha.c2
-rw-r--r--src/plugins/nat/out2in_ed.c165
-rw-r--r--src/vnet/buffer.h5
6 files changed, 170 insertions, 31 deletions
diff --git a/src/plugins/nat/nat.c b/src/plugins/nat/nat.c
index 0ca2042191d..91f14d682f4 100644
--- a/src/plugins/nat/nat.c
+++ b/src/plugins/nat/nat.c
@@ -4752,6 +4752,7 @@ VLIB_REGISTER_NODE (nat_default_node) = {
[NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
[NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
[NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
+ [NAT_NEXT_OUT2IN_ED_HANDOFF] = "nat44-ed-out2in-handoff",
[NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
[NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
},
diff --git a/src/plugins/nat/nat.h b/src/plugins/nat/nat.h
index a713f7b51cf..daebe451f4f 100644
--- a/src/plugins/nat/nat.h
+++ b/src/plugins/nat/nat.h
@@ -88,6 +88,7 @@ typedef enum
NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH,
NAT_NEXT_OUT2IN_ED_FAST_PATH,
NAT_NEXT_OUT2IN_ED_SLOW_PATH,
+ NAT_NEXT_OUT2IN_ED_HANDOFF,
NAT_NEXT_IN2OUT_CLASSIFY,
NAT_NEXT_OUT2IN_CLASSIFY,
NAT_N_NEXT,
@@ -162,6 +163,19 @@ typedef enum
NAT_IN2OUT_ED_N_ERROR,
} nat_in2out_ed_error_t;
+#define foreach_nat44_handoff_error \
+_(CONGESTION_DROP, "congestion drop") \
+_(SAME_WORKER, "same worker") \
+_(DO_HANDOFF, "do handoff")
+
+typedef enum
+{
+#define _(sym,str) NAT44_HANDOFF_ERROR_##sym,
+ foreach_nat44_handoff_error
+#undef _
+ NAT44_HANDOFF_N_ERROR,
+} nat44_handoff_error_t;
+
#define foreach_nat_out2in_ed_error \
_(UNSUPPORTED_PROTOCOL, "unsupported protocol") \
_(OUT_OF_PORTS, "out of ports") \
diff --git a/src/plugins/nat/nat44_handoff.c b/src/plugins/nat/nat44_handoff.c
index 69cdc42870f..651c8d1f4ac 100644
--- a/src/plugins/nat/nat44_handoff.c
+++ b/src/plugins/nat/nat44_handoff.c
@@ -33,26 +33,12 @@ typedef struct
u8 output;
} nat44_handoff_trace_t;
-#define foreach_nat44_handoff_error \
-_(CONGESTION_DROP, "congestion drop") \
-_(SAME_WORKER, "same worker") \
-_(DO_HANDOFF, "do handoff")
-
-typedef enum
-{
-#define _(sym,str) NAT44_HANDOFF_ERROR_##sym,
- foreach_nat44_handoff_error
-#undef _
- NAT44_HANDOFF_N_ERROR,
-} nat44_handoff_error_t;
-
static char *nat44_handoff_error_strings[] = {
#define _(sym,string) string,
foreach_nat44_handoff_error
#undef _
};
-
static u8 *
format_nat44_handoff_trace (u8 * s, va_list * args)
{
diff --git a/src/plugins/nat/nat_ha.c b/src/plugins/nat/nat_ha.c
index 3ae572ba930..08b065dec1d 100644
--- a/src/plugins/nat/nat_ha.c
+++ b/src/plugins/nat/nat_ha.c
@@ -1057,7 +1057,7 @@ typedef enum
#define _(sym,str) NAT_HA_HANDOFF_ERROR_##sym,
foreach_nat_ha_handoff_error
#undef _
- NAT44_HANDOFF_N_ERROR,
+ NAT_HA_HANDOFF_N_ERROR,
} nat_ha_handoff_error_t;
static char *nat_ha_handoff_error_strings[] = {
diff --git a/src/plugins/nat/out2in_ed.c b/src/plugins/nat/out2in_ed.c
index b0e8892f77a..95e2ea972af 100644
--- a/src/plugins/nat/out2in_ed.c
+++ b/src/plugins/nat/out2in_ed.c
@@ -46,6 +46,11 @@ typedef struct
u32 is_slow_path;
} nat44_ed_out2in_trace_t;
+typedef struct
+{
+ u16 thread_next;
+} nat44_ed_out2in_handoff_trace_t;
+
static u8 *
format_nat44_ed_out2in_trace (u8 * s, va_list * args)
{
@@ -513,16 +518,24 @@ create_bypass_for_fwd (snat_main_t * sm, vlib_buffer_t * b, ip4_header_t * ip,
nat44_session_update_lru (sm, s, thread_index);
}
-static inline void
-create_bypass_for_fwd_worker (snat_main_t * sm, vlib_buffer_t * b,
- ip4_header_t * ip, u32 rx_fib_index)
+static_always_inline int
+create_bypass_for_fwd_worker (snat_main_t * sm,
+ vlib_buffer_t * b, ip4_header_t * ip,
+ u32 rx_fib_index, u32 thread_index)
{
- ip4_header_t ip_wkr = {
+ ip4_header_t tmp = {
.src_address = ip->dst_address,
};
- u32 thread_index = sm->worker_in2out_cb (&ip_wkr, rx_fib_index, 0);
+ u32 index = sm->worker_in2out_cb (&tmp, rx_fib_index, 0);
+
+ if (index != thread_index)
+ {
+ vnet_buffer2 (b)->nat.thread_next = index;
+ return 1;
+ }
create_bypass_for_fwd (sm, b, ip, rx_fib_index, thread_index);
+ return 0;
}
#ifndef CLIB_MARCH_VARIANT
@@ -590,10 +603,19 @@ icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
else
{
if (sm->num_workers > 1)
- create_bypass_for_fwd_worker (sm, b, ip, rx_fib_index);
+ {
+ if (create_bypass_for_fwd_worker (sm, b, ip,
+ rx_fib_index,
+ thread_index))
+ {
+ next = NAT_NEXT_OUT2IN_ED_HANDOFF;
+ }
+ }
else
- create_bypass_for_fwd (sm, b, ip, rx_fib_index,
- thread_index);
+ {
+ create_bypass_for_fwd (sm, b, ip, rx_fib_index,
+ thread_index);
+ }
}
}
goto out;
@@ -1208,12 +1230,18 @@ nat44_ed_out2in_slow_path_node_fn_inline (vlib_main_t * vm,
}
else
{
- if (sm->num_workers > 1)
- create_bypass_for_fwd_worker (sm, b0, ip0,
- rx_fib_index0);
+ if ((sm->num_workers > 1)
+ && create_bypass_for_fwd_worker (sm, b0, ip0,
+ rx_fib_index0,
+ thread_index))
+ {
+ next[0] = NAT_NEXT_OUT2IN_ED_HANDOFF;
+ }
else
- create_bypass_for_fwd (sm, b0, ip0, rx_fib_index0,
- thread_index);
+ {
+ create_bypass_for_fwd (sm, b0, ip0, rx_fib_index0,
+ thread_index);
+ }
}
}
goto trace0;
@@ -1383,6 +1411,88 @@ nat44_ed_out2in_slow_path_node_fn_inline (vlib_main_t * vm,
return frame->n_vectors;
}
+static inline uword
+nat_handoff_node_fn_inline (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ vlib_frame_t * frame, u32 fq_index)
+{
+ u32 n_enq, n_left_from, *from;
+
+ u16 thread_indices[VLIB_FRAME_SIZE], *ti = thread_indices;
+ vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
+
+ from = vlib_frame_vector_args (frame);
+ n_left_from = frame->n_vectors;
+
+ vlib_get_buffers (vm, from, b, n_left_from);
+
+ while (n_left_from >= 4)
+ {
+ if (PREDICT_TRUE (n_left_from >= 8))
+ {
+ vlib_prefetch_buffer_header (b[4], LOAD);
+ vlib_prefetch_buffer_header (b[5], LOAD);
+ vlib_prefetch_buffer_header (b[6], LOAD);
+ vlib_prefetch_buffer_header (b[7], LOAD);
+ CLIB_PREFETCH (&b[4]->data, CLIB_CACHE_LINE_BYTES, LOAD);
+ CLIB_PREFETCH (&b[5]->data, CLIB_CACHE_LINE_BYTES, LOAD);
+ CLIB_PREFETCH (&b[6]->data, CLIB_CACHE_LINE_BYTES, LOAD);
+ CLIB_PREFETCH (&b[7]->data, CLIB_CACHE_LINE_BYTES, LOAD);
+ }
+
+ ti[0] = vnet_buffer2 (b[0])->nat.thread_next;
+ ti[1] = vnet_buffer2 (b[1])->nat.thread_next;
+ ti[2] = vnet_buffer2 (b[2])->nat.thread_next;
+ ti[3] = vnet_buffer2 (b[3])->nat.thread_next;
+
+ b += 4;
+ ti += 4;
+ n_left_from -= 4;
+ }
+
+ while (n_left_from > 0)
+ {
+ ti[0] = vnet_buffer2 (b[0])->nat.thread_next;
+
+ b += 1;
+ ti += 1;
+ n_left_from -= 1;
+ }
+
+ if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+ {
+ u32 i;
+ b = bufs;
+ ti = thread_indices;
+
+ for (i = 0; i < frame->n_vectors; i++)
+ {
+ if (b[0]->flags & VLIB_BUFFER_IS_TRACED)
+ {
+ nat44_ed_out2in_handoff_trace_t *t =
+ vlib_add_trace (vm, node, b[0], sizeof (*t));
+ t->thread_next = ti[0];
+ b += 1;
+ ti += 1;
+ }
+ else
+ break;
+ }
+ }
+
+ n_enq = vlib_buffer_enqueue_to_thread (vm, fq_index, from, thread_indices,
+ frame->n_vectors, 1);
+
+ if (n_enq < frame->n_vectors)
+ {
+ vlib_node_increment_counter (vm, node->node_index,
+ NAT44_HANDOFF_ERROR_CONGESTION_DROP,
+ frame->n_vectors - n_enq);
+ }
+
+ return frame->n_vectors;
+}
+
VLIB_NODE_FN (nat44_ed_out2in_node) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
@@ -1431,6 +1541,35 @@ VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
/* *INDENT-ON* */
static u8 *
+format_nat44_ed_out2in_handoff_trace (u8 * s, va_list * args)
+{
+ CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+ CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+ nat44_ed_out2in_handoff_trace_t *t =
+ va_arg (*args, nat44_ed_out2in_handoff_trace_t *);
+ return format (s, "out2in ed handoff thread_next index %d", t->thread_next);
+}
+
+VLIB_NODE_FN (nat44_ed_out2in_handoff_node) (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ vlib_frame_t * frame)
+{
+ return nat_handoff_node_fn_inline (vm, node, frame,
+ snat_main.ed_out2in_node_index);
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (nat44_ed_out2in_handoff_node) = {
+ .name = "nat44-ed-out2in-handoff",
+ .vector_size = sizeof (u32),
+ .sibling_of = "nat-default",
+ .format_trace = format_nat44_ed_out2in_handoff_trace,
+ .type = VLIB_NODE_TYPE_INTERNAL,
+ .n_errors = 0,
+};
+/* *INDENT-ON* */
+
+static u8 *
format_nat_pre_trace (u8 * s, va_list * args)
{
CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
diff --git a/src/vnet/buffer.h b/src/vnet/buffer.h
index 50515c435e2..9e997b81c52 100644
--- a/src/vnet/buffer.h
+++ b/src/vnet/buffer.h
@@ -452,11 +452,10 @@ typedef struct
/* size of L4 prototol header */
u16 gso_l4_hdr_sz;
- /* The union below has a u64 alignment, so this space is unused */
- u32 __unused2[1];
-
struct
{
+ u16 unused;
+ u16 thread_next;
u32 arc_next;
u32 ed_out2in_nat_session_index;
} nat;