aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rwxr-xr-xsrc/plugins/nat/nat.c58
-rw-r--r--src/plugins/nat/nat44_handoff.c165
2 files changed, 163 insertions, 60 deletions
diff --git a/src/plugins/nat/nat.c b/src/plugins/nat/nat.c
index 1f27fb8c788..85c42027be0 100755
--- a/src/plugins/nat/nat.c
+++ b/src/plugins/nat/nat.c
@@ -2554,26 +2554,49 @@ snat_static_mapping_match (snat_main_t * sm,
{
if (PREDICT_FALSE (lb != 0))
*lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
- if (m->affinity)
+ if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
+ match.addr,
+ match.protocol,
+ match.port,
+ &backend_index))
{
- if (nat_affinity_find_and_lock (ext_host_addr[0], match.addr,
- match.protocol, match.port,
- &backend_index))
- goto get_local;
-
local = pool_elt_at_index (m->locals, backend_index);
mapping->addr = local->addr;
mapping->port = clib_host_to_net_u16 (local->port);
mapping->fib_index = local->fib_index;
goto end;
}
- get_local:
- /* *INDENT-OFF* */
- pool_foreach_index (i, m->locals,
- ({
- vec_add1 (tmp, i);
- }));
- /* *INDENT-ON* */
+ // pick locals matching this worker
+ if (PREDICT_FALSE (sm->num_workers > 1))
+ {
+ u32 thread_index = vlib_get_thread_index ();
+ /* *INDENT-OFF* */
+ pool_foreach_index (i, m->locals,
+ ({
+ local = pool_elt_at_index (m->locals, i);
+
+ ip4_header_t ip = {
+ .src_address = local->addr,
+ };
+
+ if (sm->worker_in2out_cb (&ip, m->fib_index) ==
+ thread_index)
+ {
+ vec_add1 (tmp, i);
+ }
+ }));
+ /* *INDENT-ON* */
+ ASSERT (vec_len (tmp) != 0);
+ }
+ else
+ {
+ /* *INDENT-OFF* */
+ pool_foreach_index (i, m->locals,
+ ({
+ vec_add1 (tmp, i);
+ }));
+ /* *INDENT-ON* */
+ }
hi = vec_len (tmp) - 1;
local = pool_elt_at_index (m->locals, tmp[hi]);
rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
@@ -2586,15 +2609,6 @@ snat_static_mapping_match (snat_main_t * sm,
local = pool_elt_at_index (m->locals, tmp[lo]);
if (!(local->prefix >= rand))
return 1;
- if (PREDICT_FALSE (sm->num_workers > 1))
- {
- ip4_header_t ip = {
- .src_address = local->addr,
- };
- if (sm->worker_in2out_cb (&ip, m->fib_index) !=
- vlib_get_thread_index ())
- goto get_local;
- }
mapping->addr = local->addr;
mapping->port = clib_host_to_net_u16 (local->port);
mapping->fib_index = local->fib_index;
diff --git a/src/plugins/nat/nat44_handoff.c b/src/plugins/nat/nat44_handoff.c
index e4cb483a458..04590e409ac 100644
--- a/src/plugins/nat/nat44_handoff.c
+++ b/src/plugins/nat/nat44_handoff.c
@@ -68,35 +68,29 @@ format_nat44_handoff_trace (u8 * s, va_list * args)
}
static inline uword
-nat44_worker_handoff_fn_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
+nat44_worker_handoff_fn_inline (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
vlib_frame_t * frame, u8 is_output,
u8 is_in2out)
{
+ u32 n_enq, n_left_from, *from, do_handoff = 0, same_worker = 0;
+
+ u16 thread_indices[VLIB_FRAME_SIZE], *ti = thread_indices;
+ vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
snat_main_t *sm = &snat_main;
- vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
- u32 n_enq, n_left_from, *from;
- u16 thread_indices[VLIB_FRAME_SIZE], *ti;
- u32 fq_index;
+
snat_get_worker_function_t *get_worker;
- u32 thread_index = vm->thread_index;
- u32 do_handoff = 0, same_worker = 0;
+ u32 fq_index, thread_index = vm->thread_index;
from = vlib_frame_vector_args (frame);
n_left_from = frame->n_vectors;
- vlib_get_buffers (vm, from, bufs, n_left_from);
- b = bufs;
- ti = thread_indices;
-
- ASSERT (vec_len (sm->workers));
+ vlib_get_buffers (vm, from, b, n_left_from);
if (is_in2out)
{
+ fq_index = is_output ? sm->fq_in2out_output_index : sm->fq_in2out_index;
get_worker = sm->worker_in2out_cb;
- if (is_output)
- fq_index = sm->fq_in2out_output_index;
- else
- fq_index = sm->fq_in2out_index;
}
else
{
@@ -104,45 +98,138 @@ nat44_worker_handoff_fn_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
get_worker = sm->worker_out2in_cb;
}
- while (n_left_from > 0)
+ while (n_left_from >= 4)
{
- u32 sw_if_index0;
- u32 rx_fib_index0;
- ip4_header_t *ip0;
+ u32 sw_if_index0, sw_if_index1, sw_if_index2, sw_if_index3;
+ u32 rx_fib_index0 = 0, rx_fib_index1 = 0,
+ rx_fib_index2 = 0, rx_fib_index3 = 0;
+ ip4_header_t *ip0, *ip1, *ip2, *ip3;
+
+ if (PREDICT_TRUE (n_left_from >= 8))
+ {
+ vlib_prefetch_buffer_header (b[4], STORE);
+ vlib_prefetch_buffer_header (b[5], STORE);
+ vlib_prefetch_buffer_header (b[6], STORE);
+ vlib_prefetch_buffer_header (b[7], STORE);
+ CLIB_PREFETCH (&b[4]->data, CLIB_CACHE_LINE_BYTES, STORE);
+ CLIB_PREFETCH (&b[5]->data, CLIB_CACHE_LINE_BYTES, STORE);
+ CLIB_PREFETCH (&b[6]->data, CLIB_CACHE_LINE_BYTES, STORE);
+ CLIB_PREFETCH (&b[7]->data, CLIB_CACHE_LINE_BYTES, STORE);
+ }
- sw_if_index0 = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
- rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
ip0 = vlib_buffer_get_current (b[0]);
+ ip1 = vlib_buffer_get_current (b[1]);
+ ip2 = vlib_buffer_get_current (b[2]);
+ ip3 = vlib_buffer_get_current (b[3]);
+
+ if (PREDICT_FALSE (is_in2out))
+ {
+ sw_if_index0 = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
+ sw_if_index1 = vnet_buffer (b[1])->sw_if_index[VLIB_RX];
+ sw_if_index2 = vnet_buffer (b[2])->sw_if_index[VLIB_RX];
+ sw_if_index3 = vnet_buffer (b[3])->sw_if_index[VLIB_RX];
+
+ rx_fib_index0 =
+ ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
+ rx_fib_index1 =
+ ip4_fib_table_get_index_for_sw_if_index (sw_if_index1);
+ rx_fib_index2 =
+ ip4_fib_table_get_index_for_sw_if_index (sw_if_index2);
+ rx_fib_index3 =
+ ip4_fib_table_get_index_for_sw_if_index (sw_if_index3);
+ }
+
ti[0] = get_worker (ip0, rx_fib_index0);
+ ti[1] = get_worker (ip1, rx_fib_index1);
+ ti[2] = get_worker (ip2, rx_fib_index2);
+ ti[3] = get_worker (ip3, rx_fib_index3);
- if (ti[0] != thread_index)
+ if (ti[0] == thread_index)
+ same_worker++;
+ else
do_handoff++;
+
+ if (ti[1] == thread_index)
+ same_worker++;
+ else
+ do_handoff++;
+
+ if (ti[2] == thread_index)
+ same_worker++;
else
+ do_handoff++;
+
+ if (ti[3] == thread_index)
same_worker++;
+ else
+ do_handoff++;
+
+ b += 4;
+ ti += 4;
+ n_left_from -= 4;
+ }
- if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
- && (b[0]->flags & VLIB_BUFFER_IS_TRACED)))
+ while (n_left_from > 0)
+ {
+ u32 sw_if_index0;
+ u32 rx_fib_index0 = 0;
+ ip4_header_t *ip0;
+
+ ip0 = vlib_buffer_get_current (b[0]);
+
+ if (PREDICT_FALSE (is_in2out))
{
- nat44_handoff_trace_t *t =
- vlib_add_trace (vm, node, b[0], sizeof (*t));
- t->next_worker_index = ti[0];
- t->trace_index = vlib_buffer_get_trace_index (b[0]);
- t->in2out = is_in2out;
+ sw_if_index0 = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
+ rx_fib_index0 =
+ ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
}
- n_left_from -= 1;
- ti += 1;
+ ti[0] = get_worker (ip0, rx_fib_index0);
+
+ if (ti[0] == thread_index)
+ same_worker++;
+ else
+ do_handoff++;
+
b += 1;
+ ti += 1;
+ n_left_from -= 1;
+ }
+
+ if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+ {
+ u32 i;
+ b = bufs;
+ ti = thread_indices;
+
+ for (i = 0; i < frame->n_vectors; i++)
+ {
+ if (b[0]->flags & VLIB_BUFFER_IS_TRACED)
+ {
+ nat44_handoff_trace_t *t =
+ vlib_add_trace (vm, node, b[0], sizeof (*t));
+ t->next_worker_index = ti[0];
+ t->trace_index = vlib_buffer_get_trace_index (b[0]);
+ t->in2out = is_in2out;
+
+ b++;
+ ti++;
+ }
+ else
+ break;
+ }
}
- n_enq =
- vlib_buffer_enqueue_to_thread (vm, fq_index, from, thread_indices,
- frame->n_vectors, 1);
+ n_enq = vlib_buffer_enqueue_to_thread (vm, fq_index, from, thread_indices,
+ frame->n_vectors, 1);
if (n_enq < frame->n_vectors)
- vlib_node_increment_counter (vm, node->node_index,
- NAT44_HANDOFF_ERROR_CONGESTION_DROP,
- frame->n_vectors - n_enq);
+ {
+ vlib_node_increment_counter (vm, node->node_index,
+ NAT44_HANDOFF_ERROR_CONGESTION_DROP,
+ frame->n_vectors - n_enq);
+ }
+
vlib_node_increment_counter (vm, node->node_index,
NAT44_HANDOFF_ERROR_SAME_WORKER, same_worker);
vlib_node_increment_counter (vm, node->node_index,
@@ -150,6 +237,8 @@ nat44_worker_handoff_fn_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
return frame->n_vectors;
}
+
+
VLIB_NODE_FN (snat_in2out_worker_handoff_node) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)