diff options
| -rwxr-xr-x | src/plugins/nat/nat.c | 58 | ||||
| -rw-r--r-- | src/plugins/nat/nat44_handoff.c | 165 |
2 files changed, 163 insertions, 60 deletions
diff --git a/src/plugins/nat/nat.c b/src/plugins/nat/nat.c index 1f27fb8c788..85c42027be0 100755 --- a/src/plugins/nat/nat.c +++ b/src/plugins/nat/nat.c @@ -2554,26 +2554,49 @@ snat_static_mapping_match (snat_main_t * sm, { if (PREDICT_FALSE (lb != 0)) *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT; - if (m->affinity) + if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0], + match.addr, + match.protocol, + match.port, + &backend_index)) { - if (nat_affinity_find_and_lock (ext_host_addr[0], match.addr, - match.protocol, match.port, - &backend_index)) - goto get_local; - local = pool_elt_at_index (m->locals, backend_index); mapping->addr = local->addr; mapping->port = clib_host_to_net_u16 (local->port); mapping->fib_index = local->fib_index; goto end; } - get_local: - /* *INDENT-OFF* */ - pool_foreach_index (i, m->locals, - ({ - vec_add1 (tmp, i); - })); - /* *INDENT-ON* */ + // pick locals matching this worker + if (PREDICT_FALSE (sm->num_workers > 1)) + { + u32 thread_index = vlib_get_thread_index (); + /* *INDENT-OFF* */ + pool_foreach_index (i, m->locals, + ({ + local = pool_elt_at_index (m->locals, i); + + ip4_header_t ip = { + .src_address = local->addr, + }; + + if (sm->worker_in2out_cb (&ip, m->fib_index) == + thread_index) + { + vec_add1 (tmp, i); + } + })); + /* *INDENT-ON* */ + ASSERT (vec_len (tmp) != 0); + } + else + { + /* *INDENT-OFF* */ + pool_foreach_index (i, m->locals, + ({ + vec_add1 (tmp, i); + })); + /* *INDENT-ON* */ + } hi = vec_len (tmp) - 1; local = pool_elt_at_index (m->locals, tmp[hi]); rand = 1 + (random_u32 (&sm->random_seed) % local->prefix); @@ -2586,15 +2609,6 @@ snat_static_mapping_match (snat_main_t * sm, local = pool_elt_at_index (m->locals, tmp[lo]); if (!(local->prefix >= rand)) return 1; - if (PREDICT_FALSE (sm->num_workers > 1)) - { - ip4_header_t ip = { - .src_address = local->addr, - }; - if (sm->worker_in2out_cb (&ip, m->fib_index) != - vlib_get_thread_index ()) - goto get_local; - } mapping->addr = local->addr; mapping->port = clib_host_to_net_u16 (local->port); mapping->fib_index = local->fib_index; diff --git a/src/plugins/nat/nat44_handoff.c b/src/plugins/nat/nat44_handoff.c index e4cb483a458..04590e409ac 100644 --- a/src/plugins/nat/nat44_handoff.c +++ b/src/plugins/nat/nat44_handoff.c @@ -68,35 +68,29 @@ format_nat44_handoff_trace (u8 * s, va_list * args) } static inline uword -nat44_worker_handoff_fn_inline (vlib_main_t * vm, vlib_node_runtime_t * node, +nat44_worker_handoff_fn_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * frame, u8 is_output, u8 is_in2out) { + u32 n_enq, n_left_from, *from, do_handoff = 0, same_worker = 0; + + u16 thread_indices[VLIB_FRAME_SIZE], *ti = thread_indices; + vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs; snat_main_t *sm = &snat_main; - vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b; - u32 n_enq, n_left_from, *from; - u16 thread_indices[VLIB_FRAME_SIZE], *ti; - u32 fq_index; + snat_get_worker_function_t *get_worker; - u32 thread_index = vm->thread_index; - u32 do_handoff = 0, same_worker = 0; + u32 fq_index, thread_index = vm->thread_index; from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; - vlib_get_buffers (vm, from, bufs, n_left_from); - b = bufs; - ti = thread_indices; - - ASSERT (vec_len (sm->workers)); + vlib_get_buffers (vm, from, b, n_left_from); if (is_in2out) { + fq_index = is_output ? sm->fq_in2out_output_index : sm->fq_in2out_index; get_worker = sm->worker_in2out_cb; - if (is_output) - fq_index = sm->fq_in2out_output_index; - else - fq_index = sm->fq_in2out_index; } else { @@ -104,45 +98,138 @@ nat44_worker_handoff_fn_inline (vlib_main_t * vm, vlib_node_runtime_t * node, get_worker = sm->worker_out2in_cb; } - while (n_left_from > 0) + while (n_left_from >= 4) { - u32 sw_if_index0; - u32 rx_fib_index0; - ip4_header_t *ip0; + u32 sw_if_index0, sw_if_index1, sw_if_index2, sw_if_index3; + u32 rx_fib_index0 = 0, rx_fib_index1 = 0, + rx_fib_index2 = 0, rx_fib_index3 = 0; + ip4_header_t *ip0, *ip1, *ip2, *ip3; + + if (PREDICT_TRUE (n_left_from >= 8)) + { + vlib_prefetch_buffer_header (b[4], STORE); + vlib_prefetch_buffer_header (b[5], STORE); + vlib_prefetch_buffer_header (b[6], STORE); + vlib_prefetch_buffer_header (b[7], STORE); + CLIB_PREFETCH (&b[4]->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (&b[5]->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (&b[6]->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (&b[7]->data, CLIB_CACHE_LINE_BYTES, STORE); + } - sw_if_index0 = vnet_buffer (b[0])->sw_if_index[VLIB_RX]; - rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0); ip0 = vlib_buffer_get_current (b[0]); + ip1 = vlib_buffer_get_current (b[1]); + ip2 = vlib_buffer_get_current (b[2]); + ip3 = vlib_buffer_get_current (b[3]); + + if (PREDICT_FALSE (is_in2out)) + { + sw_if_index0 = vnet_buffer (b[0])->sw_if_index[VLIB_RX]; + sw_if_index1 = vnet_buffer (b[1])->sw_if_index[VLIB_RX]; + sw_if_index2 = vnet_buffer (b[2])->sw_if_index[VLIB_RX]; + sw_if_index3 = vnet_buffer (b[3])->sw_if_index[VLIB_RX]; + + rx_fib_index0 = + ip4_fib_table_get_index_for_sw_if_index (sw_if_index0); + rx_fib_index1 = + ip4_fib_table_get_index_for_sw_if_index (sw_if_index1); + rx_fib_index2 = + ip4_fib_table_get_index_for_sw_if_index (sw_if_index2); + rx_fib_index3 = + ip4_fib_table_get_index_for_sw_if_index (sw_if_index3); + } + ti[0] = get_worker (ip0, rx_fib_index0); + ti[1] = get_worker (ip1, rx_fib_index1); + ti[2] = get_worker (ip2, rx_fib_index2); + ti[3] = get_worker (ip3, rx_fib_index3); - if (ti[0] != thread_index) + if (ti[0] == thread_index) + same_worker++; + else do_handoff++; + + if (ti[1] == thread_index) + same_worker++; + else + do_handoff++; + + if (ti[2] == thread_index) + same_worker++; else + do_handoff++; + + if (ti[3] == thread_index) same_worker++; + else + do_handoff++; + + b += 4; + ti += 4; + n_left_from -= 4; + } - if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) - && (b[0]->flags & VLIB_BUFFER_IS_TRACED))) + while (n_left_from > 0) + { + u32 sw_if_index0; + u32 rx_fib_index0 = 0; + ip4_header_t *ip0; + + ip0 = vlib_buffer_get_current (b[0]); + + if (PREDICT_FALSE (is_in2out)) { - nat44_handoff_trace_t *t = - vlib_add_trace (vm, node, b[0], sizeof (*t)); - t->next_worker_index = ti[0]; - t->trace_index = vlib_buffer_get_trace_index (b[0]); - t->in2out = is_in2out; + sw_if_index0 = vnet_buffer (b[0])->sw_if_index[VLIB_RX]; + rx_fib_index0 = + ip4_fib_table_get_index_for_sw_if_index (sw_if_index0); } - n_left_from -= 1; - ti += 1; + ti[0] = get_worker (ip0, rx_fib_index0); + + if (ti[0] == thread_index) + same_worker++; + else + do_handoff++; + b += 1; + ti += 1; + n_left_from -= 1; + } + + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE))) + { + u32 i; + b = bufs; + ti = thread_indices; + + for (i = 0; i < frame->n_vectors; i++) + { + if (b[0]->flags & VLIB_BUFFER_IS_TRACED) + { + nat44_handoff_trace_t *t = + vlib_add_trace (vm, node, b[0], sizeof (*t)); + t->next_worker_index = ti[0]; + t->trace_index = vlib_buffer_get_trace_index (b[0]); + t->in2out = is_in2out; + + b++; + ti++; + } + else + break; + } } - n_enq = - vlib_buffer_enqueue_to_thread (vm, fq_index, from, thread_indices, - frame->n_vectors, 1); + n_enq = vlib_buffer_enqueue_to_thread (vm, fq_index, from, thread_indices, + frame->n_vectors, 1); if (n_enq < frame->n_vectors) - vlib_node_increment_counter (vm, node->node_index, - NAT44_HANDOFF_ERROR_CONGESTION_DROP, - frame->n_vectors - n_enq); + { + vlib_node_increment_counter (vm, node->node_index, + NAT44_HANDOFF_ERROR_CONGESTION_DROP, + frame->n_vectors - n_enq); + } + vlib_node_increment_counter (vm, node->node_index, NAT44_HANDOFF_ERROR_SAME_WORKER, same_worker); vlib_node_increment_counter (vm, node->node_index, @@ -150,6 +237,8 @@ nat44_worker_handoff_fn_inline (vlib_main_t * vm, vlib_node_runtime_t * node, return frame->n_vectors; } + + VLIB_NODE_FN (snat_in2out_worker_handoff_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) |