diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/plugins/nat/in2out_ed.c | 380 | ||||
| -rwxr-xr-x | src/plugins/nat/nat.c | 119 | ||||
| -rw-r--r-- | src/plugins/nat/nat.h | 54 | ||||
| -rw-r--r-- | src/plugins/nat/nat44_classify.c | 326 | ||||
| -rw-r--r-- | src/plugins/nat/nat44_handoff.c | 16 | ||||
| -rw-r--r-- | src/plugins/nat/nat_inlines.h | 134 | ||||
| -rw-r--r-- | src/plugins/nat/out2in_ed.c | 277 |
7 files changed, 887 insertions, 419 deletions
diff --git a/src/plugins/nat/in2out_ed.c b/src/plugins/nat/in2out_ed.c index 77d620f11cb..733e4f0c0e5 100644 --- a/src/plugins/nat/in2out_ed.c +++ b/src/plugins/nat/in2out_ed.c @@ -37,16 +37,6 @@ static char *nat_in2out_ed_error_strings[] = { #undef _ }; -typedef enum -{ - NAT_IN2OUT_ED_NEXT_LOOKUP, - NAT_IN2OUT_ED_NEXT_DROP, - NAT_IN2OUT_ED_NEXT_ICMP_ERROR, - NAT_IN2OUT_ED_NEXT_SLOW_PATH, - NAT_IN2OUT_ED_NEXT_REASS, - NAT_IN2OUT_ED_N_NEXT, -} nat_in2out_ed_next_t; - typedef struct { u32 sw_if_index; @@ -179,7 +169,7 @@ icmp_in2out_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0, next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, next0, thread_index, p_s0, 0); snat_session_t *s0 = *p_s0; - if (PREDICT_TRUE (next0 != NAT_IN2OUT_ED_NEXT_DROP && s0)) + if (PREDICT_TRUE (next0 != NAT_NEXT_DROP && s0)) { /* Accounting */ nat44_session_update_counters (s0, now, @@ -224,7 +214,7 @@ slow_path_ed (snat_main_t * sm, b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED]; nat_ipfix_logging_max_sessions (thread_index, sm->max_translations); nat_elog_notice ("maximum sessions exceeded"); - return NAT_IN2OUT_ED_NEXT_DROP; + return NAT_NEXT_DROP; } key0.addr = key->l_addr; @@ -244,7 +234,7 @@ slow_path_ed (snat_main_t * sm, { nat_elog_notice ("addresses exhausted"); b->error = node->errors[NAT_IN2OUT_ED_ERROR_OUT_OF_PORTS]; - return NAT_IN2OUT_ED_NEXT_DROP; + return NAT_NEXT_DROP; } } else @@ -263,7 +253,7 @@ slow_path_ed (snat_main_t * sm, if (!tcp_is_init (tcp)) { b->error = node->errors[NAT_IN2OUT_ED_ERROR_NON_SYN]; - return NAT_IN2OUT_ED_NEXT_DROP; + return NAT_NEXT_DROP; } } @@ -274,7 +264,7 @@ slow_path_ed (snat_main_t * sm, if (!is_sm) snat_free_outside_address_and_port (sm->addresses, thread_index, &key1); - return NAT_IN2OUT_ED_NEXT_DROP; + return NAT_NEXT_DROP; } s = nat_ed_session_alloc (sm, u, thread_index, now); @@ -285,7 +275,7 @@ slow_path_ed (snat_main_t * sm, if (!is_sm) snat_free_outside_address_and_port (sm->addresses, thread_index, &key1); - return NAT_IN2OUT_ED_NEXT_DROP; + return NAT_NEXT_DROP; } user_session_increment (sm, u, is_sm); @@ -544,7 +534,7 @@ icmp_match_in2out_ed (snat_main_t * sm, vlib_node_runtime_t * node, if (err != 0) { b->error = node->errors[err]; - next = NAT_IN2OUT_ED_NEXT_DROP; + next = NAT_NEXT_DROP; goto out; } key.fib_index = rx_fib_index; @@ -588,14 +578,14 @@ icmp_match_in2out_ed (snat_main_t * sm, vlib_node_runtime_t * node, if (PREDICT_FALSE (icmp_is_error_message (icmp))) { b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE]; - next = NAT_IN2OUT_ED_NEXT_DROP; + next = NAT_NEXT_DROP; goto out; } next = slow_path_ed (sm, b, rx_fib_index, &kv, &s, node, next, thread_index, vlib_time_now (sm->vlib_main), 0); - if (PREDICT_FALSE (next == NAT_IN2OUT_ED_NEXT_DROP)) + if (PREDICT_FALSE (next == NAT_NEXT_DROP)) goto out; if (!s) @@ -611,7 +601,7 @@ icmp_match_in2out_ed (snat_main_t * sm, vlib_node_runtime_t * node, !icmp_is_error_message (icmp))) { b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE]; - next = NAT_IN2OUT_ED_NEXT_DROP; + next = NAT_NEXT_DROP; goto out; } @@ -837,13 +827,19 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, int is_output_feature) { u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index; - nat_in2out_ed_next_t next_index; + nat_next_t next_index; snat_main_t *sm = &snat_main; f64 now = vlib_time_now (vm); u32 thread_index = vm->thread_index; snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index]; u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets = - 0, fragments = 0; + 0, fragments = 0, def_slow, def_reass; + + def_slow = is_output_feature ? NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH : + NAT_NEXT_IN2OUT_ED_SLOW_PATH; + + def_reass = is_output_feature ? NAT_NEXT_IN2OUT_ED_OUTPUT_REASS : + NAT_NEXT_IN2OUT_ED_REASS; stats_node_index = is_slow_path ? sm->ed_in2out_slowpath_node_index : sm->ed_in2out_node_index; @@ -900,10 +896,22 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, b0 = vlib_get_buffer (vm, bi0); b1 = vlib_get_buffer (vm, bi1); - next0 = NAT_IN2OUT_ED_NEXT_LOOKUP; - if (is_output_feature) - iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length; + { + // output feature fast path is enabled on the arc + // we need new arc_next feature + if (PREDICT_TRUE (!is_slow_path)) + { + vnet_feature_next (&nat_buffer_opaque (b0)->arc_next, b0); + vnet_feature_next (&nat_buffer_opaque (b1)->arc_next, b1); + } + + iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length; + iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length; + } + + next0 = nat_buffer_opaque (b0)->arc_next; + next1 = nat_buffer_opaque (b1)->arc_next; ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) + iph_offset0); @@ -919,7 +927,7 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded, ICMP4_time_exceeded_ttl_exceeded_in_transit, 0); - next0 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR; + next0 = NAT_NEXT_ICMP_ERROR; goto trace00; } @@ -937,7 +945,7 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, thread_index, now, vm, node); if (!s0) - next0 = NAT_IN2OUT_ED_NEXT_DROP; + next0 = NAT_NEXT_DROP; other_packets++; goto trace00; } @@ -955,13 +963,13 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, { if (PREDICT_FALSE (proto0 == ~0)) { - next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH; + next0 = def_slow; goto trace00; } if (ip4_is_fragment (ip0)) { - next0 = NAT_IN2OUT_ED_NEXT_REASS; + next0 = def_reass; fragments++; goto trace00; } @@ -976,7 +984,7 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP)) { - next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH; + next0 = def_slow; goto trace00; } } @@ -1012,7 +1020,7 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node, next0, thread_index, now, tcp0); - if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP)) + if (PREDICT_FALSE (next0 == NAT_NEXT_DROP)) goto trace00; if (PREDICT_FALSE (!s0)) @@ -1020,7 +1028,7 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, } else { - next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH; + next0 = def_slow; goto trace00; } } @@ -1126,13 +1134,7 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, t->session_index = s0 - tsm->sessions; } - pkts_processed += next0 == NAT_IN2OUT_ED_NEXT_LOOKUP; - - - next1 = NAT_IN2OUT_ED_NEXT_LOOKUP; - - if (is_output_feature) - iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length; + pkts_processed += next0 == nat_buffer_opaque (b0)->arc_next; ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) + iph_offset1); @@ -1148,7 +1150,7 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded, ICMP4_time_exceeded_ttl_exceeded_in_transit, 0); - next1 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR; + next1 = NAT_NEXT_ICMP_ERROR; goto trace01; } @@ -1166,7 +1168,7 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, thread_index, now, vm, node); if (!s1) - next1 = NAT_IN2OUT_ED_NEXT_DROP; + next1 = NAT_NEXT_DROP; other_packets++; goto trace01; } @@ -1184,13 +1186,13 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, { if (PREDICT_FALSE (proto1 == ~0)) { - next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH; + next1 = def_slow; goto trace01; } if (ip4_is_fragment (ip1)) { - next1 = NAT_IN2OUT_ED_NEXT_REASS; + next1 = def_reass; fragments++; goto trace01; } @@ -1205,7 +1207,7 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP)) { - next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH; + next1 = def_slow; goto trace01; } } @@ -1241,7 +1243,7 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, slow_path_ed (sm, b1, rx_fib_index1, &kv1, &s1, node, next1, thread_index, now, tcp1); - if (PREDICT_FALSE (next1 == NAT_IN2OUT_ED_NEXT_DROP)) + if (PREDICT_FALSE (next1 == NAT_NEXT_DROP)) goto trace01; if (PREDICT_FALSE (!s1)) @@ -1249,7 +1251,7 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, } else { - next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH; + next1 = def_slow; goto trace01; } } @@ -1355,7 +1357,8 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, t->session_index = s1 - tsm->sessions; } - pkts_processed += next1 == NAT_IN2OUT_ED_NEXT_LOOKUP; + pkts_processed += next1 == nat_buffer_opaque (b1)->arc_next; + /* verify speculative enqueues, maybe switch current next frame */ vlib_validate_buffer_enqueue_x2 (vm, node, next_index, @@ -1387,10 +1390,18 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, n_left_to_next -= 1; b0 = vlib_get_buffer (vm, bi0); - next0 = NAT_IN2OUT_ED_NEXT_LOOKUP; if (is_output_feature) - iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length; + { + // output feature fast path is enabled on the arc + // we need new arc_next feature + if (PREDICT_TRUE (!is_slow_path)) + vnet_feature_next (&nat_buffer_opaque (b0)->arc_next, b0); + + iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length; + } + + next0 = nat_buffer_opaque (b0)->arc_next; ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) + iph_offset0); @@ -1406,7 +1417,7 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded, ICMP4_time_exceeded_ttl_exceeded_in_transit, 0); - next0 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR; + next0 = NAT_NEXT_ICMP_ERROR; goto trace0; } @@ -1424,7 +1435,7 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, thread_index, now, vm, node); if (!s0) - next0 = NAT_IN2OUT_ED_NEXT_DROP; + next0 = NAT_NEXT_DROP; other_packets++; goto trace0; } @@ -1442,13 +1453,13 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, { if (PREDICT_FALSE (proto0 == ~0)) { - next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH; + next0 = def_slow; goto trace0; } if (ip4_is_fragment (ip0)) { - next0 = NAT_IN2OUT_ED_NEXT_REASS; + next0 = def_reass; fragments++; goto trace0; } @@ -1463,7 +1474,7 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP)) { - next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH; + next0 = def_slow; goto trace0; } } @@ -1499,7 +1510,7 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node, next0, thread_index, now, tcp0); - if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP)) + if (PREDICT_FALSE (next0 == NAT_NEXT_DROP)) goto trace0; if (PREDICT_FALSE (!s0)) @@ -1507,7 +1518,7 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, } else { - next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH; + next0 = def_slow; goto trace0; } } @@ -1612,7 +1623,7 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, t->session_index = s0 - tsm->sessions; } - pkts_processed += next0 == NAT_IN2OUT_ED_NEXT_LOOKUP; + pkts_processed += next0 == nat_buffer_opaque (b0)->arc_next; /* verify speculative enqueue, maybe switch current next frame */ vlib_validate_buffer_enqueue_x1 (vm, node, next_index, @@ -1642,115 +1653,6 @@ nat44_ed_in2out_node_fn_inline (vlib_main_t * vm, return frame->n_vectors; } -VLIB_NODE_FN (nat44_ed_in2out_node) (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) -{ - return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 0); -} - -/* *INDENT-OFF* */ -VLIB_REGISTER_NODE (nat44_ed_in2out_node) = { - .name = "nat44-ed-in2out", - .vector_size = sizeof (u32), - .format_trace = format_nat_in2out_ed_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings), - .error_strings = nat_in2out_ed_error_strings, - .runtime_data_bytes = sizeof (snat_runtime_t), - .n_next_nodes = NAT_IN2OUT_ED_N_NEXT, - .next_nodes = { - [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop", - [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup", - [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath", - [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error", - [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass", - }, -}; -/* *INDENT-ON* */ - -VLIB_NODE_FN (nat44_ed_in2out_output_node) (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) -{ - return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 1); -} - -/* *INDENT-OFF* */ -VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = { - .name = "nat44-ed-in2out-output", - .vector_size = sizeof (u32), - .format_trace = format_nat_in2out_ed_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings), - .error_strings = nat_in2out_ed_error_strings, - .runtime_data_bytes = sizeof (snat_runtime_t), - .n_next_nodes = NAT_IN2OUT_ED_N_NEXT, - .next_nodes = { - [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop", - [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output", - [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath", - [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error", - [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass-output", - }, -}; -/* *INDENT-ON* */ - -VLIB_NODE_FN (nat44_ed_in2out_slowpath_node) (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) -{ - return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 0); -} - -/* *INDENT-OFF* */ -VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = { - .name = "nat44-ed-in2out-slowpath", - .vector_size = sizeof (u32), - .format_trace = format_nat_in2out_ed_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings), - .error_strings = nat_in2out_ed_error_strings, - .runtime_data_bytes = sizeof (snat_runtime_t), - .n_next_nodes = NAT_IN2OUT_ED_N_NEXT, - .next_nodes = { - [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop", - [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup", - [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath", - [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error", - [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass", - }, -}; -/* *INDENT-ON* */ - -VLIB_NODE_FN (nat44_ed_in2out_output_slowpath_node) (vlib_main_t * vm, - vlib_node_runtime_t * - node, - vlib_frame_t * frame) -{ - return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 1); -} - -/* *INDENT-OFF* */ -VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = { - .name = "nat44-ed-in2out-output-slowpath", - .vector_size = sizeof (u32), - .format_trace = format_nat_in2out_ed_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings), - .error_strings = nat_in2out_ed_error_strings, - .runtime_data_bytes = sizeof (snat_runtime_t), - .n_next_nodes = NAT_IN2OUT_ED_N_NEXT, - .next_nodes = { - [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop", - [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output", - [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath", - [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error", - [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass", - }, -}; -/* *INDENT-ON* */ - static inline uword nat44_ed_in2out_reass_node_fn_inline (vlib_main_t * vm, vlib_node_runtime_t * node, @@ -1758,7 +1660,7 @@ nat44_ed_in2out_reass_node_fn_inline (vlib_main_t * vm, int is_output_feature) { u32 n_left_from, *from, *to_next; - nat_in2out_ed_next_t next_index; + nat_next_t next_index; u32 pkts_processed = 0, cached_fragments = 0; snat_main_t *sm = &snat_main; f64 now = vlib_time_now (vm); @@ -1805,7 +1707,7 @@ nat44_ed_in2out_reass_node_fn_inline (vlib_main_t * vm, b0 = vlib_get_buffer (vm, bi0); - next0 = NAT_IN2OUT_ED_NEXT_LOOKUP; + next0 = nat_buffer_opaque (b0)->arc_next; sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; rx_fib_index0 = @@ -1814,7 +1716,7 @@ nat44_ed_in2out_reass_node_fn_inline (vlib_main_t * vm, if (PREDICT_FALSE (nat_reass_is_drop_frag (0))) { - next0 = NAT_IN2OUT_ED_NEXT_DROP; + next0 = NAT_NEXT_DROP; b0->error = node->errors[NAT_IN2OUT_ED_ERROR_DROP_FRAGMENT]; goto trace0; } @@ -1838,7 +1740,7 @@ nat44_ed_in2out_reass_node_fn_inline (vlib_main_t * vm, if (PREDICT_FALSE (!reass0)) { - next0 = NAT_IN2OUT_ED_NEXT_DROP; + next0 = NAT_NEXT_DROP; b0->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_REASS]; nat_elog_notice ("maximum reassemblies exceeded"); goto trace0; @@ -1861,7 +1763,7 @@ nat44_ed_in2out_reass_node_fn_inline (vlib_main_t * vm, (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, next0, now, thread_index, &s0); - if (PREDICT_TRUE (next0 != NAT_IN2OUT_ED_NEXT_DROP)) + if (PREDICT_TRUE (next0 != NAT_NEXT_DROP)) { if (s0) reass0->sess_index = s0 - per_thread_data->sessions; @@ -1914,7 +1816,7 @@ nat44_ed_in2out_reass_node_fn_inline (vlib_main_t * vm, &s0, node, next0, thread_index, now, tcp0); - if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP)) + if (PREDICT_FALSE (next0 == NAT_NEXT_DROP)) goto trace0; if (PREDICT_FALSE (!s0)) @@ -1945,7 +1847,7 @@ nat44_ed_in2out_reass_node_fn_inline (vlib_main_t * vm, b0->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_FRAG]; nat_elog_notice ("maximum fragments per reassembly exceeded"); - next0 = NAT_IN2OUT_ED_NEXT_DROP; + next0 = NAT_NEXT_DROP; goto trace0; } cached0 = 1; @@ -2060,7 +1962,7 @@ nat44_ed_in2out_reass_node_fn_inline (vlib_main_t * vm, } else { - pkts_processed += next0 != NAT_IN2OUT_ED_NEXT_DROP; + pkts_processed += next0 == nat_buffer_opaque (b0)->arc_next; /* verify speculative enqueue, maybe switch current next frame */ vlib_validate_buffer_enqueue_x1 (vm, node, next_index, @@ -2102,13 +2004,95 @@ nat44_ed_in2out_reass_node_fn_inline (vlib_main_t * vm, nat_send_all_to_node (vm, fragments_to_drop, node, &node->errors[NAT_IN2OUT_ED_ERROR_DROP_FRAGMENT], - NAT_IN2OUT_ED_NEXT_DROP); + NAT_NEXT_DROP); vec_free (fragments_to_drop); vec_free (fragments_to_loopback); return frame->n_vectors; } +VLIB_NODE_FN (nat44_ed_in2out_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 0); +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (nat44_ed_in2out_node) = { + .name = "nat44-ed-in2out", + .vector_size = sizeof (u32), + .sibling_of = "nat-default", + .format_trace = format_nat_in2out_ed_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings), + .error_strings = nat_in2out_ed_error_strings, + .runtime_data_bytes = sizeof (snat_runtime_t), +}; +/* *INDENT-ON* */ + +VLIB_NODE_FN (nat44_ed_in2out_output_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 1); +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = { + .name = "nat44-ed-in2out-output", + .vector_size = sizeof (u32), + .sibling_of = "nat-default", + .format_trace = format_nat_in2out_ed_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings), + .error_strings = nat_in2out_ed_error_strings, + .runtime_data_bytes = sizeof (snat_runtime_t), +}; +/* *INDENT-ON* */ + +VLIB_NODE_FN (nat44_ed_in2out_slowpath_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 0); +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = { + .name = "nat44-ed-in2out-slowpath", + .vector_size = sizeof (u32), + .sibling_of = "nat-default", + .format_trace = format_nat_in2out_ed_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings), + .error_strings = nat_in2out_ed_error_strings, + .runtime_data_bytes = sizeof (snat_runtime_t), +}; +/* *INDENT-ON* */ + +VLIB_NODE_FN (nat44_ed_in2out_output_slowpath_node) (vlib_main_t * vm, + vlib_node_runtime_t * + node, + vlib_frame_t * frame) +{ + return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 1); +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = { + .name = "nat44-ed-in2out-output-slowpath", + .vector_size = sizeof (u32), + .sibling_of = "nat-default", + .format_trace = format_nat_in2out_ed_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings), + .error_strings = nat_in2out_ed_error_strings, + .runtime_data_bytes = sizeof (snat_runtime_t), +}; +/* *INDENT-ON* */ + + VLIB_NODE_FN (nat44_ed_in2out_reass_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) @@ -2120,18 +2104,11 @@ VLIB_NODE_FN (nat44_ed_in2out_reass_node) (vlib_main_t * vm, VLIB_REGISTER_NODE (nat44_ed_in2out_reass_node) = { .name = "nat44-ed-in2out-reass", .vector_size = sizeof (u32), + .sibling_of = "nat-default", .format_trace = format_nat44_reass_trace, .type = VLIB_NODE_TYPE_INTERNAL, .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings), .error_strings = nat_in2out_ed_error_strings, - .n_next_nodes = NAT_IN2OUT_ED_N_NEXT, - .next_nodes = { - [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop", - [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup", - [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-in2out-slowpath", - [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error", - [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass", - }, }; /* *INDENT-ON* */ @@ -2146,18 +2123,39 @@ VLIB_NODE_FN (nat44_ed_in2out_reass_output_node) (vlib_main_t * vm, VLIB_REGISTER_NODE (nat44_ed_in2out_reass_output_node) = { .name = "nat44-ed-in2out-reass-output", .vector_size = sizeof (u32), + .sibling_of = "nat-default", .format_trace = format_nat44_reass_trace, .type = VLIB_NODE_TYPE_INTERNAL, .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings), .error_strings = nat_in2out_ed_error_strings, - .n_next_nodes = NAT_IN2OUT_ED_N_NEXT, - .next_nodes = { - [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop", - [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output", - [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-in2out-slowpath", - [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error", - [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass", - }, +}; +/* *INDENT-ON* */ + +static u8 * +format_nat_pre_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *); + return format (s, "in2out next_index %d", t->next_index); +} + +VLIB_NODE_FN (nat_pre_in2out_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + return nat_pre_node_fn_inline (vm, node, frame, + NAT_NEXT_IN2OUT_ED_FAST_PATH); +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (nat_pre_in2out_node) = { + .name = "nat-pre-in2out", + .vector_size = sizeof (u32), + .sibling_of = "nat-default", + .format_trace = format_nat_pre_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = 0, }; /* *INDENT-ON* */ diff --git a/src/plugins/nat/nat.c b/src/plugins/nat/nat.c index 85072bcda31..0a30caf0f21 100755 --- a/src/plugins/nat/nat.c +++ b/src/plugins/nat/nat.c @@ -39,8 +39,29 @@ snat_main_t snat_main; /* *INDENT-OFF* */ - /* Hook up input features */ +VNET_FEATURE_INIT (nat_pre_in2out, static) = { + .arc_name = "ip4-unicast", + .node_name = "nat-pre-in2out", + .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"), +}; +VNET_FEATURE_INIT (nat_pre_out2in, static) = { + .arc_name = "ip4-unicast", + .node_name = "nat-pre-out2in", + .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa", + "ip4-dhcp-client-detect"), +}; +VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = { + .arc_name = "ip4-unicast", + .node_name = "nat44-in2out-worker-handoff", + .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"), +}; +VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = { + .arc_name = "ip4-unicast", + .node_name = "nat44-out2in-worker-handoff", + .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa", + "ip4-dhcp-client-detect"), +}; VNET_FEATURE_INIT (ip4_snat_in2out, static) = { .arc_name = "ip4-unicast", .node_name = "nat44-in2out", @@ -89,17 +110,6 @@ VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = { .node_name = "nat44-ed-classify", .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"), }; -VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = { - .arc_name = "ip4-unicast", - .node_name = "nat44-in2out-worker-handoff", - .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"), -}; -VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = { - .arc_name = "ip4-unicast", - .node_name = "nat44-out2in-worker-handoff", - .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa", - "ip4-dhcp-client-detect"), -}; VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = { .arc_name = "ip4-unicast", .node_name = "nat44-handoff-classify", @@ -1775,18 +1785,20 @@ snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del) else if (sm->deterministic) feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in"; else if (sm->endpoint_dependent) - feature_name = is_inside ? "nat44-ed-in2out" : "nat44-ed-out2in"; + { + feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in"; + } else feature_name = is_inside ? "nat44-in2out" : "nat44-out2in"; } if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1) - sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index, - NAT_FQ_NELTS); + sm->fq_in2out_index = + vlib_frame_queue_main_init (sm->handoff_in2out_index, NAT_FQ_NELTS); if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1) - sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index, - NAT_FQ_NELTS); + sm->fq_out2in_index = + vlib_frame_queue_main_init (sm->handoff_out2in_index, NAT_FQ_NELTS); if (!is_inside) { @@ -1844,8 +1856,8 @@ feature_set: else if (sm->endpoint_dependent) { del_feature_name = "nat44-ed-classify"; - feature_name = !is_inside ? "nat44-ed-in2out" : - "nat44-ed-out2in"; + feature_name = !is_inside ? "nat-pre-in2out" : + "nat-pre-out2in"; } else { @@ -1907,8 +1919,9 @@ feature_set: } else if (sm->endpoint_dependent) { - del_feature_name = !is_inside ? "nat44-ed-in2out" : - "nat44-ed-out2in"; + del_feature_name = !is_inside ? "nat-pre-in2out" : + "nat-pre-out2in"; + feature_name = "nat44-ed-classify"; } else @@ -2074,7 +2087,7 @@ feature_set: { if (sm->endpoint_dependent) { - vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-out2in", + vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in", sw_if_index, !is_del, 0, 0); vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output", sw_if_index, !is_del, 0, 0); @@ -2091,11 +2104,11 @@ feature_set: fq: if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1) sm->fq_in2out_output_index = - vlib_frame_queue_main_init (sm->in2out_output_node_index, 0); + vlib_frame_queue_main_init (sm->handoff_in2out_output_index, 0); if (sm->fq_out2in_index == ~0 && sm->num_workers > 1) sm->fq_out2in_index = - vlib_frame_queue_main_init (sm->out2in_node_index, 0); + vlib_frame_queue_main_init (sm->handoff_out2in_index, 0); /* *INDENT-OFF* */ pool_foreach (i, sm->output_feature_interfaces, @@ -2316,6 +2329,19 @@ snat_init (vlib_main_t * vm) node = vlib_get_node_by_name (vm, (u8 *) "error-drop"); sm->error_node_index = node->index; + node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out"); + sm->pre_in2out_node_index = node->index; + node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in"); + sm->pre_out2in_node_index = node->index; + + node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out"); + sm->pre_in2out_node_index = node->index; + + node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in"); + sm->pre_out2in_node_index = node->index; + + // TODO: output ?? (special node) + node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out"); sm->in2out_node_index = node->index; node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output"); @@ -3839,9 +3865,16 @@ snat_config (vlib_main_t * vm, unformat_input_t * input) { sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb; sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb; + + sm->handoff_out2in_index = nat_pre_out2in_node.index; + sm->handoff_in2out_index = nat_pre_in2out_node.index; + // TODO: test + sm->handoff_in2out_output_index = nat44_ed_in2out_output_node.index; + sm->in2out_node_index = nat44_ed_in2out_node.index; sm->in2out_output_node_index = nat44_ed_in2out_output_node.index; sm->out2in_node_index = nat44_ed_out2in_node.index; + sm->icmp_match_in2out_cb = icmp_match_in2out_ed; sm->icmp_match_out2in_cb = icmp_match_out2in_ed; nat_affinity_init (vm); @@ -3852,6 +3885,12 @@ snat_config (vlib_main_t * vm, unformat_input_t * input) { sm->worker_in2out_cb = snat_get_worker_in2out_cb; sm->worker_out2in_cb = snat_get_worker_out2in_cb; + + sm->handoff_out2in_index = snat_in2out_node.index; + sm->handoff_in2out_index = snat_out2in_node.index; + // TODO: test + sm->handoff_in2out_output_index = snat_in2out_output_node.index; + sm->in2out_node_index = snat_in2out_node.index; sm->in2out_output_node_index = snat_in2out_output_node.index; sm->out2in_node_index = snat_out2in_node.index; @@ -4263,6 +4302,38 @@ nat_set_alloc_addr_and_port_default (void) sm->alloc_addr_and_port = nat_alloc_addr_and_port_default; } +VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + return 0; +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (nat_default_node) = { + .name = "nat-default", + .vector_size = sizeof (u32), + .format_trace = 0, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = 0, + .n_next_nodes = NAT_N_NEXT, + .next_nodes = { + [NAT_NEXT_DROP] = "error-drop", + [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error", + [NAT_NEXT_IN2OUT_PRE] = "nat-pre-in2out", + [NAT_NEXT_OUT2IN_PRE] = "nat-pre-out2in", + [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out", + [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath", + [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath", + [NAT_NEXT_IN2OUT_ED_REASS] = "nat44-ed-in2out-reass", + [NAT_NEXT_IN2OUT_ED_OUTPUT_REASS] = "nat44-ed-in2out-reass-output", + [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in", + [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath", + [NAT_NEXT_OUT2IN_ED_REASS] = "nat44-ed-out2in-reass", + }, +}; +/* *INDENT-ON* */ + /* * fd.io coding-style-patch-verification: ON * diff --git a/src/plugins/nat/nat.h b/src/plugins/nat/nat.h index f7ce5f977d6..b65ad1b676f 100644 --- a/src/plugins/nat/nat.h +++ b/src/plugins/nat/nat.h @@ -43,6 +43,45 @@ /* NAT buffer flags */ #define SNAT_FLAG_HAIRPINNING (1 << 0) +typedef struct +{ + u32 arc_next; +} nat_buffer_opaque_t; + +typedef enum +{ + NAT_NEXT_DROP, + NAT_NEXT_ICMP_ERROR, + NAT_NEXT_IN2OUT_PRE, + NAT_NEXT_OUT2IN_PRE, + NAT_NEXT_IN2OUT_ED_FAST_PATH, + NAT_NEXT_IN2OUT_ED_SLOW_PATH, + NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH, + NAT_NEXT_IN2OUT_ED_REASS, + NAT_NEXT_IN2OUT_ED_OUTPUT_REASS, + NAT_NEXT_OUT2IN_ED_FAST_PATH, + NAT_NEXT_OUT2IN_ED_SLOW_PATH, + NAT_NEXT_OUT2IN_ED_REASS, + NAT_N_NEXT, +} nat_next_t; + +typedef struct +{ + u32 next_index; +} nat_pre_trace_t; + +#define nat_buffer_opaque(b) \ + ((nat_buffer_opaque_t *)((vnet_buffer_opaque2_t *)b->opaque2)->__unused2) + +/* +STATIC_ASSERT (sizeof (nat_buffer_opaque_t) <= + STRUCT_SIZE_OF (vnet_buffer_opaque_t, unused), + "Custom meta-data too large for vnet_buffer_opaque_t"); + +#define nat_buffer_opaque(b) \ + ((nat_buffer_opaque_t *)((u8 *)((b)->opaque) + \ + STRUCT_OFFSET_OF (vnet_buffer_opaque_t, unused)))*/ + /* session key (4-tuple) */ typedef struct { @@ -575,6 +614,15 @@ typedef struct snat_main_s /* node indexes */ u32 error_node_index; + /* handoff fq nodes */ + u32 handoff_out2in_index; + u32 handoff_in2out_index; + u32 handoff_in2out_output_index; + + /* respect feature arc nodes */ + u32 pre_out2in_node_index; + u32 pre_in2out_node_index; + u32 in2out_node_index; u32 in2out_output_node_index; u32 in2out_fast_node_index; @@ -667,6 +715,12 @@ typedef struct } snat_runtime_t; extern snat_main_t snat_main; + +// nat pre ed next_node feature classification +extern vlib_node_registration_t nat_default_node; +extern vlib_node_registration_t nat_pre_in2out_node; +extern vlib_node_registration_t nat_pre_out2in_node; + extern vlib_node_registration_t snat_in2out_node; extern vlib_node_registration_t snat_in2out_output_node; extern vlib_node_registration_t snat_out2in_node; diff --git a/src/plugins/nat/nat44_classify.c b/src/plugins/nat/nat44_classify.c index aa2bf73e94c..b6ce4d7b494 100644 --- a/src/plugins/nat/nat44_classify.c +++ b/src/plugins/nat/nat44_classify.c @@ -81,14 +81,13 @@ format_nat44_classify_trace (u8 * s, va_list * args) static inline uword nat44_classify_node_fn_inline (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_frame_t * frame, int is_ed) + vlib_frame_t * frame) { u32 n_left_from, *from, *to_next; nat44_classify_next_t next_index; snat_main_t *sm = &snat_main; snat_static_mapping_t *m; u32 thread_index = vm->thread_index; - snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index]; u32 *fragments_to_drop = 0; u32 *fragments_to_loopback = 0; u32 next_in2out = 0, next_out2in = 0, frag_cached = 0; @@ -107,12 +106,11 @@ nat44_classify_node_fn_inline (vlib_main_t * vm, { u32 bi0; vlib_buffer_t *b0; - u32 next0 = NAT44_CLASSIFY_NEXT_IN2OUT, sw_if_index0, rx_fib_index0; + u32 next0 = NAT44_CLASSIFY_NEXT_IN2OUT; ip4_header_t *ip0; snat_address_t *ap; snat_session_key_t m_key0; clib_bihash_kv_8_8_t kv0, value0; - clib_bihash_kv_16_8_t ed_kv0, ed_value0; udp_header_t *udp0; nat_reass_ip4_t *reass0; u8 cached0 = 0; @@ -129,18 +127,47 @@ nat44_classify_node_fn_inline (vlib_main_t * vm, ip0 = vlib_buffer_get_current (b0); udp0 = ip4_next_header (ip0); - if (is_ed && ip0->protocol != IP_PROTOCOL_ICMP) + /* *INDENT-OFF* */ + vec_foreach (ap, sm->addresses) + { + if (ip0->dst_address.as_u32 == ap->addr.as_u32) + { + next0 = NAT44_CLASSIFY_NEXT_OUT2IN; + goto enqueue0; + } + } + /* *INDENT-ON* */ + + if (PREDICT_FALSE (pool_elts (sm->static_mappings))) { + m_key0.addr = ip0->dst_address; + m_key0.port = 0; + m_key0.protocol = 0; + m_key0.fib_index = 0; + kv0.key = m_key0.as_u64; + /* try to classify the fragment based on IP header alone */ + if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, + &kv0, &value0)) + { + m = pool_elt_at_index (sm->static_mappings, value0.value); + if (m->local_addr.as_u32 != m->external_addr.as_u32) + next0 = NAT44_CLASSIFY_NEXT_OUT2IN; + goto enqueue0; + } if (!ip4_is_fragment (ip0) || ip4_is_first_fragment (ip0)) { /* process leading fragment/whole packet (with L4 header) */ - sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; - rx_fib_index0 = - fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, - sw_if_index0); - make_ed_kv (&ed_kv0, &ip0->src_address, &ip0->dst_address, - ip0->protocol, rx_fib_index0, udp0->src_port, - udp0->dst_port); + m_key0.port = clib_net_to_host_u16 (udp0->dst_port); + m_key0.protocol = ip_proto_to_snat_proto (ip0->protocol); + kv0.key = m_key0.as_u64; + if (!clib_bihash_search_8_8 + (&sm->static_mapping_by_external, &kv0, &value0)) + { + m = + pool_elt_at_index (sm->static_mappings, value0.value); + if (m->local_addr.as_u32 != m->external_addr.as_u32) + next0 = NAT44_CLASSIFY_NEXT_OUT2IN; + } if (ip4_is_fragment (ip0)) { reass0 = nat_ip4_reass_find_or_create (ip0->src_address, @@ -157,8 +184,219 @@ nat44_classify_node_fn_inline (vlib_main_t * vm, nat_elog_notice ("maximum reassemblies exceeded"); goto enqueue0; } - if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &ed_kv0, - &ed_value0)) + /* save classification for future fragments and set past + * fragments to be looped over and reprocessed */ + if (next0 == NAT44_CLASSIFY_NEXT_OUT2IN) + reass0->classify_next = + NAT_REASS_IP4_CLASSIFY_NEXT_OUT2IN; + else + reass0->classify_next = + NAT_REASS_IP4_CLASSIFY_NEXT_IN2OUT; + nat_ip4_reass_get_frags (reass0, + &fragments_to_loopback); + } + } + else + { + /* process non-first fragment */ + reass0 = nat_ip4_reass_find_or_create (ip0->src_address, + ip0->dst_address, + ip0->fragment_id, + ip0->protocol, + 1, + &fragments_to_drop); + if (PREDICT_FALSE (!reass0)) + { + next0 = NAT44_CLASSIFY_NEXT_DROP; + b0->error = + node->errors[NAT44_CLASSIFY_ERROR_MAX_REASS]; + nat_elog_notice ("maximum reassemblies exceeded"); + goto enqueue0; + } + if (reass0->classify_next == NAT_REASS_IP4_CLASSIFY_NONE) + /* first fragment still hasn't arrived */ + { + if (nat_ip4_reass_add_fragment + (thread_index, reass0, bi0, &fragments_to_drop)) + { + b0->error = + node->errors[NAT44_CLASSIFY_ERROR_MAX_FRAG]; + nat_elog_notice + ("maximum fragments per reassembly exceeded"); + next0 = NAT44_CLASSIFY_NEXT_DROP; + goto enqueue0; + } + cached0 = 1; + goto enqueue0; + } + else if (reass0->classify_next == + NAT_REASS_IP4_CLASSIFY_NEXT_OUT2IN) + next0 = NAT44_CLASSIFY_NEXT_OUT2IN; + else if (reass0->classify_next == + NAT_REASS_IP4_CLASSIFY_NEXT_IN2OUT) + next0 = NAT44_CLASSIFY_NEXT_IN2OUT; + } + } + + enqueue0: + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) + && (b0->flags & VLIB_BUFFER_IS_TRACED))) + { + nat44_classify_trace_t *t = + vlib_add_trace (vm, node, b0, sizeof (*t)); + t->cached = cached0; + if (!cached0) + t->next_in2out = next0 == NAT44_CLASSIFY_NEXT_IN2OUT ? 1 : 0; + } + + if (cached0) + { + n_left_to_next++; + to_next--; + frag_cached++; + } + else + { + next_in2out += next0 == NAT44_CLASSIFY_NEXT_IN2OUT; + next_out2in += next0 == NAT44_CLASSIFY_NEXT_OUT2IN; + + /* verify speculative enqueue, maybe switch current next frame */ + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, + to_next, n_left_to_next, + bi0, next0); + } + + if (n_left_from == 0 && vec_len (fragments_to_loopback)) + { + from = vlib_frame_vector_args (frame); + u32 len = vec_len (fragments_to_loopback); + if (len <= VLIB_FRAME_SIZE) + { + clib_memcpy_fast (from, fragments_to_loopback, + sizeof (u32) * len); + n_left_from = len; + vec_reset_length (fragments_to_loopback); + } + else + { + clib_memcpy_fast (from, fragments_to_loopback + + (len - VLIB_FRAME_SIZE), + sizeof (u32) * VLIB_FRAME_SIZE); + n_left_from = VLIB_FRAME_SIZE; + _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE; + } + } + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + nat_send_all_to_node (vm, fragments_to_drop, node, 0, + NAT44_CLASSIFY_NEXT_DROP); + + vec_free (fragments_to_drop); + + vlib_node_increment_counter (vm, node->node_index, + NAT44_CLASSIFY_ERROR_NEXT_IN2OUT, next_in2out); + vlib_node_increment_counter (vm, node->node_index, + NAT44_CLASSIFY_ERROR_NEXT_OUT2IN, next_out2in); + vlib_node_increment_counter (vm, node->node_index, + NAT44_CLASSIFY_ERROR_FRAG_CACHED, frag_cached); + + return frame->n_vectors; +} + +static inline uword +nat44_ed_classify_node_fn_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + u32 n_left_from, *from, *to_next; + nat44_classify_next_t next_index; + snat_main_t *sm = &snat_main; + snat_static_mapping_t *m; + u32 thread_index = vm->thread_index; + snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index]; + u32 *fragments_to_drop = 0; + u32 *fragments_to_loopback = 0; + u32 next_in2out = 0, next_out2in = 0, frag_cached = 0; + u8 in_loopback = 0; + + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + next_index = node->cached_next_index; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t *b0; + u32 next0 = + NAT_NEXT_IN2OUT_ED_FAST_PATH, sw_if_index0, rx_fib_index0; + ip4_header_t *ip0; + snat_address_t *ap; + snat_session_key_t m_key0; + clib_bihash_kv_8_8_t kv0, value0; + clib_bihash_kv_16_8_t ed_kv0, ed_value0; + udp_header_t *udp0; + nat_reass_ip4_t *reass0; + u8 cached0 = 0; + + /* speculatively enqueue b0 to the current next frame */ + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + ip0 = vlib_buffer_get_current (b0); + udp0 = ip4_next_header (ip0); + + if (!in_loopback) + { + u32 arc_next = 0; + + vnet_feature_next (&arc_next, b0); + nat_buffer_opaque (b0)->arc_next = arc_next; + } + + if (ip0->protocol != IP_PROTOCOL_ICMP) + { + if (!ip4_is_fragment (ip0) || ip4_is_first_fragment (ip0)) + { + /* process leading fragment/whole packet (with L4 header) */ + sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; + rx_fib_index0 = + fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, + sw_if_index0); + make_ed_kv (&ed_kv0, &ip0->src_address, + &ip0->dst_address, ip0->protocol, + rx_fib_index0, udp0->src_port, udp0->dst_port); + if (ip4_is_fragment (ip0)) + { + reass0 = + nat_ip4_reass_find_or_create (ip0->src_address, + ip0->dst_address, + ip0->fragment_id, + ip0->protocol, 1, + &fragments_to_drop); + if (PREDICT_FALSE (!reass0)) + { + next0 = NAT_NEXT_DROP; + b0->error = + node->errors[NAT44_CLASSIFY_ERROR_MAX_REASS]; + nat_elog_notice ("maximum reassemblies exceeded"); + goto enqueue0; + } + if (!clib_bihash_search_16_8 + (&tsm->in2out_ed, &ed_kv0, &ed_value0)) { /* session exists so classify as IN2OUT, * save this information for future fragments and set @@ -184,8 +422,8 @@ nat44_classify_node_fn_inline (vlib_main_t * vm, else { /* process whole packet */ - if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &ed_kv0, - &ed_value0)) + if (!clib_bihash_search_16_8 + (&tsm->in2out_ed, &ed_kv0, &ed_value0)) goto enqueue0; /* session doesn't exist so continue in code */ } @@ -201,15 +439,16 @@ nat44_classify_node_fn_inline (vlib_main_t * vm, &fragments_to_drop); if (PREDICT_FALSE (!reass0)) { - next0 = NAT44_CLASSIFY_NEXT_DROP; + next0 = NAT_NEXT_DROP; b0->error = node->errors[NAT44_CLASSIFY_ERROR_MAX_REASS]; nat_elog_notice ("maximum reassemblies exceeded"); goto enqueue0; } /* check if first fragment has arrived */ - if (reass0->classify_next == NAT_REASS_IP4_CLASSIFY_NONE && - !(reass0->flags & NAT_REASS_FLAG_CLASSIFY_ED_CONTINUE)) + if (reass0->classify_next == NAT_REASS_IP4_CLASSIFY_NONE + && !(reass0->flags & + NAT_REASS_FLAG_CLASSIFY_ED_CONTINUE)) { /* first fragment still hasn't arrived, cache this fragment */ if (nat_ip4_reass_add_fragment @@ -219,7 +458,7 @@ nat44_classify_node_fn_inline (vlib_main_t * vm, node->errors[NAT44_CLASSIFY_ERROR_MAX_FRAG]; nat_elog_notice ("maximum fragments per reassembly exceeded"); - next0 = NAT44_CLASSIFY_NEXT_DROP; + next0 = NAT_NEXT_DROP; goto enqueue0; } cached0 = 1; @@ -239,7 +478,7 @@ nat44_classify_node_fn_inline (vlib_main_t * vm, { if (ip0->dst_address.as_u32 == ap->addr.as_u32) { - next0 = NAT44_CLASSIFY_NEXT_OUT2IN; + next0 = NAT_NEXT_OUT2IN_ED_FAST_PATH; goto enqueue0; } } @@ -258,7 +497,7 @@ nat44_classify_node_fn_inline (vlib_main_t * vm, { m = pool_elt_at_index (sm->static_mappings, value0.value); if (m->local_addr.as_u32 != m->external_addr.as_u32) - next0 = NAT44_CLASSIFY_NEXT_OUT2IN; + next0 = NAT_NEXT_OUT2IN_ED_FAST_PATH; goto enqueue0; } if (!ip4_is_fragment (ip0) || ip4_is_first_fragment (ip0)) @@ -273,7 +512,7 @@ nat44_classify_node_fn_inline (vlib_main_t * vm, m = pool_elt_at_index (sm->static_mappings, value0.value); if (m->local_addr.as_u32 != m->external_addr.as_u32) - next0 = NAT44_CLASSIFY_NEXT_OUT2IN; + next0 = NAT_NEXT_OUT2IN_ED_FAST_PATH; } if (ip4_is_fragment (ip0)) { @@ -285,7 +524,7 @@ nat44_classify_node_fn_inline (vlib_main_t * vm, &fragments_to_drop); if (PREDICT_FALSE (!reass0)) { - next0 = NAT44_CLASSIFY_NEXT_DROP; + next0 = NAT_NEXT_DROP; b0->error = node->errors[NAT44_CLASSIFY_ERROR_MAX_REASS]; nat_elog_notice ("maximum reassemblies exceeded"); @@ -293,12 +532,10 @@ nat44_classify_node_fn_inline (vlib_main_t * vm, } /* save classification for future fragments and set past * fragments to be looped over and reprocessed */ - if (next0 == NAT44_CLASSIFY_NEXT_OUT2IN) - reass0->classify_next = - NAT_REASS_IP4_CLASSIFY_NEXT_OUT2IN; + if (next0 == NAT_NEXT_OUT2IN_ED_FAST_PATH) + reass0->classify_next = NAT_NEXT_OUT2IN_ED_REASS; else - reass0->classify_next = - NAT_REASS_IP4_CLASSIFY_NEXT_IN2OUT; + reass0->classify_next = NAT_NEXT_IN2OUT_ED_REASS; nat_ip4_reass_get_frags (reass0, &fragments_to_loopback); } @@ -314,7 +551,7 @@ nat44_classify_node_fn_inline (vlib_main_t * vm, &fragments_to_drop); if (PREDICT_FALSE (!reass0)) { - next0 = NAT44_CLASSIFY_NEXT_DROP; + next0 = NAT_NEXT_DROP; b0->error = node->errors[NAT44_CLASSIFY_ERROR_MAX_REASS]; nat_elog_notice ("maximum reassemblies exceeded"); @@ -330,7 +567,7 @@ nat44_classify_node_fn_inline (vlib_main_t * vm, node->errors[NAT44_CLASSIFY_ERROR_MAX_FRAG]; nat_elog_notice ("maximum fragments per reassembly exceeded"); - next0 = NAT44_CLASSIFY_NEXT_DROP; + next0 = NAT_NEXT_DROP; goto enqueue0; } cached0 = 1; @@ -338,10 +575,10 @@ nat44_classify_node_fn_inline (vlib_main_t * vm, } else if (reass0->classify_next == NAT_REASS_IP4_CLASSIFY_NEXT_OUT2IN) - next0 = NAT44_CLASSIFY_NEXT_OUT2IN; + next0 = NAT_NEXT_OUT2IN_ED_FAST_PATH; else if (reass0->classify_next == NAT_REASS_IP4_CLASSIFY_NEXT_IN2OUT) - next0 = NAT44_CLASSIFY_NEXT_IN2OUT; + next0 = NAT_NEXT_IN2OUT_ED_FAST_PATH; } } @@ -353,7 +590,8 @@ nat44_classify_node_fn_inline (vlib_main_t * vm, vlib_add_trace (vm, node, b0, sizeof (*t)); t->cached = cached0; if (!cached0) - t->next_in2out = next0 == NAT44_CLASSIFY_NEXT_IN2OUT ? 1 : 0; + t->next_in2out = + next0 == NAT_NEXT_IN2OUT_ED_FAST_PATH ? 1 : 0; } if (cached0) @@ -364,8 +602,8 @@ nat44_classify_node_fn_inline (vlib_main_t * vm, } else { - next_in2out += next0 == NAT44_CLASSIFY_NEXT_IN2OUT; - next_out2in += next0 == NAT44_CLASSIFY_NEXT_OUT2IN; + next_in2out += next0 == NAT_NEXT_IN2OUT_ED_FAST_PATH; + next_out2in += next0 == NAT_NEXT_OUT2IN_ED_FAST_PATH; /* verify speculative enqueue, maybe switch current next frame */ vlib_validate_buffer_enqueue_x1 (vm, node, next_index, @@ -375,6 +613,7 @@ nat44_classify_node_fn_inline (vlib_main_t * vm, if (n_left_from == 0 && vec_len (fragments_to_loopback)) { + in_loopback = 1; from = vlib_frame_vector_args (frame); u32 len = vec_len (fragments_to_loopback); if (len <= VLIB_FRAME_SIZE) @@ -417,7 +656,7 @@ VLIB_NODE_FN (nat44_classify_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) { - return nat44_classify_node_fn_inline (vm, node, frame, 0); + return nat44_classify_node_fn_inline (vm, node, frame); } /* *INDENT-OFF* */ @@ -441,21 +680,16 @@ VLIB_NODE_FN (nat44_ed_classify_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) { - return nat44_classify_node_fn_inline (vm, node, frame, 1); + return nat44_ed_classify_node_fn_inline (vm, node, frame); } /* *INDENT-OFF* */ VLIB_REGISTER_NODE (nat44_ed_classify_node) = { .name = "nat44-ed-classify", .vector_size = sizeof (u32), + .sibling_of = "nat-default", .format_trace = format_nat44_classify_trace, .type = VLIB_NODE_TYPE_INTERNAL, - .n_next_nodes = NAT44_CLASSIFY_N_NEXT, - .next_nodes = { - [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-ed-in2out", - [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-ed-out2in", - [NAT44_CLASSIFY_NEXT_DROP] = "error-drop", - }, }; /* *INDENT-ON* */ @@ -463,7 +697,7 @@ VLIB_NODE_FN (nat44_det_classify_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) { - return nat44_classify_node_fn_inline (vm, node, frame, 0); + return nat44_classify_node_fn_inline (vm, node, frame); } /* *INDENT-OFF* */ @@ -485,7 +719,7 @@ VLIB_NODE_FN (nat44_handoff_classify_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) { - return nat44_classify_node_fn_inline (vm, node, frame, 0); + return nat44_classify_node_fn_inline (vm, node, frame); } /* *INDENT-OFF* */ diff --git a/src/plugins/nat/nat44_handoff.c b/src/plugins/nat/nat44_handoff.c index c97c958d2a9..277f2de57fe 100644 --- a/src/plugins/nat/nat44_handoff.c +++ b/src/plugins/nat/nat44_handoff.c @@ -23,6 +23,7 @@ #include <vnet/fib/ip4_fib.h> #include <vppinfra/error.h> #include <nat/nat.h> +#include <nat/nat_inlines.h> typedef struct { @@ -260,14 +261,11 @@ VLIB_NODE_FN (snat_in2out_worker_handoff_node) (vlib_main_t * vm, VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = { .name = "nat44-in2out-worker-handoff", .vector_size = sizeof (u32), + .sibling_of = "nat-default", .format_trace = format_nat44_handoff_trace, .type = VLIB_NODE_TYPE_INTERNAL, .n_errors = ARRAY_LEN(nat44_handoff_error_strings), .error_strings = nat44_handoff_error_strings, - .n_next_nodes = 1, - .next_nodes = { - [0] = "error-drop", - }, }; /* *INDENT-ON* */ @@ -283,14 +281,11 @@ VLIB_NODE_FN (snat_in2out_output_worker_handoff_node) (vlib_main_t * vm, VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = { .name = "nat44-in2out-output-worker-handoff", .vector_size = sizeof (u32), + .sibling_of = "nat-default", .format_trace = format_nat44_handoff_trace, .type = VLIB_NODE_TYPE_INTERNAL, .n_errors = ARRAY_LEN(nat44_handoff_error_strings), .error_strings = nat44_handoff_error_strings, - .n_next_nodes = 1, - .next_nodes = { - [0] = "error-drop", - }, }; /* *INDENT-ON* */ @@ -305,14 +300,11 @@ VLIB_NODE_FN (snat_out2in_worker_handoff_node) (vlib_main_t * vm, VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = { .name = "nat44-out2in-worker-handoff", .vector_size = sizeof (u32), + .sibling_of = "nat-default", .format_trace = format_nat44_handoff_trace, .type = VLIB_NODE_TYPE_INTERNAL, .n_errors = ARRAY_LEN(nat44_handoff_error_strings), .error_strings = nat44_handoff_error_strings, - .n_next_nodes = 1, - .next_nodes = { - [0] = "error-drop", - }, }; /* *INDENT-ON* */ diff --git a/src/plugins/nat/nat_inlines.h b/src/plugins/nat/nat_inlines.h index a7ab235f40e..2f68ed4a700 100644 --- a/src/plugins/nat/nat_inlines.h +++ b/src/plugins/nat/nat_inlines.h @@ -23,6 +23,126 @@ #include <nat/nat.h> #include <nat/nat_ha.h> +static inline uword +nat_pre_node_fn_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame, u32 def_next) +{ + u32 n_left_from, *from, *to_next; + u16 next_index; + + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + next_index = node->cached_next_index; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + + while (n_left_from >= 4 && n_left_to_next >= 2) + { + u32 next0, next1; + u32 arc_next0, arc_next1; + u32 bi0, bi1; + vlib_buffer_t *b0, *b1; + + /* Prefetch next iteration. */ + { + vlib_buffer_t *p2, *p3; + + p2 = vlib_get_buffer (vm, from[2]); + p3 = vlib_get_buffer (vm, from[3]); + + vlib_prefetch_buffer_header (p2, LOAD); + vlib_prefetch_buffer_header (p3, LOAD); + + CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE); + } + + /* speculatively enqueue b0 and b1 to the current next frame */ + to_next[0] = bi0 = from[0]; + to_next[1] = bi1 = from[1]; + from += 2; + to_next += 2; + n_left_from -= 2; + n_left_to_next -= 2; + + b0 = vlib_get_buffer (vm, bi0); + b1 = vlib_get_buffer (vm, bi1); + + next0 = def_next; + next1 = def_next; + + vnet_feature_next (&arc_next0, b0); + vnet_feature_next (&arc_next1, b1); + + nat_buffer_opaque (b0)->arc_next = arc_next0; + nat_buffer_opaque (b1)->arc_next = arc_next1; + + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE))) + { + if (b0->flags & VLIB_BUFFER_IS_TRACED) + { + nat_pre_trace_t *t = + vlib_add_trace (vm, node, b0, sizeof (*t)); + t->next_index = next0; + } + if (b1->flags & VLIB_BUFFER_IS_TRACED) + { + nat_pre_trace_t *t = + vlib_add_trace (vm, node, b0, sizeof (*t)); + t->next_index = next0; + } + } + + /* verify speculative enqueues, maybe switch current next frame */ + vlib_validate_buffer_enqueue_x2 (vm, node, next_index, + to_next, n_left_to_next, + bi0, bi1, next0, next1); + } + + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 next0; + u32 arc_next0; + u32 bi0; + vlib_buffer_t *b0; + + /* speculatively enqueue b0 to the current next frame */ + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + next0 = def_next; + vnet_feature_next (&arc_next0, b0); + nat_buffer_opaque (b0)->arc_next = arc_next0; + + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) + && (b0->flags & VLIB_BUFFER_IS_TRACED))) + { + nat_pre_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t)); + t->next_index = next0; + } + + /* verify speculative enqueue, maybe switch current next frame */ + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, + to_next, n_left_to_next, + bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + return frame->n_vectors; +} + always_inline u32 ip_proto_to_snat_proto (u8 ip_proto) { @@ -552,13 +672,13 @@ snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t * node, return 1; snat_interface_t *i; - pool_foreach (i, sm->interfaces, ( - { - /* NAT packet aimed at outside interface */ - if ((nat_interface_is_outside (i)) - && (sw_if_index == - i->sw_if_index)) return 0;} - )); + /* *INDENT-OFF* */ + pool_foreach (i, sm->interfaces, ({ + /* NAT packet aimed at outside interface */ + if ((nat_interface_is_outside (i)) && (sw_if_index == i->sw_if_index)) + return 0; + })); + /* *INDENT-ON* */ } return 1; diff --git a/src/plugins/nat/out2in_ed.c b/src/plugins/nat/out2in_ed.c index 2067c512b92..a81d58e2fa8 100644 --- a/src/plugins/nat/out2in_ed.c +++ b/src/plugins/nat/out2in_ed.c @@ -38,17 +38,6 @@ static char *nat_out2in_ed_error_strings[] = { #undef _ }; -typedef enum -{ - NAT44_ED_OUT2IN_NEXT_DROP, - NAT44_ED_OUT2IN_NEXT_LOOKUP, - NAT44_ED_OUT2IN_NEXT_ICMP_ERROR, - NAT44_ED_OUT2IN_NEXT_IN2OUT, - NAT44_ED_OUT2IN_NEXT_SLOW_PATH, - NAT44_ED_OUT2IN_NEXT_REASS, - NAT44_ED_OUT2IN_N_NEXT, -} nat44_ed_out2in_next_t; - typedef struct { u32 sw_if_index; @@ -85,7 +74,7 @@ icmp_out2in_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0, next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, next0, thread_index, p_s0, 0); snat_session_t *s0 = *p_s0; - if (PREDICT_TRUE (next0 != NAT44_ED_OUT2IN_NEXT_DROP && s0)) + if (PREDICT_TRUE (next0 != NAT_NEXT_DROP && s0)) { /* Accounting */ nat44_session_update_counters (s0, now, @@ -467,7 +456,7 @@ icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node, if (get_icmp_o2i_ed_key (ip, &key)) { b->error = node->errors[NAT_OUT2IN_ED_ERROR_UNSUPPORTED_PROTOCOL]; - next = NAT44_ED_OUT2IN_NEXT_DROP; + next = NAT_NEXT_DROP; goto out; } key.fib_index = rx_fib_index; @@ -494,7 +483,7 @@ icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node, goto out; } b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION]; - next = NAT44_ED_OUT2IN_NEXT_DROP; + next = NAT_NEXT_DROP; goto out; } else @@ -503,7 +492,7 @@ icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node, if (next_src_nat (sm, ip, key.proto, key.l_port, key.r_port, thread_index, rx_fib_index)) { - next = NAT44_ED_OUT2IN_NEXT_IN2OUT; + next = NAT_NEXT_IN2OUT_ED_FAST_PATH; goto out; } if (sm->num_workers > 1) @@ -518,7 +507,7 @@ icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node, (icmp->type != ICMP4_echo_request || !is_addr_only))) { b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE]; - next = NAT44_ED_OUT2IN_NEXT_DROP; + next = NAT_NEXT_DROP; goto out; } @@ -536,7 +525,7 @@ icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node, if (!s) { - next = NAT44_ED_OUT2IN_NEXT_DROP; + next = NAT_NEXT_DROP; goto out; } } @@ -547,7 +536,7 @@ icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node, !icmp_is_error_message (icmp))) { b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE]; - next = NAT44_ED_OUT2IN_NEXT_DROP; + next = NAT_NEXT_DROP; goto out; } @@ -676,7 +665,7 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, vlib_frame_t * frame, int is_slow_path) { u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index; - nat44_ed_out2in_next_t next_index; + nat_next_t next_index; snat_main_t *sm = &snat_main; f64 now = vlib_time_now (vm); u32 thread_index = vm->thread_index; @@ -743,7 +732,9 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, b0 = vlib_get_buffer (vm, bi0); b1 = vlib_get_buffer (vm, bi1); - next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP; + next0 = nat_buffer_opaque (b0)->arc_next; + next1 = nat_buffer_opaque (b1)->arc_next; + vnet_buffer (b0)->snat.flags = 0; ip0 = vlib_buffer_get_current (b0); @@ -758,7 +749,7 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded, ICMP4_time_exceeded_ttl_exceeded_in_transit, 0); - next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR; + next0 = NAT_NEXT_ICMP_ERROR; goto trace00; } @@ -779,7 +770,7 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, if (!sm->forwarding_enabled) { if (!s0) - next0 = NAT44_ED_OUT2IN_NEXT_DROP; + next0 = NAT_NEXT_DROP; goto trace00; } } @@ -797,20 +788,20 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, { if (PREDICT_FALSE (proto0 == ~0)) { - next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH; + next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH; goto trace00; } if (ip4_is_fragment (ip0)) { - next0 = NAT44_ED_OUT2IN_NEXT_REASS; + next0 = NAT_NEXT_OUT2IN_ED_REASS; fragments++; goto trace00; } if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP)) { - next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH; + next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH; goto trace00; } } @@ -843,7 +834,6 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, clib_host_to_net_u16 (UDP_DST_PORT_dhcp_to_client)))) { - vnet_feature_next (&next0, b0); goto trace00; } @@ -851,7 +841,7 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, { b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION]; - next0 = NAT44_ED_OUT2IN_NEXT_DROP; + next0 = NAT_NEXT_DROP; } else { @@ -859,7 +849,7 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, udp0->src_port, udp0->dst_port, thread_index, rx_fib_index0)) { - next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT; + next0 = NAT_NEXT_IN2OUT_ED_FAST_PATH; goto trace00; } if (sm->num_workers > 1) @@ -878,7 +868,7 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0)) { b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN]; - next0 = NAT44_ED_OUT2IN_NEXT_DROP; + next0 = NAT_NEXT_DROP; goto trace00; } @@ -891,13 +881,13 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, if (!s0) { - next0 = NAT44_ED_OUT2IN_NEXT_DROP; + next0 = NAT_NEXT_DROP; goto trace00; } } else { - next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH; + next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH; goto trace00; } } @@ -998,9 +988,8 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, t->session_index = s0 - tsm->sessions; } - pkts_processed += next0 == NAT44_ED_OUT2IN_NEXT_LOOKUP; + pkts_processed += next0 == nat_buffer_opaque (b0)->arc_next; - next1 = NAT44_ED_OUT2IN_NEXT_LOOKUP; vnet_buffer (b1)->snat.flags = 0; ip1 = vlib_buffer_get_current (b1); @@ -1015,7 +1004,7 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded, ICMP4_time_exceeded_ttl_exceeded_in_transit, 0); - next1 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR; + next1 = NAT_NEXT_ICMP_ERROR; goto trace01; } @@ -1036,7 +1025,7 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, if (!sm->forwarding_enabled) { if (!s1) - next1 = NAT44_ED_OUT2IN_NEXT_DROP; + next1 = NAT_NEXT_DROP; goto trace01; } } @@ -1054,20 +1043,20 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, { if (PREDICT_FALSE (proto1 == ~0)) { - next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH; + next1 = NAT_NEXT_OUT2IN_ED_SLOW_PATH; goto trace01; } if (ip4_is_fragment (ip1)) { - next1 = NAT44_ED_OUT2IN_NEXT_REASS; + next1 = NAT_NEXT_OUT2IN_ED_REASS; fragments++; goto trace01; } if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP)) { - next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH; + next1 = NAT_NEXT_OUT2IN_ED_SLOW_PATH; goto trace01; } } @@ -1100,7 +1089,6 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, clib_host_to_net_u16 (UDP_DST_PORT_dhcp_to_client)))) { - vnet_feature_next (&next1, b1); goto trace01; } @@ -1108,7 +1096,7 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, { b1->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION]; - next1 = NAT44_ED_OUT2IN_NEXT_DROP; + next1 = NAT_NEXT_DROP; } else { @@ -1116,7 +1104,7 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, udp1->src_port, udp1->dst_port, thread_index, rx_fib_index1)) { - next1 = NAT44_ED_OUT2IN_NEXT_IN2OUT; + next1 = NAT_NEXT_IN2OUT_ED_FAST_PATH; goto trace01; } if (sm->num_workers > 1) @@ -1135,7 +1123,7 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, if ((proto1 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp1)) { b1->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN]; - next1 = NAT44_ED_OUT2IN_NEXT_DROP; + next1 = NAT_NEXT_DROP; goto trace01; } @@ -1148,13 +1136,13 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, if (!s1) { - next1 = NAT44_ED_OUT2IN_NEXT_DROP; + next1 = NAT_NEXT_DROP; goto trace01; } } else { - next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH; + next1 = NAT_NEXT_OUT2IN_ED_SLOW_PATH; goto trace01; } } @@ -1255,7 +1243,7 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, t->session_index = s1 - tsm->sessions; } - pkts_processed += next1 == NAT44_ED_OUT2IN_NEXT_LOOKUP; + pkts_processed += next1 == nat_buffer_opaque (b1)->arc_next; /* verify speculative enqueues, maybe switch current next frame */ vlib_validate_buffer_enqueue_x2 (vm, node, next_index, @@ -1291,7 +1279,8 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, n_left_to_next -= 1; b0 = vlib_get_buffer (vm, bi0); - next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP; + next0 = nat_buffer_opaque (b0)->arc_next; + vnet_buffer (b0)->snat.flags = 0; ip0 = vlib_buffer_get_current (b0); @@ -1306,7 +1295,7 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded, ICMP4_time_exceeded_ttl_exceeded_in_transit, 0); - next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR; + next0 = NAT_NEXT_ICMP_ERROR; goto trace0; } @@ -1327,7 +1316,7 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, if (!sm->forwarding_enabled) { if (!s0) - next0 = NAT44_ED_OUT2IN_NEXT_DROP; + next0 = NAT_NEXT_DROP; goto trace0; } } @@ -1345,20 +1334,20 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, { if (PREDICT_FALSE (proto0 == ~0)) { - next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH; + next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH; goto trace0; } if (ip4_is_fragment (ip0)) { - next0 = NAT44_ED_OUT2IN_NEXT_REASS; + next0 = NAT_NEXT_OUT2IN_ED_REASS; fragments++; goto trace0; } if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP)) { - next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH; + next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH; goto trace0; } } @@ -1391,7 +1380,6 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, clib_host_to_net_u16 (UDP_DST_PORT_dhcp_to_client)))) { - vnet_feature_next (&next0, b0); goto trace0; } @@ -1399,7 +1387,7 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, { b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION]; - next0 = NAT44_ED_OUT2IN_NEXT_DROP; + next0 = NAT_NEXT_DROP; } else { @@ -1407,7 +1395,7 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, udp0->src_port, udp0->dst_port, thread_index, rx_fib_index0)) { - next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT; + next0 = NAT_NEXT_IN2OUT_ED_FAST_PATH; goto trace0; } if (sm->num_workers > 1) @@ -1426,7 +1414,7 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0)) { b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN]; - next0 = NAT44_ED_OUT2IN_NEXT_DROP; + next0 = NAT_NEXT_DROP; goto trace0; } @@ -1439,13 +1427,13 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, if (!s0) { - next0 = NAT44_ED_OUT2IN_NEXT_DROP; + next0 = NAT_NEXT_DROP; goto trace0; } } else { - next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH; + next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH; goto trace0; } } @@ -1546,7 +1534,7 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, t->session_index = s0 - tsm->sessions; } - pkts_processed += next0 == NAT44_ED_OUT2IN_NEXT_LOOKUP; + pkts_processed += next0 == nat_buffer_opaque (b0)->arc_next; /* verify speculative enqueue, maybe switch current next frame */ vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, n_left_to_next, @@ -1574,68 +1562,13 @@ nat44_ed_out2in_node_fn_inline (vlib_main_t * vm, return frame->n_vectors; } -VLIB_NODE_FN (nat44_ed_out2in_node) (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) -{ - return nat44_ed_out2in_node_fn_inline (vm, node, frame, 0); -} - -/* *INDENT-OFF* */ -VLIB_REGISTER_NODE (nat44_ed_out2in_node) = { - .name = "nat44-ed-out2in", - .vector_size = sizeof (u32), - .format_trace = format_nat44_ed_out2in_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings), - .error_strings = nat_out2in_ed_error_strings, - .runtime_data_bytes = sizeof (snat_runtime_t), - .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT, - .next_nodes = { - [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop", - [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup", - [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath", - [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error", - [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out", - [NAT44_ED_OUT2IN_NEXT_REASS] = "nat44-ed-out2in-reass", - }, -}; -/* *INDENT-ON* */ - -VLIB_NODE_FN (nat44_ed_out2in_slowpath_node) (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) -{ - return nat44_ed_out2in_node_fn_inline (vm, node, frame, 1); -} - -/* *INDENT-OFF* */ -VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = { - .name = "nat44-ed-out2in-slowpath", - .vector_size = sizeof (u32), - .format_trace = format_nat44_ed_out2in_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings), - .error_strings = nat_out2in_ed_error_strings, - .runtime_data_bytes = sizeof (snat_runtime_t), - .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT, - .next_nodes = { - [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop", - [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup", - [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath", - [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error", - [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out", - [NAT44_ED_OUT2IN_NEXT_REASS] = "nat44-ed-out2in-reass", - }, -}; -/* *INDENT-ON* */ - -VLIB_NODE_FN (nat44_ed_out2in_reass_node) (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) +static inline uword +nat44_ed_out2in_reass_node_fn_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) { u32 n_left_from, *from, *to_next; - nat44_ed_out2in_next_t next_index; + nat_next_t next_index; u32 pkts_processed = 0; snat_main_t *sm = &snat_main; f64 now = vlib_time_now (vm); @@ -1684,7 +1617,7 @@ VLIB_NODE_FN (nat44_ed_out2in_reass_node) (vlib_main_t * vm, n_left_to_next -= 1; b0 = vlib_get_buffer (vm, bi0); - next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP; + next0 = nat_buffer_opaque (b0)->arc_next; sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; rx_fib_index0 = @@ -1693,7 +1626,7 @@ VLIB_NODE_FN (nat44_ed_out2in_reass_node) (vlib_main_t * vm, if (PREDICT_FALSE (nat_reass_is_drop_frag (0))) { - next0 = NAT44_ED_OUT2IN_NEXT_DROP; + next0 = NAT_NEXT_DROP; b0->error = node->errors[NAT_OUT2IN_ED_ERROR_DROP_FRAGMENT]; goto trace0; } @@ -1712,7 +1645,7 @@ VLIB_NODE_FN (nat44_ed_out2in_reass_node) (vlib_main_t * vm, if (PREDICT_FALSE (!reass0)) { - next0 = NAT44_ED_OUT2IN_NEXT_DROP; + next0 = NAT_NEXT_DROP; b0->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_REASS]; nat_elog_notice ("maximum reassemblies exceeded"); goto trace0; @@ -1726,7 +1659,7 @@ VLIB_NODE_FN (nat44_ed_out2in_reass_node) (vlib_main_t * vm, (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, next0, now, thread_index, &s0); - if (PREDICT_TRUE (next0 != NAT44_ED_OUT2IN_NEXT_DROP)) + if (PREDICT_TRUE (next0 != NAT_NEXT_DROP)) { if (s0) reass0->sess_index = s0 - per_thread_data->sessions; @@ -1767,7 +1700,6 @@ VLIB_NODE_FN (nat44_ed_out2in_reass_node) (vlib_main_t * vm, clib_host_to_net_u16 (UDP_DST_PORT_dhcp_to_client)))) { - vnet_feature_next (&next0, b0); goto trace0; } @@ -1775,7 +1707,7 @@ VLIB_NODE_FN (nat44_ed_out2in_reass_node) (vlib_main_t * vm, { b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION]; - next0 = NAT44_ED_OUT2IN_NEXT_DROP; + next0 = NAT_NEXT_DROP; } else { @@ -1783,7 +1715,7 @@ VLIB_NODE_FN (nat44_ed_out2in_reass_node) (vlib_main_t * vm, udp0->src_port, udp0->dst_port, thread_index, rx_fib_index0)) { - next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT; + next0 = NAT_NEXT_IN2OUT_ED_FAST_PATH; goto trace0; } if (sm->num_workers > 1) @@ -1808,7 +1740,7 @@ VLIB_NODE_FN (nat44_ed_out2in_reass_node) (vlib_main_t * vm, if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0)) { b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN]; - next0 = NAT44_ED_OUT2IN_NEXT_DROP; + next0 = NAT_NEXT_DROP; goto trace0; } @@ -1822,7 +1754,7 @@ VLIB_NODE_FN (nat44_ed_out2in_reass_node) (vlib_main_t * vm, { b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION]; - next0 = NAT44_ED_OUT2IN_NEXT_DROP; + next0 = NAT_NEXT_DROP; goto trace0; } reass0->sess_index = s0 - per_thread_data->sessions; @@ -1848,7 +1780,7 @@ VLIB_NODE_FN (nat44_ed_out2in_reass_node) (vlib_main_t * vm, b0->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_FRAG]; nat_elog_notice ("maximum fragments per reassembly exceeded"); - next0 = NAT44_ED_OUT2IN_NEXT_DROP; + next0 = NAT_NEXT_DROP; goto trace0; } cached0 = 1; @@ -1958,7 +1890,7 @@ VLIB_NODE_FN (nat44_ed_out2in_reass_node) (vlib_main_t * vm, } else { - pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP; + pkts_processed += next0 != NAT_NEXT_DROP; /* verify speculative enqueue, maybe switch current next frame */ vlib_validate_buffer_enqueue_x1 (vm, node, next_index, @@ -1997,33 +1929,100 @@ VLIB_NODE_FN (nat44_ed_out2in_reass_node) (vlib_main_t * vm, nat_send_all_to_node (vm, fragments_to_drop, node, &node->errors[NAT_OUT2IN_ED_ERROR_DROP_FRAGMENT], - NAT44_ED_OUT2IN_NEXT_DROP); + NAT_NEXT_DROP); vec_free (fragments_to_drop); vec_free (fragments_to_loopback); return frame->n_vectors; } +VLIB_NODE_FN (nat44_ed_out2in_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + return nat44_ed_out2in_node_fn_inline (vm, node, frame, 0); +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (nat44_ed_out2in_node) = { + .name = "nat44-ed-out2in", + .vector_size = sizeof (u32), + .sibling_of = "nat-default", + .format_trace = format_nat44_ed_out2in_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings), + .error_strings = nat_out2in_ed_error_strings, + .runtime_data_bytes = sizeof (snat_runtime_t), +}; +/* *INDENT-ON* */ + +VLIB_NODE_FN (nat44_ed_out2in_slowpath_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + return nat44_ed_out2in_node_fn_inline (vm, node, frame, 1); +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = { + .name = "nat44-ed-out2in-slowpath", + .vector_size = sizeof (u32), + .sibling_of = "nat-default", + .format_trace = format_nat44_ed_out2in_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings), + .error_strings = nat_out2in_ed_error_strings, + .runtime_data_bytes = sizeof (snat_runtime_t), +}; +/* *INDENT-ON* */ + +VLIB_NODE_FN (nat44_ed_out2in_reass_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + return nat44_ed_out2in_reass_node_fn_inline (vm, node, frame); +} + /* *INDENT-OFF* */ VLIB_REGISTER_NODE (nat44_ed_out2in_reass_node) = { .name = "nat44-ed-out2in-reass", .vector_size = sizeof (u32), + .sibling_of = "nat-default", .format_trace = format_nat44_reass_trace, .type = VLIB_NODE_TYPE_INTERNAL, .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings), .error_strings = nat_out2in_ed_error_strings, - .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT, - .next_nodes = { - [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop", - [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup", - [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath", - [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error", - [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out", - [NAT44_ED_OUT2IN_NEXT_REASS] = "nat44-ed-out2in-reass", - }, }; /* *INDENT-ON* */ +static u8 * +format_nat_pre_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *); + return format (s, "out2in next_index %d", t->next_index); +} + +VLIB_NODE_FN (nat_pre_out2in_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + return nat_pre_node_fn_inline (vm, node, frame, + NAT_NEXT_OUT2IN_ED_FAST_PATH); +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (nat_pre_out2in_node) = { + .name = "nat-pre-out2in", + .vector_size = sizeof (u32), + .sibling_of = "nat-default", + .format_trace = format_nat_pre_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = 0, + }; +/* *INDENT-ON* */ + /* * fd.io coding-style-patch-verification: ON * |