diff options
| -rwxr-xr-x | src/plugins/nat/in2out.c | 3 | ||||
| -rw-r--r-- | src/plugins/nat/in2out_ed.c | 55 | ||||
| -rwxr-xr-x | src/plugins/nat/nat.c | 214 | ||||
| -rw-r--r-- | src/plugins/nat/nat.h | 31 | ||||
| -rw-r--r-- | src/plugins/nat/nat44/inlines.h | 218 | ||||
| -rw-r--r-- | src/plugins/nat/nat44_cli.c | 195 | ||||
| -rw-r--r-- | src/plugins/nat/nat_api.c | 8 | ||||
| -rw-r--r-- | src/plugins/nat/nat_format.c | 91 | ||||
| -rwxr-xr-x | src/plugins/nat/out2in.c | 3 | ||||
| -rw-r--r-- | src/plugins/nat/out2in_ed.c | 17 | ||||
| -rw-r--r-- | src/plugins/nat/test/test_nat.py | 73 |
11 files changed, 615 insertions, 293 deletions
diff --git a/src/plugins/nat/in2out.c b/src/plugins/nat/in2out.c index 7b712523f29..10acf187208 100755 --- a/src/plugins/nat/in2out.c +++ b/src/plugins/nat/in2out.c @@ -264,9 +264,6 @@ slow_path (snat_main_t * sm, vlib_buffer_t * b0, }; nat44_is_idle_session_ctx_t ctx0; - nat44_session_try_cleanup (&ip0->src_address, rx_fib_index0, thread_index, - now); - if (PREDICT_FALSE (nat44_maximum_sessions_exceeded (sm, thread_index))) { b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED]; diff --git a/src/plugins/nat/in2out_ed.c b/src/plugins/nat/in2out_ed.c index 581d7b4ef60..ca737d56663 100644 --- a/src/plugins/nat/in2out_ed.c +++ b/src/plugins/nat/in2out_ed.c @@ -209,14 +209,18 @@ slow_path_ed (snat_main_t * sm, }; nat44_is_idle_session_ctx_t ctx; - nat44_session_try_cleanup (&key->l_addr, rx_fib_index, thread_index, now); + u32 cleared = 0; if (PREDICT_FALSE (nat44_maximum_sessions_exceeded (sm, thread_index))) { - b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED]; - nat_ipfix_logging_max_sessions (thread_index, sm->max_translations); - nat_elog_notice ("maximum sessions exceeded"); - return NAT_NEXT_DROP; + if (PREDICT_FALSE + (!(cleared = nat44_users_cleanup (thread_index, now)))) + { + b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED]; + nat_ipfix_logging_max_sessions (thread_index, sm->max_translations); + nat_elog_notice ("maximum sessions exceeded"); + return NAT_NEXT_DROP; + } } key0.addr = key->l_addr; @@ -224,6 +228,7 @@ slow_path_ed (snat_main_t * sm, key1.protocol = key0.protocol = proto; key0.fib_index = rx_fib_index; key1.fib_index = sm->outside_fib_index; + /* First try to match static mapping by local address and port */ if (snat_static_mapping_match (sm, key0, &key1, 0, 0, 0, &lb, 0, &identity_nat)) @@ -234,9 +239,16 @@ slow_path_ed (snat_main_t * sm, sm->port_per_thread, tsm->snat_thread_index)) { - nat_elog_notice ("addresses exhausted"); - b->error = node->errors[NAT_IN2OUT_ED_ERROR_OUT_OF_PORTS]; - return NAT_NEXT_DROP; + if (cleared || !nat44_out_of_ports_cleanup (thread_index, now) || + snat_alloc_outside_address_and_port (sm->addresses, + rx_fib_index, thread_index, + &key1, sm->port_per_thread, + tsm->snat_thread_index)) + { + nat_elog_notice ("addresses exhausted"); + b->error = node->errors[NAT_IN2OUT_ED_ERROR_OUT_OF_PORTS]; + return NAT_NEXT_DROP; + } } } else @@ -246,16 +258,19 @@ slow_path_ed (snat_main_t * sm, *sessionp = s; return next; } - is_sm = 1; } - if (proto == SNAT_PROTOCOL_TCP) + if (PREDICT_TRUE (proto == SNAT_PROTOCOL_TCP)) { - if (!tcp_flags_is_init - (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags)) + if (PREDICT_FALSE + (!tcp_flags_is_init + (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))) { b->error = node->errors[NAT_IN2OUT_ED_ERROR_NON_SYN]; + if (!is_sm) + snat_free_outside_address_and_port (sm->addresses, + thread_index, &key1); return NAT_NEXT_DROP; } } @@ -932,9 +947,23 @@ nat44_ed_in2out_fast_path_node_fn_inline (vlib_main_t * vm, next0 = def_slow; goto trace0; } - s0 = pool_elt_at_index (tsm->sessions, value0.value); + // drop if session expired + u64 sess_timeout_time; + sess_timeout_time = s0->last_heard + + (f64) nat44_session_get_timeout (sm, s0); + if (now >= sess_timeout_time) + { + // delete session + nat_free_session_data (sm, s0, thread_index, 0); + nat44_delete_session (sm, s0, thread_index); + + next0 = NAT_NEXT_DROP; + goto trace0; + } + // + b0->flags |= VNET_BUFFER_F_IS_NATED; if (!is_output_feature) diff --git a/src/plugins/nat/nat.c b/src/plugins/nat/nat.c index 07147083e52..4803eeb64de 100755 --- a/src/plugins/nat/nat.c +++ b/src/plugins/nat/nat.c @@ -26,6 +26,7 @@ #include <nat/nat64.h> #include <nat/nat66.h> #include <nat/nat_inlines.h> +#include <nat/nat44/inlines.h> #include <nat/nat_affinity.h> #include <nat/nat_syslog.h> #include <nat/nat_ha.h> @@ -325,6 +326,133 @@ nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index, &s->out2in); } +void +nat44_free_session_data (snat_main_t * sm, snat_session_t * s, + u32 thread_index, u8 is_ha) +{ + snat_session_key_t key; + nat_ed_ses_key_t ed_key; + clib_bihash_kv_16_8_t ed_kv; + snat_main_per_thread_data_t *tsm = + vec_elt_at_index (sm->per_thread_data, thread_index); + + if (is_fwd_bypass_session (s)) + { + if (snat_is_unk_proto_session (s)) + { + ed_key.proto = s->in2out.port; + ed_key.r_port = 0; + ed_key.l_port = 0; + } + else + { + ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol); + ed_key.l_port = s->in2out.port; + ed_key.r_port = s->ext_host_port; + } + + ed_key.l_addr = s->in2out.addr; + ed_key.r_addr = s->ext_host_addr; + ed_key.fib_index = 0; + ed_kv.key[0] = ed_key.as_u64[0]; + ed_kv.key[1] = ed_key.as_u64[1]; + + if (PREDICT_FALSE + (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))) + nat_elog_warn ("in2out_ed key del failed"); + return; + } + + /* session lookup tables */ + if (is_affinity_sessions (s)) + nat_affinity_unlock (s->ext_host_addr, s->out2in.addr, + s->in2out.protocol, s->out2in.port); + ed_key.l_addr = s->out2in.addr; + ed_key.r_addr = s->ext_host_addr; + ed_key.fib_index = s->out2in.fib_index; + if (snat_is_unk_proto_session (s)) + { + ed_key.proto = s->in2out.port; + ed_key.r_port = 0; + ed_key.l_port = 0; + } + else + { + ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol); + ed_key.l_port = s->out2in.port; + ed_key.r_port = s->ext_host_port; + } + ed_kv.key[0] = ed_key.as_u64[0]; + ed_kv.key[1] = ed_key.as_u64[1]; + + if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))) + nat_elog_warn ("out2in_ed key del failed"); + + ed_key.l_addr = s->in2out.addr; + ed_key.fib_index = s->in2out.fib_index; + + if (!snat_is_unk_proto_session (s)) + ed_key.l_port = s->in2out.port; + + if (is_twice_nat_session (s)) + { + ed_key.r_addr = s->ext_host_nat_addr; + ed_key.r_port = s->ext_host_nat_port; + } + + ed_kv.key[0] = ed_key.as_u64[0]; + ed_kv.key[1] = ed_key.as_u64[1]; + + if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))) + nat_elog_warn ("in2out_ed key del failed"); + + if (!is_ha) + { + nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index, + &s->in2out.addr, s->in2out.port, + &s->ext_host_nat_addr, s->ext_host_nat_port, + &s->out2in.addr, s->out2in.port, + &s->ext_host_addr, s->ext_host_port, + s->in2out.protocol, is_twice_nat_session (s)); + } + + if (snat_is_unk_proto_session (s)) + return; + + // is this correct ? + if (!is_ha) + { + snat_ipfix_logging_nat44_ses_delete (thread_index, + s->in2out.addr.as_u32, + s->out2in.addr.as_u32, + s->in2out.protocol, + s->in2out.port, + s->out2in.port, + s->in2out.fib_index); + nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr, + s->ext_host_port, s->out2in.protocol, s->out2in.fib_index, + thread_index); + } + + /* Twice NAT address and port for external host */ + if (is_twice_nat_session (s)) + { + key.protocol = s->in2out.protocol; + key.port = s->ext_host_nat_port; + key.addr.as_u32 = s->ext_host_nat_addr.as_u32; + snat_free_outside_address_and_port (sm->twice_nat_addresses, + thread_index, &key); + } + + if (snat_is_session_static (s)) + return; + + // should be called for every dynamic session + snat_free_outside_address_and_port (sm->addresses, thread_index, + &s->out2in); +} + + snat_user_t * nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index, u32 thread_index) @@ -345,6 +473,9 @@ nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index, /* no, make a new one */ pool_get (tsm->users, u); clib_memset (u, 0, sizeof (*u)); + + u->min_session_timeout = 0; + u->addr.as_u32 = addr->as_u32; u->fib_index = fib_index; @@ -453,13 +584,30 @@ nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u, u32 thread_index, { snat_session_t *s; snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index]; - dlist_elt_t *per_user_translation_list_elt, *oldest_elt; - u32 oldest_index; + + dlist_elt_t *oldest_elt; u64 sess_timeout_time; + u32 oldest_index; + // no sessions if (PREDICT_FALSE (!(u->nsessions) && !(u->nstaticsessions))) goto alloc_new; + // no free sessions + if (PREDICT_FALSE + ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)) + { + if (nat44_max_translations_per_user_cleanup (u, thread_index, now)) + goto alloc_new; + + nat_elog_addr (SNAT_LOG_WARNING, "[warn] max translations per user", + clib_net_to_host_u32 (u->addr.as_u32)); + snat_ipfix_logging_max_entries_per_user (thread_index, + sm->max_translations_per_user, + u->addr.as_u32); + return 0; + } + oldest_index = clib_dlist_remove_head (tsm->list_pool, u->sessions_per_user_list_head_index); @@ -469,61 +617,21 @@ nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u, u32 thread_index, sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s); if (now >= sess_timeout_time) { + // reuse old session clib_dlist_addtail (tsm->list_pool, u->sessions_per_user_list_head_index, oldest_index); - nat_free_session_data (sm, s, thread_index, 0); - if (snat_is_session_static (s)) - u->nstaticsessions--; - else - u->nsessions--; - s->flags = 0; - s->total_bytes = 0; - s->total_pkts = 0; - s->state = 0; - s->ext_host_addr.as_u32 = 0; - s->ext_host_port = 0; - s->ext_host_nat_addr.as_u32 = 0; - s->ext_host_nat_port = 0; + s = nat44_session_reuse_old (sm, u, s, thread_index, now); } else { + // alloc new session clib_dlist_addhead (tsm->list_pool, u->sessions_per_user_list_head_index, oldest_index); - if ((u->nsessions + u->nstaticsessions) >= - sm->max_translations_per_user) - { - nat_elog_addr (SNAT_LOG_WARNING, "[warn] max translations per user", - clib_net_to_host_u32 (u->addr.as_u32)); - snat_ipfix_logging_max_entries_per_user - (thread_index, sm->max_translations_per_user, u->addr.as_u32); - return 0; - } - else - { - alloc_new: - pool_get (tsm->sessions, s); - clib_memset (s, 0, sizeof (*s)); - - /* Create list elts */ - pool_get (tsm->list_pool, per_user_translation_list_elt); - clib_dlist_init (tsm->list_pool, - per_user_translation_list_elt - tsm->list_pool); - - per_user_translation_list_elt->value = s - tsm->sessions; - s->per_user_index = per_user_translation_list_elt - tsm->list_pool; - s->per_user_list_head_index = u->sessions_per_user_list_head_index; - - clib_dlist_addtail (tsm->list_pool, - s->per_user_list_head_index, - per_user_translation_list_elt - tsm->list_pool); - } - + alloc_new: + s = nat44_session_alloc_new (tsm, u, now); vlib_set_simple_counter (&sm->total_sessions, thread_index, 0, pool_elts (tsm->sessions)); } - - s->ha_last_refreshed = now; - return s; } @@ -2370,7 +2478,6 @@ snat_init (vlib_main_t * vm) sm->fq_in2out_output_index = ~0; sm->fq_out2in_index = ~0; - sm->alloc_addr_and_port = nat_alloc_addr_and_port_default; sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT; sm->forwarding_enabled = 0; @@ -3768,9 +3875,9 @@ snat_config (vlib_main_t * vm, unformat_input_t * input) u8 static_mapping_only = 0; u8 static_mapping_connection_tracking = 0; + // configurable timeouts u32 udp_timeout = SNAT_UDP_TIMEOUT; u32 icmp_timeout = SNAT_ICMP_TIMEOUT; - u32 tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT; u32 tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT; @@ -3851,9 +3958,11 @@ snat_config (vlib_main_t * vm, unformat_input_t * input) /* optionally configurable timeouts for testing purposes */ sm->udp_timeout = udp_timeout; - sm->icmp_timeout = icmp_timeout; sm->tcp_transitory_timeout = tcp_transitory_timeout; sm->tcp_established_timeout = tcp_established_timeout; + sm->icmp_timeout = icmp_timeout; + + sm->min_timeout = nat44_minimal_timeout (sm); sm->user_buckets = user_buckets; sm->user_memory_size = user_memory_size; @@ -3936,6 +4045,11 @@ snat_config (vlib_main_t * vm, unformat_input_t * input) vec_foreach (tsm, sm->per_thread_data) { tsm->min_session_timeout = 0; + + tsm->cleared = 0; + tsm->cleanup_runs = 0; + tsm->cleanup_timeout = 0; + if (sm->endpoint_dependent) { clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed", diff --git a/src/plugins/nat/nat.h b/src/plugins/nat/nat.h index 9bcce9d43cc..647bec0cd07 100644 --- a/src/plugins/nat/nat.h +++ b/src/plugins/nat/nat.h @@ -349,6 +349,8 @@ typedef struct u32 sessions_per_user_list_head_index; u32 nsessions; u32 nstaticsessions; + /* discovered minimum session timeout time */ + u64 min_session_timeout; } snat_user_t; typedef struct @@ -519,6 +521,12 @@ typedef struct /* discovered minimum session timeout time */ u64 min_session_timeout; + + /* session scavenging */ + u32 cleared; + u32 cleanup_runs; + f64 cleanup_timeout; + } snat_main_per_thread_data_t; struct snat_main_s; @@ -676,10 +684,14 @@ typedef struct snat_main_s u32 inside_fib_index; /* values of various timeouts */ + + // min timeout of all proto timeouts + u32 min_timeout; + // proto timeouts u32 udp_timeout; - u32 icmp_timeout; u32 tcp_transitory_timeout; u32 tcp_established_timeout; + u32 icmp_timeout; /* TCP MSS clamping */ u16 mss_clamping; @@ -703,6 +715,7 @@ typedef struct snat_main_s ip4_main_t *ip4_main; ip_lookup_main_t *ip4_lookup_main; api_main_t *api_main; + } snat_main_t; typedef struct @@ -750,6 +763,7 @@ extern fib_source_t nat_fib_src_low; /* format functions */ format_function_t format_snat_user; +format_function_t format_snat_user_v2; format_function_t format_snat_static_mapping; format_function_t format_snat_static_map_to_resolve; format_function_t format_snat_session; @@ -1294,6 +1308,16 @@ void nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index, u8 is_ha); /** + * @brief Free NAT44 ED session data (lookup keys, external addrres port) + * + * @param s NAT session + * @param thread_index thread index + * @param is_ha is HA event + */ +void +nat44_free_session_data (snat_main_t * sm, snat_session_t * s, + u32 thread_index, u8 is_ha); +/** * @brief Find or create NAT user * * @param addr IPv4 address @@ -1302,8 +1326,9 @@ void nat_free_session_data (snat_main_t * sm, snat_session_t * s, * * @return NAT user data structure on success otherwise zero value */ -snat_user_t *nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, - u32 fib_index, u32 thread_index); +snat_user_t *nat_user_get_or_create (snat_main_t * sm, + ip4_address_t * addr, u32 fib_index, + u32 thread_index); /** * @brief Allocate new NAT session or recycle last used diff --git a/src/plugins/nat/nat44/inlines.h b/src/plugins/nat/nat44/inlines.h index 0c8487b473d..7cc24750423 100644 --- a/src/plugins/nat/nat44/inlines.h +++ b/src/plugins/nat/nat44/inlines.h @@ -31,13 +31,66 @@ nat44_maximum_sessions_exceeded (snat_main_t * sm, u32 thread_index) return 0; } -static_always_inline void -nat44_session_cleanup (snat_session_t * s, u32 thread_index) +static_always_inline f64 +nat44_minimal_timeout (snat_main_t * sm) { - snat_main_t *sm = &snat_main; + f64 min_timeout; + + min_timeout = clib_min (sm->udp_timeout, sm->icmp_timeout); + min_timeout = clib_min (min_timeout, sm->icmp_timeout); + min_timeout = clib_min (min_timeout, sm->tcp_transitory_timeout); + min_timeout = clib_min (min_timeout, sm->tcp_established_timeout); + + return min_timeout; +} + +static_always_inline snat_session_t * +nat44_session_reuse_old (snat_main_t * sm, snat_user_t * u, + snat_session_t * s, u32 thread_index, f64 now) +{ + nat44_free_session_data (sm, s, thread_index, 0); + if (snat_is_session_static (s)) + u->nstaticsessions--; + else + u->nsessions--; + s->flags = 0; + s->total_bytes = 0; + s->total_pkts = 0; + s->state = 0; + s->ext_host_addr.as_u32 = 0; + s->ext_host_port = 0; + s->ext_host_nat_addr.as_u32 = 0; + s->ext_host_nat_port = 0; + // + s->ha_last_refreshed = now; + return s; +} + + +static_always_inline snat_session_t * +nat44_session_alloc_new (snat_main_per_thread_data_t * tsm, snat_user_t * u, + f64 now) +{ + snat_session_t *s; + dlist_elt_t *per_user_translation_list_elt; - nat_free_session_data (sm, s, thread_index, 0); - nat44_delete_session (sm, s, thread_index); + pool_get (tsm->sessions, s); + clib_memset (s, 0, sizeof (*s)); + /* Create list elts */ + pool_get (tsm->list_pool, per_user_translation_list_elt); + clib_dlist_init (tsm->list_pool, + per_user_translation_list_elt - tsm->list_pool); + + per_user_translation_list_elt->value = s - tsm->sessions; + s->per_user_index = per_user_translation_list_elt - tsm->list_pool; + s->per_user_list_head_index = u->sessions_per_user_list_head_index; + + clib_dlist_addtail (tsm->list_pool, + s->per_user_list_head_index, + per_user_translation_list_elt - tsm->list_pool); + + s->ha_last_refreshed = now; + return s; } static_always_inline void @@ -59,7 +112,10 @@ nat44_user_del_sessions (snat_user_t * u, u32 thread_index) { s = pool_elt_at_index (tsm->sessions, elt->value); elt = pool_elt_at_index (tsm->list_pool, elt->next); - nat44_session_cleanup (s, thread_index); + + nat44_free_session_data (sm, s, thread_index, 0); + // TODO: needs refactoring as in nat44_user_session_cleanup + nat44_delete_session (sm, s, thread_index); } } @@ -108,9 +164,10 @@ nat44_user_del (ip4_address_t * addr, u32 fib_index) return rv; } -static_always_inline void -nat44_user_try_cleanup (snat_user_t * u, u32 thread_index, f64 now) +static_always_inline u32 +nat44_user_session_cleanup (snat_user_t * u, u32 thread_index, f64 now) { + u32 cleared = 0; dlist_elt_t *elt; snat_session_t *s; u64 sess_timeout_time; @@ -118,6 +175,10 @@ nat44_user_try_cleanup (snat_user_t * u, u32 thread_index, f64 now) snat_main_t *sm = &snat_main; snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index]; + if (now < u->min_session_timeout) + goto done; + u->min_session_timeout = now + sm->min_timeout; + // get head elt = pool_elt_at_index (tsm->list_pool, u->sessions_per_user_list_head_index); @@ -133,102 +194,121 @@ nat44_user_try_cleanup (snat_user_t * u, u32 thread_index, f64 now) (f64) nat44_session_get_timeout (sm, s); if (now < sess_timeout_time) - { - tsm->min_session_timeout = - clib_min (sess_timeout_time, tsm->min_session_timeout); - continue; - } + continue; + + // do cleanup of this call (refactor for ED NAT44 only) + nat44_free_session_data (sm, s, thread_index, 0); + + clib_dlist_remove (tsm->list_pool, s->per_user_index); + pool_put_index (tsm->list_pool, s->per_user_index); + pool_put (tsm->sessions, s); + vlib_set_simple_counter (&sm->total_sessions, thread_index, 0, + pool_elts (tsm->sessions)); - nat44_session_cleanup (s, thread_index); + if (snat_is_session_static (s)) + u->nstaticsessions--; + else + u->nsessions--; + + cleared++; } +done: + return cleared; } -static_always_inline void -nat44_session_try_cleanup (ip4_address_t * addr, - u32 fib_index, u32 thread_index, f64 now) +static_always_inline u32 +nat44_users_cleanup (u32 thread_index, f64 now) { + snat_main_t *sm = &snat_main; + snat_main_per_thread_data_t *tsm; + + u32 cleared = 0; + + snat_user_key_t u_key; + clib_bihash_kv_8_8_t kv; + snat_user_t *u = 0; - snat_user_key_t user_key; - clib_bihash_kv_8_8_t kv, value; + u32 pool_index; - snat_main_t *sm = &snat_main; - snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index]; + tsm = vec_elt_at_index (sm->per_thread_data, thread_index); - user_key.addr.as_u32 = addr->as_u32; - user_key.fib_index = fib_index; - kv.key = user_key.as_u64; + if (now < tsm->min_session_timeout) + goto done; + tsm->min_session_timeout = now + sm->min_timeout; + // consider + tsm->cleanup_timeout = tsm->min_session_timeout; - // lookup user for this traffic - if (PREDICT_FALSE (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))) + pool_index = ~0; + do { - // there is still place and a new user can be created - if (PREDICT_TRUE (pool_elts (tsm->sessions) < sm->max_translations)) - return; + pool_index = pool_next_index (tsm->users, pool_index); + if (pool_index == ~0) + break; + u = pool_elt_at_index (tsm->users, pool_index); - if (now >= tsm->min_session_timeout) - { - tsm->min_session_timeout = ~0; - // there is no place so we try to cleanup all users in this thread - /* *INDENT-OFF* */ - pool_foreach (u, tsm->users, - ({ nat44_user_try_cleanup (u, thread_index, now); })); - /* *INDENT-ON* */ - if (~0 == tsm->min_session_timeout) - { - tsm->min_session_timeout = 0; - } - } - return; - } + cleared += nat44_user_session_cleanup (u, thread_index, now); - if (now >= tsm->min_session_timeout) - { - tsm->min_session_timeout = ~0; - // each time user creates a new session we try to cleanup expired sessions - nat44_user_try_cleanup (pool_elt_at_index (tsm->users, value.value), - thread_index, now); - if (~0 == tsm->min_session_timeout) + if (u->nstaticsessions == 0 && u->nsessions == 0) { - tsm->min_session_timeout = 0; + u_key.addr.as_u32 = u->addr.as_u32; + u_key.fib_index = u->fib_index; + kv.key = u_key.as_u64; + + // delete user + pool_put_index (tsm->list_pool, + u->sessions_per_user_list_head_index); + pool_put (tsm->users, u); + clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 0); + + // update total users counter + vlib_set_simple_counter (&sm->total_users, thread_index, 0, + pool_elts (tsm->users)); } } + while (1); + tsm->cleared += cleared; + tsm->cleanup_runs++; +done: + return cleared; } -static_always_inline void -nat44_force_session_cleanup (void) +static_always_inline u32 +nat44_out_of_ports_cleanup (u32 thread_index, f64 now) { - snat_user_t *u = 0; + return nat44_users_cleanup (thread_index, now); +} + +static_always_inline u32 +nat44_max_translations_per_user_cleanup (snat_user_t * u, u32 thread_index, + f64 now) +{ + return nat44_user_session_cleanup (u, thread_index, now); +} +static_always_inline u32 +nat44_force_users_cleanup (void) +{ snat_main_t *sm = &snat_main; snat_main_per_thread_data_t *tsm; - vlib_main_t *vm = vlib_get_main (); - f64 now = vlib_time_now (vm); - - // TODO: consider own timeouts + f64 now = vlib_time_now (vlib_get_main ()); + u32 cleared = 0; if (sm->num_workers > 1) { /* *INDENT-OFF* */ vec_foreach (tsm, sm->per_thread_data) { - pool_foreach (u, tsm->users, - ({ - nat44_user_try_cleanup (u, tsm->thread_index, now); - })); + cleared += nat44_users_cleanup (tsm->thread_index, now); } /* *INDENT-ON* */ } else { tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers); - /* *INDENT-OFF* */ - pool_foreach (u, tsm->users, - ({ - nat44_user_try_cleanup (u, tsm->thread_index, now); - })); - /* *INDENT-ON* */ + cleared += nat44_users_cleanup (tsm->thread_index, now); } + return cleared; } #endif /* included_nat44_inlines_h__ */ diff --git a/src/plugins/nat/nat44_cli.c b/src/plugins/nat/nat44_cli.c index 11e87092ae7..45e00693676 100644 --- a/src/plugins/nat/nat44_cli.c +++ b/src/plugins/nat/nat44_cli.c @@ -122,9 +122,7 @@ nat44_session_cleanup_command_fn (vlib_main_t * vm, vlib_cli_command_t * cmd) { clib_error_t *error = 0; - - nat44_force_session_cleanup (); - + nat44_force_users_cleanup (); return error; } @@ -645,56 +643,26 @@ nat44_show_summary_command_fn (vlib_main_t * vm, unformat_input_t * input, { snat_main_per_thread_data_t *tsm; snat_main_t *sm = &snat_main; - snat_address_t *a; snat_session_t *s; - u32 free_ports, free_ports_addr; - if (sm->deterministic) + if (sm->deterministic || !sm->endpoint_dependent) return clib_error_return (0, UNSUPPORTED_IN_DET_MODE_STR); - if (sm->endpoint_dependent) - vlib_cli_output (vm, "mode: endpoint-depenent"); - - // print timeouts - vlib_cli_output (vm, "icmp timeout: %u", sm->icmp_timeout); - vlib_cli_output (vm, "udp timeout: %u", sm->udp_timeout); - - vlib_cli_output (vm, "tcp established timeout: %u", - sm->tcp_established_timeout); - vlib_cli_output (vm, "tcp transitory timeout: %u", - sm->tcp_transitory_timeout); - // print session configuration values vlib_cli_output (vm, "max translations: %u", sm->max_translations); vlib_cli_output (vm, "max translations per user: %u", sm->max_translations_per_user); - // do we need also twice-nat addresses ?? - if (vec_len (sm->addresses)) - { - free_ports = 0; - /* *INDENT-OFF* */ - vec_foreach (a, sm->addresses) - { - free_ports_addr = sm->port_per_thread; - - #define _(N, i, n, s) \ - free_ports_addr -= a->busy_##n##_ports; - foreach_snat_protocol - #undef _ + u32 count = 0; - vlib_cli_output (vm, "addr: %U free ports: %u out of: %u", - format_ip4_address, &a->addr, free_ports_addr, sm->port_per_thread); + u64 now = vlib_time_now (sm->vlib_main); + u64 sess_timeout_time; - free_ports += free_ports_addr; - } - /* *INDENT-ON* */ - vlib_cli_output (vm, "free ports overall: %u out of: %u", - free_ports, - vec_len (sm->addresses) * sm->port_per_thread); - } + u32 udp_sessions = 0; + u32 tcp_sessions = 0; + u32 icmp_sessions = 0; - u32 count = 0; + u32 timed_out = 0; u32 transitory = 0; u32 established = 0; @@ -703,14 +671,44 @@ nat44_show_summary_command_fn (vlib_main_t * vm, unformat_input_t * input, /* *INDENT-OFF* */ vec_foreach (tsm, sm->per_thread_data) { - count += vec_len (tsm->sessions); pool_foreach (s, tsm->sessions, ({ - if (s->state) - transitory++; - else - established++; + sess_timeout_time = s->last_heard + + (f64) nat44_session_get_timeout (sm, s); + if (now >= sess_timeout_time) + timed_out++; + + switch (s->in2out.protocol) + { + case SNAT_PROTOCOL_ICMP: + icmp_sessions++; + break; + case SNAT_PROTOCOL_TCP: + tcp_sessions++; + if (s->state) + transitory++; + else + established++; + break; + case SNAT_PROTOCOL_UDP: + default: + udp_sessions++; + break; + } })); + count += pool_elts (tsm->sessions); + + vlib_cli_output (vm, "tid[%u] session scavenging cleared: %u", + tsm->thread_index, tsm->cleared); + vlib_cli_output (vm, "tid[%u] session scavenging cleanup runs: %u", + tsm->thread_index, tsm->cleanup_runs); + + if (now < tsm->cleanup_timeout) + vlib_cli_output (vm, "tid[%u] session scavenging next run in: %f", + tsm->thread_index, tsm->cleanup_timeout - now); + else + vlib_cli_output (vm, "tid[%u] session scavenging next run in: 0", + tsm->thread_index); } /* *INDENT-ON* */ } @@ -720,17 +718,51 @@ nat44_show_summary_command_fn (vlib_main_t * vm, unformat_input_t * input, /* *INDENT-OFF* */ pool_foreach (s, tsm->sessions, ({ - if (s->state) - transitory++; - else - established++; + sess_timeout_time = s->last_heard + + (f64) nat44_session_get_timeout (sm, s); + if (now >= sess_timeout_time) + timed_out++; + + switch (s->in2out.protocol) + { + case SNAT_PROTOCOL_ICMP: + icmp_sessions++; + break; + case SNAT_PROTOCOL_TCP: + tcp_sessions++; + if (s->state) + transitory++; + else + established++; + break; + case SNAT_PROTOCOL_UDP: + default: + udp_sessions++; + break; + } })); /* *INDENT-ON* */ - count = vec_len (tsm->sessions); + count = pool_elts (tsm->sessions); + + vlib_cli_output (vm, "tid[0] session scavenging cleared: %u", + tsm->cleared); + vlib_cli_output (vm, "tid[0] session scavenging cleanup runs: %u", + tsm->cleanup_runs); + + if (now < tsm->cleanup_timeout) + vlib_cli_output (vm, "tid[0] session scavenging next run in: %f", + tsm->cleanup_timeout - now); + else + vlib_cli_output (vm, "tid[0] session scavenging next run in: 0"); } - vlib_cli_output (vm, "established sessions: %u", established); - vlib_cli_output (vm, "transitory sessions: %u", transitory); - vlib_cli_output (vm, "sessions: %u", count); + + vlib_cli_output (vm, "total timed out sessions: %u", timed_out); + vlib_cli_output (vm, "total sessions: %u", count); + vlib_cli_output (vm, "total tcp sessions: %u", tcp_sessions); + vlib_cli_output (vm, "total tcp established sessions: %u", established); + vlib_cli_output (vm, "total tcp transitory sessions: %u", transitory); + vlib_cli_output (vm, "total udp sessions: %u", udp_sessions); + vlib_cli_output (vm, "total icmp sessions: %u", icmp_sessions); return 0; } @@ -1439,20 +1471,38 @@ static clib_error_t * nat44_show_sessions_command_fn (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { - int verbose = 0; + unformat_input_t _line_input, *line_input = &_line_input; + clib_error_t *error = 0; snat_main_t *sm = &snat_main; snat_main_per_thread_data_t *tsm; + + int detail = 0, metrics = 0; snat_user_t *u; int i = 0; if (sm->deterministic) return clib_error_return (0, UNSUPPORTED_IN_DET_MODE_STR); - if (unformat (input, "detail")) - verbose = 1; + if (!unformat_user (input, unformat_line_input, line_input)) + goto print; - vlib_cli_output (vm, "NAT44 sessions:"); + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "detail")) + detail = 1; + else if (unformat (line_input, "metrics")) + metrics = 1; + else + { + error = clib_error_return (0, "unknown input '%U'", + format_unformat_error, line_input); + break; + } + } + unformat_free (line_input); +print: + vlib_cli_output (vm, "NAT44 sessions:"); /* *INDENT-OFF* */ vec_foreach_index (i, sm->per_thread_data) { @@ -1461,14 +1511,24 @@ nat44_show_sessions_command_fn (vlib_main_t * vm, unformat_input_t * input, vlib_cli_output (vm, "-------- thread %d %s: %d sessions --------\n", i, vlib_worker_threads[i].name, pool_elts (tsm->sessions)); - pool_foreach (u, tsm->users, - ({ - vlib_cli_output (vm, " %U", format_snat_user, tsm, u, verbose); - })); + if (metrics) + { + u64 now = vlib_time_now (sm->vlib_main); + pool_foreach (u, tsm->users, + ({ + vlib_cli_output (vm, " %U", format_snat_user_v2, tsm, u, now); + })); + } + else + { + pool_foreach (u, tsm->users, + ({ + vlib_cli_output (vm, " %U", format_snat_user, tsm, u, detail); + })); + } } /* *INDENT-ON* */ - - return 0; + return error; } static clib_error_t * @@ -1896,10 +1956,9 @@ set_timeout_command_fn (vlib_main_t * vm, goto done; } } - done: unformat_free (line_input); - + sm->min_timeout = nat44_minimal_timeout (sm); return error; } @@ -1910,6 +1969,8 @@ nat_show_timeouts_command_fn (vlib_main_t * vm, { snat_main_t *sm = &snat_main; + // fix text + vlib_cli_output (vm, "min session cleanup timeout: %dsec", sm->min_timeout); vlib_cli_output (vm, "udp timeout: %dsec", sm->udp_timeout); vlib_cli_output (vm, "tcp-established timeout: %dsec", sm->tcp_established_timeout); @@ -2535,7 +2596,7 @@ VLIB_CLI_COMMAND (nat44_show_interface_address_command, static) = { ?*/ VLIB_CLI_COMMAND (nat44_show_sessions_command, static) = { .path = "show nat44 sessions", - .short_help = "show nat44 sessions [detail]", + .short_help = "show nat44 sessions [detail|metrics]", .function = nat44_show_sessions_command_fn, }; diff --git a/src/plugins/nat/nat_api.c b/src/plugins/nat/nat_api.c index d8c2c5fd1c9..a71c888b6c6 100644 --- a/src/plugins/nat/nat_api.c +++ b/src/plugins/nat/nat_api.c @@ -247,9 +247,7 @@ vl_api_nat44_session_cleanup_t_handler (vl_api_nat44_session_cleanup_t * mp) snat_main_t *sm = &snat_main; vl_api_nat44_session_cleanup_reply_t *rmp; int rv = 0; - - nat44_force_session_cleanup (); - + nat44_force_users_cleanup (); REPLY_MACRO (VL_API_NAT44_SESSION_CLEANUP_REPLY); } @@ -338,6 +336,8 @@ vl_api_nat_set_timeouts_t_handler (vl_api_nat_set_timeouts_t * mp) sm->tcp_transitory_timeout = ntohl (mp->tcp_transitory); sm->icmp_timeout = ntohl (mp->icmp); + sm->min_timeout = nat44_minimal_timeout (sm); + rv = nat64_set_icmp_timeout (ntohl (mp->icmp)); if (rv) goto send_reply; @@ -747,10 +747,8 @@ vl_api_nat44_del_user_t_handler (vl_api_nat44_del_user_t * mp) vl_api_nat44_del_user_reply_t *rmp; ip4_address_t addr; int rv; - memcpy (&addr.as_u8, mp->ip_address, 4); rv = nat44_user_del (&addr, ntohl (mp->fib_index)); - REPLY_MACRO (VL_API_NAT44_DEL_USER_REPLY); } diff --git a/src/plugins/nat/nat_format.c b/src/plugins/nat/nat_format.c index 17f64b9b222..71bdaa6662c 100644 --- a/src/plugins/nat/nat_format.c +++ b/src/plugins/nat/nat_format.c @@ -18,6 +18,7 @@ */ #include <nat/nat.h> +#include <nat/nat_inlines.h> #include <nat/nat_det.h> uword @@ -173,7 +174,9 @@ format_snat_session (u8 * s, va_list * args) u8 * format_snat_user (u8 * s, va_list * args) { - snat_main_per_thread_data_t *sm = + + snat_main_t *sm = &snat_main; + snat_main_per_thread_data_t *tsm = va_arg (*args, snat_main_per_thread_data_t *); snat_user_t *u = va_arg (*args, snat_user_t *); int verbose = va_arg (*args, int); @@ -191,20 +194,20 @@ format_snat_user (u8 * s, va_list * args) if (u->nsessions || u->nstaticsessions) { head_index = u->sessions_per_user_list_head_index; - head = pool_elt_at_index (sm->list_pool, head_index); + head = pool_elt_at_index (tsm->list_pool, head_index); elt_index = head->next; - elt = pool_elt_at_index (sm->list_pool, elt_index); + elt = pool_elt_at_index (tsm->list_pool, elt_index); session_index = elt->value; while (session_index != ~0) { - sess = pool_elt_at_index (sm->sessions, session_index); + sess = pool_elt_at_index (tsm->sessions, session_index); s = format (s, " %U\n", format_snat_session, sm, sess); elt_index = elt->next; - elt = pool_elt_at_index (sm->list_pool, elt_index); + elt = pool_elt_at_index (tsm->list_pool, elt_index); session_index = elt->value; } } @@ -213,6 +216,84 @@ format_snat_user (u8 * s, va_list * args) } u8 * +format_snat_user_v2 (u8 * s, va_list * args) +{ + + snat_main_t *sm = &snat_main; + snat_main_per_thread_data_t *tsm = + va_arg (*args, snat_main_per_thread_data_t *); + snat_user_t *u = va_arg (*args, snat_user_t *); + u64 now = va_arg (*args, u64); + + dlist_elt_t *head, *elt; + u32 elt_index, head_index; + u32 session_index; + snat_session_t *sess; + + u32 udp_sessions = 0; + u32 tcp_sessions = 0; + u32 icmp_sessions = 0; + + u32 timed_out = 0; + u32 transitory = 0; + u32 established = 0; + + u64 sess_timeout_time; + + if (u->nsessions || u->nstaticsessions) + { + head_index = u->sessions_per_user_list_head_index; + head = pool_elt_at_index (tsm->list_pool, head_index); + + elt_index = head->next; + elt = pool_elt_at_index (tsm->list_pool, elt_index); + session_index = elt->value; + + while (session_index != ~0) + { + sess = pool_elt_at_index (tsm->sessions, session_index); + + sess_timeout_time = sess->last_heard + + (f64) nat44_session_get_timeout (sm, sess); + if (now >= sess_timeout_time) + timed_out++; + + switch (sess->in2out.protocol) + { + case SNAT_PROTOCOL_ICMP: + icmp_sessions++; + break; + case SNAT_PROTOCOL_TCP: + tcp_sessions++; + if (sess->state) + transitory++; + else + established++; + break; + case SNAT_PROTOCOL_UDP: + default: + udp_sessions++; + break; + + } + + elt_index = elt->next; + elt = pool_elt_at_index (tsm->list_pool, elt_index); + session_index = elt->value; + } + } + + s = format (s, "%U: %d dynamic translations, %d static translations\n", + format_ip4_address, &u->addr, u->nsessions, u->nstaticsessions); + s = format (s, "\t%u timed out, %u transitory, %u established\n", + timed_out, transitory, established); + s = format (s, "\t%u tcp sessions, %u udp sessions, %u icmp sessions\n", + tcp_sessions, udp_sessions, icmp_sessions); + + return s; +} + +u8 * format_snat_static_mapping (u8 * s, va_list * args) { snat_static_mapping_t *m = va_arg (*args, snat_static_mapping_t *); diff --git a/src/plugins/nat/out2in.c b/src/plugins/nat/out2in.c index 144f324dbf4..f920599b059 100755 --- a/src/plugins/nat/out2in.c +++ b/src/plugins/nat/out2in.c @@ -188,9 +188,6 @@ create_session_for_static_mapping (snat_main_t * sm, udp_header_t *udp0; nat44_is_idle_session_ctx_t ctx0; - nat44_session_try_cleanup (&in2out.addr, in2out.fib_index, thread_index, - now); - if (PREDICT_FALSE (nat44_maximum_sessions_exceeded (sm, thread_index))) { b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED]; diff --git a/src/plugins/nat/out2in_ed.c b/src/plugins/nat/out2in_ed.c index 91b8d5a8aae..420b7b7a863 100644 --- a/src/plugins/nat/out2in_ed.c +++ b/src/plugins/nat/out2in_ed.c @@ -203,8 +203,6 @@ create_session_for_static_mapping_ed (snat_main_t * sm, snat_session_key_t eh_key; nat44_is_idle_session_ctx_t ctx; - nat44_session_try_cleanup (&l_key.addr, l_key.fib_index, thread_index, now); - if (PREDICT_FALSE (nat44_maximum_sessions_exceeded (sm, thread_index))) { b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED]; @@ -764,6 +762,21 @@ nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm, } s0 = pool_elt_at_index (tsm->sessions, value0.value); + // drop if session expired + u64 sess_timeout_time; + sess_timeout_time = s0->last_heard + + (f64) nat44_session_get_timeout (sm, s0); + if (now >= sess_timeout_time) + { + // delete session + nat_free_session_data (sm, s0, thread_index, 0); + nat44_delete_session (sm, s0, thread_index); + + next0 = NAT_NEXT_DROP; + goto trace0; + } + // + old_addr0 = ip0->dst_address.as_u32; new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32; vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index; diff --git a/src/plugins/nat/test/test_nat.py b/src/plugins/nat/test/test_nat.py index d16204e7217..89af40c4d18 100644 --- a/src/plugins/nat/test/test_nat.py +++ b/src/plugins/nat/test/test_nat.py @@ -6886,79 +6886,6 @@ class TestNAT44EndpointDependent(MethodHolder): self.pg0.remote_ip4) self.assertEqual(users[0].nsessions, 1) - @unittest.skipUnless(running_extended_tests, "part of extended tests") - def test_session_limit_per_user(self): - """ Maximum sessions per user limit """ - self.nat44_add_address(self.nat_addr) - flags = self.config_flags.NAT_IS_INSIDE - self.vapi.nat44_interface_add_del_feature( - sw_if_index=self.pg0.sw_if_index, - flags=flags, is_add=1) - self.vapi.nat44_interface_add_del_feature( - sw_if_index=self.pg1.sw_if_index, - is_add=1) - self.vapi.set_ipfix_exporter(collector_address=self.pg2.remote_ip4, - src_address=self.pg2.local_ip4, - path_mtu=512, - template_interval=10) - self.vapi.nat_set_timeouts(udp=5, tcp_established=7440, - tcp_transitory=240, icmp=60) - - # get maximum number of translations per user - nat44_config = self.vapi.nat_show_config() - - pkts = [] - for port in range(0, nat44_config.max_translations_per_user): - p = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) / - IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) / - UDP(sport=1025 + port, dport=1025 + port)) - pkts.append(p) - - self.pg0.add_stream(pkts) - self.pg_enable_capture(self.pg_interfaces) - self.pg_start() - capture = self.pg1.get_capture(len(pkts)) - - self.vapi.nat_ipfix_enable_disable(domain_id=self.ipfix_domain_id, - src_port=self.ipfix_src_port, - enable=1) - - p = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) / - IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) / - UDP(sport=3001, dport=3002)) - self.pg0.add_stream(p) - self.pg_enable_capture(self.pg_interfaces) - self.pg_start() - capture = self.pg1.assert_nothing_captured() - - # verify IPFIX logging - self.vapi.ipfix_flush() - sleep(1) - capture = self.pg2.get_capture(10) - ipfix = IPFIXDecoder() - # first load template - for p in capture: - self.assertTrue(p.haslayer(IPFIX)) - if p.haslayer(Template): - ipfix.add_template(p.getlayer(Template)) - # verify events in data set - for p in capture: - if p.haslayer(Data): - data = ipfix.decode_data_set(p.getlayer(Set)) - self.verify_ipfix_max_entries_per_user( - data, - nat44_config.max_translations_per_user, - self.pg0.remote_ip4) - - sleep(6) - p = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) / - IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) / - UDP(sport=3001, dport=3002)) - self.pg0.add_stream(p) - self.pg_enable_capture(self.pg_interfaces) - self.pg_start() - self.pg1.get_capture(1) - def test_syslog_sess(self): """ Test syslog session creation and deletion """ self.vapi.syslog_set_filter( |