NAT: VPP-1537 IPFIX per worker processing

Change-Id: I428bd25a513eb9fe65bea56572fea8cab7c51681
Signed-off-by: Filip Varga <fivarga@cisco.com>

21 files changed, 545 insertions, 533 deletions

diff --git a/src/plugins/nat/in2out.c b/src/plugins/nat/in2out.c
index 15dae650844..abb916060f1 100755
--- a/src/plugins/nat/in2out.c
+++ b/src/plugins/nat/in2out.c
@@ -217,7 +217,8 @@ nat44_i2o_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg)
       if (clib_bihash_add_del_8_8 (&tsm->out2in, &s_kv, 0))
 	nat_log_warn ("out2in key del failed");
 
-      snat_ipfix_logging_nat44_ses_delete (s->in2out.addr.as_u32,
+      snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
+					   s->in2out.addr.as_u32,
 					   s->out2in.addr.as_u32,
 					   s->in2out.protocol,
 					   s->in2out.port,
@@ -269,7 +270,7 @@ slow_path (snat_main_t * sm, vlib_buffer_t * b0,
   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
     {
       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
-      nat_ipfix_logging_max_sessions (sm->max_translations);
+      nat_ipfix_logging_max_sessions (thread_index, sm->max_translations);
       nat_log_notice ("maximum sessions exceeded");
       return SNAT_IN2OUT_NEXT_DROP;
     }
@@ -373,7 +374,8 @@ slow_path (snat_main_t * sm, vlib_buffer_t * b0,
     nat_log_notice ("out2in key add failed");
 
   /* log NAT event */
-  snat_ipfix_logging_nat44_ses_create (s->in2out.addr.as_u32,
+  snat_ipfix_logging_nat44_ses_create (thread_index,
+				       s->in2out.addr.as_u32,
 				       s->out2in.addr.as_u32,
 				       s->in2out.protocol,
 				       s->in2out.port,
@@ -1844,7 +1846,7 @@ nat44_in2out_reass_node_fn (vlib_main_t * vm,
 	      if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
 		{
 		  if (nat_ip4_reass_add_fragment
-		      (reass0, bi0, &fragments_to_drop))
+		      (thread_index, reass0, bi0, &fragments_to_drop))
 		    {
 		      b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
 		      nat_log_notice
diff --git a/src/plugins/nat/in2out_ed.c b/src/plugins/nat/in2out_ed.c
index 9a61af9dad9..2cde378e801 100644
--- a/src/plugins/nat/in2out_ed.c
+++ b/src/plugins/nat/in2out_ed.c
@@ -199,7 +199,8 @@ nat44_i2o_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
       if (snat_is_unk_proto_session (s))
 	goto delete;
 
-      snat_ipfix_logging_nat44_ses_delete (s->in2out.addr.as_u32,
+      snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
+					   s->in2out.addr.as_u32,
 					   s->out2in.addr.as_u32,
 					   s->in2out.protocol,
 					   s->in2out.port,
@@ -296,7 +297,7 @@ slow_path_ed (snat_main_t * sm,
   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
     {
       b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
-      nat_ipfix_logging_max_sessions (sm->max_translations);
+      nat_ipfix_logging_max_sessions (thread_index, sm->max_translations);
       nat_log_notice ("maximum sessions exceeded");
       return NAT_IN2OUT_ED_NEXT_DROP;
     }
@@ -420,7 +421,8 @@ slow_path_ed (snat_main_t * sm,
   *sessionp = s;
 
   /* log NAT event */
-  snat_ipfix_logging_nat44_ses_create (s->in2out.addr.as_u32,
+  snat_ipfix_logging_nat44_ses_create (thread_index,
+				       s->in2out.addr.as_u32,
 				       s->out2in.addr.as_u32,
 				       s->in2out.protocol,
 				       s->in2out.port,
@@ -767,7 +769,7 @@ nat44_ed_in2out_unknown_proto (snat_main_t * sm,
       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 	{
 	  b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
-	  nat_ipfix_logging_max_sessions (sm->max_translations);
+	  nat_ipfix_logging_max_sessions (thread_index, sm->max_translations);
 	  nat_log_notice ("maximum sessions exceeded");
 	  return 0;
 	}
@@ -1965,7 +1967,7 @@ nat44_ed_in2out_reass_node_fn_inline (vlib_main_t * vm,
 	      if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
 		{
 		  if (nat_ip4_reass_add_fragment
-		      (reass0, bi0, &fragments_to_drop))
+		      (thread_index, reass0, bi0, &fragments_to_drop))
 		    {
 		      b0->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_FRAG];
 		      nat_log_notice
diff --git a/src/plugins/nat/nat.c b/src/plugins/nat/nat.c
index 4f5a8a68dd2..8d02e5607c3 100755
--- a/src/plugins/nat/nat.c
+++ b/src/plugins/nat/nat.c
@@ -264,7 +264,8 @@ nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index)
     return;
 
   /* log NAT event */
-  snat_ipfix_logging_nat44_ses_delete (s->in2out.addr.as_u32,
+  snat_ipfix_logging_nat44_ses_delete (thread_index,
+				       s->in2out.addr.as_u32,
 				       s->out2in.addr.as_u32,
 				       s->in2out.protocol,
 				       s->in2out.port,
@@ -454,7 +455,7 @@ nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u, u32 thread_index,
 	  nat_log_warn ("max translations per user %U", format_ip4_address,
 			&u->addr);
 	  snat_ipfix_logging_max_entries_per_user
-	    (sm->max_translations_per_user, u->addr.as_u32);
+	    (thread_index, sm->max_translations_per_user, u->addr.as_u32);
 	  return 0;
 	}
       else
@@ -2557,7 +2558,7 @@ nat_alloc_addr_and_port_default (snat_address_t * addresses,
     }
 
   /* Totally out of translations to use... */
-  snat_ipfix_logging_addresses_exhausted (0);
+  snat_ipfix_logging_addresses_exhausted (thread_index, 0);
   return 1;
 }
 
@@ -2607,7 +2608,7 @@ nat_alloc_addr_and_port_mape (snat_address_t * addresses,
 
 exhausted:
   /* Totally out of translations to use... */
-  snat_ipfix_logging_addresses_exhausted (0);
+  snat_ipfix_logging_addresses_exhausted (thread_index, 0);
   return 1;
 }
 
@@ -2655,7 +2656,7 @@ nat_alloc_addr_and_port_range (snat_address_t * addresses,
 
 exhausted:
   /* Totally out of translations to use... */
-  snat_ipfix_logging_addresses_exhausted (0);
+  snat_ipfix_logging_addresses_exhausted (thread_index, 0);
   return 1;
 }
 
diff --git a/src/plugins/nat/nat44_classify.c b/src/plugins/nat/nat44_classify.c
index 8a417dcf5d9..ed0c37ea21a 100644
--- a/src/plugins/nat/nat44_classify.c
+++ b/src/plugins/nat/nat44_classify.c
@@ -217,8 +217,8 @@ nat44_classify_node_fn_inline (vlib_main_t * vm,
 		      !(reass0->flags & NAT_REASS_FLAG_CLASSIFY_ED_CONTINUE))
 		    {
 		      /* first fragment still hasn't arrived, cache this fragment */
-		      if (nat_ip4_reass_add_fragment (reass0, bi0,
-						      &fragments_to_drop))
+		      if (nat_ip4_reass_add_fragment
+			  (thread_index, reass0, bi0, &fragments_to_drop))
 			{
 			  b0->error =
 			    node->errors[NAT44_CLASSIFY_ERROR_MAX_FRAG];
@@ -328,8 +328,8 @@ nat44_classify_node_fn_inline (vlib_main_t * vm,
 		  if (reass0->classify_next == NAT_REASS_IP4_CLASSIFY_NONE)
 		    /* first fragment still hasn't arrived */
 		    {
-		      if (nat_ip4_reass_add_fragment (reass0, bi0,
-						      &fragments_to_drop))
+		      if (nat_ip4_reass_add_fragment
+			  (thread_index, reass0, bi0, &fragments_to_drop))
 			{
 			  b0->error =
 			    node->errors[NAT44_CLASSIFY_ERROR_MAX_FRAG];
diff --git a/src/plugins/nat/nat44_cli.c b/src/plugins/nat/nat44_cli.c
index 9aaa8498e22..d33ee5f353b 100644
--- a/src/plugins/nat/nat44_cli.c
+++ b/src/plugins/nat/nat44_cli.c
@@ -126,6 +126,9 @@ snat_ipfix_logging_enable_disable_command_fn (vlib_main_t * vm,
   int rv = 0;
   clib_error_t *error = 0;
 
+  // TODO: fix if you don't add domain id, "nat ipfix logging", won't
+  // enable logging
+
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
diff --git a/src/plugins/nat/nat64.c b/src/plugins/nat/nat64.c
index 4dbfb05c78c..2f665ab4cfe 100644
--- a/src/plugins/nat/nat64.c
+++ b/src/plugins/nat/nat64.c
@@ -82,12 +82,14 @@ nat64_ip4_add_del_interface_address_cb (ip4_main_t * im, uword opaque,
 		if (nm->addr_pool[j].addr.as_u32 == address->as_u32)
 		  return;
 
-	      (void) nat64_add_del_pool_addr (address, ~0, 1);
+	      (void) nat64_add_del_pool_addr (vlib_get_thread_index (),
+					      address, ~0, 1);
 	      return;
 	    }
 	  else
 	    {
-	      (void) nat64_add_del_pool_addr (address, ~0, 0);
+	      (void) nat64_add_del_pool_addr (vlib_get_thread_index (),
+					      address, ~0, 0);
 	      return;
 	    }
 	}
@@ -280,7 +282,8 @@ nat64_set_hash (u32 bib_buckets, u32 bib_memory_size, u32 st_buckets,
 }
 
 int
-nat64_add_del_pool_addr (ip4_address_t * addr, u32 vrf_id, u8 is_add)
+nat64_add_del_pool_addr (u32 thread_index,
+			 ip4_address_t * addr, u32 vrf_id, u8 is_add)
 {
   nat64_main_t *nm = &nat64_main;
   snat_address_t *a = 0;
@@ -330,7 +333,7 @@ nat64_add_del_pool_addr (ip4_address_t * addr, u32 vrf_id, u8 is_add)
         /* *INDENT-OFF* */
         vec_foreach (db, nm->db)
           {
-            nat64_db_free_out_addr (db, &a->addr);
+            nat64_db_free_out_addr (thread_index, db, &a->addr);
             vlib_set_simple_counter (&nm->total_bibs, db - nm->db, 0,
                                      db->bib.bib_entries_num);
             vlib_set_simple_counter (&nm->total_sessions, db - nm->db, 0,
@@ -394,8 +397,8 @@ nat64_add_interface_address (u32 sw_if_index, int is_add)
 	    {
 	      /* if have address remove it */
 	      if (first_int_addr)
-		(void) nat64_add_del_pool_addr (first_int_addr, ~0, 0);
-
+		(void) nat64_add_del_pool_addr (vlib_get_thread_index (),
+						first_int_addr, ~0, 0);
 	      vec_del1 (nm->auto_add_sw_if_indices, i);
 	      return 0;
 	    }
@@ -410,7 +413,8 @@ nat64_add_interface_address (u32 sw_if_index, int is_add)
 
   /* If the address is already bound - or static - add it now */
   if (first_int_addr)
-    (void) nat64_add_del_pool_addr (first_int_addr, ~0, 1);
+    (void) nat64_add_del_pool_addr (vlib_get_thread_index (),
+				    first_int_addr, ~0, 1);
 
   return 0;
 }
@@ -601,7 +605,8 @@ nat64_static_bib_worker_fn (vlib_main_t * vm, vlib_node_runtime_t * rt,
 
     if (static_bib->is_add)
       {
-          (void) nat64_db_bib_entry_create (db, &static_bib->in_addr,
+          (void) nat64_db_bib_entry_create (thread_index, db,
+                                            &static_bib->in_addr,
                                             &static_bib->out_addr,
                                             static_bib->in_port,
                                             static_bib->out_port,
@@ -619,7 +624,7 @@ nat64_static_bib_worker_fn (vlib_main_t * vm, vlib_node_runtime_t * rt,
                                         static_bib->fib_index, 1);
         if (bibe)
           {
-            nat64_db_bib_entry_free (db, bibe);
+            nat64_db_bib_entry_free (thread_index, db, bibe);
             vlib_set_simple_counter (&nm->total_bibs, thread_index, 0,
                                      db->bib.bib_entries_num);
             vlib_set_simple_counter (&nm->total_sessions, thread_index, 0,
@@ -723,7 +728,7 @@ nat64_add_del_static_bib_entry (ip6_address_t * in_addr,
       if (!nm->sm->num_workers)
 	{
 	  bibe =
-	    nat64_db_bib_entry_create (db, in_addr, out_addr,
+	    nat64_db_bib_entry_create (thread_index, db, in_addr, out_addr,
 				       clib_host_to_net_u16 (in_port),
 				       clib_host_to_net_u16 (out_port),
 				       fib_index, proto, 1);
@@ -741,7 +746,7 @@ nat64_add_del_static_bib_entry (ip6_address_t * in_addr,
 
       if (!nm->sm->num_workers)
 	{
-	  nat64_db_bib_entry_free (db, bibe);
+	  nat64_db_bib_entry_free (thread_index, db, bibe);
 	  vlib_set_simple_counter (&nm->total_bibs, thread_index, 0,
 				   db->bib.bib_entries_num);
 	}
@@ -1175,7 +1180,7 @@ nat64_expire_worker_walk_fn (vlib_main_t * vm, vlib_node_runtime_t * rt,
   nat64_db_t *db = &nm->db[thread_index];
   u32 now = (u32) vlib_time_now (vm);
 
-  nad64_db_st_free_expired (db, now);
+  nad64_db_st_free_expired (thread_index, db, now);
   vlib_set_simple_counter (&nm->total_bibs, thread_index, 0,
 			   db->bib.bib_entries_num);
   vlib_set_simple_counter (&nm->total_sessions, thread_index, 0,
diff --git a/src/plugins/nat/nat64.h b/src/plugins/nat/nat64.h
index 0a60571a069..eca5e7dbe96 100644
--- a/src/plugins/nat/nat64.h
+++ b/src/plugins/nat/nat64.h
@@ -123,13 +123,15 @@ extern vlib_node_registration_t nat64_out2in_node;
 /**
  * @brief Add/delete address to NAT64 pool.
  *
+ * @param thread_index Thread index.
  * @param addr   IPv4 address.
  * @param vrf_id VRF id of tenant, ~0 means independent of VRF.
  * @param is_add 1 if add, 0 if delete.
  *
  * @returns 0 on success, non-zero value otherwise.
  */
-int nat64_add_del_pool_addr (ip4_address_t * addr, u32 vrf_id, u8 is_add);
+int nat64_add_del_pool_addr (u32 thread_index,
+			     ip4_address_t * addr, u32 vrf_id, u8 is_add);
 
 /**
  * @brief Call back function when walking addresses in NAT64 pool, non-zero
diff --git a/src/plugins/nat/nat64_cli.c b/src/plugins/nat/nat64_cli.c
index efeaa0be0c2..3f785c2a621 100644
--- a/src/plugins/nat/nat64_cli.c
+++ b/src/plugins/nat/nat64_cli.c
@@ -73,7 +73,8 @@ nat64_add_del_pool_addr_command_fn (vlib_main_t * vm,
 
   for (i = 0; i < count; i++)
     {
-      rv = nat64_add_del_pool_addr (&this_addr, vrf_id, is_add);
+      rv = nat64_add_del_pool_addr (vm->thread_index,
+				    &this_addr, vrf_id, is_add);
 
       switch (rv)
 	{
diff --git a/src/plugins/nat/nat64_db.c b/src/plugins/nat/nat64_db.c
index ca8358ef8a2..178e483ff98 100644
--- a/src/plugins/nat/nat64_db.c
+++ b/src/plugins/nat/nat64_db.c
@@ -50,7 +50,8 @@ nat64_db_init (nat64_db_t * db, u32 bib_buckets, u32 bib_memory_size,
 }
 
 nat64_db_bib_entry_t *
-nat64_db_bib_entry_create (nat64_db_t * db, ip6_address_t * in_addr,
+nat64_db_bib_entry_create (u32 thread_index, nat64_db_t * db,
+			   ip6_address_t * in_addr,
 			   ip4_address_t * out_addr, u16 in_port,
 			   u16 out_port, u32 fib_index, u8 proto,
 			   u8 is_static)
@@ -63,7 +64,7 @@ nat64_db_bib_entry_create (nat64_db_t * db, ip6_address_t * in_addr,
   if (db->bib.bib_entries_num >= db->bib.limit)
     {
       db->free_addr_port_cb (db, out_addr, out_port, proto);
-      nat_ipfix_logging_max_bibs (db->bib.limit);
+      nat_ipfix_logging_max_bibs (thread_index, db->bib.limit);
       return 0;
     }
 
@@ -119,13 +120,14 @@ nat64_db_bib_entry_create (nat64_db_t * db, ip6_address_t * in_addr,
   clib_bihash_add_del_24_8 (&db->bib.out2in, &kv, 1);
 
   fib = fib_table_get (bibe->fib_index, FIB_PROTOCOL_IP6);
-  nat_ipfix_logging_nat64_bib (in_addr, out_addr, proto, in_port, out_port,
-			       fib->ft_table_id, 1);
+  nat_ipfix_logging_nat64_bib (thread_index, in_addr, out_addr, proto,
+			       in_port, out_port, fib->ft_table_id, 1);
   return bibe;
 }
 
 void
-nat64_db_bib_entry_free (nat64_db_t * db, nat64_db_bib_entry_t * bibe)
+nat64_db_bib_entry_free (u32 thread_index, nat64_db_t * db,
+			 nat64_db_bib_entry_t * bibe)
 {
   nat64_db_bib_entry_key_t bibe_key;
   clib_bihash_kv_24_8_t kv;
@@ -164,7 +166,8 @@ nat64_db_bib_entry_free (nat64_db_t * db, nat64_db_bib_entry_t * bibe)
 			       vec_add1 (ste_to_be_free, ste - st);}
 		    ));
       vec_foreach (ste_index, ste_to_be_free)
-	nat64_db_st_entry_free (db, pool_elt_at_index (st, ste_index[0]));
+	nat64_db_st_entry_free (thread_index, db,
+				pool_elt_at_index (st, ste_index[0]));
       vec_free (ste_to_be_free);
     }
 
@@ -193,8 +196,8 @@ nat64_db_bib_entry_free (nat64_db_t * db, nat64_db_bib_entry_t * bibe)
     db->free_addr_port_cb (db, &bibe->out_addr, bibe->out_port, bibe->proto);
 
   fib = fib_table_get (bibe->fib_index, FIB_PROTOCOL_IP6);
-  nat_ipfix_logging_nat64_bib (&bibe->in_addr, &bibe->out_addr, bibe->proto,
-			       bibe->in_port, bibe->out_port,
+  nat_ipfix_logging_nat64_bib (thread_index, &bibe->in_addr, &bibe->out_addr,
+			       bibe->proto, bibe->in_port, bibe->out_port,
 			       fib->ft_table_id, 0);
 
   /* delete from pool */
@@ -370,7 +373,8 @@ nat64_db_st_walk (nat64_db_t * db, u8 proto,
 }
 
 nat64_db_st_entry_t *
-nat64_db_st_entry_create (nat64_db_t * db, nat64_db_bib_entry_t * bibe,
+nat64_db_st_entry_create (u32 thread_index, nat64_db_t * db,
+			  nat64_db_bib_entry_t * bibe,
 			  ip6_address_t * in_r_addr,
 			  ip4_address_t * out_r_addr, u16 r_port)
 {
@@ -382,7 +386,7 @@ nat64_db_st_entry_create (nat64_db_t * db, nat64_db_bib_entry_t * bibe,
 
   if (db->st.st_entries_num >= db->st.limit)
     {
-      nat_ipfix_logging_max_sessions (db->st.limit);
+      nat_ipfix_logging_max_sessions (thread_index, db->st.limit);
       return 0;
     }
 
@@ -452,8 +456,9 @@ nat64_db_st_entry_create (nat64_db_t * db, nat64_db_bib_entry_t * bibe,
   clib_bihash_add_del_48_8 (&db->st.out2in, &kv, 1);
 
   fib = fib_table_get (bibe->fib_index, FIB_PROTOCOL_IP6);
-  nat_ipfix_logging_nat64_session (&bibe->in_addr, &bibe->out_addr,
-				   bibe->proto, bibe->in_port, bibe->out_port,
+  nat_ipfix_logging_nat64_session (thread_index, &bibe->in_addr,
+				   &bibe->out_addr, bibe->proto,
+				   bibe->in_port, bibe->out_port,
 				   &ste->in_r_addr, &ste->out_r_addr,
 				   ste->r_port, ste->r_port, fib->ft_table_id,
 				   1);
@@ -464,7 +469,8 @@ nat64_db_st_entry_create (nat64_db_t * db, nat64_db_bib_entry_t * bibe,
 }
 
 void
-nat64_db_st_entry_free (nat64_db_t * db, nat64_db_st_entry_t * ste)
+nat64_db_st_entry_free (u32 thread_index,
+			nat64_db_t * db, nat64_db_st_entry_t * ste)
 {
   nat64_db_st_entry_t *st;
   nat64_db_bib_entry_t *bib, *bibe;
@@ -526,8 +532,9 @@ nat64_db_st_entry_free (nat64_db_t * db, nat64_db_st_entry_t * ste)
   clib_bihash_add_del_48_8 (&db->st.out2in, &kv, 0);
 
   fib = fib_table_get (bibe->fib_index, FIB_PROTOCOL_IP6);
-  nat_ipfix_logging_nat64_session (&bibe->in_addr, &bibe->out_addr,
-				   bibe->proto, bibe->in_port, bibe->out_port,
+  nat_ipfix_logging_nat64_session (thread_index, &bibe->in_addr,
+				   &bibe->out_addr, bibe->proto,
+				   bibe->in_port, bibe->out_port,
 				   &ste->in_r_addr, &ste->out_r_addr,
 				   ste->r_port, ste->r_port, fib->ft_table_id,
 				   0);
@@ -543,7 +550,7 @@ nat64_db_st_entry_free (nat64_db_t * db, nat64_db_st_entry_t * ste)
 
   /* delete BIB entry if last session and dynamic */
   if (!bibe->is_static && !bibe->ses_num)
-    nat64_db_bib_entry_free (db, bibe);
+    nat64_db_bib_entry_free (thread_index, db, bibe);
 }
 
 nat64_db_st_entry_t *
@@ -641,7 +648,7 @@ nat64_db_st_entry_by_index (nat64_db_t * db, u8 proto, u32 ste_index)
 }
 
 void
-nad64_db_st_free_expired (nat64_db_t * db, u32 now)
+nad64_db_st_free_expired (u32 thread_index, nat64_db_t * db, u32 now)
 {
   u32 *ste_to_be_free = 0, *ste_index;
   nat64_db_st_entry_t *st, *ste;
@@ -656,7 +663,8 @@ nad64_db_st_free_expired (nat64_db_t * db, u32 now)
       vec_add1 (ste_to_be_free, ste - st); \
   })); \
   vec_foreach (ste_index, ste_to_be_free) \
-    nat64_db_st_entry_free (db, pool_elt_at_index(st, ste_index[0])); \
+    nat64_db_st_entry_free (thread_index, db, \
+                            pool_elt_at_index(st, ste_index[0])); \
   vec_free (ste_to_be_free); \
   ste_to_be_free = 0;
   foreach_snat_protocol
@@ -667,13 +675,15 @@ nad64_db_st_free_expired (nat64_db_t * db, u32 now)
       vec_add1 (ste_to_be_free, ste - st);
   }));
   vec_foreach (ste_index, ste_to_be_free)
-    nat64_db_st_entry_free (db, pool_elt_at_index(st, ste_index[0]));
+    nat64_db_st_entry_free (thread_index, db,
+                            pool_elt_at_index(st, ste_index[0]));
   vec_free (ste_to_be_free);
 /* *INDENT-ON* */
 }
 
 void
-nat64_db_free_out_addr (nat64_db_t * db, ip4_address_t * out_addr)
+nat64_db_free_out_addr (u32 thread_index,
+			nat64_db_t * db, ip4_address_t * out_addr)
 {
   u32 *ste_to_be_free = 0, *ste_index;
   nat64_db_st_entry_t *st, *ste;
@@ -689,7 +699,8 @@ nat64_db_free_out_addr (nat64_db_t * db, ip4_address_t * out_addr)
       vec_add1 (ste_to_be_free, ste - st); \
   })); \
   vec_foreach (ste_index, ste_to_be_free) \
-    nat64_db_st_entry_free (db, pool_elt_at_index(st, ste_index[0])); \
+    nat64_db_st_entry_free (thread_index, db, \
+                            pool_elt_at_index(st, ste_index[0])); \
   vec_free (ste_to_be_free); \
   ste_to_be_free = 0;
   foreach_snat_protocol
@@ -701,7 +712,8 @@ nat64_db_free_out_addr (nat64_db_t * db, ip4_address_t * out_addr)
       vec_add1 (ste_to_be_free, ste - st);
   }));
   vec_foreach (ste_index, ste_to_be_free)
-    nat64_db_st_entry_free (db, pool_elt_at_index(st, ste_index[0]));
+    nat64_db_st_entry_free (thread_index, db,
+                            pool_elt_at_index(st, ste_index[0]));
   vec_free (ste_to_be_free);
   db->addr_free = 0;
 /* *INDENT-ON* */
diff --git a/src/plugins/nat/nat64_db.h b/src/plugins/nat/nat64_db.h
index f1b93cf8928..e6fa8e727ad 100644
--- a/src/plugins/nat/nat64_db.h
+++ b/src/plugins/nat/nat64_db.h
@@ -160,6 +160,7 @@ int nat64_db_init (nat64_db_t * db, u32 bib_buckets, u32 bib_memory_size,
 /**
  * @brief Create new NAT64 BIB entry.
  *
+ * @param thread_index thread index.
  * @param db NAT64 DB.
  * @param in_addr Inside IPv6 address.
  * @param out_addr Outside IPv4 address.
@@ -171,20 +172,23 @@ int nat64_db_init (nat64_db_t * db, u32 bib_buckets, u32 bib_memory_size,
  *
  * @returns BIB entry on success, 0 otherwise.
  */
-nat64_db_bib_entry_t *nat64_db_bib_entry_create (nat64_db_t * db,
+nat64_db_bib_entry_t *nat64_db_bib_entry_create (u32 thread_index,
+						 nat64_db_t * db,
 						 ip6_address_t * in_addr,
 						 ip4_address_t * out_addr,
 						 u16 in_port, u16 out_port,
-						 u32 fib_index,
-						 u8 proto, u8 is_static);
+						 u32 fib_index, u8 proto,
+						 u8 is_static);
 
 /**
  * @brief Free NAT64 BIB entry.
  *
+ * @param thread_index thread index.
  * @param db NAT64 DB.
  * @param bibe BIB entry.
  */
-void nat64_db_bib_entry_free (nat64_db_t * db, nat64_db_bib_entry_t * bibe);
+void nat64_db_bib_entry_free (u32 thread_index, nat64_db_t * db,
+			      nat64_db_bib_entry_t * bibe);
 
 /**
  * @brief Call back function when walking NAT64 BIB, non-zero
@@ -201,7 +205,8 @@ typedef int (*nat64_db_bib_walk_fn_t) (nat64_db_bib_entry_t * bibe,
  *  - 6 TCP BIB
  *  - 17 UDP BIB
  *  - 1/58 ICMP BIB
- *  - otherwise "unknown" protocol BIB
+ *
+ * u - otherwise "unknown" protocol BIB
  * @param fn The function to invoke on each entry visited.
  * @param ctx A context passed in the visit function.
  */
@@ -240,6 +245,7 @@ nat64_db_bib_entry_t *nat64_db_bib_entry_by_index (nat64_db_t * db,
 /**
  * @brief Create new NAT64 session table entry.
  *
+ * @param thread_index thread index.
  * @param db NAT64 DB.
  * @param bibe Corresponding BIB entry.
  * @param in_r_addr Inside IPv6 address of the remote host.
@@ -248,7 +254,8 @@ nat64_db_bib_entry_t *nat64_db_bib_entry_by_index (nat64_db_t * db,
  *
  * @returns BIB entry on success, 0 otherwise.
  */
-nat64_db_st_entry_t *nat64_db_st_entry_create (nat64_db_t * db,
+nat64_db_st_entry_t *nat64_db_st_entry_create (u32 thread_index,
+					       nat64_db_t * db,
 					       nat64_db_bib_entry_t * bibe,
 					       ip6_address_t * in_r_addr,
 					       ip4_address_t * out_r_addr,
@@ -257,10 +264,12 @@ nat64_db_st_entry_t *nat64_db_st_entry_create (nat64_db_t * db,
 /**
  * @brief Free NAT64 session table entry.
  *
+ * @param thread_index thread index.
  * @param db NAT64 DB.
  * @param ste Session table entry.
  */
-void nat64_db_st_entry_free (nat64_db_t * db, nat64_db_st_entry_t * ste);
+void nat64_db_st_entry_free (u32 thread_index, nat64_db_t * db,
+			     nat64_db_st_entry_t * ste);
 
 /**
  * @brief Find NAT64 session table entry.
@@ -308,18 +317,21 @@ void nat64_db_st_walk (nat64_db_t * db, u8 proto,
 /**
  * @brief Free expired session entries in session tables.
  *
+ * @param thread_index thread index.
  * @param db NAT64 DB.
  * @param now Current time.
  */
-void nad64_db_st_free_expired (nat64_db_t * db, u32 now);
+void nad64_db_st_free_expired (u32 thread_index, nat64_db_t * db, u32 now);
 
 /**
  * @brief Free sessions using specific outside address.
  *
+ * @param thread_index thread index.
  * @param db NAT64 DB.
  * @param out_addr Outside address to match.
  */
-void nat64_db_free_out_addr (nat64_db_t * db, ip4_address_t * out_addr);
+void nat64_db_free_out_addr (u32 thread_index, nat64_db_t * db,
+			     ip4_address_t * out_addr);
 
 /*
  * @brief Get ST entry index.
diff --git a/src/plugins/nat/nat64_in2out.c b/src/plugins/nat/nat64_in2out.c
index 660df093bf9..d30a9cbe519 100644
--- a/src/plugins/nat/nat64_in2out.c
+++ b/src/plugins/nat/nat64_in2out.c
@@ -217,8 +217,9 @@ nat64_in2out_tcp_udp_set_cb (ip6_header_t * ip6, ip4_header_t * ip4,
 	    return -1;
 
 	  bibe =
-	    nat64_db_bib_entry_create (db, &ip6->src_address, &out_addr,
-				       sport, out_port, fib_index, proto, 0);
+	    nat64_db_bib_entry_create (ctx->thread_index, db,
+				       &ip6->src_address, &out_addr, sport,
+				       out_port, fib_index, proto, 0);
 	  if (!bibe)
 	    return -1;
 
@@ -228,8 +229,8 @@ nat64_in2out_tcp_udp_set_cb (ip6_header_t * ip6, ip4_header_t * ip4,
 
       nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
       ste =
-	nat64_db_st_entry_create (db, bibe, &ip6->dst_address,
-				  &daddr.ip4, dport);
+	nat64_db_st_entry_create (ctx->thread_index, db, bibe,
+				  &ip6->dst_address, &daddr.ip4, dport);
       if (!ste)
 	return -1;
 
@@ -313,9 +314,10 @@ nat64_in2out_icmp_set_cb (ip6_header_t * ip6, ip4_header_t * ip4, void *arg)
 		return -1;
 
 	      bibe =
-		nat64_db_bib_entry_create (db, &ip6->src_address,
-					   &out_addr, in_id, out_id,
-					   fib_index, IP_PROTOCOL_ICMP, 0);
+		nat64_db_bib_entry_create (ctx->thread_index, db,
+					   &ip6->src_address, &out_addr,
+					   in_id, out_id, fib_index,
+					   IP_PROTOCOL_ICMP, 0);
 	      if (!bibe)
 		return -1;
 
@@ -325,8 +327,8 @@ nat64_in2out_icmp_set_cb (ip6_header_t * ip6, ip4_header_t * ip4, void *arg)
 
 	  nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
 	  ste =
-	    nat64_db_st_entry_create (db, bibe, &ip6->dst_address,
-				      &daddr.ip4, 0);
+	    nat64_db_st_entry_create (ctx->thread_index, db, bibe,
+				      &ip6->dst_address, &daddr.ip4, 0);
 	  if (!ste)
 	    return -1;
 
@@ -556,9 +558,9 @@ nat64_in2out_unk_proto_set_cb (ip6_header_t * ip6, ip4_header_t * ip4,
 	    return -1;
 
 	  bibe =
-	    nat64_db_bib_entry_create (db, &ip6->src_address,
-				       &ctx.out_addr, 0, 0, fib_index, proto,
-				       0);
+	    nat64_db_bib_entry_create (s_ctx->thread_index, db,
+				       &ip6->src_address, &ctx.out_addr,
+				       0, 0, fib_index, proto, 0);
 	  if (!bibe)
 	    return -1;
 
@@ -568,7 +570,8 @@ nat64_in2out_unk_proto_set_cb (ip6_header_t * ip6, ip4_header_t * ip4,
 
       nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
       ste =
-	nat64_db_st_entry_create (db, bibe, &ip6->dst_address, &daddr.ip4, 0);
+	nat64_db_st_entry_create (s_ctx->thread_index, db, bibe,
+				  &ip6->dst_address, &daddr.ip4, 0);
       if (!ste)
 	return -1;
 
@@ -649,8 +652,9 @@ nat64_in2out_tcp_udp_hairpinning (vlib_main_t * vm, vlib_buffer_t * b,
 	    return -1;
 
 	  bibe =
-	    nat64_db_bib_entry_create (db, &ip6->src_address, &out_addr,
-				       sport, out_port, fib_index, proto, 0);
+	    nat64_db_bib_entry_create (thread_index, db, &ip6->src_address,
+				       &out_addr, sport, out_port, fib_index,
+				       proto, 0);
 	  if (!bibe)
 	    return -1;
 
@@ -660,7 +664,7 @@ nat64_in2out_tcp_udp_hairpinning (vlib_main_t * vm, vlib_buffer_t * b,
 
       nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
       ste =
-	nat64_db_st_entry_create (db, bibe, &ip6->dst_address,
+	nat64_db_st_entry_create (thread_index, db, bibe, &ip6->dst_address,
 				  &daddr.ip4, dport);
       if (!ste)
 	return -1;
@@ -909,7 +913,7 @@ nat64_in2out_unk_proto_hairpinning (vlib_main_t * vm, vlib_buffer_t * b,
 	    return -1;
 
 	  bibe =
-	    nat64_db_bib_entry_create (db, &ip6->src_address,
+	    nat64_db_bib_entry_create (thread_index, db, &ip6->src_address,
 				       &ctx.out_addr, 0, 0, fib_index, proto,
 				       0);
 	  if (!bibe)
@@ -921,7 +925,8 @@ nat64_in2out_unk_proto_hairpinning (vlib_main_t * vm, vlib_buffer_t * b,
 
       nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
       ste =
-	nat64_db_st_entry_create (db, bibe, &ip6->dst_address, &daddr.ip4, 0);
+	nat64_db_st_entry_create (thread_index, db, bibe, &ip6->dst_address,
+				  &daddr.ip4, 0);
       if (!ste)
 	return -1;
 
@@ -1485,7 +1490,7 @@ nat64_in2out_reass_node_fn (vlib_main_t * vm,
 	      if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
 		{
 		  if (nat_ip6_reass_add_fragment
-		      (reass0, bi0, &fragments_to_drop))
+		      (thread_index, reass0, bi0, &fragments_to_drop))
 		    {
 		      b0->error = node->errors[NAT64_IN2OUT_ERROR_MAX_FRAG];
 		      next0 = NAT64_IN2OUT_NEXT_DROP;
@@ -1528,7 +1533,7 @@ nat64_in2out_reass_node_fn (vlib_main_t * vm,
 			}
 
 		      bibe0 =
-			nat64_db_bib_entry_create (db,
+			nat64_db_bib_entry_create (thread_index, db,
 						   &ip60->src_address,
 						   &out_addr0, udp0->src_port,
 						   out_port0, fib_index0,
@@ -1546,7 +1551,7 @@ nat64_in2out_reass_node_fn (vlib_main_t * vm,
 		  nat64_extract_ip4 (&ip60->dst_address, &daddr0.ip4,
 				     fib_index0);
 		  ste0 =
-		    nat64_db_st_entry_create (db, bibe0,
+		    nat64_db_st_entry_create (thread_index, db, bibe0,
 					      &ip60->dst_address, &daddr0.ip4,
 					      udp0->dst_port);
 		  if (!ste0)
diff --git a/src/plugins/nat/nat64_out2in.c b/src/plugins/nat/nat64_out2in.c
index 437052fc441..6c9e216d96e 100644
--- a/src/plugins/nat/nat64_out2in.c
+++ b/src/plugins/nat/nat64_out2in.c
@@ -164,7 +164,8 @@ nat64_out2in_tcp_udp_set_cb (ip4_header_t * ip4, ip6_header_t * ip6,
 
       nat64_compose_ip6 (&ip6_saddr, &ip4->src_address, bibe->fib_index);
       ste =
-	nat64_db_st_entry_create (db, bibe, &ip6_saddr, &saddr.ip4, sport);
+	nat64_db_st_entry_create (ctx->thread_index, db, bibe, &ip6_saddr,
+				  &saddr.ip4, sport);
 
       if (!ste)
 	return -1;
@@ -245,7 +246,8 @@ nat64_out2in_icmp_set_cb (ip4_header_t * ip4, ip6_header_t * ip6, void *arg)
 
 	  nat64_compose_ip6 (&ip6_saddr, &ip4->src_address, bibe->fib_index);
 	  ste =
-	    nat64_db_st_entry_create (db, bibe, &ip6_saddr, &saddr.ip4, 0);
+	    nat64_db_st_entry_create (ctx->thread_index, db,
+				      bibe, &ip6_saddr, &saddr.ip4, 0);
 
 	  if (!ste)
 	    return -1;
@@ -408,7 +410,8 @@ nat64_out2in_unk_proto_set_cb (ip4_header_t * ip4, ip6_header_t * ip6,
 	return -1;
 
       nat64_compose_ip6 (&ip6_saddr, &ip4->src_address, bibe->fib_index);
-      ste = nat64_db_st_entry_create (db, bibe, &ip6_saddr, &saddr.ip4, 0);
+      ste = nat64_db_st_entry_create (ctx->thread_index, db,
+				      bibe, &ip6_saddr, &saddr.ip4, 0);
 
       if (!ste)
 	return -1;
@@ -810,7 +813,8 @@ nat64_out2in_reass_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 		  nat64_compose_ip6 (&ip6_saddr0, &ip40->src_address,
 				     bibe0->fib_index);
 		  ste0 =
-		    nat64_db_st_entry_create (db, bibe0, &ip6_saddr0,
+		    nat64_db_st_entry_create (thread_index,
+					      db, bibe0, &ip6_saddr0,
 					      &saddr0.ip4, udp0->src_port);
 
 		  if (!ste0)
@@ -836,7 +840,7 @@ nat64_out2in_reass_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 	      if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
 		{
 		  if (nat_ip4_reass_add_fragment
-		      (reass0, bi0, &fragments_to_drop))
+		      (thread_index, reass0, bi0, &fragments_to_drop))
 		    {
 		      b0->error = node->errors[NAT64_OUT2IN_ERROR_MAX_FRAG];
 		      next0 = NAT64_OUT2IN_NEXT_DROP;
diff --git a/src/plugins/nat/nat_api.c b/src/plugins/nat/nat_api.c
index b1aa3243ab9..55e6bf9e90f 100644
--- a/src/plugins/nat/nat_api.c
+++ b/src/plugins/nat/nat_api.c
@@ -2324,7 +2324,8 @@ static void
 
   for (i = 0; i < count; i++)
     {
-      if ((rv = nat64_add_del_pool_addr (&this_addr, vrf_id, mp->is_add)))
+      if ((rv = nat64_add_del_pool_addr (vlib_get_thread_index (),
+					 &this_addr, vrf_id, mp->is_add)))
 	goto send_reply;
 
       increment_v4_address (&this_addr);
diff --git a/src/plugins/nat/nat_det.h b/src/plugins/nat/nat_det.h
index 7878a4c4ed6..d2966356d8a 100644
--- a/src/plugins/nat/nat_det.h
+++ b/src/plugins/nat/nat_det.h
@@ -147,8 +147,9 @@ snat_det_find_ses_by_in (snat_det_map_t * dm, ip4_address_t * in_addr,
 }
 
 always_inline snat_det_session_t *
-snat_det_ses_create (snat_det_map_t * dm, ip4_address_t * in_addr,
-		     u16 in_port, snat_det_out_key_t * out)
+snat_det_ses_create (u32 thread_index, snat_det_map_t * dm,
+		     ip4_address_t * in_addr, u16 in_port,
+		     snat_det_out_key_t * out)
 {
   u32 user_offset;
   u16 i;
@@ -171,7 +172,8 @@ snat_det_ses_create (snat_det_map_t * dm, ip4_address_t * in_addr,
 	}
     }
 
-  snat_ipfix_logging_max_entries_per_user (SNAT_DET_SES_PER_USER,
+  snat_ipfix_logging_max_entries_per_user (thread_index,
+					   SNAT_DET_SES_PER_USER,
 					   in_addr->as_u32);
   return 0;
 }
diff --git a/src/plugins/nat/nat_det_in2out.c b/src/plugins/nat/nat_det_in2out.c
index 1366c2fce69..e4aa0463a06 100644
--- a/src/plugins/nat/nat_det_in2out.c
+++ b/src/plugins/nat/nat_det_in2out.c
@@ -202,7 +202,8 @@ icmp_match_in2out_det (snat_main_t * sm, vlib_node_runtime_t * node,
 	    continue;
 
 	  ses0 =
-	    snat_det_ses_create (dm0, &in_addr, echo0->identifier, &key0);
+	    snat_det_ses_create (thread_index, dm0,
+				 &in_addr, echo0->identifier, &key0);
 	  break;
 	}
       if (PREDICT_FALSE (!ses0))
@@ -374,8 +375,8 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
 		    continue;
 
 		  ses0 =
-		    snat_det_ses_create (dm0, &ip0->src_address, tcp0->src,
-					 &key0);
+		    snat_det_ses_create (thread_index, dm0, &ip0->src_address,
+					 tcp0->src, &key0);
 		  break;
 		}
 	      if (PREDICT_FALSE (!ses0))
@@ -538,8 +539,8 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
 		    continue;
 
 		  ses1 =
-		    snat_det_ses_create (dm1, &ip1->src_address, tcp1->src,
-					 &key1);
+		    snat_det_ses_create (thread_index, dm1, &ip1->src_address,
+					 tcp1->src, &key1);
 		  break;
 		}
 	      if (PREDICT_FALSE (!ses1))
@@ -738,8 +739,8 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
 		    continue;
 
 		  ses0 =
-		    snat_det_ses_create (dm0, &ip0->src_address, tcp0->src,
-					 &key0);
+		    snat_det_ses_create (thread_index, dm0, &ip0->src_address,
+					 tcp0->src, &key0);
 		  break;
 		}
 	      if (PREDICT_FALSE (!ses0))
diff --git a/src/plugins/nat/nat_ipfix_logging.c b/src/plugins/nat/nat_ipfix_logging.c
index 8bee8e444dc..4bb96bcf4a0 100644
--- a/src/plugins/nat/nat_ipfix_logging.c
+++ b/src/plugins/nat/nat_ipfix_logging.c
@@ -19,7 +19,9 @@
 #include <vlibmemory/api.h>
 #include <nat/nat_inlines.h>
 #include <nat/nat_ipfix_logging.h>
+#include <vppinfra/atomics.h>
 
+vlib_node_registration_t snat_ipfix_flush_node;
 snat_ipfix_logging_main_t snat_ipfix_logging_main;
 
 #define NAT44_SESSION_CREATE_LEN 26
@@ -111,11 +113,17 @@ typedef struct
   u32 vrf_id;
 } nat_ipfix_logging_nat64_bib_args_t;
 
-#define skip_if_disabled()                                    \
-do {                                                          \
-  snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main; \
-  if (PREDICT_TRUE (!silm->enabled))                          \
-    return;                                                   \
+#define skip_if_disabled()                                        \
+do {                                                              \
+  snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;     \
+  if (PREDICT_TRUE (!clib_atomic_fetch_or(&silm->enabled, 0)))    \
+    return;                                                       \
+} while (0)
+
+#define update_template_id(old_id, new_id)                \
+do {                                                      \
+  u16 template_id = clib_atomic_fetch_or(old_id, 0);      \
+  clib_atomic_cmp_and_swap(old_id, template_id, new_id);  \
 } while (0)
 
 /**
@@ -151,56 +159,79 @@ snat_template_rewrite (flow_report_main_t * frm,
   ip4_ipfix_template_packet_t *tp;
   u32 field_count = 0;
   flow_report_stream_t *stream;
+  u32 stream_index;
 
   stream = &frm->streams[fr->stream_index];
-  silm->stream_index = fr->stream_index;
+
+  stream_index = clib_atomic_fetch_or(&silm->stream_index, 0);
+  clib_atomic_cmp_and_swap (&silm->stream_index,
+                            stream_index, fr->stream_index);
 
   if (event == NAT_ADDRESSES_EXHAUTED)
     {
       field_count = NAT_ADDRESSES_EXHAUTED_FIELD_COUNT;
-      silm->addr_exhausted_template_id = fr->template_id;
+
+      update_template_id(&silm->addr_exhausted_template_id,
+                         fr->template_id);
     }
   else if (event == NAT44_SESSION_CREATE)
     {
       field_count = NAT44_SESSION_CREATE_FIELD_COUNT;
-      silm->nat44_session_template_id = fr->template_id;
+
+      update_template_id(&silm->nat44_session_template_id,
+                         fr->template_id);
     }
   else if (event == NAT64_BIB_CREATE)
     {
       field_count = NAT64_BIB_FIELD_COUNT;
-      silm->nat64_bib_template_id = fr->template_id;
+
+      update_template_id(&silm->nat64_bib_template_id,
+                         fr->template_id);
     }
   else if (event == NAT64_SESSION_CREATE)
     {
       field_count = NAT64_SES_FIELD_COUNT;
-      silm->nat64_ses_template_id = fr->template_id;
+
+      update_template_id(&silm->nat64_ses_template_id,
+                         fr->template_id);
     }
   else if (event == QUOTA_EXCEEDED)
     {
       if (quota_event == MAX_ENTRIES_PER_USER)
 	{
 	  field_count = MAX_ENTRIES_PER_USER_FIELD_COUNT;
-	  silm->max_entries_per_user_template_id = fr->template_id;
+
+          update_template_id(&silm->max_entries_per_user_template_id,
+                             fr->template_id);
+
 	}
       else if (quota_event == MAX_SESSION_ENTRIES)
 	{
 	  field_count = MAX_SESSIONS_FIELD_COUNT;
-	  silm->max_sessions_template_id = fr->template_id;
+
+          update_template_id(&silm->max_sessions_template_id,
+                             fr->template_id);
 	}
       else if (quota_event == MAX_BIB_ENTRIES)
 	{
 	  field_count = MAX_BIBS_FIELD_COUNT;
-	  silm->max_bibs_template_id = fr->template_id;
+
+          update_template_id(&silm->max_bibs_template_id,
+                             fr->template_id);
 	}
       else if (quota_event == MAX_FRAGMENTS_PENDING_REASSEMBLY)
 	{
 	  field_count = MAX_FRAGMENTS_FIELD_COUNT;
-	  silm->max_frags_ip4_template_id = fr->template_id;
+
+          update_template_id(&silm->max_frags_ip4_template_id,
+                             fr->template_id);
 	}
       else if (quota_event == MAX_FRAGMENTS_PENDING_REASSEMBLY_IP6)
         {
           field_count = MAX_FRAGMENTS_FIELD_COUNT;
-          silm->max_frags_ip6_template_id = fr->template_id;
+
+          update_template_id(&silm->max_frags_ip6_template_id,
+                             fr->template_id);
         }
     }
 
@@ -530,10 +561,13 @@ snat_ipfix_header_create (flow_report_main_t * frm,
   ip4_ipfix_template_packet_t *tp;
   ipfix_message_header_t *h = 0;
   ipfix_set_header_t *s = 0;
+  u32 sequence_number;
+  u32 stream_index;
   ip4_header_t *ip;
   udp_header_t *udp;
-
-  stream = &frm->streams[silm->stream_index];
+  
+  stream_index = clib_atomic_fetch_or(&silm->stream_index, 0);
+  stream = &frm->streams[stream_index];
 
   b0->current_data = 0;
   b0->current_length = sizeof (*ip) + sizeof (*udp) + sizeof (*h) +
@@ -561,7 +595,9 @@ snat_ipfix_header_create (flow_report_main_t * frm,
 					 (((f64) frm->unix_time_0) +
 					  (vlib_time_now (frm->vlib_main) -
 					   frm->vlib_time_0)));
-  h->sequence_number = clib_host_to_net_u32 (stream->sequence_number++);
+
+  sequence_number = clib_atomic_fetch_add (&stream->sequence_number, 1);
+  h->sequence_number = clib_host_to_net_u32 (sequence_number);
   h->domain_id = clib_host_to_net_u32 (stream->domain_id);
 
   *offset = (u32) (((u8 *) (s + 1)) - (u8 *) tp);
@@ -608,11 +644,13 @@ snat_ipfix_send (flow_report_main_t * frm,
 }
 
 static void
-snat_ipfix_logging_nat44_ses (u8 nat_event, u32 src_ip, u32 nat_src_ip,
-			      snat_protocol_t snat_proto, u16 src_port,
-			      u16 nat_src_port, u32 vrf_id, int do_flush)
+snat_ipfix_logging_nat44_ses (u32 thread_index, u8 nat_event, u32 src_ip,
+                              u32 nat_src_ip, snat_protocol_t snat_proto,
+                              u16 src_port, u16 nat_src_port, u32 vrf_id,
+                              int do_flush)
 {
   snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
+  snat_ipfix_per_thread_data_t *sitd = &silm->per_thread_data[thread_index];
   flow_report_main_t *frm = &flow_report_main;
   vlib_frame_t *f;
   vlib_buffer_t *b0 = 0;
@@ -621,16 +659,14 @@ snat_ipfix_logging_nat44_ses (u8 nat_event, u32 src_ip, u32 nat_src_ip,
   vlib_main_t *vm = frm->vlib_main;
   u64 now;
   u8 proto = ~0;
-
-  if (!silm->enabled)
-    return;
+  u16 template_id;
 
   proto = snat_proto_to_ip_proto (snat_proto);
 
   now = (u64) ((vlib_time_now (vm) - silm->vlib_time_0) * 1e3);
   now += silm->milisecond_time_0;
 
-  b0 = silm->nat44_session_buffer;
+  b0 = sitd->nat44_session_buffer;
 
   if (PREDICT_FALSE (b0 == 0))
     {
@@ -643,22 +679,22 @@ snat_ipfix_logging_nat44_ses (u8 nat_event, u32 src_ip, u32 nat_src_ip,
 	  return;
 	}
 
-      b0 = silm->nat44_session_buffer = vlib_get_buffer (vm, bi0);
+      b0 = sitd->nat44_session_buffer = vlib_get_buffer (vm, bi0);
       VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
       offset = 0;
     }
   else
     {
       bi0 = vlib_get_buffer_index (vm, b0);
-      offset = silm->nat44_session_next_record_offset;
+      offset = sitd->nat44_session_next_record_offset;
     }
 
-  f = silm->nat44_session_frame;
+  f = sitd->nat44_session_frame;
   if (PREDICT_FALSE (f == 0))
     {
       u32 *to_next;
       f = vlib_get_frame_to_node (vm, ip4_lookup_node.index);
-      silm->nat44_session_frame = f;
+      sitd->nat44_session_frame = f;
       to_next = vlib_frame_vector_args (f);
       to_next[0] = bi0;
       f->n_vectors = 1;
@@ -700,18 +736,22 @@ snat_ipfix_logging_nat44_ses (u8 nat_event, u32 src_ip, u32 nat_src_ip,
   if (PREDICT_FALSE
       (do_flush || (offset + NAT44_SESSION_CREATE_LEN) > frm->path_mtu))
     {
-      snat_ipfix_send (frm, f, b0, silm->nat44_session_template_id);
-      silm->nat44_session_frame = 0;
-      silm->nat44_session_buffer = 0;
+      template_id = clib_atomic_fetch_or (
+        &silm->nat44_session_template_id,
+        0);
+      snat_ipfix_send (frm, f, b0, template_id);
+      sitd->nat44_session_frame = 0;
+      sitd->nat44_session_buffer = 0;
       offset = 0;
     }
-  silm->nat44_session_next_record_offset = offset;
+  sitd->nat44_session_next_record_offset = offset;
 }
 
 static void
-snat_ipfix_logging_addr_exhausted (u32 pool_id, int do_flush)
+snat_ipfix_logging_addr_exhausted (u32 thread_index, u32 pool_id, int do_flush)
 {
   snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
+  snat_ipfix_per_thread_data_t *sitd = &silm->per_thread_data[thread_index];
   flow_report_main_t *frm = &flow_report_main;
   vlib_frame_t *f;
   vlib_buffer_t *b0 = 0;
@@ -720,14 +760,12 @@ snat_ipfix_logging_addr_exhausted (u32 pool_id, int do_flush)
   vlib_main_t *vm = frm->vlib_main;
   u64 now;
   u8 nat_event = NAT_ADDRESSES_EXHAUTED;
-
-  if (!silm->enabled)
-    return;
+  u16 template_id;
 
   now = (u64) ((vlib_time_now (vm) - silm->vlib_time_0) * 1e3);
   now += silm->milisecond_time_0;
 
-  b0 = silm->addr_exhausted_buffer;
+  b0 = sitd->addr_exhausted_buffer;
 
   if (PREDICT_FALSE (b0 == 0))
     {
@@ -740,22 +778,22 @@ snat_ipfix_logging_addr_exhausted (u32 pool_id, int do_flush)
 	  return;
 	}
 
-      b0 = silm->addr_exhausted_buffer = vlib_get_buffer (vm, bi0);
+      b0 = sitd->addr_exhausted_buffer = vlib_get_buffer (vm, bi0);
       VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
       offset = 0;
     }
   else
     {
       bi0 = vlib_get_buffer_index (vm, b0);
-      offset = silm->addr_exhausted_next_record_offset;
+      offset = sitd->addr_exhausted_next_record_offset;
     }
 
-  f = silm->addr_exhausted_frame;
+  f = sitd->addr_exhausted_frame;
   if (PREDICT_FALSE (f == 0))
     {
       u32 *to_next;
       f = vlib_get_frame_to_node (vm, ip4_lookup_node.index);
-      silm->addr_exhausted_frame = f;
+      sitd->addr_exhausted_frame = f;
       to_next = vlib_frame_vector_args (f);
       to_next[0] = bi0;
       f->n_vectors = 1;
@@ -782,18 +820,23 @@ snat_ipfix_logging_addr_exhausted (u32 pool_id, int do_flush)
   if (PREDICT_FALSE
       (do_flush || (offset + NAT_ADDRESSES_EXHAUTED_LEN) > frm->path_mtu))
     {
-      snat_ipfix_send (frm, f, b0, silm->addr_exhausted_template_id);
-      silm->addr_exhausted_frame = 0;
-      silm->addr_exhausted_buffer = 0;
+      template_id = clib_atomic_fetch_or (
+          &silm->addr_exhausted_template_id,
+          0);
+      snat_ipfix_send (frm, f, b0, template_id);
+      sitd->addr_exhausted_frame = 0;
+      sitd->addr_exhausted_buffer = 0;
       offset = 0;
     }
-  silm->addr_exhausted_next_record_offset = offset;
+  sitd->addr_exhausted_next_record_offset = offset;
 }
 
 static void
-snat_ipfix_logging_max_entries_per_usr (u32 limit, u32 src_ip, int do_flush)
+snat_ipfix_logging_max_entries_per_usr (u32 thread_index,
+                                        u32 limit, u32 src_ip, int do_flush)
 {
   snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
+  snat_ipfix_per_thread_data_t *sitd = &silm->per_thread_data[thread_index];
   flow_report_main_t *frm = &flow_report_main;
   vlib_frame_t *f;
   vlib_buffer_t *b0 = 0;
@@ -803,14 +846,12 @@ snat_ipfix_logging_max_entries_per_usr (u32 limit, u32 src_ip, int do_flush)
   u64 now;
   u8 nat_event = QUOTA_EXCEEDED;
   u32 quota_event = MAX_ENTRIES_PER_USER;
-
-  if (!silm->enabled)
-    return;
+  u16 template_id;
 
   now = (u64) ((vlib_time_now (vm) - silm->vlib_time_0) * 1e3);
   now += silm->milisecond_time_0;
 
-  b0 = silm->max_entries_per_user_buffer;
+  b0 = sitd->max_entries_per_user_buffer;
 
   if (PREDICT_FALSE (b0 == 0))
     {
@@ -823,22 +864,22 @@ snat_ipfix_logging_max_entries_per_usr (u32 limit, u32 src_ip, int do_flush)
 	  return;
 	}
 
-      b0 = silm->max_entries_per_user_buffer = vlib_get_buffer (vm, bi0);
+      b0 = sitd->max_entries_per_user_buffer = vlib_get_buffer (vm, bi0);
       VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
       offset = 0;
     }
   else
     {
       bi0 = vlib_get_buffer_index (vm, b0);
-      offset = silm->max_entries_per_user_next_record_offset;
+      offset = sitd->max_entries_per_user_next_record_offset;
     }
 
-  f = silm->max_entries_per_user_frame;
+  f = sitd->max_entries_per_user_frame;
   if (PREDICT_FALSE (f == 0))
     {
       u32 *to_next;
       f = vlib_get_frame_to_node (vm, ip4_lookup_node.index);
-      silm->max_entries_per_user_frame = f;
+      sitd->max_entries_per_user_frame = f;
       to_next = vlib_frame_vector_args (f);
       to_next[0] = bi0;
       f->n_vectors = 1;
@@ -871,18 +912,22 @@ snat_ipfix_logging_max_entries_per_usr (u32 limit, u32 src_ip, int do_flush)
   if (PREDICT_FALSE
       (do_flush || (offset + MAX_ENTRIES_PER_USER_LEN) > frm->path_mtu))
     {
-      snat_ipfix_send (frm, f, b0, silm->max_entries_per_user_template_id);
-      silm->max_entries_per_user_frame = 0;
-      silm->max_entries_per_user_buffer = 0;
+      template_id = clib_atomic_fetch_or (
+          &silm->max_entries_per_user_template_id,
+          0);
+      snat_ipfix_send (frm, f, b0, template_id);
+      sitd->max_entries_per_user_frame = 0;
+      sitd->max_entries_per_user_buffer = 0;
       offset = 0;
     }
-  silm->max_entries_per_user_next_record_offset = offset;
+  sitd->max_entries_per_user_next_record_offset = offset;
 }
 
 static void
-nat_ipfix_logging_max_ses (u32 limit, int do_flush)
+nat_ipfix_logging_max_ses (u32 thread_index, u32 limit, int do_flush)
 {
   snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
+  snat_ipfix_per_thread_data_t *sitd = &silm->per_thread_data[thread_index];
   flow_report_main_t *frm = &flow_report_main;
   vlib_frame_t *f;
   vlib_buffer_t *b0 = 0;
@@ -892,14 +937,12 @@ nat_ipfix_logging_max_ses (u32 limit, int do_flush)
   u64 now;
   u8 nat_event = QUOTA_EXCEEDED;
   u32 quota_event = MAX_SESSION_ENTRIES;
-
-  if (!silm->enabled)
-    return;
+  u16 template_id;
 
   now = (u64) ((vlib_time_now (vm) - silm->vlib_time_0) * 1e3);
   now += silm->milisecond_time_0;
 
-  b0 = silm->max_sessions_buffer;
+  b0 = sitd->max_sessions_buffer;
 
   if (PREDICT_FALSE (b0 == 0))
     {
@@ -912,22 +955,22 @@ nat_ipfix_logging_max_ses (u32 limit, int do_flush)
 	  return;
 	}
 
-      b0 = silm->max_sessions_buffer = vlib_get_buffer (vm, bi0);
+      b0 = sitd->max_sessions_buffer = vlib_get_buffer (vm, bi0);
       VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
       offset = 0;
     }
   else
     {
       bi0 = vlib_get_buffer_index (vm, b0);
-      offset = silm->max_sessions_next_record_offset;
+      offset = sitd->max_sessions_next_record_offset;
     }
 
-  f = silm->max_sessions_frame;
+  f = sitd->max_sessions_frame;
   if (PREDICT_FALSE (f == 0))
     {
       u32 *to_next;
       f = vlib_get_frame_to_node (vm, ip4_lookup_node.index);
-      silm->max_sessions_frame = f;
+      sitd->max_sessions_frame = f;
       to_next = vlib_frame_vector_args (f);
       to_next[0] = bi0;
       f->n_vectors = 1;
@@ -957,18 +1000,22 @@ nat_ipfix_logging_max_ses (u32 limit, int do_flush)
   if (PREDICT_FALSE
       (do_flush || (offset + MAX_SESSIONS_LEN) > frm->path_mtu))
     {
-      snat_ipfix_send (frm, f, b0, silm->max_sessions_template_id);
-      silm->max_sessions_frame = 0;
-      silm->max_sessions_buffer = 0;
+      template_id = clib_atomic_fetch_or (
+        &silm->max_sessions_template_id,
+        0);
+      snat_ipfix_send (frm, f, b0, template_id);
+      sitd->max_sessions_frame = 0;
+      sitd->max_sessions_buffer = 0;
       offset = 0;
     }
-  silm->max_sessions_next_record_offset = offset;
+  sitd->max_sessions_next_record_offset = offset;
 }
 
 static void
-nat_ipfix_logging_max_bib (u32 limit, int do_flush)
+nat_ipfix_logging_max_bib (u32 thread_index, u32 limit, int do_flush)
 {
   snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
+  snat_ipfix_per_thread_data_t *sitd = &silm->per_thread_data[thread_index];
   flow_report_main_t *frm = &flow_report_main;
   vlib_frame_t *f;
   vlib_buffer_t *b0 = 0;
@@ -978,14 +1025,12 @@ nat_ipfix_logging_max_bib (u32 limit, int do_flush)
   u64 now;
   u8 nat_event = QUOTA_EXCEEDED;
   u32 quota_event = MAX_BIB_ENTRIES;
-
-  if (!silm->enabled)
-    return;
+  u16 template_id;
 
   now = (u64) ((vlib_time_now (vm) - silm->vlib_time_0) * 1e3);
   now += silm->milisecond_time_0;
 
-  b0 = silm->max_bibs_buffer;
+  b0 = sitd->max_bibs_buffer;
 
   if (PREDICT_FALSE (b0 == 0))
     {
@@ -998,22 +1043,22 @@ nat_ipfix_logging_max_bib (u32 limit, int do_flush)
 	  return;
 	}
 
-      b0 = silm->max_bibs_buffer = vlib_get_buffer (vm, bi0);
+      b0 = sitd->max_bibs_buffer = vlib_get_buffer (vm, bi0);
       VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
       offset = 0;
     }
   else
     {
       bi0 = vlib_get_buffer_index (vm, b0);
-      offset = silm->max_bibs_next_record_offset;
+      offset = sitd->max_bibs_next_record_offset;
     }
 
-  f = silm->max_bibs_frame;
+  f = sitd->max_bibs_frame;
   if (PREDICT_FALSE (f == 0))
     {
       u32 *to_next;
       f = vlib_get_frame_to_node (vm, ip4_lookup_node.index);
-      silm->max_bibs_frame = f;
+      sitd->max_bibs_frame = f;
       to_next = vlib_frame_vector_args (f);
       to_next[0] = bi0;
       f->n_vectors = 1;
@@ -1043,18 +1088,23 @@ nat_ipfix_logging_max_bib (u32 limit, int do_flush)
   if (PREDICT_FALSE
       (do_flush || (offset + MAX_BIBS_LEN) > frm->path_mtu))
     {
-      snat_ipfix_send (frm, f, b0, silm->max_bibs_template_id);
-      silm->max_bibs_frame = 0;
-      silm->max_bibs_buffer = 0;
+      template_id = clib_atomic_fetch_or (
+        &silm->max_bibs_template_id,
+        0);
+      snat_ipfix_send (frm, f, b0, template_id);
+      sitd->max_bibs_frame = 0;
+      sitd->max_bibs_buffer = 0;
       offset = 0;
     }
-  silm->max_bibs_next_record_offset = offset;
+  sitd->max_bibs_next_record_offset = offset;
 }
 
 static void
-nat_ipfix_logging_max_frag_ip4 (u32 limit, u32 src, int do_flush)
+nat_ipfix_logging_max_frag_ip4 (u32 thread_index,
+                                u32 limit, u32 src, int do_flush)
 {
   snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
+  snat_ipfix_per_thread_data_t *sitd = &silm->per_thread_data[thread_index];
   flow_report_main_t *frm = &flow_report_main;
   vlib_frame_t *f;
   vlib_buffer_t *b0 = 0;
@@ -1064,14 +1114,12 @@ nat_ipfix_logging_max_frag_ip4 (u32 limit, u32 src, int do_flush)
   u64 now;
   u8 nat_event = QUOTA_EXCEEDED;
   u32 quota_event = MAX_FRAGMENTS_PENDING_REASSEMBLY;
-
-  if (!silm->enabled)
-    return;
+  u16 template_id;
 
   now = (u64) ((vlib_time_now (vm) - silm->vlib_time_0) * 1e3);
   now += silm->milisecond_time_0;
 
-  b0 = silm->max_frags_ip4_buffer;
+  b0 = sitd->max_frags_ip4_buffer;
 
   if (PREDICT_FALSE (b0 == 0))
     {
@@ -1084,22 +1132,22 @@ nat_ipfix_logging_max_frag_ip4 (u32 limit, u32 src, int do_flush)
 	  return;
 	}
 
-      b0 = silm->max_frags_ip4_buffer = vlib_get_buffer (vm, bi0);
+      b0 = sitd->max_frags_ip4_buffer = vlib_get_buffer (vm, bi0);
       VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
       offset = 0;
     }
   else
     {
       bi0 = vlib_get_buffer_index (vm, b0);
-      offset = silm->max_frags_ip4_next_record_offset;
+      offset = sitd->max_frags_ip4_next_record_offset;
     }
 
-  f = silm->max_frags_ip4_frame;
+  f = sitd->max_frags_ip4_frame;
   if (PREDICT_FALSE (f == 0))
     {
       u32 *to_next;
       f = vlib_get_frame_to_node (vm, ip4_lookup_node.index);
-      silm->max_frags_ip4_frame = f;
+      sitd->max_frags_ip4_frame = f;
       to_next = vlib_frame_vector_args (f);
       to_next[0] = bi0;
       f->n_vectors = 1;
@@ -1132,18 +1180,23 @@ nat_ipfix_logging_max_frag_ip4 (u32 limit, u32 src, int do_flush)
   if (PREDICT_FALSE
       (do_flush || (offset + MAX_BIBS_LEN) > frm->path_mtu))
     {
-      snat_ipfix_send (frm, f, b0, silm->max_frags_ip4_template_id);
-      silm->max_frags_ip4_frame = 0;
-      silm->max_frags_ip4_buffer = 0;
+      template_id = clib_atomic_fetch_or (
+        &silm->max_frags_ip4_template_id,
+        0);
+      snat_ipfix_send (frm, f, b0, template_id);
+      sitd->max_frags_ip4_frame = 0;
+      sitd->max_frags_ip4_buffer = 0;
       offset = 0;
     }
-  silm->max_frags_ip4_next_record_offset = offset;
+  sitd->max_frags_ip4_next_record_offset = offset;
 }
 
 static void
-nat_ipfix_logging_max_frag_ip6 (u32 limit, ip6_address_t * src, int do_flush)
+nat_ipfix_logging_max_frag_ip6 (u32 thread_index,
+                                u32 limit, ip6_address_t * src, int do_flush)
 {
   snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
+  snat_ipfix_per_thread_data_t *sitd = &silm->per_thread_data[thread_index];
   flow_report_main_t *frm = &flow_report_main;
   vlib_frame_t *f;
   vlib_buffer_t *b0 = 0;
@@ -1153,14 +1206,12 @@ nat_ipfix_logging_max_frag_ip6 (u32 limit, ip6_address_t * src, int do_flush)
   u64 now;
   u8 nat_event = QUOTA_EXCEEDED;
   u32 quota_event = MAX_FRAGMENTS_PENDING_REASSEMBLY;
-
-  if (!silm->enabled)
-    return;
+  u16 template_id;
 
   now = (u64) ((vlib_time_now (vm) - silm->vlib_time_0) * 1e3);
   now += silm->milisecond_time_0;
 
-  b0 = silm->max_frags_ip6_buffer;
+  b0 = sitd->max_frags_ip6_buffer;
 
   if (PREDICT_FALSE (b0 == 0))
     {
@@ -1173,22 +1224,22 @@ nat_ipfix_logging_max_frag_ip6 (u32 limit, ip6_address_t * src, int do_flush)
 	  return;
 	}
 
-      b0 = silm->max_frags_ip6_buffer = vlib_get_buffer (vm, bi0);
+      b0 = sitd->max_frags_ip6_buffer = vlib_get_buffer (vm, bi0);
       VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
       offset = 0;
     }
   else
     {
       bi0 = vlib_get_buffer_index (vm, b0);
-      offset = silm->max_frags_ip6_next_record_offset;
+      offset = sitd->max_frags_ip6_next_record_offset;
     }
 
-  f = silm->max_frags_ip6_frame;
+  f = sitd->max_frags_ip6_frame;
   if (PREDICT_FALSE (f == 0))
     {
       u32 *to_next;
       f = vlib_get_frame_to_node (vm, ip4_lookup_node.index);
-      silm->max_frags_ip6_frame = f;
+      sitd->max_frags_ip6_frame = f;
       to_next = vlib_frame_vector_args (f);
       to_next[0] = bi0;
       f->n_vectors = 1;
@@ -1221,20 +1272,25 @@ nat_ipfix_logging_max_frag_ip6 (u32 limit, ip6_address_t * src, int do_flush)
   if (PREDICT_FALSE
       (do_flush || (offset + MAX_BIBS_LEN) > frm->path_mtu))
     {
-      snat_ipfix_send (frm, f, b0, silm->max_frags_ip6_template_id);
-      silm->max_frags_ip6_frame = 0;
-      silm->max_frags_ip6_buffer = 0;
+      template_id = clib_atomic_fetch_or (
+        &silm->max_frags_ip6_template_id,
+        0);
+      snat_ipfix_send (frm, f, b0, template_id);
+      sitd->max_frags_ip6_frame = 0;
+      sitd->max_frags_ip6_buffer = 0;
       offset = 0;
     }
-  silm->max_frags_ip6_next_record_offset = offset;
+  sitd->max_frags_ip6_next_record_offset = offset;
 }
 
 static void
-nat_ipfix_logging_nat64_bibe (u8 nat_event, ip6_address_t * src_ip,
-                              u32 nat_src_ip, u8 proto, u16 src_port,
-                              u16 nat_src_port, u32 vrf_id, int do_flush)
+nat_ipfix_logging_nat64_bibe (u32 thread_index, u8 nat_event,
+                              ip6_address_t * src_ip, u32 nat_src_ip,
+                              u8 proto, u16 src_port, u16 nat_src_port,
+                              u32 vrf_id, int do_flush)
 {
   snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
+  snat_ipfix_per_thread_data_t *sitd = &silm->per_thread_data[thread_index];
   flow_report_main_t *frm = &flow_report_main;
   vlib_frame_t *f;
   vlib_buffer_t *b0 = 0;
@@ -1242,14 +1298,12 @@ nat_ipfix_logging_nat64_bibe (u8 nat_event, ip6_address_t * src_ip,
   u32 offset;
   vlib_main_t *vm = frm->vlib_main;
   u64 now;
-
-  if (!silm->enabled)
-    return;
+  u16 template_id;
 
   now = (u64) ((vlib_time_now (vm) - silm->vlib_time_0) * 1e3);
   now += silm->milisecond_time_0;
 
-  b0 = silm->nat64_bib_buffer;
+  b0 = sitd->nat64_bib_buffer;
 
   if (PREDICT_FALSE (b0 == 0))
     {
@@ -1262,22 +1316,22 @@ nat_ipfix_logging_nat64_bibe (u8 nat_event, ip6_address_t * src_ip,
 	  return;
 	}
 
-      b0 = silm->nat64_bib_buffer = vlib_get_buffer (vm, bi0);
+      b0 = sitd->nat64_bib_buffer = vlib_get_buffer (vm, bi0);
       VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
       offset = 0;
     }
   else
     {
       bi0 = vlib_get_buffer_index (vm, b0);
-      offset = silm->nat64_bib_next_record_offset;
+      offset = sitd->nat64_bib_next_record_offset;
     }
 
-  f = silm->nat64_bib_frame;
+  f = sitd->nat64_bib_frame;
   if (PREDICT_FALSE (f == 0))
     {
       u32 *to_next;
       f = vlib_get_frame_to_node (vm, ip4_lookup_node.index);
-      silm->nat64_bib_frame = f;
+      sitd->nat64_bib_frame = f;
       to_next = vlib_frame_vector_args (f);
       to_next[0] = bi0;
       f->n_vectors = 1;
@@ -1319,22 +1373,27 @@ nat_ipfix_logging_nat64_bibe (u8 nat_event, ip6_address_t * src_ip,
   if (PREDICT_FALSE
       (do_flush || (offset + NAT64_BIB_LEN) > frm->path_mtu))
     {
-      snat_ipfix_send (frm, f, b0, silm->nat64_bib_template_id);
-      silm->nat64_bib_frame = 0;
-      silm->nat64_bib_buffer = 0;
+      template_id = clib_atomic_fetch_or (
+        &silm->nat64_bib_template_id,
+        0);
+      snat_ipfix_send (frm, f, b0, template_id);
+      sitd->nat64_bib_frame = 0;
+      sitd->nat64_bib_buffer = 0;
       offset = 0;
     }
-  silm->nat64_bib_next_record_offset = offset;
+  sitd->nat64_bib_next_record_offset = offset;
 }
 
 static void
-nat_ipfix_logging_nat64_ses (u8 nat_event, ip6_address_t * src_ip,
-                             u32 nat_src_ip, u8 proto, u16 src_port,
-                             u16 nat_src_port, ip6_address_t * dst_ip,
-                             u32 nat_dst_ip, u16 dst_port, u16 nat_dst_port,
+nat_ipfix_logging_nat64_ses (u32 thread_index, u8 nat_event,
+                             ip6_address_t * src_ip, u32 nat_src_ip,
+                             u8 proto, u16 src_port, u16 nat_src_port,
+                             ip6_address_t * dst_ip, u32 nat_dst_ip,
+                             u16 dst_port, u16 nat_dst_port,
                              u32 vrf_id, int do_flush)
 {
   snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
+  snat_ipfix_per_thread_data_t *sitd = &silm->per_thread_data[thread_index];
   flow_report_main_t *frm = &flow_report_main;
   vlib_frame_t *f;
   vlib_buffer_t *b0 = 0;
@@ -1342,14 +1401,12 @@ nat_ipfix_logging_nat64_ses (u8 nat_event, ip6_address_t * src_ip,
   u32 offset;
   vlib_main_t *vm = frm->vlib_main;
   u64 now;
-
-  if (!silm->enabled)
-    return;
+  u16 template_id;
 
   now = (u64) ((vlib_time_now (vm) - silm->vlib_time_0) * 1e3);
   now += silm->milisecond_time_0;
 
-  b0 = silm->nat64_ses_buffer;
+  b0 = sitd->nat64_ses_buffer;
 
   if (PREDICT_FALSE (b0 == 0))
     {
@@ -1362,22 +1419,22 @@ nat_ipfix_logging_nat64_ses (u8 nat_event, ip6_address_t * src_ip,
 	  return;
 	}
 
-      b0 = silm->nat64_ses_buffer = vlib_get_buffer (vm, bi0);
+      b0 = sitd->nat64_ses_buffer = vlib_get_buffer (vm, bi0);
       VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
       offset = 0;
     }
   else
     {
       bi0 = vlib_get_buffer_index (vm, b0);
-      offset = silm->nat64_ses_next_record_offset;
+      offset = sitd->nat64_ses_next_record_offset;
     }
 
-  f = silm->nat64_ses_frame;
+  f = sitd->nat64_ses_frame;
   if (PREDICT_FALSE (f == 0))
     {
       u32 *to_next;
       f = vlib_get_frame_to_node (vm, ip4_lookup_node.index);
-      silm->nat64_ses_frame = f;
+      sitd->nat64_ses_frame = f;
       to_next = vlib_frame_vector_args (f);
       to_next[0] = bi0;
       f->n_vectors = 1;
@@ -1431,25 +1488,73 @@ nat_ipfix_logging_nat64_ses (u8 nat_event, ip6_address_t * src_ip,
   if (PREDICT_FALSE
       (do_flush || (offset + NAT64_SES_LEN) > frm->path_mtu))
     {
-      snat_ipfix_send (frm, f, b0, silm->nat64_ses_template_id);
-      silm->nat64_ses_frame = 0;
-      silm->nat64_ses_buffer = 0;
+      template_id = clib_atomic_fetch_or (
+        &silm->nat64_ses_template_id,
+        0);
+      snat_ipfix_send (frm, f, b0, template_id);
+      sitd->nat64_ses_frame = 0;
+      sitd->nat64_ses_buffer = 0;
       offset = 0;
     }
-  silm->nat64_ses_next_record_offset = offset;
+  sitd->nat64_ses_next_record_offset = offset;
 }
 
-static void
-snat_ipfix_logging_nat44_ses_rpc_cb (snat_ipfix_logging_nat44_ses_args_t * a)
+void
+snat_ipfix_flush (u32 thread_index)
 {
-  snat_ipfix_logging_nat44_ses (a->nat_event, a->src_ip, a->nat_src_ip,
-				a->snat_proto, a->src_port, a->nat_src_port,
-				a->vrf_id, 0);
+  int do_flush = 1;
+
+  snat_ipfix_logging_nat44_ses (thread_index,
+                                0, 0, 0, 0, 0, 0, 0, do_flush);
+  snat_ipfix_logging_addr_exhausted (thread_index, 0, do_flush);
+  snat_ipfix_logging_max_entries_per_usr (thread_index, 0, 0, do_flush);
+  nat_ipfix_logging_max_ses (thread_index, 0, do_flush);
+  nat_ipfix_logging_max_bib (thread_index, 0, do_flush);
+  nat_ipfix_logging_max_frag_ip4 (thread_index, 0, 0, do_flush);
+  nat_ipfix_logging_max_frag_ip6 (thread_index, 0, 0, do_flush);
+  nat_ipfix_logging_nat64_bibe (thread_index,
+                                0, 0, 0, 0, 0, 0, 0, do_flush);
+  nat_ipfix_logging_nat64_ses (thread_index,
+                               0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, do_flush);
+}
+
+void
+snat_ipfix_flush_from_main (void)
+{
+  snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
+  vlib_main_t *worker_vm;
+  int i;
+
+  if (PREDICT_TRUE (!clib_atomic_fetch_or(&silm->enabled, 0)))
+    return;
+
+  if (PREDICT_FALSE (!silm->worker_vms))
+    {
+      for (i = 1; i < vec_len (vlib_mains); i++)
+        {
+          worker_vm = vlib_mains[i];
+          if (worker_vm)
+            vec_add1 (silm->worker_vms, worker_vm);
+        }
+    }
+
+  /* Trigger flush for each worker thread */
+  for (i = 1; i < vec_len (silm->worker_vms); i++)
+    {
+      worker_vm = silm->worker_vms[i];
+      if (worker_vm)
+	vlib_node_set_interrupt_pending (worker_vm,
+					 snat_ipfix_flush_node.index);
+    }
+
+  /* Finally flush main thread */
+  snat_ipfix_flush (0);
 }
 
 /**
  * @brief Generate NAT44 session create event
  *
+ * @param thread_index thread index
  * @param src_ip       source IPv4 address
  * @param nat_src_ip   transaltes source IPv4 address
  * @param snat_proto   NAT transport protocol
@@ -1458,31 +1563,24 @@ snat_ipfix_logging_nat44_ses_rpc_cb (snat_ipfix_logging_nat44_ses_args_t * a)
  * @param vrf_id       VRF ID
  */
 void
-snat_ipfix_logging_nat44_ses_create (u32 src_ip,
+snat_ipfix_logging_nat44_ses_create (u32 thread_index,
+                                     u32 src_ip,
 				     u32 nat_src_ip,
 				     snat_protocol_t snat_proto,
 				     u16 src_port,
 				     u16 nat_src_port, u32 vrf_id)
 {
-  snat_ipfix_logging_nat44_ses_args_t a;
-
   skip_if_disabled ();
 
-  a.nat_event = NAT44_SESSION_CREATE;
-  a.src_ip = src_ip;
-  a.nat_src_ip = nat_src_ip;
-  a.snat_proto = snat_proto;
-  a.src_port = src_port;
-  a.nat_src_port = nat_src_port;
-  a.vrf_id = vrf_id;
-
-  vl_api_rpc_call_main_thread (snat_ipfix_logging_nat44_ses_rpc_cb,
-			       (u8 *) & a, sizeof (a));
+  snat_ipfix_logging_nat44_ses (thread_index, NAT44_SESSION_CREATE, src_ip,
+                                nat_src_ip, snat_proto, src_port, nat_src_port,
+				vrf_id, 0);
 }
 
 /**
  * @brief Generate NAT44 session delete event
  *
+ * @param thread_index thread index
  * @param src_ip       source IPv4 address
  * @param nat_src_ip   transaltes source IPv4 address
  * @param snat_proto   NAT transport protocol
@@ -1491,274 +1589,131 @@ snat_ipfix_logging_nat44_ses_create (u32 src_ip,
  * @param vrf_id       VRF ID
  */
 void
-snat_ipfix_logging_nat44_ses_delete (u32 src_ip,
+snat_ipfix_logging_nat44_ses_delete (u32 thread_index,
+                                     u32 src_ip,
 				     u32 nat_src_ip,
 				     snat_protocol_t snat_proto,
 				     u16 src_port,
 				     u16 nat_src_port, u32 vrf_id)
 {
-  snat_ipfix_logging_nat44_ses_args_t a;
-
   skip_if_disabled ();
 
-  a.nat_event = NAT44_SESSION_DELETE;
-  a.src_ip = src_ip;
-  a.nat_src_ip = nat_src_ip;
-  a.snat_proto = snat_proto;
-  a.src_port = src_port;
-  a.nat_src_port = nat_src_port;
-  a.vrf_id = vrf_id;
-
-  vl_api_rpc_call_main_thread (snat_ipfix_logging_nat44_ses_rpc_cb,
-			       (u8 *) & a, sizeof (a));
-}
-
-vlib_frame_t *
-snat_data_callback_nat44_session (flow_report_main_t * frm,
-				  flow_report_t * fr,
-				  vlib_frame_t * f,
-				  u32 * to_next, u32 node_index)
-{
-  snat_ipfix_logging_nat44_ses (0, 0, 0, 0, 0, 0, 0, 1);
-  return f;
-}
-
-static void
-  snat_ipfix_logging_addr_exhausted_rpc_cb
-  (snat_ipfix_logging_addr_exhausted_args_t * a)
-{
-  snat_ipfix_logging_addr_exhausted (a->pool_id, 0);
+  snat_ipfix_logging_nat44_ses (thread_index, NAT44_SESSION_DELETE, src_ip,
+                                nat_src_ip, snat_proto, src_port, nat_src_port,
+				vrf_id, 0);
 }
 
 /**
  * @brief Generate NAT addresses exhausted event
  *
+ * @param thread_index thread index
  * @param pool_id NAT pool ID
  */
 void
-snat_ipfix_logging_addresses_exhausted (u32 pool_id)
+snat_ipfix_logging_addresses_exhausted (u32 thread_index, u32 pool_id)
 {
   //TODO: This event SHOULD be rate limited
-  snat_ipfix_logging_addr_exhausted_args_t a;
-
   skip_if_disabled ();
 
-  a.pool_id = pool_id;
-
-  vl_api_rpc_call_main_thread (snat_ipfix_logging_addr_exhausted_rpc_cb,
-			       (u8 *) & a, sizeof (a));
-}
-
-vlib_frame_t *
-snat_data_callback_addr_exhausted (flow_report_main_t * frm,
-				   flow_report_t * fr,
-				   vlib_frame_t * f,
-				   u32 * to_next, u32 node_index)
-{
-  snat_ipfix_logging_addr_exhausted (0, 1);
-  return f;
-}
-
-static void
-  snat_ipfix_logging_max_entries_per_usr_rpc_cb
-  (snat_ipfix_logging_max_entries_per_user_args_t * a)
-{
-  snat_ipfix_logging_max_entries_per_usr (a->limit, a->src_ip, 0);
+  snat_ipfix_logging_addr_exhausted (thread_index, pool_id, 0);
 }
 
 /**
  * @brief Generate maximum entries per user exceeded event
  *
+ * @param thread_index thread index
  * @param limit maximum NAT entries that can be created per user
  * @param src_ip source IPv4 address
  */
 void
-snat_ipfix_logging_max_entries_per_user (u32 limit, u32 src_ip)
+snat_ipfix_logging_max_entries_per_user (u32 thread_index, u32 limit, u32 src_ip)
 {
   //TODO: This event SHOULD be rate limited
-  snat_ipfix_logging_max_entries_per_user_args_t a;
-
   skip_if_disabled ();
 
-  a.limit = limit;
-  a.src_ip = src_ip;
-
-  vl_api_rpc_call_main_thread (snat_ipfix_logging_max_entries_per_usr_rpc_cb,
-			       (u8 *) & a, sizeof (a));
+  snat_ipfix_logging_max_entries_per_usr (thread_index, limit, src_ip, 0);
 }
 
 vlib_frame_t *
-snat_data_callback_max_entries_per_usr (flow_report_main_t * frm,
+deterministic_nat_data_callback
+(flow_report_main_t * frm,
 					flow_report_t * fr,
 					vlib_frame_t * f,
 					u32 * to_next, u32 node_index)
 {
-  snat_ipfix_logging_max_entries_per_usr (0, 0, 1);
-  return f;
-}
+  snat_ipfix_flush_from_main();
 
-static void
-nat_ipfix_logging_max_ses_rpc_cb (nat_ipfix_logging_max_sessions_args_t * a)
-{
-  nat_ipfix_logging_max_ses (a->limit, 0);
+  return f;
 }
 
 /**
  * @brief Generate maximum session entries exceeded event
  *
+ * @param thread_index thread index
  * @param limit configured limit
  */
 void
-nat_ipfix_logging_max_sessions (u32 limit)
+nat_ipfix_logging_max_sessions (u32 thread_index, u32 limit)
 {
   //TODO: This event SHOULD be rate limited
-  nat_ipfix_logging_max_sessions_args_t a;
-
   skip_if_disabled ();
 
-  a.limit = limit;
-
-  vl_api_rpc_call_main_thread (nat_ipfix_logging_max_ses_rpc_cb,
-			       (u8 *) & a, sizeof (a));
-}
-
-vlib_frame_t *
-nat_data_callback_max_sessions (flow_report_main_t * frm,
-				flow_report_t * fr,
-				vlib_frame_t * f,
-				u32 * to_next, u32 node_index)
-{
-  nat_ipfix_logging_max_ses (0, 1);
-  return f;
-}
-
-static void
-nat_ipfix_logging_max_bib_rpc_cb (nat_ipfix_logging_max_bibs_args_t * a)
-{
-  nat_ipfix_logging_max_bib (a->limit, 0);
+  nat_ipfix_logging_max_ses (thread_index, limit, 0);
 }
 
 /**
  * @brief Generate maximum BIB entries exceeded event
  *
+ * @param thread_index thread index
  * @param limit configured limit
  */
 void
-nat_ipfix_logging_max_bibs (u32 limit)
+nat_ipfix_logging_max_bibs (u32 thread_index, u32 limit)
 {
   //TODO: This event SHOULD be rate limited
-  nat_ipfix_logging_max_bibs_args_t a;
-
   skip_if_disabled ();
 
-  a.limit = limit;
-
-  vl_api_rpc_call_main_thread (nat_ipfix_logging_max_bib_rpc_cb,
-			       (u8 *) & a, sizeof (a));
-}
-
-vlib_frame_t *
-nat_data_callback_max_bibs (flow_report_main_t * frm,
-			    flow_report_t * fr,
-			    vlib_frame_t * f,
-			    u32 * to_next, u32 node_index)
-{
-  nat_ipfix_logging_max_bib (0, 1);
-  return f;
-}
-
-static void
-nat_ipfix_logging_max_frag_ip4_rpc_cb (nat_ipfix_logging_max_frags_ip4_args_t * a)
-{
-  nat_ipfix_logging_max_frag_ip4 (a->limit, a->src, 0);
+  nat_ipfix_logging_max_bib (thread_index, limit, 0);
 }
 
 /**
  * @brief Generate maximum IPv4 fragments pending reassembly exceeded event
  *
+ * @param thread_index thread index
  * @param limit configured limit
  * @param src source IPv4 address
  */
 void
-nat_ipfix_logging_max_fragments_ip4 (u32 limit, ip4_address_t * src)
+nat_ipfix_logging_max_fragments_ip4 (u32 thread_index,
+                                     u32 limit, ip4_address_t * src)
 {
   //TODO: This event SHOULD be rate limited
-  nat_ipfix_logging_max_frags_ip4_args_t a;
-
   skip_if_disabled ();
 
-  a.limit = limit;
-  a.src = src->as_u32;
-
-  vl_api_rpc_call_main_thread (nat_ipfix_logging_max_frag_ip4_rpc_cb,
-			       (u8 *) & a, sizeof (a));
-}
-
-vlib_frame_t *
-nat_data_callback_max_frags_ip4 (flow_report_main_t * frm,
-			         flow_report_t * fr,
-			         vlib_frame_t * f,
-			         u32 * to_next, u32 node_index)
-{
-  nat_ipfix_logging_max_frag_ip4 (0, 0, 1);
-  return f;
-}
-
-static void
-nat_ipfix_logging_max_frag_ip6_rpc_cb (nat_ipfix_logging_max_frags_ip6_args_t * a)
-{
-  ip6_address_t src;
-  src.as_u64[0] = a->src[0];
-  src.as_u64[1] = a->src[1];
-  nat_ipfix_logging_max_frag_ip6 (a->limit, &src, 0);
+  nat_ipfix_logging_max_frag_ip4 (thread_index, limit, src->as_u32, 0);
 }
 
 /**
  * @brief Generate maximum IPv6 fragments pending reassembly exceeded event
  *
+ * @param thread_index thread index
  * @param limit configured limit
  * @param src source IPv6 address
  */
 void
-nat_ipfix_logging_max_fragments_ip6 (u32 limit, ip6_address_t * src)
+nat_ipfix_logging_max_fragments_ip6 (u32 thread_index,
+                                     u32 limit, ip6_address_t * src)
 {
   //TODO: This event SHOULD be rate limited
-  nat_ipfix_logging_max_frags_ip6_args_t a;
-
   skip_if_disabled ();
 
-  a.limit = limit;
-  a.src[0] = src->as_u64[0];
-  a.src[1] = src->as_u64[1];
-
-  vl_api_rpc_call_main_thread (nat_ipfix_logging_max_frag_ip6_rpc_cb,
-			       (u8 *) & a, sizeof (a));
-}
-
-vlib_frame_t *
-nat_data_callback_max_frags_ip6 (flow_report_main_t * frm,
-			         flow_report_t * fr,
-			         vlib_frame_t * f,
-			         u32 * to_next, u32 node_index)
-{
-  nat_ipfix_logging_max_frag_ip6 (0, 0, 1);
-  return f;
-}
-
-static void
-nat_ipfix_logging_nat64_bib_rpc_cb (nat_ipfix_logging_nat64_bib_args_t * a)
-{
-  ip6_address_t src_ip;
-  src_ip.as_u64[0] = a->src_ip[0];
-  src_ip.as_u64[1] = a->src_ip[1];
-  nat_ipfix_logging_nat64_bibe (a->nat_event, &src_ip, a->nat_src_ip,
-                                a->proto, a->src_port, a->nat_src_port,
-                                a->vrf_id, 0);
+  nat_ipfix_logging_max_frag_ip6 (thread_index, limit, src, 0);
 }
 
 /**
  * @brief Generate NAT64 BIB create and delete events
  *
+ * @param thread_index thread index
  * @param src_ip       source IPv6 address
  * @param nat_src_ip   transaltes source IPv4 address
  * @param proto        L4 protocol
@@ -1768,55 +1723,26 @@ nat_ipfix_logging_nat64_bib_rpc_cb (nat_ipfix_logging_nat64_bib_args_t * a)
  * @param is_create    non-zero value if create event otherwise delete event
  */
 void
-nat_ipfix_logging_nat64_bib (ip6_address_t * src_ip,
+nat_ipfix_logging_nat64_bib (u32 thread_index, ip6_address_t * src_ip,
                              ip4_address_t * nat_src_ip, u8 proto,
                              u16 src_port, u16 nat_src_port, u32 vrf_id,
                              u8 is_create)
 {
-  nat_ipfix_logging_nat64_bib_args_t a;
+  u8 nat_event;
 
   skip_if_disabled ();
 
-  a.src_ip[0] = src_ip->as_u64[0];
-  a.src_ip[1] = src_ip->as_u64[1];
-  a.nat_src_ip = nat_src_ip->as_u32;
-  a.proto = proto;
-  a.src_port = src_port;
-  a.nat_src_port = nat_src_port;
-  a.vrf_id = vrf_id;
-  a.nat_event = is_create ? NAT64_BIB_CREATE : NAT64_BIB_DELETE;
-
-  vl_api_rpc_call_main_thread (nat_ipfix_logging_nat64_bib_rpc_cb,
-			       (u8 *) & a, sizeof (a));
-}
+  nat_event = is_create ? NAT64_BIB_CREATE : NAT64_BIB_DELETE;
 
-vlib_frame_t *
-nat_data_callback_nat64_bib (flow_report_main_t * frm,
-			     flow_report_t * fr,
-			     vlib_frame_t * f,
-			     u32 * to_next, u32 node_index)
-{
-  nat_ipfix_logging_nat64_bibe (0, 0, 0, 0, 0, 0, 0, 1);
-  return f;
-}
-
-static void
-nat_ipfix_logging_nat64_ses_rpc_cb (nat_ipfix_logging_nat64_ses_args_t * a)
-{
-  ip6_address_t src_ip, dst_ip;
-  src_ip.as_u64[0] = a->src_ip[0];
-  src_ip.as_u64[1] = a->src_ip[1];
-  dst_ip.as_u64[0] = a->dst_ip[0];
-  dst_ip.as_u64[1] = a->dst_ip[1];
-  nat_ipfix_logging_nat64_ses (a->nat_event, &src_ip, a->nat_src_ip,
-                               a->proto, a->src_port, a->nat_src_port,
-                               &dst_ip, a->nat_dst_ip, a->dst_port,
-                               a->nat_dst_port, a->vrf_id, 0);
+  nat_ipfix_logging_nat64_bibe (thread_index, nat_event, src_ip,
+                                nat_src_ip->as_u32, proto, src_port,
+                                nat_src_port, vrf_id, 0);
 }
 
 /**
  * @brief Generate NAT64 session create and delete events
  *
+ * @param thread_index thread index
  * @param src_ip       source IPv6 address
  * @param nat_src_ip   transaltes source IPv4 address
  * @param proto        L4 protocol
@@ -1830,42 +1756,38 @@ nat_ipfix_logging_nat64_ses_rpc_cb (nat_ipfix_logging_nat64_ses_args_t * a)
  * @param is_create    non-zero value if create event otherwise delete event
  */
 void
-nat_ipfix_logging_nat64_session (ip6_address_t * src_ip,
+nat_ipfix_logging_nat64_session (u32 thread_index,
+                                 ip6_address_t * src_ip,
                                  ip4_address_t * nat_src_ip, u8 proto,
                                  u16 src_port, u16 nat_src_port,
                                  ip6_address_t * dst_ip,
                                  ip4_address_t * nat_dst_ip, u16 dst_port,
                                  u16 nat_dst_port, u32 vrf_id, u8 is_create)
 {
-  nat_ipfix_logging_nat64_ses_args_t a;
+  u8 nat_event;
 
   skip_if_disabled ();
 
-  a.src_ip[0] = src_ip->as_u64[0];
-  a.src_ip[1] = src_ip->as_u64[1];
-  a.nat_src_ip = nat_src_ip->as_u32;
-  a.proto = proto;
-  a.src_port = src_port;
-  a.nat_src_port = nat_src_port;
-  a.dst_ip[0] = dst_ip->as_u64[0];
-  a.dst_ip[1] = dst_ip->as_u64[1];
-  a.nat_dst_ip = nat_dst_ip->as_u32;
-  a.dst_port = dst_port;
-  a.nat_dst_port = nat_dst_port;
-  a.vrf_id = vrf_id;
-  a.nat_event = is_create ? NAT64_SESSION_CREATE : NAT64_SESSION_DELETE;
-
-  vl_api_rpc_call_main_thread (nat_ipfix_logging_nat64_ses_rpc_cb,
-			       (u8 *) & a, sizeof (a));
+  nat_event = is_create ? NAT64_SESSION_CREATE : NAT64_SESSION_DELETE;
+
+  nat_ipfix_logging_nat64_ses (thread_index, nat_event, src_ip,
+                               nat_src_ip->as_u32, proto, src_port,
+                               nat_src_port, dst_ip, nat_dst_ip->as_u32,
+                               dst_port, nat_dst_port, vrf_id, 0);
 }
 
 vlib_frame_t *
-nat_data_callback_nat64_session (flow_report_main_t * frm,
-			         flow_report_t * fr,
-			         vlib_frame_t * f,
-			         u32 * to_next, u32 node_index)
+data_callback (flow_report_main_t * frm, flow_report_t * fr,
+               vlib_frame_t * f, u32 * to_next, u32 node_index)
 {
-  nat_ipfix_logging_nat64_ses (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1);
+  snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
+
+  if (PREDICT_FALSE (++silm->call_counter >= vec_len (frm->reports)))
+    {
+      snat_ipfix_flush_from_main();
+      silm->call_counter = 0;
+    }
+
   return f;
 }
 
@@ -1888,20 +1810,18 @@ snat_ipfix_logging_enable_disable (int enable, u32 domain_id, u16 src_port)
   int rv;
   u8 e = enable ? 1 : 0;
 
-  if (silm->enabled == e)
+  if (clib_atomic_cmp_and_swap (&silm->enabled, e ^ 1, e) == e)
     return 0;
 
-  silm->enabled = e;
-
   clib_memset (&a, 0, sizeof (a));
   a.is_add = enable;
   a.domain_id = domain_id ? domain_id : 1;
   a.src_port = src_port ? src_port : UDP_DST_PORT_ipfix;
+  a.flow_data_callback = data_callback;
 
   if (sm->deterministic)
     {
       a.rewrite_callback = snat_template_rewrite_max_entries_per_usr;
-      a.flow_data_callback = snat_data_callback_max_entries_per_usr;
 
       rv = vnet_flow_report_add_del (frm, &a, NULL);
       if (rv)
@@ -1913,7 +1833,6 @@ snat_ipfix_logging_enable_disable (int enable, u32 domain_id, u16 src_port)
   else
     {
       a.rewrite_callback = snat_template_rewrite_nat44_session;
-      a.flow_data_callback = snat_data_callback_nat44_session;
 
       rv = vnet_flow_report_add_del (frm, &a, NULL);
       if (rv)
@@ -1923,7 +1842,6 @@ snat_ipfix_logging_enable_disable (int enable, u32 domain_id, u16 src_port)
 	}
 
       a.rewrite_callback = snat_template_rewrite_addr_exhausted;
-      a.flow_data_callback = snat_data_callback_addr_exhausted;
 
       rv = vnet_flow_report_add_del (frm, &a, NULL);
       if (rv)
@@ -1933,7 +1851,6 @@ snat_ipfix_logging_enable_disable (int enable, u32 domain_id, u16 src_port)
 	}
 
       a.rewrite_callback = nat_template_rewrite_max_sessions;
-      a.flow_data_callback = nat_data_callback_max_sessions;
 
       rv = vnet_flow_report_add_del (frm, &a, NULL);
       if (rv)
@@ -1943,7 +1860,6 @@ snat_ipfix_logging_enable_disable (int enable, u32 domain_id, u16 src_port)
 	}
 
       a.rewrite_callback = nat_template_rewrite_max_bibs;
-      a.flow_data_callback = nat_data_callback_max_bibs;
 
       rv = vnet_flow_report_add_del (frm, &a, NULL);
       if (rv)
@@ -1953,7 +1869,6 @@ snat_ipfix_logging_enable_disable (int enable, u32 domain_id, u16 src_port)
 	}
 
       a.rewrite_callback = nat_template_rewrite_max_frags_ip4;
-      a.flow_data_callback = nat_data_callback_max_frags_ip4;
 
       rv = vnet_flow_report_add_del (frm, &a, NULL);
       if (rv)
@@ -1963,7 +1878,6 @@ snat_ipfix_logging_enable_disable (int enable, u32 domain_id, u16 src_port)
 	}
 
       a.rewrite_callback = nat_template_rewrite_max_frags_ip6;
-      a.flow_data_callback = nat_data_callback_max_frags_ip6;
 
       rv = vnet_flow_report_add_del (frm, &a, NULL);
       if (rv)
@@ -1973,7 +1887,6 @@ snat_ipfix_logging_enable_disable (int enable, u32 domain_id, u16 src_port)
 	}
 
       a.rewrite_callback = nat_template_rewrite_nat64_bib;
-      a.flow_data_callback = nat_data_callback_nat64_bib;
 
       rv = vnet_flow_report_add_del (frm, &a, NULL);
       if (rv)
@@ -1983,7 +1896,6 @@ snat_ipfix_logging_enable_disable (int enable, u32 domain_id, u16 src_port)
 	}
 
       a.rewrite_callback = nat_template_rewrite_nat64_session;
-      a.flow_data_callback = nat_data_callback_nat64_session;
 
       rv = vnet_flow_report_add_del (frm, &a, NULL);
       if (rv)
@@ -1995,7 +1907,6 @@ snat_ipfix_logging_enable_disable (int enable, u32 domain_id, u16 src_port)
       if (sm->endpoint_dependent)
         {
           a.rewrite_callback = snat_template_rewrite_max_entries_per_usr;
-          a.flow_data_callback = snat_data_callback_max_entries_per_usr;
 
           rv = vnet_flow_report_add_del (frm, &a, NULL);
           if (rv)
@@ -2018,10 +1929,33 @@ void
 snat_ipfix_logging_init (vlib_main_t * vm)
 {
   snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
 
   silm->enabled = 0;
+  silm->worker_vms = 0;
+  silm->call_counter = 0;
 
   /* Set up time reference pair */
   silm->vlib_time_0 = vlib_time_now (vm);
   silm->milisecond_time_0 = unix_time_now_nsec () * 1e-6;
+
+  vec_validate (silm->per_thread_data, tm->n_vlib_mains - 1);
 }
+
+static uword
+ipfix_flush_process (vlib_main_t *vm,
+		     vlib_node_runtime_t *rt,
+                     vlib_frame_t *f)
+{
+  snat_ipfix_flush(vm->thread_index);
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (snat_ipfix_flush_node) = {
+  .function = ipfix_flush_process,
+  .name = "snat-ipfix-flush",
+  .type = VLIB_NODE_TYPE_INPUT,
+  .state = VLIB_NODE_STATE_INTERRUPT,
+};
+/* *INDENT-ON* */
diff --git a/src/plugins/nat/nat_ipfix_logging.h b/src/plugins/nat/nat_ipfix_logging.h
index 0750149d7a5..a5cdb1a3c1a 100644
--- a/src/plugins/nat/nat_ipfix_logging.h
+++ b/src/plugins/nat/nat_ipfix_logging.h
@@ -40,8 +40,6 @@ typedef enum {
 } quota_exceed_event_t;
 
 typedef struct {
-  /** NAT plugin IPFIX logging enabled */
-  u8 enabled;
 
   /** ipfix buffers under construction */
   vlib_buffer_t *nat44_session_buffer;
@@ -76,10 +74,19 @@ typedef struct {
   u32 nat64_bib_next_record_offset;
   u32 nat64_ses_next_record_offset;
 
+} snat_ipfix_per_thread_data_t;
+
+typedef struct {
+  /** NAT plugin IPFIX logging enabled */
+  u8 enabled;
+
   /** Time reference pair */
   u64 milisecond_time_0;
   f64 vlib_time_0;
 
+  /* Per thread data */
+  snat_ipfix_per_thread_data_t *per_thread_data;
+
   /** template IDs */
   u16 nat44_session_template_id;
   u16 addr_exhausted_template_id;
@@ -93,34 +100,48 @@ typedef struct {
 
   /** stream index */
   u32 stream_index;
+
+  /** vector of worker vlib mains */
+  vlib_main_t **worker_vms;
+
+  /** nat data callbacks call counter */
+  u16 call_counter;
+
 } snat_ipfix_logging_main_t;
 
 extern snat_ipfix_logging_main_t snat_ipfix_logging_main;
 
 void snat_ipfix_logging_init (vlib_main_t * vm);
 int snat_ipfix_logging_enable_disable (int enable, u32 domain_id, u16 src_port);
-void snat_ipfix_logging_nat44_ses_create (u32 src_ip, u32 nat_src_ip,
+void snat_ipfix_logging_nat44_ses_create (u32 thread_index, u32 src_ip,
+                                          u32 nat_src_ip,
                                           snat_protocol_t snat_proto,
                                           u16 src_port, u16 nat_src_port,
                                           u32 vrf_id);
-void snat_ipfix_logging_nat44_ses_delete (u32 src_ip, u32 nat_src_ip,
+void snat_ipfix_logging_nat44_ses_delete (u32 thread_index, u32 src_ip,
+                                          u32 nat_src_ip,
                                           snat_protocol_t snat_proto,
                                           u16 src_port, u16 nat_src_port,
                                           u32 vrf_id);
-void snat_ipfix_logging_addresses_exhausted(u32 pool_id);
-void snat_ipfix_logging_max_entries_per_user(u32 limit, u32 src_ip);
-void nat_ipfix_logging_max_sessions(u32 limit);
-void nat_ipfix_logging_max_bibs(u32 limit);
-void nat_ipfix_logging_max_fragments_ip4(u32 limit, ip4_address_t * src);
-void nat_ipfix_logging_max_fragments_ip6(u32 limit, ip6_address_t * src);
-void nat_ipfix_logging_nat64_session(ip6_address_t * src_ip,
+void snat_ipfix_logging_addresses_exhausted(u32 thread_index, u32 pool_id);
+void snat_ipfix_logging_max_entries_per_user(u32 thread_index,
+                                             u32 limit, u32 src_ip);
+void nat_ipfix_logging_max_sessions(u32 thread_index, u32 limit);
+void nat_ipfix_logging_max_bibs(u32 thread_index, u32 limit);
+void nat_ipfix_logging_max_fragments_ip4(u32 thread_index,
+                                         u32 limit, ip4_address_t * src);
+void nat_ipfix_logging_max_fragments_ip6(u32 thread_index,
+                                         u32 limit, ip6_address_t * src);
+void nat_ipfix_logging_nat64_session(u32 thread_index,
+                                     ip6_address_t * src_ip,
                                      ip4_address_t * nat_src_ip, u8 proto,
                                      u16 src_port, u16 nat_src_port,
                                      ip6_address_t * dst_ip,
                                      ip4_address_t * nat_dst_ip,
                                      u16 dst_port, u16 nat_dst_port,
                                      u32 vrf_id, u8 is_create);
-void nat_ipfix_logging_nat64_bib(ip6_address_t * src_ip,
+void nat_ipfix_logging_nat64_bib(u32 thread_index,
+                                 ip6_address_t * src_ip,
                                  ip4_address_t * nat_src_ip, u8 proto,
                                  u16 src_port, u16 nat_src_port,
                                  u32 vrf_id, u8 is_create);
diff --git a/src/plugins/nat/nat_reass.c b/src/plugins/nat/nat_reass.c
index 1185e1bb4af..ed827a9038b 100755
--- a/src/plugins/nat/nat_reass.c
+++ b/src/plugins/nat/nat_reass.c
@@ -335,8 +335,8 @@ unlock:
 }
 
 int
-nat_ip4_reass_add_fragment (nat_reass_ip4_t * reass, u32 bi,
-			    u32 ** bi_to_drop)
+nat_ip4_reass_add_fragment (u32 thread_index, nat_reass_ip4_t * reass,
+			    u32 bi, u32 ** bi_to_drop)
 {
   nat_reass_main_t *srm = &nat_reass_main;
   dlist_elt_t *elt;
@@ -344,7 +344,7 @@ nat_ip4_reass_add_fragment (nat_reass_ip4_t * reass, u32 bi,
 
   if (reass->frag_n >= srm->ip4_max_frag)
     {
-      nat_ipfix_logging_max_fragments_ip4 (srm->ip4_max_frag,
+      nat_ipfix_logging_max_fragments_ip4 (thread_index, srm->ip4_max_frag,
 					   &reass->key.src);
       reass->flags |= NAT_REASS_FLAG_MAX_FRAG_DROP;
       nat_ip4_reass_get_frags_inline (reass, bi_to_drop);
@@ -541,8 +541,8 @@ unlock:
 }
 
 int
-nat_ip6_reass_add_fragment (nat_reass_ip6_t * reass, u32 bi,
-			    u32 ** bi_to_drop)
+nat_ip6_reass_add_fragment (u32 thread_index, nat_reass_ip6_t * reass,
+			    u32 bi, u32 ** bi_to_drop)
 {
   nat_reass_main_t *srm = &nat_reass_main;
   dlist_elt_t *elt;
@@ -550,7 +550,7 @@ nat_ip6_reass_add_fragment (nat_reass_ip6_t * reass, u32 bi,
 
   if (reass->frag_n >= srm->ip6_max_frag)
     {
-      nat_ipfix_logging_max_fragments_ip6 (srm->ip6_max_frag,
+      nat_ipfix_logging_max_fragments_ip6 (thread_index, srm->ip6_max_frag,
 					   &reass->key.src);
       reass->flags |= NAT_REASS_FLAG_MAX_FRAG_DROP;
       nat_ip6_reass_get_frags_inline (reass, bi_to_drop);
diff --git a/src/plugins/nat/nat_reass.h b/src/plugins/nat/nat_reass.h
index 579961d72c9..e58db445e6e 100644
--- a/src/plugins/nat/nat_reass.h
+++ b/src/plugins/nat/nat_reass.h
@@ -240,8 +240,8 @@ nat_reass_ip4_t *nat_ip4_reass_find_or_create (ip4_address_t src,
  *
  * @returns 0 on success, non-zero value otherwise.
  */
-int nat_ip4_reass_add_fragment (nat_reass_ip4_t * reass, u32 bi,
-				u32 ** bi_to_drop);
+int nat_ip4_reass_add_fragment (u32 thread_index, nat_reass_ip4_t * reass,
+				u32 bi, u32 ** bi_to_drop);
 
 /**
  * @brief Get cached fragments.
@@ -291,8 +291,8 @@ nat_reass_ip6_t *nat_ip6_reass_find_or_create (ip6_address_t src,
  *
  * @returns 0 on success, non-zero value otherwise.
  */
-int nat_ip6_reass_add_fragment (nat_reass_ip6_t * reass, u32 bi,
-				u32 ** bi_to_drop);
+int nat_ip6_reass_add_fragment (u32 thread_index, nat_reass_ip6_t * reass,
+				u32 bi, u32 ** bi_to_drop);
 
 /**
  * @brief Get cached fragments.
diff --git a/src/plugins/nat/out2in.c b/src/plugins/nat/out2in.c
index 18595e9f0ad..c687a06f498 100755
--- a/src/plugins/nat/out2in.c
+++ b/src/plugins/nat/out2in.c
@@ -133,7 +133,8 @@ nat44_o2i_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg)
       if (clib_bihash_add_del_8_8 (&tsm->in2out, &s_kv, 0))
 	nat_log_warn ("out2in key del failed");
 
-      snat_ipfix_logging_nat44_ses_delete (s->in2out.addr.as_u32,
+      snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
+					   s->in2out.addr.as_u32,
 					   s->out2in.addr.as_u32,
 					   s->in2out.protocol,
 					   s->in2out.port,
@@ -237,7 +238,8 @@ create_session_for_static_mapping (snat_main_t * sm,
     nat_log_notice ("out2in key add failed");
 
   /* log NAT event */
-  snat_ipfix_logging_nat44_ses_create (s->in2out.addr.as_u32,
+  snat_ipfix_logging_nat44_ses_create (thread_index,
+				       s->in2out.addr.as_u32,
 				       s->out2in.addr.as_u32,
 				       s->in2out.protocol,
 				       s->in2out.port,
@@ -1532,7 +1534,7 @@ nat44_out2in_reass_node_fn (vlib_main_t * vm,
 	      if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
 		{
 		  if (nat_ip4_reass_add_fragment
-		      (reass0, bi0, &fragments_to_drop))
+		      (thread_index, reass0, bi0, &fragments_to_drop))
 		    {
 		      b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
 		      nat_log_notice
diff --git a/src/plugins/nat/out2in_ed.c b/src/plugins/nat/out2in_ed.c
index c3f05592acc..c53d6d0066c 100644
--- a/src/plugins/nat/out2in_ed.c
+++ b/src/plugins/nat/out2in_ed.c
@@ -174,7 +174,8 @@ nat44_o2i_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
       if (snat_is_unk_proto_session (s))
 	goto delete;
 
-      snat_ipfix_logging_nat44_ses_delete (s->in2out.addr.as_u32,
+      snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
+					   s->in2out.addr.as_u32,
 					   s->out2in.addr.as_u32,
 					   s->in2out.protocol,
 					   s->in2out.port,
@@ -318,7 +319,8 @@ create_session_for_static_mapping_ed (snat_main_t * sm,
 					       &ctx))
     nat_log_notice ("in2out-ed key add failed");
 
-  snat_ipfix_logging_nat44_ses_create (s->in2out.addr.as_u32,
+  snat_ipfix_logging_nat44_ses_create (thread_index,
+				       s->in2out.addr.as_u32,
 				       s->out2in.addr.as_u32,
 				       s->in2out.protocol,
 				       s->in2out.port,
@@ -1819,7 +1821,7 @@ nat44_ed_out2in_reass_node_fn (vlib_main_t * vm,
 	      if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
 		{
 		  if (nat_ip4_reass_add_fragment
-		      (reass0, bi0, &fragments_to_drop))
+		      (thread_index, reass0, bi0, &fragments_to_drop))
 		    {
 		      b0->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_FRAG];
 		      nat_log_notice