vppinfra: add atomic macros for __sync builtins

This is first part of addition of atomic macros with only macros for
__sync builtins.

- Based on earlier patch by Damjan (https://gerrit.fd.io/r/#/c/10729/)
Additionally
- clib_atomic_release macro added and used in the absence
of any memory barrier.
- clib_atomic_bool_cmp_and_swap added

Change-Id: Ie4e48c1e184a652018d1d0d87c4be80ddd180a3b
Original-patch-by: Damjan Marion <damarion@cisco.com>
Signed-off-by: Sirshak Das <sirshak.das@arm.com>
Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>
Reviewed-by: Ola Liljedahl <ola.liljedahl@arm.com>
Reviewed-by: Steve Capper <steve.capper@arm.com>

9 files changed, 43 insertions, 41 deletions

diff --git a/src/plugins/acl/session_inlines.h b/src/plugins/acl/session_inlines.h
index 6ac5983e086..cd23f390b28 100644
--- a/src/plugins/acl/session_inlines.h
+++ b/src/plugins/acl/session_inlines.h
@@ -410,7 +410,7 @@ acl_fa_deactivate_session (acl_main_t * am, u32 sw_if_index,
     }
 
   sess->deleted = 1;
-  clib_smp_atomic_add (&am->fa_session_total_deactivations, 1);
+  clib_atomic_fetch_add (&am->fa_session_total_deactivations, 1);
   clib_mem_set_heap (oldheap);
 }
 
@@ -432,7 +432,7 @@ acl_fa_put_session (acl_main_t * am, u32 sw_if_index,
   vec_validate (pw->fa_session_dels_by_sw_if_index, sw_if_index);
   clib_mem_set_heap (oldheap);
   pw->fa_session_dels_by_sw_if_index[sw_if_index]++;
-  clib_smp_atomic_add (&am->fa_session_total_dels, 1);
+  clib_atomic_fetch_add (&am->fa_session_total_dels, 1);
 }
 
 always_inline int
@@ -571,7 +571,7 @@ acl_fa_add_session (acl_main_t * am, int is_input, int is_ip6,
   vec_validate (pw->fa_session_adds_by_sw_if_index, sw_if_index);
   clib_mem_set_heap (oldheap);
   pw->fa_session_adds_by_sw_if_index[sw_if_index]++;
-  clib_smp_atomic_add (&am->fa_session_total_adds, 1);
+  clib_atomic_fetch_add (&am->fa_session_total_adds, 1);
   return sess;
 }
 
diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c
index b4001337aca..ea00df25a25 100644
--- a/src/plugins/dpdk/device/device.c
+++ b/src/plugins/dpdk/device/device.c
@@ -162,7 +162,7 @@ static_always_inline
       if (PREDICT_FALSE (xd->lockp != 0))
 	{
 	  queue_id = queue_id % xd->tx_q_used;
-	  while (__sync_lock_test_and_set (xd->lockp[queue_id], 1))
+	  while (clib_atomic_test_and_set (xd->lockp[queue_id]))
 	    /* zzzz */
 	    queue_id = (queue_id + 1) % xd->tx_q_used;
 	}
@@ -191,7 +191,7 @@ static_always_inline
 	}
 
       if (PREDICT_FALSE (xd->lockp != 0))
-	*xd->lockp[queue_id] = 0;
+	clib_atomic_release (xd->lockp[queue_id]);
 
       if (PREDICT_FALSE (n_sent < 0))
 	{
diff --git a/src/plugins/ioam/analyse/ioam_analyse.h b/src/plugins/ioam/analyse/ioam_analyse.h
index 881b0451721..3aec7a792d8 100644
--- a/src/plugins/ioam/analyse/ioam_analyse.h
+++ b/src/plugins/ioam/analyse/ioam_analyse.h
@@ -191,7 +191,7 @@ ip6_ioam_analyse_set_paths_down (ioam_analyser_data_t * data)
   ioam_path_map_t *path;
   u8 k, i;
 
-  while (__sync_lock_test_and_set (data->writer_lock, 1))
+  while (clib_atomic_test_and_set (data->writer_lock))
     ;
 
   trace_data = &data->trace_data;
@@ -208,7 +208,7 @@ ip6_ioam_analyse_set_paths_down (ioam_analyser_data_t * data)
       for (k = 0; k < trace_record->num_nodes; k++)
 	path[k].state_up = 0;
     }
-  *(data->writer_lock) = 0;
+  clib_atomic_release (data->writer_lock);
 }
 
 always_inline void
@@ -225,7 +225,7 @@ ip6_ioam_analyse_hbh_trace_loopback (ioam_analyser_data_t * data,
   u16 size_of_traceopt_per_node;
   u16 size_of_all_traceopts;
 
-  while (__sync_lock_test_and_set (data->writer_lock, 1))
+  while (clib_atomic_test_and_set (data->writer_lock))
     ;
 
   trace_data = &data->trace_data;
@@ -277,7 +277,7 @@ ip6_ioam_analyse_hbh_trace_loopback (ioam_analyser_data_t * data,
 	}
     }
 end:
-  *(data->writer_lock) = 0;
+  clib_atomic_release (data->writer_lock);
 }
 
 always_inline int
@@ -295,7 +295,7 @@ ip6_ioam_analyse_hbh_trace (ioam_analyser_data_t * data,
   ioam_path_map_t *path = NULL;
   ioam_analyse_trace_record *trace_record;
 
-  while (__sync_lock_test_and_set (data->writer_lock, 1))
+  while (clib_atomic_test_and_set (data->writer_lock))
     ;
 
   trace_data = &data->trace_data;
@@ -409,7 +409,7 @@ found_match:
 	(u32) ((sum + delay) / (data->seqno_data.rx_packets + 1));
     }
 DONE:
-  *(data->writer_lock) = 0;
+  clib_atomic_release (data->writer_lock);
   return 0;
 }
 
@@ -417,13 +417,14 @@ always_inline int
 ip6_ioam_analyse_hbh_e2e (ioam_analyser_data_t * data,
 			  ioam_e2e_packet_t * e2e, u16 len)
 {
-  while (__sync_lock_test_and_set (data->writer_lock, 1))
+  while (clib_atomic_test_and_set (data->writer_lock))
     ;
 
   ioam_analyze_seqno (&data->seqno_data,
 		      (u64) clib_net_to_host_u32 (e2e->e2e_data));
 
-  *(data->writer_lock) = 0;
+  clib_atomic_release (data->writer_lock);
+
   return 0;
 }
 
@@ -510,7 +511,8 @@ ioam_analyse_init_data (ioam_analyser_data_t * data)
 
   data->writer_lock = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
 					      CLIB_CACHE_LINE_BYTES);
-  *(data->writer_lock) = 0;
+
+  clib_atomic_release (data->writer_lock);
 
   trace_data = &(data->trace_data);
   for (j = 0; j < IOAM_MAX_PATHS_PER_FLOW; j++)
diff --git a/src/plugins/ioam/analyse/ioam_summary_export.c b/src/plugins/ioam/analyse/ioam_summary_export.c
index c118bfa3529..440dcca8369 100644
--- a/src/plugins/ioam/analyse/ioam_summary_export.c
+++ b/src/plugins/ioam/analyse/ioam_summary_export.c
@@ -150,7 +150,7 @@ ioam_analyse_add_ipfix_record (flow_report_t * fr,
 			       ip6_address_t * src, ip6_address_t * dst,
 			       u16 src_port, u16 dst_port)
 {
-  while (__sync_lock_test_and_set (record->writer_lock, 1))
+  while (clib_atomic_test_and_set (record->writer_lock))
     ;
 
   int field_index = 0;
@@ -259,7 +259,7 @@ ioam_analyse_add_ipfix_record (flow_report_t * fr,
   *(record->chached_data_list) = *record;
   record->chached_data_list->chached_data_list = NULL;
 
-  *(record->writer_lock) = 0;
+  clib_atomic_release (record->writer_lock);
   return offset;
 }
 
diff --git a/src/plugins/ioam/analyse/ip6/node.c b/src/plugins/ioam/analyse/ip6/node.c
index 902fb9b09ce..7a8d71de93b 100644
--- a/src/plugins/ioam/analyse/ip6/node.c
+++ b/src/plugins/ioam/analyse/ip6/node.c
@@ -256,17 +256,17 @@ ip6_ioam_analyse_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 		  data0 = ioam_analyse_get_data_from_flow_id (flow_id0);
 		  data1 = ioam_analyse_get_data_from_flow_id (flow_id1);
 
-		  while (__sync_lock_test_and_set (data0->writer_lock, 1))
+		  while (clib_atomic_test_and_set (data0->writer_lock))
 		    ;
 		  data0->pkt_counter++;
 		  data0->bytes_counter += p_len0;
-		  *(data0->writer_lock) = 0;
+		  clib_atomic_release (data0->writer_lock);
 
-		  while (__sync_lock_test_and_set (data1->writer_lock, 1))
+		  while (clib_atomic_test_and_set (data1->writer_lock))
 		    ;
 		  data1->pkt_counter++;
 		  data1->bytes_counter += p_len1;
-		  *(data1->writer_lock) = 0;
+		  clib_atomic_release (data1->writer_lock);
 		}
 	      else if (error0 == 0)
 		{
@@ -274,11 +274,11 @@ ip6_ioam_analyse_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 		  pkts_failed++;
 
 		  data0 = ioam_analyse_get_data_from_flow_id (flow_id0);
-		  while (__sync_lock_test_and_set (data0->writer_lock, 1))
+		  while (clib_atomic_test_and_set (data0->writer_lock))
 		    ;
 		  data0->pkt_counter++;
 		  data0->bytes_counter += p_len0;
-		  *(data0->writer_lock) = 0;
+		  clib_atomic_release (data0->writer_lock);
 		}
 	      else if (error1 == 0)
 		{
@@ -286,11 +286,11 @@ ip6_ioam_analyse_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 		  pkts_failed++;
 
 		  data1 = ioam_analyse_get_data_from_flow_id (flow_id1);
-		  while (__sync_lock_test_and_set (data1->writer_lock, 1))
+		  while (clib_atomic_test_and_set (data1->writer_lock))
 		    ;
 		  data1->pkt_counter++;
 		  data1->bytes_counter += p_len1;
-		  *(data1->writer_lock) = 0;
+		  clib_atomic_release (data1->writer_lock);
 		}
 	      else
 		pkts_failed += 2;
@@ -327,12 +327,12 @@ ip6_ioam_analyse_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 		{
 		  pkts_analysed++;
 		  data0 = ioam_analyse_get_data_from_flow_id (flow_id0);
-		  while (__sync_lock_test_and_set (data0->writer_lock, 1))
+		  while (clib_atomic_test_and_set (data0->writer_lock))
 		    ;
 		  data0->pkt_counter++;
 		  data0->bytes_counter +=
 		    clib_net_to_host_u16 (ip60->payload_length);
-		  *(data0->writer_lock) = 0;
+		  clib_atomic_release (data0->writer_lock);
 		}
 	      else
 		pkts_failed++;
@@ -393,13 +393,13 @@ ip6_ioam_analyse_hbh_pot (u32 flow_id, ip6_hop_by_hop_option_t * opt0,
   pot_profile = pot_profile_get_active ();
   ret = pot_validate (pot_profile, cumulative, random);
 
-  while (__sync_lock_test_and_set (data->writer_lock, 1))
+  while (clib_atomic_test_and_set (data->writer_lock))
     ;
 
   (0 == ret) ? (data->pot_data.sfc_validated_count++) :
     (data->pot_data.sfc_invalidated_count++);
 
-  *(data->writer_lock) = 0;
+  clib_atomic_release (data->writer_lock);
   return 0;
 }
 
diff --git a/src/plugins/ioam/export-common/ioam_export.h b/src/plugins/ioam/export-common/ioam_export.h
index 36d71d26716..6e640951802 100644
--- a/src/plugins/ioam/export-common/ioam_export.h
+++ b/src/plugins/ioam/export-common/ioam_export.h
@@ -436,11 +436,11 @@ ioam_export_process_common (ioam_export_main_t * em, vlib_main_t * vm,
 	   */
 	  for (i = 0; i < vec_len (thread_index); i++)
 	    {
-	      while (__sync_lock_test_and_set (em->lockp[thread_index[i]], 1))
+	      while (clib_atomic_test_and_set (em->lockp[thread_index[i]]))
 		;
 	      em->buffer_per_thread[thread_index[i]] =
 		vec_pop (vec_buffer_indices);
-	      *em->lockp[thread_index[i]] = 0;
+	      clib_atomic_release (em->lockp[thread_index[i]]);
 	    }
 
 	  /* Send the buffers */
@@ -479,7 +479,7 @@ do {                                                                           \
   from = vlib_frame_vector_args (F);                                           \
   n_left_from = (F)->n_vectors;                                                \
   next_index = (N)->cached_next_index;                                         \
-  while (__sync_lock_test_and_set ((EM)->lockp[(VM)->thread_index], 1));       \
+  while (clib_atomic_test_and_set ((EM)->lockp[(VM)->thread_index]));	       \
   my_buf = ioam_export_get_my_buffer (EM, (VM)->thread_index);                 \
   my_buf->touched_at = vlib_time_now (VM);                                     \
   while (n_left_from > 0)                                                      \
diff --git a/src/plugins/lb/lb.c b/src/plugins/lb/lb.c
index 1936c1983ee..76bbfa0ac81 100644
--- a/src/plugins/lb/lb.c
+++ b/src/plugins/lb/lb.c
@@ -27,8 +27,8 @@
 
 lb_main_t lb_main;
 
-#define lb_get_writer_lock() do {} while(__sync_lock_test_and_set (lb_main.writer_lock, 1))
-#define lb_put_writer_lock() lb_main.writer_lock[0] = 0
+#define lb_get_writer_lock() do {} while(clib_atomic_test_and_set (lb_main.writer_lock))
+#define lb_put_writer_lock() clib_atomic_release (lb_main.writer_lock)
 
 static void lb_as_stack (lb_as_t *as);
 
diff --git a/src/plugins/map/map.h b/src/plugins/map/map.h
index 6587a8a455b..1e07b59f762 100644
--- a/src/plugins/map/map.h
+++ b/src/plugins/map/map.h
@@ -474,7 +474,7 @@ map_ip4_reass_get(u32 src, u32 dst, u16 fragment_id,
 void
 map_ip4_reass_free(map_ip4_reass_t *r, u32 **pi_to_drop);
 
-#define map_ip4_reass_lock() while (__sync_lock_test_and_set(map_main.ip4_reass_lock, 1)) {}
+#define map_ip4_reass_lock() while (clib_atomic_test_and_set (map_main.ip4_reass_lock)) {}
 #define map_ip4_reass_unlock() do {CLIB_MEMORY_BARRIER(); *map_main.ip4_reass_lock = 0;} while(0)
 
 static_always_inline void
@@ -499,7 +499,7 @@ map_ip6_reass_get(ip6_address_t *src, ip6_address_t *dst, u32 fragment_id,
 void
 map_ip6_reass_free(map_ip6_reass_t *r, u32 **pi_to_drop);
 
-#define map_ip6_reass_lock() while (__sync_lock_test_and_set(map_main.ip6_reass_lock, 1)) {}
+#define map_ip6_reass_lock() while (clib_atomic_test_and_set (map_main.ip6_reass_lock)) {}
 #define map_ip6_reass_unlock() do {CLIB_MEMORY_BARRIER(); *map_main.ip6_reass_lock = 0;} while(0)
 
 int
@@ -555,14 +555,14 @@ static inline void
 map_domain_counter_lock (map_main_t *mm)
 {
   if (mm->counter_lock)
-    while (__sync_lock_test_and_set(mm->counter_lock, 1))
+    while (clib_atomic_test_and_set (mm->counter_lock))
       /* zzzz */ ;
 }
 static inline void
 map_domain_counter_unlock (map_main_t *mm)
 {
   if (mm->counter_lock)
-    *mm->counter_lock = 0;
+    clib_atomic_release (mm->counter_lock);
 }
 
 
diff --git a/src/plugins/nat/nat_det.h b/src/plugins/nat/nat_det.h
index ce876bca9fb..7878a4c4ed6 100644
--- a/src/plugins/nat/nat_det.h
+++ b/src/plugins/nat/nat_det.h
@@ -159,13 +159,13 @@ snat_det_ses_create (snat_det_map_t * dm, ip4_address_t * in_addr,
     {
       if (!dm->sessions[i + user_offset].in_port)
 	{
-	  if (__sync_bool_compare_and_swap
+	  if (clib_atomic_bool_cmp_and_swap
 	      (&dm->sessions[i + user_offset].in_port, 0, in_port))
 	    {
 	      dm->sessions[i + user_offset].out.as_u64 = out->as_u64;
 	      dm->sessions[i + user_offset].state = SNAT_SESSION_UNKNOWN;
 	      dm->sessions[i + user_offset].expire = 0;
-	      __sync_add_and_fetch (&dm->ses_num, 1);
+	      clib_atomic_add_fetch (&dm->ses_num, 1);
 	      return &dm->sessions[i + user_offset];
 	    }
 	}
@@ -179,10 +179,10 @@ snat_det_ses_create (snat_det_map_t * dm, ip4_address_t * in_addr,
 always_inline void
 snat_det_ses_close (snat_det_map_t * dm, snat_det_session_t * ses)
 {
-  if (__sync_bool_compare_and_swap (&ses->in_port, ses->in_port, 0))
+  if (clib_atomic_bool_cmp_and_swap (&ses->in_port, ses->in_port, 0))
     {
       ses->out.as_u64 = 0;
-      __sync_add_and_fetch (&dm->ses_num, -1);
+      clib_atomic_add_fetch (&dm->ses_num, -1);
     }
 }