aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSirshak Das <sirshak.das@arm.com>2018-10-03 22:53:51 +0000
committerDamjan Marion <dmarion@me.com>2018-10-19 07:10:47 +0000
commit2f6d7bb93c157b874efb79a2d1583a4c368bf89a (patch)
tree05dc2867c598cbb8d711f074b4b0eb62dd464f41
parentbf3443b0f852f5a4c551d12f926defbd047f2161 (diff)
vppinfra: add atomic macros for __sync builtins
This is first part of addition of atomic macros with only macros for __sync builtins. - Based on earlier patch by Damjan (https://gerrit.fd.io/r/#/c/10729/) Additionally - clib_atomic_release macro added and used in the absence of any memory barrier. - clib_atomic_bool_cmp_and_swap added Change-Id: Ie4e48c1e184a652018d1d0d87c4be80ddd180a3b Original-patch-by: Damjan Marion <damarion@cisco.com> Signed-off-by: Sirshak Das <sirshak.das@arm.com> Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com> Reviewed-by: Ola Liljedahl <ola.liljedahl@arm.com> Reviewed-by: Steve Capper <steve.capper@arm.com>
-rw-r--r--extras/japi/java/jvpp-common/jvpp_common.h4
-rw-r--r--src/plugins/acl/session_inlines.h6
-rw-r--r--src/plugins/dpdk/device/device.c4
-rw-r--r--src/plugins/ioam/analyse/ioam_analyse.h20
-rw-r--r--src/plugins/ioam/analyse/ioam_summary_export.c4
-rw-r--r--src/plugins/ioam/analyse/ip6/node.c24
-rw-r--r--src/plugins/ioam/export-common/ioam_export.h6
-rw-r--r--src/plugins/lb/lb.c4
-rw-r--r--src/plugins/map/map.h8
-rw-r--r--src/plugins/nat/nat_det.h8
-rw-r--r--src/svm/message_queue.c6
-rw-r--r--src/svm/ssvm.h4
-rw-r--r--src/svm/svm_fifo.c10
-rw-r--r--src/svm/svm_fifo.h2
-rw-r--r--src/vlib/buffer_funcs.h6
-rw-r--r--src/vlib/threads.c26
-rw-r--r--src/vlib/threads.h10
-rw-r--r--src/vlib/unix/cj.c2
-rw-r--r--src/vnet/classify/vnet_classify.c2
-rw-r--r--src/vnet/devices/virtio/vhost_user_output.c4
-rw-r--r--src/vnet/dns/dns.h2
-rw-r--r--src/vnet/gre/gre.c6
-rw-r--r--src/vnet/interface.h4
-rwxr-xr-xsrc/vnet/ip/ip4_mtrie.c18
-rw-r--r--src/vnet/ipfix-export/flow_report_classify.c4
-rw-r--r--src/vnet/mfib/mfib_forward.c4
-rw-r--r--src/vnet/mfib/mfib_signal.c8
-rw-r--r--src/vnet/pg/output.c5
-rw-r--r--src/vnet/session-apps/echo_client.c8
-rw-r--r--src/vnet/util/refcount.h4
-rw-r--r--src/vpp/stats/stats_to_be_deprecated.c2
-rw-r--r--src/vppinfra/CMakeLists.txt1
-rw-r--r--src/vppinfra/atomics.h45
-rw-r--r--src/vppinfra/clib.h1
-rw-r--r--src/vppinfra/elog.c2
-rw-r--r--src/vppinfra/elog.h2
-rw-r--r--src/vppinfra/lock.h10
-rw-r--r--src/vppinfra/maplog.h2
-rw-r--r--src/vppinfra/mheap.c2
-rw-r--r--src/vppinfra/smp.h2
40 files changed, 162 insertions, 130 deletions
diff --git a/extras/japi/java/jvpp-common/jvpp_common.h b/extras/japi/java/jvpp-common/jvpp_common.h
index 14027a96f78..dd48138a536 100644
--- a/extras/japi/java/jvpp-common/jvpp_common.h
+++ b/extras/japi/java/jvpp-common/jvpp_common.h
@@ -43,11 +43,11 @@ typedef struct {
extern jvpp_main_t jvpp_main __attribute__((aligned (64)));
static_always_inline u32 vppjni_get_context_id(jvpp_main_t * jm) {
- return __sync_add_and_fetch(&jm->context_id, 1);
+ return clib_atomic_add_fetch(&jm->context_id, 1);
}
static_always_inline void vppjni_lock(jvpp_main_t * jm, u32 tag) {
- while (__sync_lock_test_and_set(&jm->lock, 1))
+ while (clib_atomic_test_and_set(&jm->lock))
;
jm->tag = tag;
}
diff --git a/src/plugins/acl/session_inlines.h b/src/plugins/acl/session_inlines.h
index 6ac5983e086..cd23f390b28 100644
--- a/src/plugins/acl/session_inlines.h
+++ b/src/plugins/acl/session_inlines.h
@@ -410,7 +410,7 @@ acl_fa_deactivate_session (acl_main_t * am, u32 sw_if_index,
}
sess->deleted = 1;
- clib_smp_atomic_add (&am->fa_session_total_deactivations, 1);
+ clib_atomic_fetch_add (&am->fa_session_total_deactivations, 1);
clib_mem_set_heap (oldheap);
}
@@ -432,7 +432,7 @@ acl_fa_put_session (acl_main_t * am, u32 sw_if_index,
vec_validate (pw->fa_session_dels_by_sw_if_index, sw_if_index);
clib_mem_set_heap (oldheap);
pw->fa_session_dels_by_sw_if_index[sw_if_index]++;
- clib_smp_atomic_add (&am->fa_session_total_dels, 1);
+ clib_atomic_fetch_add (&am->fa_session_total_dels, 1);
}
always_inline int
@@ -571,7 +571,7 @@ acl_fa_add_session (acl_main_t * am, int is_input, int is_ip6,
vec_validate (pw->fa_session_adds_by_sw_if_index, sw_if_index);
clib_mem_set_heap (oldheap);
pw->fa_session_adds_by_sw_if_index[sw_if_index]++;
- clib_smp_atomic_add (&am->fa_session_total_adds, 1);
+ clib_atomic_fetch_add (&am->fa_session_total_adds, 1);
return sess;
}
diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c
index b4001337aca..ea00df25a25 100644
--- a/src/plugins/dpdk/device/device.c
+++ b/src/plugins/dpdk/device/device.c
@@ -162,7 +162,7 @@ static_always_inline
if (PREDICT_FALSE (xd->lockp != 0))
{
queue_id = queue_id % xd->tx_q_used;
- while (__sync_lock_test_and_set (xd->lockp[queue_id], 1))
+ while (clib_atomic_test_and_set (xd->lockp[queue_id]))
/* zzzz */
queue_id = (queue_id + 1) % xd->tx_q_used;
}
@@ -191,7 +191,7 @@ static_always_inline
}
if (PREDICT_FALSE (xd->lockp != 0))
- *xd->lockp[queue_id] = 0;
+ clib_atomic_release (xd->lockp[queue_id]);
if (PREDICT_FALSE (n_sent < 0))
{
diff --git a/src/plugins/ioam/analyse/ioam_analyse.h b/src/plugins/ioam/analyse/ioam_analyse.h
index 881b0451721..3aec7a792d8 100644
--- a/src/plugins/ioam/analyse/ioam_analyse.h
+++ b/src/plugins/ioam/analyse/ioam_analyse.h
@@ -191,7 +191,7 @@ ip6_ioam_analyse_set_paths_down (ioam_analyser_data_t * data)
ioam_path_map_t *path;
u8 k, i;
- while (__sync_lock_test_and_set (data->writer_lock, 1))
+ while (clib_atomic_test_and_set (data->writer_lock))
;
trace_data = &data->trace_data;
@@ -208,7 +208,7 @@ ip6_ioam_analyse_set_paths_down (ioam_analyser_data_t * data)
for (k = 0; k < trace_record->num_nodes; k++)
path[k].state_up = 0;
}
- *(data->writer_lock) = 0;
+ clib_atomic_release (data->writer_lock);
}
always_inline void
@@ -225,7 +225,7 @@ ip6_ioam_analyse_hbh_trace_loopback (ioam_analyser_data_t * data,
u16 size_of_traceopt_per_node;
u16 size_of_all_traceopts;
- while (__sync_lock_test_and_set (data->writer_lock, 1))
+ while (clib_atomic_test_and_set (data->writer_lock))
;
trace_data = &data->trace_data;
@@ -277,7 +277,7 @@ ip6_ioam_analyse_hbh_trace_loopback (ioam_analyser_data_t * data,
}
}
end:
- *(data->writer_lock) = 0;
+ clib_atomic_release (data->writer_lock);
}
always_inline int
@@ -295,7 +295,7 @@ ip6_ioam_analyse_hbh_trace (ioam_analyser_data_t * data,
ioam_path_map_t *path = NULL;
ioam_analyse_trace_record *trace_record;
- while (__sync_lock_test_and_set (data->writer_lock, 1))
+ while (clib_atomic_test_and_set (data->writer_lock))
;
trace_data = &data->trace_data;
@@ -409,7 +409,7 @@ found_match:
(u32) ((sum + delay) / (data->seqno_data.rx_packets + 1));
}
DONE:
- *(data->writer_lock) = 0;
+ clib_atomic_release (data->writer_lock);
return 0;
}
@@ -417,13 +417,14 @@ always_inline int
ip6_ioam_analyse_hbh_e2e (ioam_analyser_data_t * data,
ioam_e2e_packet_t * e2e, u16 len)
{
- while (__sync_lock_test_and_set (data->writer_lock, 1))
+ while (clib_atomic_test_and_set (data->writer_lock))
;
ioam_analyze_seqno (&data->seqno_data,
(u64) clib_net_to_host_u32 (e2e->e2e_data));
- *(data->writer_lock) = 0;
+ clib_atomic_release (data->writer_lock);
+
return 0;
}
@@ -510,7 +511,8 @@ ioam_analyse_init_data (ioam_analyser_data_t * data)
data->writer_lock = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
CLIB_CACHE_LINE_BYTES);
- *(data->writer_lock) = 0;
+
+ clib_atomic_release (data->writer_lock);
trace_data = &(data->trace_data);
for (j = 0; j < IOAM_MAX_PATHS_PER_FLOW; j++)
diff --git a/src/plugins/ioam/analyse/ioam_summary_export.c b/src/plugins/ioam/analyse/ioam_summary_export.c
index c118bfa3529..440dcca8369 100644
--- a/src/plugins/ioam/analyse/ioam_summary_export.c
+++ b/src/plugins/ioam/analyse/ioam_summary_export.c
@@ -150,7 +150,7 @@ ioam_analyse_add_ipfix_record (flow_report_t * fr,
ip6_address_t * src, ip6_address_t * dst,
u16 src_port, u16 dst_port)
{
- while (__sync_lock_test_and_set (record->writer_lock, 1))
+ while (clib_atomic_test_and_set (record->writer_lock))
;
int field_index = 0;
@@ -259,7 +259,7 @@ ioam_analyse_add_ipfix_record (flow_report_t * fr,
*(record->chached_data_list) = *record;
record->chached_data_list->chached_data_list = NULL;
- *(record->writer_lock) = 0;
+ clib_atomic_release (record->writer_lock);
return offset;
}
diff --git a/src/plugins/ioam/analyse/ip6/node.c b/src/plugins/ioam/analyse/ip6/node.c
index 902fb9b09ce..7a8d71de93b 100644
--- a/src/plugins/ioam/analyse/ip6/node.c
+++ b/src/plugins/ioam/analyse/ip6/node.c
@@ -256,17 +256,17 @@ ip6_ioam_analyse_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
data0 = ioam_analyse_get_data_from_flow_id (flow_id0);
data1 = ioam_analyse_get_data_from_flow_id (flow_id1);
- while (__sync_lock_test_and_set (data0->writer_lock, 1))
+ while (clib_atomic_test_and_set (data0->writer_lock))
;
data0->pkt_counter++;
data0->bytes_counter += p_len0;
- *(data0->writer_lock) = 0;
+ clib_atomic_release (data0->writer_lock);
- while (__sync_lock_test_and_set (data1->writer_lock, 1))
+ while (clib_atomic_test_and_set (data1->writer_lock))
;
data1->pkt_counter++;
data1->bytes_counter += p_len1;
- *(data1->writer_lock) = 0;
+ clib_atomic_release (data1->writer_lock);
}
else if (error0 == 0)
{
@@ -274,11 +274,11 @@ ip6_ioam_analyse_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
pkts_failed++;
data0 = ioam_analyse_get_data_from_flow_id (flow_id0);
- while (__sync_lock_test_and_set (data0->writer_lock, 1))
+ while (clib_atomic_test_and_set (data0->writer_lock))
;
data0->pkt_counter++;
data0->bytes_counter += p_len0;
- *(data0->writer_lock) = 0;
+ clib_atomic_release (data0->writer_lock);
}
else if (error1 == 0)
{
@@ -286,11 +286,11 @@ ip6_ioam_analyse_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
pkts_failed++;
data1 = ioam_analyse_get_data_from_flow_id (flow_id1);
- while (__sync_lock_test_and_set (data1->writer_lock, 1))
+ while (clib_atomic_test_and_set (data1->writer_lock))
;
data1->pkt_counter++;
data1->bytes_counter += p_len1;
- *(data1->writer_lock) = 0;
+ clib_atomic_release (data1->writer_lock);
}
else
pkts_failed += 2;
@@ -327,12 +327,12 @@ ip6_ioam_analyse_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
{
pkts_analysed++;
data0 = ioam_analyse_get_data_from_flow_id (flow_id0);
- while (__sync_lock_test_and_set (data0->writer_lock, 1))
+ while (clib_atomic_test_and_set (data0->writer_lock))
;
data0->pkt_counter++;
data0->bytes_counter +=
clib_net_to_host_u16 (ip60->payload_length);
- *(data0->writer_lock) = 0;
+ clib_atomic_release (data0->writer_lock);
}
else
pkts_failed++;
@@ -393,13 +393,13 @@ ip6_ioam_analyse_hbh_pot (u32 flow_id, ip6_hop_by_hop_option_t * opt0,
pot_profile = pot_profile_get_active ();
ret = pot_validate (pot_profile, cumulative, random);
- while (__sync_lock_test_and_set (data->writer_lock, 1))
+ while (clib_atomic_test_and_set (data->writer_lock))
;
(0 == ret) ? (data->pot_data.sfc_validated_count++) :
(data->pot_data.sfc_invalidated_count++);
- *(data->writer_lock) = 0;
+ clib_atomic_release (data->writer_lock);
return 0;
}
diff --git a/src/plugins/ioam/export-common/ioam_export.h b/src/plugins/ioam/export-common/ioam_export.h
index 36d71d26716..6e640951802 100644
--- a/src/plugins/ioam/export-common/ioam_export.h
+++ b/src/plugins/ioam/export-common/ioam_export.h
@@ -436,11 +436,11 @@ ioam_export_process_common (ioam_export_main_t * em, vlib_main_t * vm,
*/
for (i = 0; i < vec_len (thread_index); i++)
{
- while (__sync_lock_test_and_set (em->lockp[thread_index[i]], 1))
+ while (clib_atomic_test_and_set (em->lockp[thread_index[i]]))
;
em->buffer_per_thread[thread_index[i]] =
vec_pop (vec_buffer_indices);
- *em->lockp[thread_index[i]] = 0;
+ clib_atomic_release (em->lockp[thread_index[i]]);
}
/* Send the buffers */
@@ -479,7 +479,7 @@ do { \
from = vlib_frame_vector_args (F); \
n_left_from = (F)->n_vectors; \
next_index = (N)->cached_next_index; \
- while (__sync_lock_test_and_set ((EM)->lockp[(VM)->thread_index], 1)); \
+ while (clib_atomic_test_and_set ((EM)->lockp[(VM)->thread_index])); \
my_buf = ioam_export_get_my_buffer (EM, (VM)->thread_index); \
my_buf->touched_at = vlib_time_now (VM); \
while (n_left_from > 0) \
diff --git a/src/plugins/lb/lb.c b/src/plugins/lb/lb.c
index 1936c1983ee..76bbfa0ac81 100644
--- a/src/plugins/lb/lb.c
+++ b/src/plugins/lb/lb.c
@@ -27,8 +27,8 @@
lb_main_t lb_main;
-#define lb_get_writer_lock() do {} while(__sync_lock_test_and_set (lb_main.writer_lock, 1))
-#define lb_put_writer_lock() lb_main.writer_lock[0] = 0
+#define lb_get_writer_lock() do {} while(clib_atomic_test_and_set (lb_main.writer_lock))
+#define lb_put_writer_lock() clib_atomic_release (lb_main.writer_lock)
static void lb_as_stack (lb_as_t *as);
diff --git a/src/plugins/map/map.h b/src/plugins/map/map.h
index 6587a8a455b..1e07b59f762 100644
--- a/src/plugins/map/map.h
+++ b/src/plugins/map/map.h
@@ -474,7 +474,7 @@ map_ip4_reass_get(u32 src, u32 dst, u16 fragment_id,
void
map_ip4_reass_free(map_ip4_reass_t *r, u32 **pi_to_drop);
-#define map_ip4_reass_lock() while (__sync_lock_test_and_set(map_main.ip4_reass_lock, 1)) {}
+#define map_ip4_reass_lock() while (clib_atomic_test_and_set (map_main.ip4_reass_lock)) {}
#define map_ip4_reass_unlock() do {CLIB_MEMORY_BARRIER(); *map_main.ip4_reass_lock = 0;} while(0)
static_always_inline void
@@ -499,7 +499,7 @@ map_ip6_reass_get(ip6_address_t *src, ip6_address_t *dst, u32 fragment_id,
void
map_ip6_reass_free(map_ip6_reass_t *r, u32 **pi_to_drop);
-#define map_ip6_reass_lock() while (__sync_lock_test_and_set(map_main.ip6_reass_lock, 1)) {}
+#define map_ip6_reass_lock() while (clib_atomic_test_and_set (map_main.ip6_reass_lock)) {}
#define map_ip6_reass_unlock() do {CLIB_MEMORY_BARRIER(); *map_main.ip6_reass_lock = 0;} while(0)
int
@@ -555,14 +555,14 @@ static inline void
map_domain_counter_lock (map_main_t *mm)
{
if (mm->counter_lock)
- while (__sync_lock_test_and_set(mm->counter_lock, 1))
+ while (clib_atomic_test_and_set (mm->counter_lock))
/* zzzz */ ;
}
static inline void
map_domain_counter_unlock (map_main_t *mm)
{
if (mm->counter_lock)
- *mm->counter_lock = 0;
+ clib_atomic_release (mm->counter_lock);
}
diff --git a/src/plugins/nat/nat_det.h b/src/plugins/nat/nat_det.h
index ce876bca9fb..7878a4c4ed6 100644
--- a/src/plugins/nat/nat_det.h
+++ b/src/plugins/nat/nat_det.h
@@ -159,13 +159,13 @@ snat_det_ses_create (snat_det_map_t * dm, ip4_address_t * in_addr,
{
if (!dm->sessions[i + user_offset].in_port)
{
- if (__sync_bool_compare_and_swap
+ if (clib_atomic_bool_cmp_and_swap
(&dm->sessions[i + user_offset].in_port, 0, in_port))
{
dm->sessions[i + user_offset].out.as_u64 = out->as_u64;
dm->sessions[i + user_offset].state = SNAT_SESSION_UNKNOWN;
dm->sessions[i + user_offset].expire = 0;
- __sync_add_and_fetch (&dm->ses_num, 1);
+ clib_atomic_add_fetch (&dm->ses_num, 1);
return &dm->sessions[i + user_offset];
}
}
@@ -179,10 +179,10 @@ snat_det_ses_create (snat_det_map_t * dm, ip4_address_t * in_addr,
always_inline void
snat_det_ses_close (snat_det_map_t * dm, snat_det_session_t * ses)
{
- if (__sync_bool_compare_and_swap (&ses->in_port, ses->in_port, 0))
+ if (clib_atomic_bool_cmp_and_swap (&ses->in_port, ses->in_port, 0))
{
ses->out.as_u64 = 0;
- __sync_add_and_fetch (&dm->ses_num, -1);
+ clib_atomic_add_fetch (&dm->ses_num, -1);
}
}
diff --git a/src/svm/message_queue.c b/src/svm/message_queue.c
index a73a56d8044..13d089a97cc 100644
--- a/src/svm/message_queue.c
+++ b/src/svm/message_queue.c
@@ -108,7 +108,7 @@ svm_msg_q_alloc_msg_w_ring (svm_msg_q_t * mq, u32 ring_index)
msg.ring_index = ring - mq->rings;
msg.elt_index = ring->tail;
ring->tail = (ring->tail + 1) % ring->nitems;
- __sync_fetch_and_add (&ring->cursize, 1);
+ clib_atomic_fetch_add (&ring->cursize, 1);
return msg;
}
@@ -155,7 +155,7 @@ svm_msg_q_alloc_msg (svm_msg_q_t * mq, u32 nbytes)
msg.ring_index = ring - mq->rings;
msg.elt_index = ring->tail;
ring->tail = (ring->tail + 1) % ring->nitems;
- __sync_fetch_and_add (&ring->cursize, 1);
+ clib_atomic_fetch_add (&ring->cursize, 1);
break;
}
return msg;
@@ -185,7 +185,7 @@ svm_msg_q_free_msg (svm_msg_q_t * mq, svm_msg_q_msg_t * msg)
/* for now, expect messages to be processed in order */
ASSERT (0);
}
- __sync_fetch_and_sub (&ring->cursize, 1);
+ clib_atomic_fetch_sub (&ring->cursize, 1);
}
static int
diff --git a/src/svm/ssvm.h b/src/svm/ssvm.h
index 8677f5680be..5b2bf0d202d 100644
--- a/src/svm/ssvm.h
+++ b/src/svm/ssvm.h
@@ -103,7 +103,7 @@ ssvm_lock (ssvm_shared_header_t * h, u32 my_pid, u32 tag)
return;
}
- while (__sync_lock_test_and_set (&h->lock, 1))
+ while (clib_atomic_test_and_set (&h->lock))
;
h->owner_pid = my_pid;
@@ -114,7 +114,7 @@ ssvm_lock (ssvm_shared_header_t * h, u32 my_pid, u32 tag)
always_inline void
ssvm_lock_non_recursive (ssvm_shared_header_t * h, u32 tag)
{
- while (__sync_lock_test_and_set (&h->lock, 1))
+ while (clib_atomic_test_and_set (&h->lock))
;
h->tag = tag;
diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c
index 4eae0a1c7e6..aa523c6b55c 100644
--- a/src/svm/svm_fifo.c
+++ b/src/svm/svm_fifo.c
@@ -513,7 +513,7 @@ CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 max_bytes,
/* Atomically increase the queue length */
ASSERT (cursize + total_copy_bytes <= nitems);
- __sync_fetch_and_add (&f->cursize, total_copy_bytes);
+ clib_atomic_fetch_add (&f->cursize, total_copy_bytes);
return (total_copy_bytes);
}
@@ -659,7 +659,7 @@ CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 max_bytes,
ASSERT (f->head <= nitems);
ASSERT (cursize >= total_copy_bytes);
- __sync_fetch_and_sub (&f->cursize, total_copy_bytes);
+ clib_atomic_fetch_sub (&f->cursize, total_copy_bytes);
return (total_copy_bytes);
}
@@ -757,7 +757,7 @@ svm_fifo_dequeue_drop (svm_fifo_t * f, u32 max_bytes)
ASSERT (f->head <= nitems);
ASSERT (cursize >= total_drop_bytes);
- __sync_fetch_and_sub (&f->cursize, total_drop_bytes);
+ clib_atomic_fetch_sub (&f->cursize, total_drop_bytes);
return total_drop_bytes;
}
@@ -766,7 +766,7 @@ void
svm_fifo_dequeue_drop_all (svm_fifo_t * f)
{
f->head = f->tail;
- __sync_fetch_and_sub (&f->cursize, f->cursize);
+ clib_atomic_fetch_sub (&f->cursize, f->cursize);
}
int
@@ -813,7 +813,7 @@ svm_fifo_segments_free (svm_fifo_t * f, svm_fifo_segment_t * fs)
f->head = (f->head + fs[0].len) % f->nitems;
total_drop_bytes = fs[0].len;
}
- __sync_fetch_and_sub (&f->cursize, total_drop_bytes);
+ clib_atomic_fetch_sub (&f->cursize, total_drop_bytes);
}
u32
diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h
index d7852a79b32..e049d3e3147 100644
--- a/src/svm/svm_fifo.h
+++ b/src/svm/svm_fifo.h
@@ -169,7 +169,7 @@ svm_fifo_set_event (svm_fifo_t * f)
always_inline void
svm_fifo_unset_event (svm_fifo_t * f)
{
- __sync_lock_release (&f->has_event);
+ clib_atomic_release (&f->has_event);
}
static inline void
diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h
index d8abdf31d79..438bf7e5ee4 100644
--- a/src/vlib/buffer_funcs.h
+++ b/src/vlib/buffer_funcs.h
@@ -923,7 +923,7 @@ vlib_buffer_attach_clone (vlib_main_t * vm, vlib_buffer_t * head,
tail->total_length_not_including_first_buffer;
next_segment:
- __sync_add_and_fetch (&tail->n_add_refs, 1);
+ clib_atomic_add_fetch (&tail->n_add_refs, 1);
if (tail->flags & VLIB_BUFFER_NEXT_PRESENT)
{
@@ -1153,7 +1153,7 @@ vlib_validate_buffer_in_use (vlib_buffer_t * b, u32 expected)
oldheap = clib_mem_set_heap (vlib_buffer_state_heap);
- while (__sync_lock_test_and_set (vlib_buffer_state_validation_lock, 1))
+ while (clib_atomic_test_and_set (vlib_buffer_state_validation_lock))
;
p = hash_get (vlib_buffer_state_validation_hash, b);
@@ -1196,7 +1196,7 @@ vlib_validate_buffer_set_in_use (vlib_buffer_t * b, u32 expected)
oldheap = clib_mem_set_heap (vlib_buffer_state_heap);
- while (__sync_lock_test_and_set (vlib_buffer_state_validation_lock, 1))
+ while (clib_atomic_test_and_set (vlib_buffer_state_validation_lock))
;
hash_set (vlib_buffer_state_validation_hash, b, expected);
diff --git a/src/vlib/threads.c b/src/vlib/threads.c
index 981209bf2e7..7f407e92478 100644
--- a/src/vlib/threads.c
+++ b/src/vlib/threads.c
@@ -516,7 +516,7 @@ vlib_frame_queue_enqueue (vlib_main_t * vm, u32 node_runtime_index,
ASSERT (fq);
- new_tail = __sync_add_and_fetch (&fq->tail, 1);
+ new_tail = clib_atomic_add_fetch (&fq->tail, 1);
/* Wait until a ring slot is available */
while (new_tail >= fq->head + fq->nelts)
@@ -576,12 +576,12 @@ vlib_worker_thread_init (vlib_worker_thread_t * w)
{
/* Initial barrier sync, for both worker and i/o threads */
- clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, 1);
+ clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, 1);
while (*vlib_worker_threads->wait_at_barrier)
;
- clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, -1);
+ clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, -1);
}
}
@@ -1310,22 +1310,6 @@ cpu_config (vlib_main_t * vm, unformat_input_t * input)
VLIB_EARLY_CONFIG_FUNCTION (cpu_config, "cpu");
-#if !defined (__x86_64__) && !defined (__i386__) && !defined (__aarch64__) && !defined (__powerpc64__) && !defined(__arm__)
-void
-__sync_fetch_and_add_8 (void)
-{
- fformat (stderr, "%s called\n", __FUNCTION__);
- abort ();
-}
-
-void
-__sync_add_and_fetch_8 (void)
-{
- fformat (stderr, "%s called\n", __FUNCTION__);
- abort ();
-}
-#endif
-
void vnet_main_fixup (vlib_fork_fixup_t which) __attribute__ ((weak));
void
vnet_main_fixup (vlib_fork_fixup_t which)
@@ -1493,8 +1477,8 @@ vlib_worker_thread_barrier_release (vlib_main_t * vm)
/* Do per thread rebuilds in parallel */
refork_needed = 1;
- clib_smp_atomic_add (vlib_worker_threads->node_reforks_required,
- (vec_len (vlib_mains) - 1));
+ clib_atomic_fetch_add (vlib_worker_threads->node_reforks_required,
+ (vec_len (vlib_mains) - 1));
now = vlib_time_now (vm);
t_update_main = now - vm->barrier_epoch;
}
diff --git a/src/vlib/threads.h b/src/vlib/threads.h
index bb7c164c2e3..0e9cba52103 100644
--- a/src/vlib/threads.h
+++ b/src/vlib/threads.h
@@ -414,7 +414,7 @@ vlib_worker_thread_barrier_check (void)
ed->thread_index = thread_index;
}
- clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, 1);
+ clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, 1);
if (CLIB_DEBUG > 0)
{
vm = vlib_get_main ();
@@ -424,7 +424,7 @@ vlib_worker_thread_barrier_check (void)
;
if (CLIB_DEBUG > 0)
vm->parked_at_barrier = 0;
- clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, -1);
+ clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, -1);
if (PREDICT_FALSE (*vlib_worker_threads->node_reforks_required))
{
@@ -450,8 +450,8 @@ vlib_worker_thread_barrier_check (void)
}
vlib_worker_thread_node_refork ();
- clib_smp_atomic_add (vlib_worker_threads->node_reforks_required,
- -1);
+ clib_atomic_fetch_add (vlib_worker_threads->node_reforks_required,
+ -1);
while (*vlib_worker_threads->node_reforks_required)
;
}
@@ -519,7 +519,7 @@ vlib_get_frame_queue_elt (u32 frame_queue_index, u32 index)
fq = fqm->vlib_frame_queues[index];
ASSERT (fq);
- new_tail = __sync_add_and_fetch (&fq->tail, 1);
+ new_tail = clib_atomic_add_fetch (&fq->tail, 1);
/* Wait until a ring slot is available */
while (new_tail >= fq->head_hint + fq->nelts)
diff --git a/src/vlib/unix/cj.c b/src/vlib/unix/cj.c
index 0232ea2d690..7757146c7a7 100644
--- a/src/vlib/unix/cj.c
+++ b/src/vlib/unix/cj.c
@@ -44,7 +44,7 @@ cj_log (u32 type, void *data0, void *data1)
if (cjm->enable == 0)
return;
- new_tail = __sync_add_and_fetch (&cjm->tail, 1);
+ new_tail = clib_atomic_add_fetch (&cjm->tail, 1);
r = (cj_record_t *) & (cjm->records[new_tail & (cjm->num_records - 1)]);
r->time = vlib_time_now (cjm->vlib_main);
diff --git a/src/vnet/classify/vnet_classify.c b/src/vnet/classify/vnet_classify.c
index 9d8694a4993..52cabbc7f42 100644
--- a/src/vnet/classify/vnet_classify.c
+++ b/src/vnet/classify/vnet_classify.c
@@ -444,7 +444,7 @@ vnet_classify_add_del (vnet_classify_table_t * t,
hash >>= t->log2_nbuckets;
- while (__sync_lock_test_and_set (t->writer_lock, 1))
+ while (clib_atomic_test_and_set (t->writer_lock))
;
/* First elt in the bucket? */
diff --git a/src/vnet/devices/virtio/vhost_user_output.c b/src/vnet/devices/virtio/vhost_user_output.c
index c77cdb6a2ee..dab8fa5fb3b 100644
--- a/src/vnet/devices/virtio/vhost_user_output.c
+++ b/src/vnet/devices/virtio/vhost_user_output.c
@@ -122,7 +122,7 @@ vhost_user_name_renumber (vnet_hw_interface_t * hi, u32 new_dev_instance)
static_always_inline int
vhost_user_vring_try_lock (vhost_user_intf_t * vui, u32 qid)
{
- return __sync_lock_test_and_set (vui->vring_locks[qid], 1);
+ return clib_atomic_test_and_set (vui->vring_locks[qid]);
}
/**
@@ -141,7 +141,7 @@ vhost_user_vring_lock (vhost_user_intf_t * vui, u32 qid)
static_always_inline void
vhost_user_vring_unlock (vhost_user_intf_t * vui, u32 qid)
{
- *vui->vring_locks[qid] = 0;
+ clib_atomic_release (vui->vring_locks[qid]);
}
static_always_inline void
diff --git a/src/vnet/dns/dns.h b/src/vnet/dns/dns.h
index f0edd8cf4ba..59a61ed7687 100644
--- a/src/vnet/dns/dns.h
+++ b/src/vnet/dns/dns.h
@@ -187,7 +187,7 @@ dns_cache_lock (dns_main_t * dm)
{
if (dm->cache_lock)
{
- while (__sync_lock_test_and_set (dm->cache_lock, 1))
+ while (clib_atomic_test_and_set (dm->cache_lock))
;
}
}
diff --git a/src/vnet/gre/gre.c b/src/vnet/gre/gre.c
index e82befe7918..070c78e8984 100644
--- a/src/vnet/gre/gre.c
+++ b/src/vnet/gre/gre.c
@@ -406,7 +406,7 @@ gre_interface_tx (vlib_main_t * vm,
/* Encap GRE seq# and ERSPAN type II header */
vlib_buffer_advance (b0, -sizeof (erspan_t2_t));
erspan_t2_t *h0 = vlib_buffer_get_current (b0);
- u32 seq_num = clib_smp_atomic_add (&gt0->gre_sn->seq_num, 1);
+ u32 seq_num = clib_atomic_fetch_add (&gt0->gre_sn->seq_num, 1);
u64 hdr = clib_host_to_net_u64 (ERSPAN_HDR2);
h0->seq_num = clib_host_to_net_u32 (seq_num);
h0->t2_u64 = hdr;
@@ -418,7 +418,7 @@ gre_interface_tx (vlib_main_t * vm,
/* Encap GRE seq# and ERSPAN type II header */
vlib_buffer_advance (b1, -sizeof (erspan_t2_t));
erspan_t2_t *h1 = vlib_buffer_get_current (b1);
- u32 seq_num = clib_smp_atomic_add (&gt1->gre_sn->seq_num, 1);
+ u32 seq_num = clib_atomic_fetch_add (&gt1->gre_sn->seq_num, 1);
u64 hdr = clib_host_to_net_u64 (ERSPAN_HDR2);
h1->seq_num = clib_host_to_net_u32 (seq_num);
h1->t2_u64 = hdr;
@@ -473,7 +473,7 @@ gre_interface_tx (vlib_main_t * vm,
/* Encap GRE seq# and ERSPAN type II header */
vlib_buffer_advance (b0, -sizeof (erspan_t2_t));
erspan_t2_t *h0 = vlib_buffer_get_current (b0);
- u32 seq_num = clib_smp_atomic_add (&gt0->gre_sn->seq_num, 1);
+ u32 seq_num = clib_atomic_fetch_add (&gt0->gre_sn->seq_num, 1);
u64 hdr = clib_host_to_net_u64 (ERSPAN_HDR2);
h0->seq_num = clib_host_to_net_u32 (seq_num);
h0->t2_u64 = hdr;
diff --git a/src/vnet/interface.h b/src/vnet/interface.h
index 6ca2b0d6cee..7ce6aafdf54 100644
--- a/src/vnet/interface.h
+++ b/src/vnet/interface.h
@@ -872,7 +872,7 @@ static inline void
vnet_interface_counter_lock (vnet_interface_main_t * im)
{
if (im->sw_if_counter_lock)
- while (__sync_lock_test_and_set (im->sw_if_counter_lock, 1))
+ while (clib_atomic_test_and_set (im->sw_if_counter_lock))
/* zzzz */ ;
}
@@ -880,7 +880,7 @@ static inline void
vnet_interface_counter_unlock (vnet_interface_main_t * im)
{
if (im->sw_if_counter_lock)
- *im->sw_if_counter_lock = 0;
+ clib_atomic_release (im->sw_if_counter_lock);
}
void vnet_pcap_drop_trace_filter_add_del (u32 error_index, int is_add);
diff --git a/src/vnet/ip/ip4_mtrie.c b/src/vnet/ip/ip4_mtrie.c
index fbb8a748074..e6425ca703a 100755
--- a/src/vnet/ip/ip4_mtrie.c
+++ b/src/vnet/ip/ip4_mtrie.c
@@ -254,7 +254,7 @@ set_ply_with_more_specific_leaf (ip4_fib_mtrie_t * m,
else if (new_leaf_dst_address_bits >=
ply->dst_address_bits_of_leaves[i])
{
- __sync_val_compare_and_swap (&ply->leaves[i], old_leaf, new_leaf);
+ clib_atomic_cmp_and_swap (&ply->leaves[i], old_leaf, new_leaf);
ASSERT (ply->leaves[i] == new_leaf);
ply->dst_address_bits_of_leaves[i] = new_leaf_dst_address_bits;
ply->n_non_empty_leafs += ip4_fib_mtrie_leaf_is_non_empty (ply, i);
@@ -319,8 +319,8 @@ set_leaf (ip4_fib_mtrie_t * m,
old_ply->dst_address_bits_of_leaves[i] =
a->dst_address_length;
- __sync_val_compare_and_swap (&old_ply->leaves[i], old_leaf,
- new_leaf);
+ clib_atomic_cmp_and_swap (&old_ply->leaves[i], old_leaf,
+ new_leaf);
ASSERT (old_ply->leaves[i] == new_leaf);
old_ply->n_non_empty_leafs +=
@@ -378,8 +378,8 @@ set_leaf (ip4_fib_mtrie_t * m,
/* Refetch since ply_create may move pool. */
old_ply = pool_elt_at_index (ip4_ply_pool, old_ply_index);
- __sync_val_compare_and_swap (&old_ply->leaves[dst_byte], old_leaf,
- new_leaf);
+ clib_atomic_cmp_and_swap (&old_ply->leaves[dst_byte], old_leaf,
+ new_leaf);
ASSERT (old_ply->leaves[dst_byte] == new_leaf);
old_ply->dst_address_bits_of_leaves[dst_byte] = ply_base_len;
@@ -451,8 +451,8 @@ set_root_leaf (ip4_fib_mtrie_t * m,
* the new one */
old_ply->dst_address_bits_of_leaves[slot] =
a->dst_address_length;
- __sync_val_compare_and_swap (&old_ply->leaves[slot],
- old_leaf, new_leaf);
+ clib_atomic_cmp_and_swap (&old_ply->leaves[slot],
+ old_leaf, new_leaf);
ASSERT (old_ply->leaves[slot] == new_leaf);
}
else
@@ -498,8 +498,8 @@ set_root_leaf (ip4_fib_mtrie_t * m,
ply_base_len);
new_ply = get_next_ply_for_leaf (m, new_leaf);
- __sync_val_compare_and_swap (&old_ply->leaves[dst_byte], old_leaf,
- new_leaf);
+ clib_atomic_cmp_and_swap (&old_ply->leaves[dst_byte], old_leaf,
+ new_leaf);
ASSERT (old_ply->leaves[dst_byte] == new_leaf);
old_ply->dst_address_bits_of_leaves[dst_byte] = ply_base_len;
}
diff --git a/src/vnet/ipfix-export/flow_report_classify.c b/src/vnet/ipfix-export/flow_report_classify.c
index 8fb73fc0867..196cb725ffe 100644
--- a/src/vnet/ipfix-export/flow_report_classify.c
+++ b/src/vnet/ipfix-export/flow_report_classify.c
@@ -197,7 +197,7 @@ ipfix_classify_send_flows (flow_report_main_t * frm,
t = pool_elt_at_index (vcm->tables, table->classify_table_index);
- while (__sync_lock_test_and_set (t->writer_lock, 1))
+ while (clib_atomic_test_and_set (t->writer_lock))
;
for (i = 0; i < t->nbuckets; i++)
@@ -385,7 +385,7 @@ flush:
bi0 = ~0;
}
- *(t->writer_lock) = 0;
+ clib_atomic_release (t->writer_lock);
return f;
}
diff --git a/src/vnet/mfib/mfib_forward.c b/src/vnet/mfib/mfib_forward.c
index 3d0e0d47996..4b121324fb6 100644
--- a/src/vnet/mfib/mfib_forward.c
+++ b/src/vnet/mfib/mfib_forward.c
@@ -300,8 +300,8 @@ mfib_forward_itf_signal (vlib_main_t *vm,
{
mfib_itf_flags_t old_flags;
- old_flags = __sync_fetch_and_or(&mfi->mfi_flags,
- MFIB_ITF_FLAG_SIGNAL_PRESENT);
+ old_flags = clib_atomic_fetch_or(&mfi->mfi_flags,
+ MFIB_ITF_FLAG_SIGNAL_PRESENT);
if (!(old_flags & MFIB_ITF_FLAG_SIGNAL_PRESENT))
{
diff --git a/src/vnet/mfib/mfib_signal.c b/src/vnet/mfib/mfib_signal.c
index ce9a664c548..176e8ec9114 100644
--- a/src/vnet/mfib/mfib_signal.c
+++ b/src/vnet/mfib/mfib_signal.c
@@ -71,14 +71,14 @@ mfib_signal_module_init (void)
static inline void
mfib_signal_lock_aquire (void)
{
- while (__sync_lock_test_and_set (&mfib_signal_pending.mip_lock, 1))
+ while (clib_atomic_test_and_set (&mfib_signal_pending.mip_lock))
;
}
static inline void
mfib_signal_lock_release (void)
{
- mfib_signal_pending.mip_lock = 0;
+ clib_atomic_release(&mfib_signal_pending.mip_lock);
}
#define MFIB_SIGNAL_CRITICAL_SECTION(_body) \
@@ -117,8 +117,8 @@ mfib_signal_send_one (struct vl_api_registration_ *reg,
mfs = pool_elt_at_index(mfib_signal_pool, si);
mfi = mfib_itf_get(mfs->mfs_itf);
mfi->mfi_si = INDEX_INVALID;
- __sync_fetch_and_and(&mfi->mfi_flags,
- ~MFIB_ITF_FLAG_SIGNAL_PRESENT);
+ clib_atomic_fetch_and(&mfi->mfi_flags,
+ ~MFIB_ITF_FLAG_SIGNAL_PRESENT);
vl_mfib_signal_send_one(reg, context, mfs);
diff --git a/src/vnet/pg/output.c b/src/vnet/pg/output.c
index ab57deefd24..016e5b370e5 100644
--- a/src/vnet/pg/output.c
+++ b/src/vnet/pg/output.c
@@ -54,7 +54,7 @@ pg_output (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
pg_interface_t *pif = pool_elt_at_index (pg->interfaces, rd->dev_instance);
if (PREDICT_FALSE (pif->lockp != 0))
- while (__sync_lock_test_and_set (pif->lockp, 1))
+ while (clib_atomic_test_and_set (pif->lockp))
;
while (n_left > 0)
@@ -82,7 +82,8 @@ pg_output (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
vlib_buffer_free (vm, vlib_frame_args (frame), n_buffers);
if (PREDICT_FALSE (pif->lockp != 0))
- *pif->lockp = 0;
+ clib_atomic_release (pif->lockp);
+
return n_buffers;
}
diff --git a/src/vnet/session-apps/echo_client.c b/src/vnet/session-apps/echo_client.c
index 1ece0196dde..c3d838d49cc 100644
--- a/src/vnet/session-apps/echo_client.c
+++ b/src/vnet/session-apps/echo_client.c
@@ -263,8 +263,8 @@ echo_client_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
{
stream_session_t *s;
- __sync_fetch_and_add (&ecm->tx_total, sp->bytes_sent);
- __sync_fetch_and_add (&ecm->rx_total, sp->bytes_received);
+ clib_atomic_fetch_add (&ecm->tx_total, sp->bytes_sent);
+ clib_atomic_fetch_add (&ecm->rx_total, sp->bytes_received);
s = session_get_from_handle_if_valid (sp->vpp_session_handle);
if (s)
@@ -276,7 +276,7 @@ echo_client_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
vec_delete (connections_this_batch, 1, i);
i--;
- __sync_fetch_and_add (&ecm->ready_connections, -1);
+ clib_atomic_fetch_add (&ecm->ready_connections, -1);
}
else
{
@@ -408,7 +408,7 @@ echo_clients_session_connected_callback (u32 app_index, u32 api_context,
}
vec_add1 (ecm->connection_index_by_thread[thread_index], session_index);
- __sync_fetch_and_add (&ecm->ready_connections, 1);
+ clib_atomic_fetch_add (&ecm->ready_connections, 1);
if (ecm->ready_connections == ecm->expected_connections)
{
ecm->run_test = ECHO_CLIENTS_RUNNING;
diff --git a/src/vnet/util/refcount.h b/src/vnet/util/refcount.h
index ea92148dafa..873ab6def69 100644
--- a/src/vnet/util/refcount.h
+++ b/src/vnet/util/refcount.h
@@ -52,14 +52,14 @@ typedef struct {
static_always_inline
void vlib_refcount_lock (volatile u32 *counter_lock)
{
- while (__sync_lock_test_and_set (counter_lock, 1))
+ while (clib_atomic_test_and_set (counter_lock))
;
}
static_always_inline
void vlib_refcount_unlock (volatile u32 *counter_lock)
{
- *counter_lock = 0;
+ clib_atomic_release(counter_lock);
}
void __vlib_refcount_resize(vlib_refcount_per_cpu_t *per_cpu, u32 size);
diff --git a/src/vpp/stats/stats_to_be_deprecated.c b/src/vpp/stats/stats_to_be_deprecated.c
index 94d68c839d5..241bdd978df 100644
--- a/src/vpp/stats/stats_to_be_deprecated.c
+++ b/src/vpp/stats/stats_to_be_deprecated.c
@@ -221,7 +221,7 @@ dslock (stats_main_t * sm, int release_hint, int tag)
if (release_hint)
l->release_hint++;
- while (__sync_lock_test_and_set (&l->lock, 1))
+ while (clib_atomic_test_and_set (&l->lock))
/* zzzz */ ;
l->tag = tag;
l->thread_index = thread_index;
diff --git a/src/vppinfra/CMakeLists.txt b/src/vppinfra/CMakeLists.txt
index b279f90ae2a..7103d600780 100644
--- a/src/vppinfra/CMakeLists.txt
+++ b/src/vppinfra/CMakeLists.txt
@@ -158,6 +158,7 @@ set(VPPINFRA_HEADERS
tw_timer_template.c
tw_timer_template.h
types.h
+ atomics.h
unix.h
valgrind.h
valloc.h
diff --git a/src/vppinfra/atomics.h b/src/vppinfra/atomics.h
new file mode 100644
index 00000000000..8ddf13801df
--- /dev/null
+++ b/src/vppinfra/atomics.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Copyright (c) 2018 Arm Limited. and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef included_clib_atomics_h
+#define included_clib_atomics_h
+
+/* Legacy __sync builtins */
+
+/* Full Barrier */
+#define clib_atomic_fetch_add(a, b) __sync_fetch_and_add(a, b)
+#define clib_atomic_fetch_sub(a, b) __sync_fetch_and_sub(a, b)
+#define clib_atomic_fetch_and(a, b) __sync_fetch_and_and(a, b)
+#define clib_atomic_fetch_xor(a, b) __sync_fetch_and_xor(a, b)
+#define clib_atomic_fetch_or(a, b) __sync_fetch_and_or(a, b)
+#define clib_atomic_fetch_nand(a, b) __sync_fetch_nand(a, b)
+
+#define clib_atomic_add_fetch(a, b) __sync_add_and_fetch(a, b)
+#define clib_atomic_sub_fetch(a, b) __sync_sub_and_fetch(a, b)
+#define clib_atomic_and_fetch(a, b) __sync_and_and_fetch(a, b)
+#define clib_atomic_xor_fetch(a, b) __sync_xor_and_fetch(a, b)
+#define clib_atomic_or_fetch(a, b) __sync_or_and_fetch(a, b)
+#define clib_atomic_nand_fetch(a, b) __sync_nand_and_fetch(a, b)
+
+#define clib_atomic_cmp_and_swap(addr,old,new) __sync_val_compare_and_swap(addr, old, new)
+#define clib_atomic_bool_cmp_and_swap(addr,old,new) __sync_bool_compare_and_swap(addr, old, new)
+
+/*Accquire Barrier*/
+#define clib_atomic_test_and_set(a) __sync_lock_test_and_set(a, 1)
+/*Release Barrier*/
+#define clib_atomic_release(a) __sync_lock_release(a)
+
+#endif /* included_clib_atomics_h */
diff --git a/src/vppinfra/clib.h b/src/vppinfra/clib.h
index a6f88245d79..95dadd9c09d 100644
--- a/src/vppinfra/clib.h
+++ b/src/vppinfra/clib.h
@@ -46,6 +46,7 @@
#endif
#include <vppinfra/types.h>
+#include <vppinfra/atomics.h>
/* Global DEBUG flag. Setting this to 1 or 0 turns off
ASSERT (see vppinfra/error.h) & other debugging code. */
diff --git a/src/vppinfra/elog.c b/src/vppinfra/elog.c
index a86fadeaa39..c6902eb86ea 100644
--- a/src/vppinfra/elog.c
+++ b/src/vppinfra/elog.c
@@ -46,7 +46,7 @@ static inline void
elog_lock (elog_main_t * em)
{
if (PREDICT_FALSE (em->lock != 0))
- while (__sync_lock_test_and_set (em->lock, 1))
+ while (clib_atomic_test_and_set (em->lock))
;
}
diff --git a/src/vppinfra/elog.h b/src/vppinfra/elog.h
index d50c9a6500d..322c2c63ae7 100644
--- a/src/vppinfra/elog.h
+++ b/src/vppinfra/elog.h
@@ -313,7 +313,7 @@ elog_event_data_inline (elog_main_t * em,
ASSERT (is_pow2 (vec_len (em->event_ring)));
if (em->lock)
- ei = clib_smp_atomic_add (&em->n_total_events, 1);
+ ei = clib_atomic_fetch_add (&em->n_total_events, 1);
else
ei = em->n_total_events++;
diff --git a/src/vppinfra/lock.h b/src/vppinfra/lock.h
index dd79c40b7f2..4645378360d 100644
--- a/src/vppinfra/lock.h
+++ b/src/vppinfra/lock.h
@@ -73,7 +73,7 @@ clib_spinlock_free (clib_spinlock_t * p)
static_always_inline void
clib_spinlock_lock (clib_spinlock_t * p)
{
- while (__sync_lock_test_and_set (&(*p)->lock, 1))
+ while (clib_atomic_test_and_set (&(*p)->lock))
CLIB_PAUSE ();
CLIB_LOCK_DBG (p);
}
@@ -138,13 +138,13 @@ clib_rwlock_free (clib_rwlock_t * p)
always_inline void
clib_rwlock_reader_lock (clib_rwlock_t * p)
{
- while (__sync_lock_test_and_set (&(*p)->n_readers_lock, 1))
+ while (clib_atomic_test_and_set (&(*p)->n_readers_lock))
CLIB_PAUSE ();
(*p)->n_readers += 1;
if ((*p)->n_readers == 1)
{
- while (__sync_lock_test_and_set (&(*p)->writer_lock, 1))
+ while (clib_atomic_test_and_set (&(*p)->writer_lock))
CLIB_PAUSE ();
}
CLIB_MEMORY_BARRIER ();
@@ -159,7 +159,7 @@ clib_rwlock_reader_unlock (clib_rwlock_t * p)
ASSERT ((*p)->n_readers > 0);
CLIB_LOCK_DBG_CLEAR (p);
- while (__sync_lock_test_and_set (&(*p)->n_readers_lock, 1))
+ while (clib_atomic_test_and_set (&(*p)->n_readers_lock))
CLIB_PAUSE ();
(*p)->n_readers -= 1;
@@ -176,7 +176,7 @@ clib_rwlock_reader_unlock (clib_rwlock_t * p)
always_inline void
clib_rwlock_writer_lock (clib_rwlock_t * p)
{
- while (__sync_lock_test_and_set (&(*p)->writer_lock, 1))
+ while (clib_atomic_test_and_set (&(*p)->writer_lock))
CLIB_PAUSE ();
CLIB_LOCK_DBG (p);
}
diff --git a/src/vppinfra/maplog.h b/src/vppinfra/maplog.h
index f7d2f75a95b..ea6a835c7b5 100644
--- a/src/vppinfra/maplog.h
+++ b/src/vppinfra/maplog.h
@@ -137,7 +137,7 @@ clib_maplog_get_entry (clib_maplog_main_t * mm)
ASSERT (mm->flags & CLIB_MAPLOG_FLAG_INIT);
- my_record_index = __sync_fetch_and_add (&mm->next_record_index, 1);
+ my_record_index = clib_atomic_fetch_add (&mm->next_record_index, 1);
/* Time to unmap and create a new logfile? */
if (PREDICT_FALSE ((my_record_index & (mm->file_size_in_records - 1)) == 0))
diff --git a/src/vppinfra/mheap.c b/src/vppinfra/mheap.c
index 5b71873b9ed..0a62943e2f2 100644
--- a/src/vppinfra/mheap.c
+++ b/src/vppinfra/mheap.c
@@ -63,7 +63,7 @@ mheap_maybe_lock (void *v)
return;
}
- while (__sync_lock_test_and_set (&h->lock, 1))
+ while (clib_atomic_test_and_set (&h->lock))
;
h->owner_cpu = my_cpu;
diff --git a/src/vppinfra/smp.h b/src/vppinfra/smp.h
index e4ab66ad5c3..7146e51ac77 100644
--- a/src/vppinfra/smp.h
+++ b/src/vppinfra/smp.h
@@ -41,9 +41,7 @@
#include <vppinfra/cache.h>
#include <vppinfra/os.h> /* for os_panic */
-#define clib_smp_compare_and_swap(addr,new,old) __sync_val_compare_and_swap(addr,old,new)
#define clib_smp_swap(addr,new) __sync_lock_test_and_set(addr,new)
-#define clib_smp_atomic_add(addr,increment) __sync_fetch_and_add(addr,increment)
#if defined (i386) || defined (__x86_64__)
#define clib_smp_pause() do { asm volatile ("pause"); } while (0)