/* * Copyright (c) 2017 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef included_clib_lock_h #define included_clib_lock_h #include <vppinfra/clib.h> #include <vppinfra/atomics.h> #if __x86_64__ #define CLIB_PAUSE() __builtin_ia32_pause () #elif defined (__aarch64__) || defined (__arm__) #define CLIB_PAUSE() __asm__ ("yield") #else #define CLIB_PAUSE() #endif #if CLIB_DEBUG > 1 #define CLIB_LOCK_DBG(_p) \ do { \ (*_p)->frame_address = __builtin_frame_address (0); \ (*_p)->pid = getpid (); \ (*_p)->thread_index = os_get_thread_index (); \ } while (0) #define CLIB_LOCK_DBG_CLEAR(_p) \ do { \ (*_p)->frame_address = 0; \ (*_p)->pid = 0; \ (*_p)->thread_index = 0; \ } while (0) #else #define CLIB_LOCK_DBG(_p) #define CLIB_LOCK_DBG_CLEAR(_p) #endif #define CLIB_SPINLOCK_IS_LOCKED(_p) (*(_p))->lock #define CLIB_SPINLOCK_ASSERT_LOCKED(_p) ASSERT(CLIB_SPINLOCK_IS_LOCKED((_p))) struct clib_spinlock_s { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); u32 lock; #if CLIB_DEBUG > 0 pid_t pid; uword thread_index; void *frame_address; #endif }; typedef struct clib_spinlock_s *clib_spinlock_t; static inline void clib_spinlock_init (clib_spinlock_t * p) { *p = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES, CLIB_CACHE_LINE_BYTES); clib_memset ((void *) *p, 0, CLIB_CACHE_LINE_BYTES); } static inline void clib_spinlock_free (clib_spinlock_t * p) { if (*p) { clib_mem_free ((void *) *p); *p = 0; } } static_always_inline void clib_spinlock_lock (clib_spinlock_t * p) { u32 free = 0; while (!clib_atomic_cmp_and_swap_acq_relax_n (&(*p)->lock, &free, 1, 0)) { /* atomic load limits number of compare_exchange executions */ while (clib_atomic_load_relax_n (&(*p)->lock)) CLIB_PAUSE (); /* on failure, compare_exchange writes (*p)->lock into free */ free = 0; } CLIB_LOCK_DBG (p); } static_always_inline int clib_spinlock_trylock (clib_spinlock_t * p) { if (PREDICT_FALSE (CLIB_SPINLOCK_IS_LOCKED (p))) return 0; clib_spinlock_lock (p); return 1; } static_always_inline void clib_spinlock_lock_if_init (clib_spinlock_t * p) { if (PREDICT_FALSE (*p != 0)) clib_spinlock_lock (p); } static_always_inline int clib_spinlock_trylock_if_init (clib_spinlock_t * p) { if (PREDICT_FALSE (*p != 0)) return clib_spinlock_trylock (p); return 1; } static_always_inline void clib_spinlock_unlock (clib_spinlock_t * p) { CLIB_LOCK_DBG_CLEAR (p); /* Make sure all reads/writes are complete before releasing the lock */ clib_atomic_release (&(*p)->lock); } static_always_inline void clib_spinlock_unlock_if_init (clib_spinlock_t * p) { if (PREDICT_FALSE (*p != 0)) clib_spinlock_unlock (p); } /* * Readers-Writer Lock */ typedef struct clib_rw_lock_ { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); /* -1 when W lock held, > 0 when R lock held */ volatile i32 rw_cnt; #if CLIB_DEBUG > 0 pid_t pid; uword thread_index; void *frame_address; #endif } *clib_rwlock_t; always_inline void clib_rwlock_init (clib_rwlock_t * p) { *p = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES, CLIB_CACHE_LINE_BYTES); clib_memset ((void *) *p, 0, CLIB_CACHE_LINE_BYTES); } always_inline void clib_rwlock_free (clib_rwlock_t * p) { if (*p) { clib_mem_free ((void *) *p); *p = 0; } } always_inline void clib_rwlock_reader_lock (clib_rwlock_t * p) { i32 cnt; do { /* rwlock held by a writer */ while ((cnt = clib_atomic_load_relax_n (&(*p)->rw_cnt)) < 0) CLIB_PAUSE (); } while (!clib_atomic_cmp_and_swap_acq_relax_n (&(*p)->rw_cnt, &cnt, cnt + 1, 1)); CLIB_LOCK_DBG (p); } always_inline void clib_rwlock_reader_unlock (clib_rwlock_t * p) { ASSERT ((*p)->rw_cnt > 0); CLIB_LOCK_DBG_CLEAR (p); clib_atomic_fetch_sub_rel (&(*p)->rw_cnt, 1); } always_inline void clib_rwlock_writer_lock (clib_rwlock_t * p) { i32 cnt = 0; do { /* rwlock held by writer or reader(s) */ while ((cnt = clib_atomic_load_relax_n (&(*p)->rw_cnt)) != 0) CLIB_PAUSE (); } while (!clib_atomic_cmp_and_swap_acq_relax_n (&(*p)->rw_cnt, &cnt, -1, 1)); CLIB_LOCK_DBG (p); } always_inline void clib_rwlock_writer_unlock (clib_rwlock_t * p) { CLIB_LOCK_DBG_CLEAR (p); clib_atomic_release (&(*p)->rw_cnt); } #endif /* * fd.io coding-style-patch-verification: ON * * Local Variables: * eval: (c-set-style "gnu") * End: */