summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--src/svm/fifo_segment.c85
-rw-r--r--src/vppinfra/atomics.h3
2 files changed, 51 insertions, 37 deletions
diff --git a/src/svm/fifo_segment.c b/src/svm/fifo_segment.c
index 9cbd7205853..b0f41a92258 100644
--- a/src/svm/fifo_segment.c
+++ b/src/svm/fifo_segment.c
@@ -143,8 +143,10 @@ static void
fss_chunk_free_list_push (fifo_segment_slice_t * fss, u32 fl_index,
svm_fifo_chunk_t * c)
{
+ clib_spinlock_lock (&fss->chunk_lock);
c->next = fss->free_chunks[fl_index];
fss->free_chunks[fl_index] = c;
+ clib_spinlock_unlock (&fss->chunk_lock);
}
static void
@@ -152,8 +154,10 @@ fss_chunk_free_list_push_list (fifo_segment_slice_t * fss, u32 fl_index,
svm_fifo_chunk_t * head,
svm_fifo_chunk_t * tail)
{
+ clib_spinlock_lock (&fss->chunk_lock);
tail->next = fss->free_chunks[fl_index];
fss->free_chunks[fl_index] = head;
+ clib_spinlock_unlock (&fss->chunk_lock);
}
static svm_fifo_chunk_t *
@@ -163,12 +167,19 @@ fss_chunk_free_list_pop (fifo_segment_slice_t * fss, u32 fl_index)
ASSERT (fss_chunk_fl_index_is_valid (fss, fl_index));
+ clib_spinlock_lock (&fss->chunk_lock);
+
if (!fss->free_chunks[fl_index])
- return 0;
+ {
+ clib_spinlock_unlock (&fss->chunk_lock);
+ return 0;
+ }
c = fss->free_chunks[fl_index];
fss->free_chunks[fl_index] = c->next;
+ clib_spinlock_unlock (&fss->chunk_lock);
+
return c;
}
@@ -197,6 +208,24 @@ fss_fifo_del_active_list (fifo_segment_slice_t * fss, svm_fifo_t * f)
}
}
+static inline uword
+fss_fl_chunk_bytes (fifo_segment_slice_t * fss)
+{
+ return clib_atomic_load_relax_n (&fss->n_fl_chunk_bytes);
+}
+
+static inline void
+fss_fl_chunk_bytes_add (fifo_segment_slice_t * fss, uword size)
+{
+ clib_atomic_fetch_add_relax (&fss->n_fl_chunk_bytes, size);
+}
+
+static inline void
+fss_fl_chunk_bytes_sub (fifo_segment_slice_t * fss, uword size)
+{
+ clib_atomic_fetch_sub_relax (&fss->n_fl_chunk_bytes, size);
+}
+
/**
* Initialize fifo segment shared header
*/
@@ -441,7 +470,7 @@ fs_try_alloc_multi_chunk (fifo_segment_header_t * fsh,
}
done:
- fss->n_fl_chunk_bytes -= n_alloc;
+ fss_fl_chunk_bytes_sub (fss, n_alloc);
fsh_cached_bytes_sub (fsh, n_alloc);
return first;
}
@@ -520,7 +549,7 @@ fsh_try_alloc_chunk_batch (fifo_segment_header_t * fsh,
fss_chunk_free_list_push_list (fss, fl_index, head, tail);
fss->num_chunks[fl_index] += batch_size;
- fss->n_fl_chunk_bytes += batch_size * rounded_data_size;
+ fss_fl_chunk_bytes_add (fss, batch_size * rounded_data_size);
fsh_cached_bytes_add (fsh, batch_size * rounded_data_size);
fsh_free_bytes_sub (fsh, size);
@@ -570,7 +599,7 @@ free_list:
if (c)
{
c->next = 0;
- fss->n_fl_chunk_bytes -= fs_freelist_index_to_size (fl_index);
+ fss_fl_chunk_bytes_sub (fss, fs_freelist_index_to_size (fl_index));
fsh_cached_bytes_sub (fsh, fs_freelist_index_to_size (fl_index));
}
else
@@ -589,7 +618,7 @@ free_list:
}
/* Failed to allocate larger chunk, try to allocate multi-chunk
* that is close to what was actually requested */
- if (data_bytes <= fss->n_fl_chunk_bytes)
+ if (data_bytes <= fss_fl_chunk_bytes (fss))
{
c = fs_try_alloc_multi_chunk (fsh, fss, data_bytes);
if (c)
@@ -601,11 +630,11 @@ free_list:
goto done;
}
}
- if (data_bytes <= fss->n_fl_chunk_bytes + n_free)
+ if (data_bytes <= fss_fl_chunk_bytes (fss) + n_free)
{
u32 min_size = FIFO_SEGMENT_MIN_FIFO_SIZE;
- batch = (data_bytes - fss->n_fl_chunk_bytes) / min_size;
+ batch = (data_bytes - fss_fl_chunk_bytes (fss)) / min_size;
batch = clib_min (batch + 1, n_free / min_size);
if (fsh_try_alloc_chunk_batch (fsh, fss, 0, batch))
{
@@ -634,10 +663,9 @@ static svm_fifo_t *
fs_try_alloc_fifo (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss,
u32 data_bytes)
{
- u32 fl_index;
+ u32 fl_index, min_size;
svm_fifo_chunk_t *c;
svm_fifo_t *f = 0;
- u32 min_size;
min_size = clib_max ((fsh->pct_first_alloc * data_bytes) / 100, 4096);
fl_index = fs_freelist_for_size (min_size);
@@ -645,35 +673,25 @@ fs_try_alloc_fifo (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss,
if (!fss_chunk_fl_index_is_valid (fss, fl_index))
return 0;
- clib_spinlock_lock (&fss->chunk_lock);
-
f = fsh_try_alloc_fifo_hdr (fsh, fss);
if (!f)
- goto done;
+ return 0;
c = fsh_try_alloc_chunk (fsh, fss, min_size);
- if (c)
- {
- f->start_chunk = c;
- while (c->next)
- c = c->next;
- f->end_chunk = c;
- }
- else
+ if (!c)
{
f->next = fss->free_fifos;
fss->free_fifos = f;
- f = 0;
+ return 0;
}
-done:
- clib_spinlock_unlock (&fss->chunk_lock);
+ f->start_chunk = c;
+ while (c->next)
+ c = c->next;
+ f->end_chunk = c;
+ f->size = data_bytes;
+ f->fs_hdr = fsh;
- if (f)
- {
- f->size = data_bytes;
- f->fs_hdr = fsh;
- }
return f;
}
@@ -684,10 +702,7 @@ fsh_alloc_chunk (fifo_segment_header_t * fsh, u32 slice_index, u32 chunk_size)
svm_fifo_chunk_t *c;
fss = fsh_slice_get (fsh, slice_index);
-
- clib_spinlock_lock (&fss->chunk_lock);
c = fsh_try_alloc_chunk (fsh, fss, chunk_size);
- clib_spinlock_unlock (&fss->chunk_lock);
return c;
}
@@ -699,8 +714,6 @@ fsh_slice_collect_chunks (fifo_segment_header_t * fsh,
u32 n_collect = 0, fl_index;
svm_fifo_chunk_t *next;
- clib_spinlock_lock (&fss->chunk_lock);
-
while (c)
{
CLIB_MEM_UNPOISON (c, sizeof (*c));
@@ -711,10 +724,8 @@ fsh_slice_collect_chunks (fifo_segment_header_t * fsh,
c = next;
}
- fss->n_fl_chunk_bytes += n_collect;
+ fss_fl_chunk_bytes_add (fss, n_collect);
fsh_cached_bytes_add (fsh, n_collect);
-
- clib_spinlock_unlock (&fss->chunk_lock);
}
void
@@ -1141,7 +1152,7 @@ fifo_segment_fl_chunk_bytes (fifo_segment_t * fs)
for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
{
fss = fsh_slice_get (fsh, slice_index);
- n_bytes += fss->n_fl_chunk_bytes;
+ n_bytes += fss_fl_chunk_bytes (fss);
}
return n_bytes;
diff --git a/src/vppinfra/atomics.h b/src/vppinfra/atomics.h
index ce2177df5c2..d6616a94ba3 100644
--- a/src/vppinfra/atomics.h
+++ b/src/vppinfra/atomics.h
@@ -54,4 +54,7 @@
#define clib_atomic_fetch_add_rel(a, b) __atomic_fetch_add((a), (b), __ATOMIC_RELEASE)
#define clib_atomic_fetch_sub_rel(a, b) __atomic_fetch_sub((a), (b), __ATOMIC_RELEASE)
+#define clib_atomic_fetch_add_relax(a, b) __atomic_fetch_add((a), (b), __ATOMIC_RELAXED)
+#define clib_atomic_fetch_sub_relax(a, b) __atomic_fetch_sub((a), (b), __ATOMIC_RELAXED)
+
#endif /* included_clib_atomics_h */