aboutsummaryrefslogtreecommitdiffstats
path: root/src/svm
diff options
context:
space:
mode:
Diffstat (limited to 'src/svm')
-rw-r--r--src/svm/fifo_segment.c130
-rw-r--r--src/svm/fifo_segment.h33
-rw-r--r--src/svm/svm_fifo.c40
-rw-r--r--src/svm/svm_fifo.h31
4 files changed, 217 insertions, 17 deletions
diff --git a/src/svm/fifo_segment.c b/src/svm/fifo_segment.c
index 6bf2634a290..9c332d6f1fa 100644
--- a/src/svm/fifo_segment.c
+++ b/src/svm/fifo_segment.c
@@ -237,7 +237,7 @@ fs_free_list_for_size (u32 size)
}
static inline int
-fs_fifo_size_is_valid (u32 size)
+fs_chunk_size_is_valid (u32 size)
{
/*
* 4K minimum. It's not likely that anything good will happen
@@ -293,7 +293,7 @@ fifo_segment_alloc_fifo (fifo_segment_t * fs, u32 data_bytes,
void *oldheap;
int fl_index;
- if (!fs_fifo_size_is_valid (data_bytes))
+ if (!fs_chunk_size_is_valid (data_bytes))
{
clib_warning ("fifo size out of range %d", data_bytes);
return 0;
@@ -304,7 +304,7 @@ fifo_segment_alloc_fifo (fifo_segment_t * fs, u32 data_bytes,
sh = fs->ssvm.sh;
ssvm_lock_non_recursive (sh, 1);
- fsh = (fifo_segment_header_t *) sh->opaque[0];
+ fsh = fs->h;
vec_validate_init_empty (fsh->free_fifos, fl_index, 0);
f = fsh->free_fifos[fl_index];
@@ -363,7 +363,7 @@ done:
* Free fifo allocated in fifo segment
*/
void
-fifo_segment_free_fifo (fifo_segment_t * s, svm_fifo_t * f)
+fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f)
{
fifo_segment_header_t *fsh;
ssvm_shared_header_t *sh;
@@ -374,8 +374,8 @@ fifo_segment_free_fifo (fifo_segment_t * s, svm_fifo_t * f)
if (--f->refcnt > 0)
return;
- sh = s->ssvm.sh;
- fsh = (fifo_segment_header_t *) sh->opaque[0];
+ sh = fs->ssvm.sh;
+ fsh = fs->h;
fl_index = f->freelist_index;
ASSERT (fl_index < vec_len (fsh->free_fifos));
@@ -399,6 +399,29 @@ fifo_segment_free_fifo (fifo_segment_t * s, svm_fifo_t * f)
f->prev = 0;
fsh->free_fifos[fl_index] = f;
+ /* If fifo has more chunks, free them */
+ if (f->flags & SVM_FIFO_F_MULTI_CHUNK)
+ {
+ svm_fifo_chunk_t *cur, *next;
+ void *oldheap;
+
+ next = f->start_chunk->next;
+ while (next != f->start_chunk)
+ {
+ cur = next;
+ next = next->next;
+ fl_index = fs_free_list_for_size (cur->length);
+ cur->next = fsh->free_chunks[fl_index];
+ fsh->free_chunks[fl_index] = cur;
+ }
+ oldheap = ssvm_push_heap (sh);
+ svm_fifo_free_chunk_lookup (f);
+ ssvm_pop_heap (oldheap);
+ }
+
+ /* not allocated on segment heap */
+ svm_fifo_free_ooo_data (f);
+
if (CLIB_DEBUG)
{
f->master_session_index = ~0;
@@ -431,13 +454,13 @@ fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs,
if (rx_fifo_size == 0 || tx_fifo_size == 0 || *n_fifo_pairs == 0)
return;
- if (!fs_fifo_size_is_valid (rx_fifo_size))
+ if (!fs_chunk_size_is_valid (rx_fifo_size))
{
clib_warning ("rx fifo_size out of range %d", rx_fifo_size);
return;
}
- if (!fs_fifo_size_is_valid (tx_fifo_size))
+ if (!fs_chunk_size_is_valid (tx_fifo_size))
{
clib_warning ("tx fifo_size out of range %d", tx_fifo_size);
return;
@@ -502,6 +525,51 @@ fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs,
ssvm_pop_heap (oldheap);
}
+int
+fifo_segment_grow_fifo (fifo_segment_t * fs, svm_fifo_t * f, u32 chunk_size)
+{
+ ssvm_shared_header_t *sh;
+ svm_fifo_chunk_t *c;
+ void *oldheap;
+ int fl_index;
+
+ if (!fs_chunk_size_is_valid (chunk_size))
+ {
+ clib_warning ("chunk size out of range %d", chunk_size);
+ return 0;
+ }
+
+ fl_index = fs_free_list_for_size (chunk_size);
+
+ sh = fs->ssvm.sh;
+ ssvm_lock_non_recursive (sh, 1);
+
+ vec_validate_init_empty (fs->h->free_chunks, fl_index, 0);
+ c = fs->h->free_chunks[fl_index];
+
+ oldheap = ssvm_push_heap (sh);
+
+ if (!c)
+ {
+ c = svm_fifo_chunk_alloc (chunk_size);
+ if (!c)
+ {
+ ssvm_pop_heap (oldheap);
+ return -1;
+ }
+ }
+ else
+ {
+ fs->h->free_chunks[fl_index] = c->next;
+ }
+
+ svm_fifo_add_chunk (f, c);
+
+ ssvm_pop_heap (oldheap);
+ ssvm_unlock_non_recursive (sh);
+ return 0;
+}
+
/**
* Get number of active fifos
*/
@@ -559,6 +627,52 @@ fifo_segment_num_free_fifos (fifo_segment_t * fs, u32 fifo_size_in_bytes)
return count;
}
+u32
+fifo_segment_num_free_chunks (fifo_segment_t * fs, u32 size)
+{
+ u32 count = 0, rounded_size, fl_index;
+ fifo_segment_header_t *fsh;
+ svm_fifo_chunk_t *c;
+ int i;
+
+ fsh = fs->h;
+
+ /* Count all free chunks? */
+ if (size == ~0)
+ {
+ for (i = 0; i < vec_len (fsh->free_chunks); i++)
+ {
+ c = fsh->free_chunks[i];
+ if (c == 0)
+ continue;
+
+ while (c)
+ {
+ c = c->next;
+ count++;
+ }
+ }
+ return count;
+ }
+
+ rounded_size = (1 << (max_log2 (size)));
+ fl_index = fs_free_list_for_size (rounded_size);
+
+ if (fl_index >= vec_len (fsh->free_chunks))
+ return 0;
+
+ c = fsh->free_chunks[fl_index];
+ if (c == 0)
+ return 0;
+
+ while (c)
+ {
+ c = c->next;
+ count++;
+ }
+ return count;
+}
+
u8
fifo_segment_has_fifos (fifo_segment_t * fs)
{
diff --git a/src/svm/fifo_segment.h b/src/svm/fifo_segment.h
index cc0ff3adf3a..6ff538fcc07 100644
--- a/src/svm/fifo_segment.h
+++ b/src/svm/fifo_segment.h
@@ -38,10 +38,11 @@ typedef enum fifo_segment_flags_
typedef struct
{
- svm_fifo_t *fifos; /**< Linked list of active RX fifos */
- svm_fifo_t **free_fifos; /**< Freelists, by fifo size */
- u32 n_active_fifos; /**< Number of active fifos */
- u8 flags; /**< Segment flags */
+ svm_fifo_t *fifos; /**< Linked list of active RX fifos */
+ svm_fifo_t **free_fifos; /**< Freelists by fifo size */
+ svm_fifo_chunk_t **free_chunks; /**< Freelists by chunk size */
+ u32 n_active_fifos; /**< Number of active fifos */
+ u8 flags; /**< Segment flags */
} fifo_segment_header_t;
typedef struct
@@ -82,7 +83,7 @@ void fifo_segment_info (fifo_segment_t * seg, char **address, size_t * size);
/**
* Allocate fifo in fifo segment
*
- * @param fs fifo segment
+ * @param fs fifo segment for fifo
* @param data_bytes size of default fifo chunk in bytes
* @param ftype fifo type @ref fifo_segment_ftype_t
* @return new fifo or 0 if alloc failed
@@ -94,7 +95,7 @@ svm_fifo_t *fifo_segment_alloc_fifo (fifo_segment_t * fs,
/**
* Free fifo allocated in fifo segment
*
- * @param fs fifo segment
+ * @param fs fifo segment for fifo
* @param f fifo to be freed
*/
void fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f);
@@ -107,7 +108,7 @@ void fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f);
* is hit, the number of fifo pairs requested is updated by subtracting the
* number of fifos that have been successfully allocated.
*
- * @param fs fifo segment
+ * @param fs fifo segment for fifo
* @param rx_fifo_size data size of rx fifos
* @param tx_fifo_size data size of tx fifos
* @param n_fifo_pairs number of pairs requested. Prior to returning, this
@@ -117,10 +118,28 @@ void fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs,
u32 rx_fifo_size,
u32 tx_fifo_size,
u32 * n_fifo_pairs);
+/**
+ * Grow fifo size by adding an additional chunk of memory
+ *
+ * @param fs fifo segment for fifo
+ * @param f fifo to be grown
+ * @param chunk_size number of bytes to be added to fifo
+ * @return 0 on success or a negative number otherwise
+ */
+int fifo_segment_grow_fifo (fifo_segment_t * fs, svm_fifo_t * f,
+ u32 chunk_size);
u8 fifo_segment_has_fifos (fifo_segment_t * fs);
svm_fifo_t *fifo_segment_get_fifo_list (fifo_segment_t * fs);
u32 fifo_segment_num_fifos (fifo_segment_t * fs);
u32 fifo_segment_num_free_fifos (fifo_segment_t * fs, u32 fifo_size_in_bytes);
+/**
+ * Find number of free chunks of given size
+ *
+ * @param fs fifo segment
+ * @param size chunk size of interest or ~0 if all should be counted
+ * @return number of chunks of given size
+ */
+u32 fifo_segment_num_free_chunks (fifo_segment_t * fs, u32 size);
void fifo_segment_main_init (fifo_segment_main_t * sm, u64 baseva,
u32 timeout_in_seconds);
diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c
index eafe497eec1..c8fd263c094 100644
--- a/src/svm/svm_fifo.c
+++ b/src/svm/svm_fifo.c
@@ -298,7 +298,9 @@ svm_fifo_init (svm_fifo_t * f, u32 size)
f->head_chunk = f->tail_chunk = f->ooo_enq = f->ooo_deq = f->start_chunk;
}
-/** create an svm fifo, in the current heap. Fails vs blow up the process */
+/**
+ * Creates a fifo in the current heap. Fails vs blow up the process
+ */
svm_fifo_t *
svm_fifo_create (u32 data_size_in_bytes)
{
@@ -317,6 +319,27 @@ svm_fifo_create (u32 data_size_in_bytes)
return f;
}
+/**
+ * Creates a fifo chunk in the current heap
+ */
+svm_fifo_chunk_t *
+svm_fifo_chunk_alloc (u32 size)
+{
+ svm_fifo_chunk_t *c;
+ u32 rounded_size;
+
+ /* round chunk size to the next highest power-of-two */
+ rounded_size = (1 << (max_log2 (size)));
+ c = clib_mem_alloc_aligned_or_null (sizeof (*c) + rounded_size,
+ CLIB_CACHE_LINE_BYTES);
+ if (c == 0)
+ return 0;
+
+ clib_memset (c, 0, sizeof (*c));
+ c->length = rounded_size;
+ return c;
+}
+
static inline void
svm_fifo_size_update (svm_fifo_t * f, svm_fifo_chunk_t * c)
{
@@ -462,13 +485,26 @@ svm_fifo_find_chunk (svm_fifo_t * f, u32 pos)
}
void
+svm_fifo_free_chunk_lookup (svm_fifo_t * f)
+{
+ rb_tree_free_nodes (&f->chunk_lookup);
+}
+
+void
+svm_fifo_free_ooo_data (svm_fifo_t * f)
+{
+ pool_free (f->ooo_segments);
+}
+
+void
svm_fifo_free (svm_fifo_t * f)
{
ASSERT (f->refcnt > 0);
if (--f->refcnt == 0)
{
- pool_free (f->ooo_segments);
+ /* ooo data is not allocated on segment heap */
+ svm_fifo_free_chunk_lookup (f);
clib_mem_free (f);
}
}
diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h
index 4f2f61939b7..7c84fc82198 100644
--- a/src/svm/svm_fifo.h
+++ b/src/svm/svm_fifo.h
@@ -332,6 +332,19 @@ svm_fifo_unset_event (svm_fifo_t * f)
svm_fifo_t *svm_fifo_create (u32 data_size_in_bytes);
void svm_fifo_init (svm_fifo_t * f, u32 size);
+
+/**
+ * Allocate a fifo chunk on heap
+ *
+ * If the chunk is allocated on a fifo segment, this should be called
+ * with the segment's heap pushed.
+ *
+ * @param size chunk size in bytes. Will be rounded to the next highest
+ * power-of-two
+ * @return new chunk or 0 if alloc failed
+ */
+svm_fifo_chunk_t *svm_fifo_chunk_alloc (u32 size);
+
/**
* Grow fifo size by adding chunk to chunk list
*
@@ -343,6 +356,24 @@ void svm_fifo_init (svm_fifo_t * f, u32 size);
*/
void svm_fifo_add_chunk (svm_fifo_t * f, svm_fifo_chunk_t * c);
void svm_fifo_free (svm_fifo_t * f);
+/**
+ * Cleanup fifo chunk lookup rb tree
+ *
+ * The rb tree is allocated in segment heap so this should be called
+ * with it pushed.
+ *
+ * @param f fifo to cleanup
+ */
+void svm_fifo_free_chunk_lookup (svm_fifo_t * f);
+/**
+ * Cleanup fifo ooo data
+ *
+ * The ooo data is allocated in producer process memory. The fifo
+ * segment heap should not be pushed.
+ *
+ * @param f fifo to cleanup
+ */
+void svm_fifo_free_ooo_data (svm_fifo_t * f);
int svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes,
const u8 * copy_from_here);