diff options
| -rw-r--r-- | src/svm/svm_fifo.c | 116 | ||||
| -rw-r--r-- | src/svm/svm_fifo.h | 9 |
2 files changed, 117 insertions, 8 deletions
diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c index 6bd6f9157bc..975a82026f7 100644 --- a/src/svm/svm_fifo.c +++ b/src/svm/svm_fifo.c @@ -590,6 +590,95 @@ svm_fifo_add_chunk (svm_fifo_t * f, svm_fifo_chunk_t * c) f->flags |= SVM_FIFO_F_MULTI_CHUNK; } + /* If fifo is not wrapped, update the size now */ + if (!svm_fifo_is_wrapped (f)) + { + /* Initialize chunks and add to lookup rbtree */ + cur = c; + if (f->new_chunks) + { + prev = f->new_chunks; + while (prev->next) + prev = prev->next; + prev->next = c; + } + else + prev = f->end_chunk; + + while (cur) + { + cur->start_byte = prev->start_byte + prev->length; + rb_tree_add2 (&f->chunk_lookup, cur->start_byte, + pointer_to_uword (cur)); + prev = cur; + cur = cur->next; + } + + ASSERT (!f->new_chunks); + svm_fifo_grow (f, c); + return; + } + + /* Wrapped */ + if (f->flags & SVM_FIFO_F_SINGLE_THREAD_OWNED) + { + ASSERT (f->master_thread_index == os_get_thread_index ()); + + if (!f->new_chunks && f->head_chunk != f->tail_chunk) + { + u32 head = 0, tail = 0; + f_load_head_tail_cons (f, &head, &tail); + + svm_fifo_chunk_t *tmp = f->tail_chunk->next; + + prev = f->tail_chunk; + u32 add_bytes = 0; + cur = prev->next; + while (cur != f->start_chunk) + { + /* remove any existing rb_tree entry */ + rb_tree_del (&f->chunk_lookup, cur->start_byte); + cur = cur->next; + } + + /* insert new chunk after the tail_chunk */ + f->tail_chunk->next = c; + while (c) + { + add_bytes += c->length; + c->start_byte = prev->start_byte + prev->length; + rb_tree_add2 (&f->chunk_lookup, c->start_byte, + pointer_to_uword (c)); + + prev = c; + c = c->next; + } + prev->next = tmp; + + /* shift existing chunks along */ + cur = tmp; + while (cur != f->start_chunk) + { + cur->start_byte = prev->start_byte + prev->length; + rb_tree_add2 (&f->chunk_lookup, cur->start_byte, + pointer_to_uword (cur)); + prev = cur; + cur = cur->next; + } + + f->size += add_bytes; + f->nitems = f->size - 1; + f->new_chunks = 0; + head += add_bytes; + + clib_atomic_store_rel_n (&f->head, head); + ASSERT (svm_fifo_is_sane (f)); + + return; + } + } + + /* Wrapped, and optimization of single-thread-owned fifo cannot be applied */ /* Initialize chunks and add to lookup rbtree */ cur = c; if (f->new_chunks) @@ -611,14 +700,6 @@ svm_fifo_add_chunk (svm_fifo_t * f, svm_fifo_chunk_t * c) cur = cur->next; } - /* If fifo is not wrapped, update the size now */ - if (!svm_fifo_is_wrapped (f)) - { - ASSERT (!f->new_chunks); - svm_fifo_grow (f, c); - return; - } - /* Postpone size update */ if (!f->new_chunks) { @@ -1152,6 +1233,25 @@ svm_fifo_is_sane (svm_fifo_t * f) return 1; } +u8 +svm_fifo_set_single_thread_owned (svm_fifo_t * f) +{ + if (f->flags & SVM_FIFO_F_SINGLE_THREAD_OWNED) + { + if (f->master_thread_index == os_get_thread_index ()) + { + /* just a duplicate call */ + return 0; + } + + /* already owned by another thread */ + return 1; + } + + f->flags |= SVM_FIFO_F_SINGLE_THREAD_OWNED; + return 0; +} + u8 * format_ooo_segment (u8 * s, va_list * args) { diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h index 64ed53bf827..ce4c53d9abe 100644 --- a/src/svm/svm_fifo.h +++ b/src/svm/svm_fifo.h @@ -70,6 +70,7 @@ typedef enum svm_fifo_flag_ SVM_FIFO_F_SHRINK = 1 << 2, SVM_FIFO_F_COLLECT_CHUNKS = 1 << 3, SVM_FIFO_F_LL_TRACKED = 1 << 4, + SVM_FIFO_F_SINGLE_THREAD_OWNED = 1 << 5, } svm_fifo_flag_t; typedef struct _svm_fifo @@ -477,6 +478,14 @@ ooo_segment_t *svm_fifo_first_ooo_segment (svm_fifo_t * f); * @return 1 if sane, 0 otherwise */ u8 svm_fifo_is_sane (svm_fifo_t * f); +/** + * Declare this fifo is used by only a single thread. + * In this special case, fifo-growth can be done in an efficient way without delay. + * + * @param f fifo + * @return 1 if the fifo is already owned by another thread, 0 otherwise + */ +u8 svm_fifo_set_single_thread_owned (svm_fifo_t * f); format_function_t format_svm_fifo; /** |