/* * Copyright (c) 2016-2019 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include <svm/fifo_segment.h> /** * Fifo segment free space * * Queries the underlying memory manager, dlmalloc, for free space. Since this * ends up walking the internal data structures, it should not be called * indiscriminately. * * @param fs fifo segment * @return number of free bytes */ static u32 fs_free_space (fifo_segment_t * fs) { struct dlmallinfo dlminfo; dlminfo = mspace_mallinfo (fs->ssvm.sh->heap); return dlminfo.fordblks; } /** * Initialize fifo segment shared header */ int fifo_segment_init (fifo_segment_t * fs) { fifo_segment_header_t *fsh; ssvm_shared_header_t *sh; void *oldheap; sh = fs->ssvm.sh; oldheap = ssvm_push_heap (sh); fsh = clib_mem_alloc (sizeof (*fsh)); clib_memset (fsh, 0, sizeof (*fsh)); fs->h = sh->opaque[0] = fsh; ssvm_pop_heap (oldheap); fsh->n_free_bytes = fs_free_space (fs); sh->ready = 1; return (0); } /** * Create a fifo segment and initialize as master */ int fifo_segment_create (fifo_segment_main_t * sm, fifo_segment_create_args_t * a) { fifo_segment_t *fs; uword baseva; int rv; /* Allocate a fresh segment */ pool_get_zero (sm->segments, fs); baseva = a->segment_type == SSVM_SEGMENT_PRIVATE ? ~0ULL : sm->next_baseva; fs->ssvm.ssvm_size = a->segment_size; fs->ssvm.i_am_master = 1; fs->ssvm.my_pid = getpid (); fs->ssvm.name = format (0, "%s%c", a->segment_name, 0); fs->ssvm.requested_va = baseva; if ((rv = ssvm_master_init (&fs->ssvm, a->segment_type))) { pool_put (sm->segments, fs); return (rv); } /* Note: requested_va updated due to seg base addr randomization */ sm->next_baseva = fs->ssvm.sh->ssvm_va + fs->ssvm.ssvm_size; fifo_segment_init (fs); vec_add1 (a->new_segment_indices, fs - sm->segments); return (0); } /** * Attach as slave to a fifo segment */ int fifo_segment_attach (fifo_segment_main_t * sm, fifo_segment_create_args_t * a) { fifo_segment_t *s; int rv; pool_get_zero (sm->segments, s); s->ssvm.ssvm_size = a->segment_size; s->ssvm.my_pid = getpid (); s->ssvm.name = format (0, "%s%c", a->segment_name, 0); s->ssvm.requested_va = sm->next_baseva; if (a->segment_type == SSVM_SEGMENT_MEMFD) s->ssvm.fd = a->memfd_fd; else s->ssvm.attach_timeout = sm->timeout_in_seconds; if ((rv = ssvm_slave_init (&s->ssvm, a->segment_type))) { _vec_len (s) = vec_len (s) - 1; return (rv); } /* Fish the segment header */ s->h = s->ssvm.sh->opaque[0]; vec_add1 (a->new_segment_indices, s - sm->segments); return (0); } void fifo_segment_delete (fifo_segment_main_t * sm, fifo_segment_t * s) { ssvm_delete (&s->ssvm); clib_memset (s, 0xfe, sizeof (*s)); pool_put (sm->segments, s); } u32 fifo_segment_index (fifo_segment_main_t * sm, fifo_segment_t * s) { return s - sm->segments; } void * svm_fifo_segment_heap (fifo_segment_t * seg) { return seg->ssvm.sh->heap; } fifo_segment_t * fifo_segment_get_segment (fifo_segment_main_t * sm, u32 segment_index) { return pool_elt_at_index (sm->segments, segment_index); } void fifo_segment_info (fifo_segment_t * seg, char **address, size_t * size) { *address = (char *) seg->ssvm.sh->ssvm_va; *size = seg->ssvm.ssvm_size; } void fifo_segment_main_init (fifo_segment_main_t * sm, u64 baseva, u32 timeout_in_seconds) { sm->next_baseva = baseva; sm->timeout_in_seconds = timeout_in_seconds; } static inline u32 fs_freelist_for_size (u32 size) { return max_log2 (size) - max_log2 (FIFO_SEGMENT_MIN_FIFO_SIZE); } static inline u32 fs_freelist_index_to_size (u32 fl_index) { return 1 << (fl_index + max_log2 (FIFO_SEGMENT_MIN_FIFO_SIZE)); } static inline int fs_chunk_size_is_valid (u32 size) { /* * 4K minimum. It's not likely that anything good will happen * with a smaller FIFO. */ return size >= FIFO_SEGMENT_MIN_FIFO_SIZE && size <= FIFO_SEGMENT_MAX_FIFO_SIZE; } static svm_fifo_t * fs_try_alloc_fifo_freelist (fifo_segment_t * fs, u32 fl_index, u32 data_bytes) { fifo_segment_header_t *fsh = fs->h; svm_fifo_chunk_t *c; svm_fifo_t *f; f = fsh->free_fifos; c = fsh->free_chunks[fl_index]; if (!f || !c) return 0; fsh->free_fifos = f->next; fsh->free_chunks[fl_index] = c->next; c->next = c; c->start_byte = 0; c->length = data_bytes; memset (f, 0, sizeof (*f)); f->start_chunk = c; f->end_chunk = c; fsh->n_fl_chunk_bytes -= fs_freelist_index_to_size (fl_index); return f; } static svm_fifo_t * fs_try_alloc_fifo_freelist_multi_chunk (fifo_segment_t * fs, u32 data_bytes) { svm_fifo_chunk_t *c, *first = 0, *last = 0; fifo_segment_header_t *fsh = fs->h; u32 fl_index, fl_size, n_alloc = 0; svm_fifo_t *f; f = fsh->free_fifos; if (!f) { void *oldheap = ssvm_push_heap (fs->ssvm.sh); f = clib_mem_alloc_aligned (sizeof (*f), CLIB_CACHE_LINE_BYTES); ssvm_pop_heap (oldheap); if (!f) return 0; memset (f, 0, sizeof (*f)); } else { fsh->free_fifos = f->next; } fl_index = fs_freelist_for_size (data_bytes) - 1; vec_validate_init_empty (fsh->free_chunks, fl_index, 0); fl_size = fs_freelist_index_to_size (fl_index); while (data_bytes) { c = fsh->free_chunks[fl_index]; if (c) { fsh->free_chunks[fl_index] = c->next; if (!last) last = c; c->next = first; first = c; n_alloc += fl_size; c->length = clib_min (fl_size, data_bytes); data_bytes -= c->length; } else { ASSERT (fl_index > 0); fl_index -= 1; fl_size = fl_size >> 1; } } f->start_chunk = first; f->end_chunk = last; last->next = first; fsh->n_fl_chunk_bytes -= n_alloc; return f; } static int fs_try_alloc_fifo_batch (fifo_segment_t * fs, u32 fl_index, u32 batch_size) { fifo_segment_header_t *fsh = fs->h; u32 size, hdrs, rounded_data_size; svm_fifo_chunk_t *c; svm_fifo_t *f; void *oldheap; u8 *fmem; int i; vec_validate_init_empty (fsh->free_chunks, fl_index, 0); rounded_data_size = fs_freelist_index_to_size (fl_index); hdrs = sizeof (*f) + sizeof (*c); size = (hdrs + rounded_data_size) * batch_size; oldheap = ssvm_push_heap (fs->ssvm.sh); fmem = clib_mem_alloc_aligned_at_offset (size, CLIB_CACHE_LINE_BYTES, 0 /* align_offset */ , 0 /* os_out_of_memory */ ); ssvm_pop_heap (oldheap); /* Out of space.. */ if (fmem == 0) return -1; /* Carve fifo + chunk space */ for (i = 0; i < batch_size; i++) { f = (svm_fifo_t *) fmem; memset (f, 0, sizeof (*f)); f->next = fsh->free_fifos; fsh->free_fifos = f; c = (svm_fifo_chunk_t *) (fmem + sizeof (*f)); c->start_byte = 0; c->length = rounded_data_size; c->next = fsh->free_chunks[fl_index]; fsh->free_chunks[fl_index] = c; fmem += hdrs + rounded_data_size; } fsh->n_fl_chunk_bytes += batch_size * rounded_data_size; fsh->n_free_bytes -= size; return 0; } /** * Try to allocate new fifo * * Tries the following steps in order: * - grab fifo and chunk from freelists * - batch fifo and chunk allocation * - single fifo allocation * - grab multiple fifo chunks from freelists */ static svm_fifo_t * fs_try_alloc_fifo (fifo_segment_t * fs, u32 data_bytes) { fifo_segment_header_t *fsh = fs->h; u32 fifo_sz, fl_index; svm_fifo_t *f = 0; fl_index = fs_freelist_for_size (data_bytes); vec_validate_init_empty (fsh->free_chunks, fl_index, 0); fifo_sz = sizeof (svm_fifo_t) + sizeof (svm_fifo_chunk_t); fifo_sz += 1 << max_log2 (data_bytes); if (fsh->free_fifos && fsh->free_chunks[fl_index]) { f = fs_try_alloc_fifo_freelist (fs, fl_index, data_bytes); if (f) goto done; } if (fifo_sz * FIFO_SEGMENT_ALLOC_BATCH_SIZE < fsh->n_free_bytes) { if (fs_try_alloc_fifo_batch (fs, fl_index, FIFO_SEGMENT_ALLOC_BATCH_SIZE)) goto done; f = fs_try_alloc_fifo_freelist (fs, fl_index, data_bytes); goto done; } if (fifo_sz <= fsh->n_free_bytes) { void *oldheap = ssvm_push_heap (fs->ssvm.sh); f = svm_fifo_create (data_bytes); ssvm_pop_heap (oldheap); if (f) { fsh->n_free_bytes -= fifo_sz; goto done; } } if (data_bytes <= fsh->n_fl_chunk_bytes) f = fs_try_alloc_fifo_freelist_multi_chunk (fs, data_bytes); done: return f; } /** * Allocate fifo in fifo segment */ svm_fifo_t * fifo_segment_alloc_fifo (fifo_segment_t * fs, u32 data_bytes, fifo_segment_ftype_t ftype) { fifo_segment_header_t *fsh; svm_fifo_t *f = 0; if (!fs_chunk_size_is_valid (data_bytes)) { clib_warning ("fifo size out of range %d", data_bytes); return 0; } fsh = fs->h; ssvm_lock_non_recursive (fs->ssvm.sh, 1); f = fs_try_alloc_fifo (fs, data_bytes); if (!f) goto done; /* (re)initialize the fifo, as in svm_fifo_create */ svm_fifo_init (f, data_bytes); /* Initialize chunks and rbtree for multi-chunk fifos */ if (f->start_chunk->next != f->start_chunk) { void *oldheap = ssvm_push_heap (fs->ssvm.sh); svm_fifo_init_chunks (f); ssvm_pop_heap (oldheap); } /* If rx fifo type add to active fifos list. When cleaning up segment, * we need a list of active sessions that should be disconnected. Since * both rx and tx fifos keep pointers to the session, it's enough to track * only one. */ if (ftype == FIFO_SEGMENT_RX_FIFO) { if (fsh->fifos) { fsh->fifos->prev = f; f->next = fsh->fifos; } fsh->fifos = f; f->flags |= SVM_FIFO_F_LL_TRACKED; } fsh->n_active_fifos++; done: ssvm_unlock_non_recursive (fs->ssvm.sh); return (f); } /** * Free fifo allocated in fifo segment */ void fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f) { svm_fifo_chunk_t *cur, *next; fifo_segment_header_t *fsh; ssvm_shared_header_t *sh; void *oldheap; int fl_index; ASSERT (f->refcnt > 0); if (--f->refcnt > 0) return; sh = fs->ssvm.sh; fsh = fs->h; ssvm_lock_non_recursive (sh, 2); /* Remove from active list. Only rx fifos are tracked */ if (f->flags & SVM_FIFO_F_LL_TRACKED) { if (f->prev) f->prev->next = f->next; else fsh->fifos = f->next; if (f->next) f->next->prev = f->prev; f->flags &= ~SVM_FIFO_F_LL_TRACKED; } /* Add to free list */ f->next = fsh->free_fifos; f->prev = 0; fsh->free_fifos = f; /* Free fifo chunks */ cur = f->start_chunk; do { next = cur->next; fl_index = fs_freelist_for_size (cur->length); ASSERT (fl_index < vec_len (fsh->free_chunks)); cur->next = fsh->free_chunks[fl_index]; fsh->free_chunks[fl_index] = cur; fsh->n_fl_chunk_bytes += fs_freelist_index_to_size (fl_index); cur = next; } while (cur != f->start_chunk); f->start_chunk = f->end_chunk = f->new_chunks = 0; f->head_chunk = f->tail_chunk = f->ooo_enq = f->ooo_deq = 0; oldheap = ssvm_push_heap (sh); svm_fifo_free_chunk_lookup (f); ssvm_pop_heap (oldheap); /* not allocated on segment heap */ svm_fifo_free_ooo_data (f); if (CLIB_DEBUG) { f->master_session_index = ~0; f->master_thread_index = ~0; } fsh->n_active_fifos--; ssvm_unlock_non_recursive (sh); } int fifo_segment_prealloc_fifo_hdrs (fifo_segment_t * fs, u32 batch_size) { fifo_segment_header_t *fsh = fs->h; svm_fifo_t *f; void *oldheap; u32 size; u8 *fmem; int i; size = (sizeof (*f)) * batch_size; oldheap = ssvm_push_heap (fs->ssvm.sh); fmem = clib_mem_alloc_aligned_at_offset (size, CLIB_CACHE_LINE_BYTES, 0 /* align_offset */ , 0 /* os_out_of_memory */ ); ssvm_pop_heap (oldheap); /* Out of space.. */ if (fmem == 0) return -1; /* Carve fifo + chunk space */ for (i = 0; i < batch_size; i++) { f = (svm_fifo_t *) fmem; memset (f, 0, sizeof (*f)); f->next = fsh->free_fifos; fsh->free_fifos = f; fmem += sizeof (*f); } fsh->n_free_bytes -= size; return 0; } int fifo_segment_prealloc_fifo_chunks (fifo_segment_t * fs, u32 chunk_size, u32 batch_size) { fifo_segment_header_t *fsh = fs->h; u32 size, rounded_data_size, fl_index; svm_fifo_chunk_t *c; void *oldheap; u8 *cmem; int i; if (!fs_chunk_size_is_valid (chunk_size)) { clib_warning ("chunk size out of range %d", chunk_size); return -1; } fl_index = fs_freelist_for_size (chunk_size); vec_validate_init_empty (fsh->free_chunks, fl_index, 0); rounded_data_size = fs_freelist_index_to_size (fl_index); size = (sizeof (*c) + rounded_data_size) * batch_size; oldheap = ssvm_push_heap (fs->ssvm.sh); cmem = clib_mem_alloc_aligned_at_offset (size, CLIB_CACHE_LINE_BYTES, 0 /* align_offset */ , 0 /* os_out_of_memory */ ); ssvm_pop_heap (oldheap); /* Out of space.. */ if (cmem == 0) return -1; /* Carve fifo + chunk space */ for (i = 0; i < batch_size; i++) { c = (svm_fifo_chunk_t *) cmem; c->start_byte = 0; c->length = rounded_data_size; c->next = fsh->free_chunks[fl_index]; fsh->free_chunks[fl_index] = c; cmem += sizeof (*c) + rounded_data_size; } fsh->n_fl_chunk_bytes += batch_size * rounded_data_size; fsh->n_free_bytes -= size; return 0; } /** * Pre-allocates fifo pairs in fifo segment */ void fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs, u32 rx_fifo_size, u32 tx_fifo_size, u32 * n_fifo_pairs) { u32 rx_rounded_data_size, tx_rounded_data_size, pair_size, pairs_to_alloc; int rx_fl_index, tx_fl_index; uword space_available; u32 hdrs; /* Parameter check */ if (rx_fifo_size == 0 || tx_fifo_size == 0 || *n_fifo_pairs == 0) return; if (!fs_chunk_size_is_valid (rx_fifo_size)) { clib_warning ("rx fifo_size out of range %d", rx_fifo_size); return; } if (!fs_chunk_size_is_valid (tx_fifo_size)) { clib_warning ("tx fifo_size out of range %d", tx_fifo_size); return; } rx_rounded_data_size = (1 << (max_log2 (rx_fifo_size))); rx_fl_index = fs_freelist_for_size (rx_fifo_size); tx_rounded_data_size = (1 << (max_log2 (tx_fifo_size))); tx_fl_index = fs_freelist_for_size (tx_fifo_size); hdrs = sizeof (svm_fifo_t) + sizeof (svm_fifo_chunk_t); /* Calculate space requirements */ pair_size = 2 * hdrs + rx_rounded_data_size + tx_rounded_data_size; space_available = fs_free_space (fs); pairs_to_alloc = space_available / pair_size; pairs_to_alloc = clib_min (pairs_to_alloc, *n_fifo_pairs); if (!pairs_to_alloc) return; if (fs_try_alloc_fifo_batch (fs, rx_fl_index, pairs_to_alloc)) clib_warning ("rx prealloc failed: pairs %u", pairs_to_alloc); if (fs_try_alloc_fifo_batch (fs, tx_fl_index, pairs_to_alloc)) clib_warning ("tx prealloc failed: pairs %u", pairs_to_alloc); /* Account for the pairs allocated */ *n_fifo_pairs -= pairs_to_alloc; } int fifo_segment_grow_fifo (fifo_segment_t * fs, svm_fifo_t * f, u32 chunk_size) { ssvm_shared_header_t *sh; svm_fifo_chunk_t *c; void *oldheap; int fl_index; if (!fs_chunk_size_is_valid (chunk_size)) { clib_warning ("chunk size out of range %d", chunk_size); return -1; } fl_index = fs_freelist_for_size (chunk_size); sh = fs->ssvm.sh; ssvm_lock_non_recursive (sh, 1); vec_validate_init_empty (fs->h->free_chunks, fl_index, 0); c = fs->h->free_chunks[fl_index]; oldheap = ssvm_push_heap (sh); if (!c) { c = svm_fifo_chunk_alloc (chunk_size); if (!c) { ssvm_pop_heap (oldheap); ssvm_unlock_non_recursive (sh); return -1; } } else { fs->h->free_chunks[fl_index] = c->next; c->next = 0; fs->h->n_fl_chunk_bytes -= fs_freelist_index_to_size (fl_index); } svm_fifo_add_chunk (f, c); ssvm_pop_heap (oldheap); ssvm_unlock_non_recursive (sh); return 0; } int fifo_segment_collect_fifo_chunks (fifo_segment_t * fs, svm_fifo_t * f) { svm_fifo_chunk_t *cur, *next; ssvm_shared_header_t *sh; void *oldheap; int fl_index; sh = fs->ssvm.sh; ssvm_lock_non_recursive (sh, 1); oldheap = ssvm_push_heap (sh); cur = svm_fifo_collect_chunks (f); while (cur) { next = cur->next; fl_index = fs_freelist_for_size (cur->length); cur->next = fs->h->free_chunks[fl_index]; fs->h->free_chunks[fl_index] = cur; cur = next; } ssvm_pop_heap (oldheap); ssvm_unlock_non_recursive (sh); return 0; } /** * Get number of active fifos */ u32 fifo_segment_num_fifos (fifo_segment_t * fs) { return fs->h->n_active_fifos; } u32 fifo_segment_num_free_fifos (fifo_segment_t * fs) { fifo_segment_header_t *fsh = fs->h; svm_fifo_t *f; u32 count = 0; f = fsh->free_fifos; if (f == 0) return 0; while (f) { f = f->next; count++; } return count; } u32 fifo_segment_num_free_chunks (fifo_segment_t * fs, u32 size) { u32 count = 0, rounded_size, fl_index; fifo_segment_header_t *fsh; svm_fifo_chunk_t *c; int i; fsh = fs->h; /* Count all free chunks? */ if (size == ~0) { for (i = 0; i < vec_len (fsh->free_chunks); i++) { c = fsh->free_chunks[i]; if (c == 0) continue; while (c) { c = c->next; count++; } } return count; } rounded_size = (1 << (max_log2 (size))); fl_index = fs_freelist_for_size (rounded_size); if (fl_index >= vec_len (fsh->free_chunks)) return 0; c = fsh->free_chunks[fl_index]; if (c == 0) return 0; while (c) { c = c->next; count++; } return count; } void fifo_segment_update_free_bytes (fifo_segment_t * fs) { fs->h->n_free_bytes = fs_free_space (fs); } u32 fifo_segment_free_bytes (fifo_segment_t * fs) { return fs->h->n_free_bytes; } u32 fifo_segment_fl_chunk_bytes (fifo_segment_t * fs) { return fs->h->n_fl_chunk_bytes; } u8 fifo_segment_has_fifos (fifo_segment_t * fs) { return fs->h->fifos != 0; } svm_fifo_t * fifo_segment_get_fifo_list (fifo_segment_t * fs) { return fs->h->fifos; } u8 * format_fifo_segment_type (u8 * s, va_list * args) { fifo_segment_t *sp; sp = va_arg (*args, fifo_segment_t *); ssvm_segment_type_t st = ssvm_type (&sp->ssvm); if (st == SSVM_SEGMENT_PRIVATE) s = format (s, "%s", "private-heap"); else if (st == SSVM_SEGMENT_MEMFD) s = format (s, "%s", "memfd"); else if (st == SSVM_SEGMENT_SHM) s = format (s, "%s", "shm"); else s = format (s, "%s", "unknown"); return s; } /** * Segment format function */ u8 * format_fifo_segment (u8 * s, va_list * args) { fifo_segment_t *fs = va_arg (*args, fifo_segment_t *); int verbose __attribute__ ((unused)) = va_arg (*args, int); fifo_segment_header_t *fsh; svm_fifo_chunk_t *c; u32 count, indent; u32 active_fifos; u32 free_fifos; char *address; size_t size; int i; indent = format_get_indent (s) + 2; #if USE_DLMALLOC == 0 s = format (s, "%U segment heap: %U\n", format_white_space, indent, format_mheap, svm_fifo_segment_heap (fs), verbose); s = format (s, "%U segment has %u active fifos\n", format_white_space, indent, fifo_segment_num_fifos (fs)); #endif if (fs == 0) { s = format (s, "%-15s%15s%15s%15s%15s%15s", "Name", "Type", "HeapSize (M)", "ActiveFifos", "FreeFifos", "Address"); return s; } fsh = fs->h; fifo_segment_info (fs, &address, &size); active_fifos = fifo_segment_num_fifos (fs); free_fifos = fifo_segment_num_free_fifos (fs); s = format (s, "%-15v%15U%15llu%15u%15u%15llx", ssvm_name (&fs->ssvm), format_fifo_segment_type, fs, size >> 20ULL, active_fifos, free_fifos, address); if (!verbose) return s; s = format (s, "\n"); for (i = 0; i < vec_len (fsh->free_chunks); i++) { c = fsh->free_chunks[i]; if (c == 0) continue; count = 0; while (c) { c = c->next; count++; } s = format (s, "%U%-5u Kb: %u free", format_white_space, indent + 2, 1 << (i + max_log2 (FIFO_SEGMENT_MIN_FIFO_SIZE) - 10), count); } s = format (s, "%Ufree bytes %U", format_white_space, indent + 2, format_memory_size, fsh->n_free_bytes); return s; } /* * fd.io coding-style-patch-verification: ON * * Local Variables: * eval: (c-set-style "gnu") * End: */