aboutsummaryrefslogtreecommitdiffstats
path: root/vppinfra/vppinfra/heap.c
diff options
context:
space:
mode:
authorDamjan Marion <damarion@cisco.com>2016-12-19 23:05:39 +0100
committerDamjan Marion <damarion@cisco.com>2016-12-28 12:25:14 +0100
commit7cd468a3d7dee7d6c92f69a0bb7061ae208ec727 (patch)
tree5de62f8dbd3a752f5a676ca600e43d2652d1ff1a /vppinfra/vppinfra/heap.c
parent696f1adec0df3b8f161862566dd9c86174302658 (diff)
Reorganize source tree to use single autotools instance
Change-Id: I7b51f88292e057c6443b12224486f2d0c9f8ae23 Signed-off-by: Damjan Marion <damarion@cisco.com>
Diffstat (limited to 'vppinfra/vppinfra/heap.c')
-rw-r--r--vppinfra/vppinfra/heap.c828
1 files changed, 0 insertions, 828 deletions
diff --git a/vppinfra/vppinfra/heap.c b/vppinfra/vppinfra/heap.c
deleted file mode 100644
index 2a5fb5c8d8e..00000000000
--- a/vppinfra/vppinfra/heap.c
+++ /dev/null
@@ -1,828 +0,0 @@
-/*
- * Copyright (c) 2015 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/*
- Copyright (c) 2001, 2002, 2003 Eliot Dresselhaus
-
- Permission is hereby granted, free of charge, to any person obtaining
- a copy of this software and associated documentation files (the
- "Software"), to deal in the Software without restriction, including
- without limitation the rights to use, copy, modify, merge, publish,
- distribute, sublicense, and/or sell copies of the Software, and to
- permit persons to whom the Software is furnished to do so, subject to
- the following conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
- LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-*/
-
-#include <vppinfra/cache.h> /* for CLIB_CACHE_LINE_BYTES */
-#include <vppinfra/mem.h>
-#include <vppinfra/hash.h>
-#include <vppinfra/vec.h>
-#include <vppinfra/heap.h>
-#include <vppinfra/error.h>
-
-always_inline heap_elt_t *
-elt_at (heap_header_t * h, uword i)
-{
- ASSERT (i < vec_len (h->elts));
- return h->elts + i;
-}
-
-always_inline heap_elt_t *
-last (heap_header_t * h)
-{
- return elt_at (h, h->tail);
-}
-
-always_inline heap_elt_t *
-first (heap_header_t * h)
-{
- return elt_at (h, h->head);
-}
-
-/* Objects sizes are binned into N_BINS bins.
- Objects with size <= SMALL_BINS have their own bins.
- Larger objects are grouped together in power or 2 sized
- bins.
-
- Sizes are in units of elt_bytes bytes. */
-
-/* Convert size to bin. */
-always_inline uword
-size_to_bin (uword size)
-{
- uword bin;
-
- ASSERT (size > 0);
-
- if (size <= HEAP_SMALL_BINS)
- {
- bin = size - 1;
- if (size == 0)
- bin = 0;
- }
- else
- {
- bin = HEAP_SMALL_BINS + max_log2 (size) - (HEAP_LOG2_SMALL_BINS + 1);
- if (bin >= HEAP_N_BINS)
- bin = HEAP_N_BINS - 1;
- }
-
- return bin;
-}
-
-/* Convert bin to size. */
-always_inline __attribute__ ((unused))
- uword bin_to_size (uword bin)
-{
- uword size;
-
- if (bin <= HEAP_SMALL_BINS - 1)
- size = bin + 1;
- else
- size = (uword) 1 << ((bin - HEAP_SMALL_BINS) + HEAP_LOG2_SMALL_BINS + 1);
-
- return size;
-}
-
-static void
-elt_delete (heap_header_t * h, heap_elt_t * e)
-{
- heap_elt_t *l = vec_end (h->elts) - 1;
-
- ASSERT (e >= h->elts && e <= l);
-
- /* Update doubly linked pointers. */
- {
- heap_elt_t *p = heap_prev (e);
- heap_elt_t *n = heap_next (e);
-
- if (p == e)
- {
- n->prev = 0;
- h->head = n - h->elts;
- }
- else if (n == e)
- {
- p->next = 0;
- h->tail = p - h->elts;
- }
- else
- {
- p->next = n - p;
- n->prev = p - n;
- }
- }
-
- /* Add to index free list or delete from end. */
- if (e < l)
- vec_add1 (h->free_elts, e - h->elts);
- else
- _vec_len (h->elts)--;
-}
-
-/*
- Before: P ... E
- After : P ... NEW ... E
-*/
-always_inline void
-elt_insert_before (heap_header_t * h, heap_elt_t * e, heap_elt_t * new)
-{
- heap_elt_t *p = heap_prev (e);
-
- if (p == e)
- {
- new->prev = 0;
- new->next = e - new;
- p->prev = new - p;
- h->head = new - h->elts;
- }
- else
- {
- new->prev = p - new;
- new->next = e - new;
- e->prev = new - e;
- p->next = new - p;
- }
-}
-
-/*
- Before: E ... N
- After : E ... NEW ... N
-*/
-always_inline void
-elt_insert_after (heap_header_t * h, heap_elt_t * e, heap_elt_t * new)
-{
- heap_elt_t *n = heap_next (e);
-
- if (n == e)
- {
- new->next = 0;
- new->prev = e - new;
- e->next = new - e;
- h->tail = new - h->elts;
- }
- else
- {
- new->prev = e - new;
- new->next = n - new;
- e->next = new - e;
- n->prev = new - n;
- }
-}
-
-always_inline heap_elt_t *
-elt_new (heap_header_t * h)
-{
- heap_elt_t *e;
- uword l;
- if ((l = vec_len (h->free_elts)) > 0)
- {
- e = elt_at (h, h->free_elts[l - 1]);
- _vec_len (h->free_elts) -= 1;
- }
- else
- vec_add2 (h->elts, e, 1);
- return e;
-}
-
-/* Return pointer to object at given offset.
- Used to write free list index of free objects. */
-always_inline u32 *
-elt_data (void *v, heap_elt_t * e)
-{
- heap_header_t *h = heap_header (v);
- return v + heap_offset (e) * h->elt_bytes;
-}
-
-always_inline void
-set_free_elt (void *v, heap_elt_t * e, uword fi)
-{
- heap_header_t *h = heap_header (v);
-
- e->offset |= HEAP_ELT_FREE_BIT;
- if (h->elt_bytes >= sizeof (u32))
- {
- *elt_data (v, e) = fi;
- }
- else
- {
- /* For elt_bytes < 4 we must store free index in separate
- vector. */
- uword elt_index = e - h->elts;
- vec_validate (h->small_free_elt_free_index, elt_index);
- h->small_free_elt_free_index[elt_index] = fi;
- }
-}
-
-always_inline uword
-get_free_elt (void *v, heap_elt_t * e, uword * bin_result)
-{
- heap_header_t *h = heap_header (v);
- uword fb, fi;
-
- ASSERT (heap_is_free (e));
- fb = size_to_bin (heap_elt_size (v, e));
-
- if (h->elt_bytes >= sizeof (u32))
- {
- fi = *elt_data (v, e);
- }
- else
- {
- uword elt_index = e - h->elts;
- fi = vec_elt (h->small_free_elt_free_index, elt_index);
- }
-
- *bin_result = fb;
- return fi;
-}
-
-always_inline void
-remove_free_block (void *v, uword b, uword i)
-{
- heap_header_t *h = heap_header (v);
- uword l;
-
- ASSERT (b < vec_len (h->free_lists));
- ASSERT (i < vec_len (h->free_lists[b]));
-
- l = vec_len (h->free_lists[b]);
-
- if (i < l - 1)
- {
- uword t = h->free_lists[b][l - 1];
- h->free_lists[b][i] = t;
- set_free_elt (v, elt_at (h, t), i);
- }
- _vec_len (h->free_lists[b]) = l - 1;
-}
-
-static heap_elt_t *
-search_free_list (void *v, uword size)
-{
- heap_header_t *h = heap_header (v);
- heap_elt_t *f, *u;
- uword b, fb, f_size, f_index;
- word s, l;
-
- if (!v)
- return 0;
-
- /* Search free lists for bins >= given size. */
- for (b = size_to_bin (size); b < vec_len (h->free_lists); b++)
- if ((l = vec_len (h->free_lists[b])) > 0)
- {
- /* Find an object that is large enough.
- Search list in reverse so that more recently freed objects will be
- allocated again sooner. */
- do
- {
- l--;
- f_index = h->free_lists[b][l];
- f = elt_at (h, f_index);
- f_size = heap_elt_size (v, f);
- if ((s = f_size - size) >= 0)
- break;
- }
- while (l >= 0);
-
- /* If we fail to find a large enough object, try the next larger size. */
- if (l < 0)
- continue;
-
- ASSERT (heap_is_free (f));
-
- /* Link in used object (u) after free object (f). */
- if (s == 0)
- {
- u = f;
- fb = HEAP_N_BINS;
- }
- else
- {
- u = elt_new (h);
- f = elt_at (h, f_index);
- elt_insert_after (h, f, u);
- fb = size_to_bin (s);
- }
-
- u->offset = heap_offset (f) + s;
-
- if (fb != b)
- {
- if (fb < HEAP_N_BINS)
- {
- uword i;
- vec_validate (h->free_lists, fb);
- i = vec_len (h->free_lists[fb]);
- vec_add1 (h->free_lists[fb], f - h->elts);
- set_free_elt (v, f, i);
- }
-
- remove_free_block (v, b, l);
- }
-
- return u;
- }
-
- return 0;
-}
-
-static void combine_free_blocks (void *v, heap_elt_t * e0, heap_elt_t * e1);
-
-static inline void
-dealloc_elt (void *v, heap_elt_t * e)
-{
- heap_header_t *h = heap_header (v);
- uword b, l;
- heap_elt_t *n, *p;
-
- b = size_to_bin (heap_elt_size (v, e));
- vec_validate (h->free_lists, b);
- l = vec_len (h->free_lists[b]);
- vec_add1 (h->free_lists[b], e - h->elts);
- set_free_elt (v, e, l);
-
- /* See if we can combine the block we just freed with neighboring free blocks. */
- p = heap_prev (e);
- if (!heap_is_free (p))
- p = e;
-
- n = heap_next (e);
- if (!heap_is_free (n))
- n = e;
-
- if (p != n)
- combine_free_blocks (v, p, n);
-}
-
-void *
-_heap_alloc (void *v,
- uword size,
- uword align,
- uword elt_bytes, uword * offset_return, uword * handle_return)
-{
- uword offset = 0, align_size;
- heap_header_t *h;
- heap_elt_t *e;
-
- if (size == 0)
- goto error;
-
- /* Round up alignment to power of 2. */
- if (align <= 1)
- {
- align = 0;
- align_size = size;
- }
- else
- {
- align = max_pow2 (align);
- align_size = size + align - 1;
- }
-
- e = search_free_list (v, align_size);
-
- /* If nothing found on free list, allocate object from end of vector. */
- if (!e)
- {
- uword max_len;
-
- offset = vec_len (v);
- max_len = heap_get_max_len (v);
-
- if (max_len && offset + align_size > max_len)
- goto error;
-
- h = heap_header (v);
- if (!v || !(h->flags & HEAP_IS_STATIC))
- v = _vec_resize (v,
- align_size,
- (offset + align_size) * elt_bytes,
- sizeof (h[0]), HEAP_DATA_ALIGN);
- else
- _vec_len (v) += align_size;
-
- if (offset == 0)
- {
- h = heap_header (v);
- h->elt_bytes = elt_bytes;
- }
- }
-
- h = heap_header (v);
-
- /* Add new element to doubly linked chain of elements. */
- if (!e)
- {
- e = elt_new (h);
- e->offset = offset;
- elt_insert_after (h, last (h), e);
- }
-
- if (align > 0)
- {
- uword e_index;
- uword new_offset, old_offset;
-
- old_offset = e->offset;
- new_offset = (old_offset + align - 1) & ~(align - 1);
- e->offset = new_offset;
- e_index = e - h->elts;
-
- /* Free fragments before and after aligned object. */
- if (new_offset > old_offset)
- {
- heap_elt_t *before_e = elt_new (h);
- before_e->offset = old_offset;
- elt_insert_before (h, h->elts + e_index, before_e);
- dealloc_elt (v, before_e);
- }
-
- if (new_offset + size < old_offset + align_size)
- {
- heap_elt_t *after_e = elt_new (h);
- after_e->offset = new_offset + size;
- elt_insert_after (h, h->elts + e_index, after_e);
- dealloc_elt (v, after_e);
- }
-
- e = h->elts + e_index;
- }
-
- h->used_count++;
-
- /* Keep track of used elements when debugging.
- This allows deallocation to check that passed objects are valid. */
- if (CLIB_DEBUG > 0)
- {
- uword handle = e - h->elts;
- ASSERT (!clib_bitmap_get (h->used_elt_bitmap, handle));
- h->used_elt_bitmap = clib_bitmap_ori (h->used_elt_bitmap, handle);
- }
-
- *offset_return = e->offset;
- *handle_return = e - h->elts;
- return v;
-
-error:
- *offset_return = *handle_return = ~0;
- return v;
-}
-
-void
-heap_dealloc (void *v, uword handle)
-{
- heap_header_t *h = heap_header (v);
- heap_elt_t *e;
-
- ASSERT (handle < vec_len (h->elts));
-
- /* For debugging we keep track of indices for valid objects.
- We make sure user is not trying to free object with an invalid index. */
- if (CLIB_DEBUG > 0)
- {
- ASSERT (clib_bitmap_get (h->used_elt_bitmap, handle));
- h->used_elt_bitmap = clib_bitmap_andnoti (h->used_elt_bitmap, handle);
- }
-
- h->used_count--;
-
- e = h->elts + handle;
- ASSERT (!heap_is_free (e));
-
- dealloc_elt (v, e);
-}
-
-/* While freeing objects at INDEX we noticed free blocks i0 <= index and
- i1 >= index. We combine these two or three blocks into one big free block. */
-static void
-combine_free_blocks (void *v, heap_elt_t * e0, heap_elt_t * e1)
-{
- heap_header_t *h = heap_header (v);
- uword total_size, i, b, tb, ti, i_last, g_offset;
- heap_elt_t *e;
-
- struct
- {
- u32 index;
- u32 bin;
- u32 bin_index;
- } f[3], g;
-
- /* Compute total size of free objects i0 through i1. */
- total_size = 0;
- for (i = 0, e = e0; 1; e = heap_next (e), i++)
- {
- ASSERT (i < ARRAY_LEN (f));
-
- ti = get_free_elt (v, e, &tb);
-
- ASSERT (tb < vec_len (h->free_lists));
- ASSERT (ti < vec_len (h->free_lists[tb]));
-
- f[i].index = h->free_lists[tb][ti];
- f[i].bin = tb;
- f[i].bin_index = ti;
-
- total_size += heap_elt_size (v, elt_at (h, f[i].index));
-
- if (e == e1)
- {
- i_last = i;
- break;
- }
- }
-
- /* Compute combined bin. See if all objects can be
- combined into existing bin. */
- b = size_to_bin (total_size);
- g.index = g.bin_index = 0;
- for (i = 0; i <= i_last; i++)
- if (b == f[i].bin)
- {
- g = f[i];
- break;
- }
-
- /* Make sure we found a bin. */
- if (i > i_last)
- {
- g.index = elt_new (h) - h->elts;
- vec_validate (h->free_lists, b);
- g.bin_index = vec_len (h->free_lists[b]);
- vec_add1 (h->free_lists[b], g.index);
- elt_insert_before (h, elt_at (h, f[0].index), elt_at (h, g.index));
- }
-
- g_offset = elt_at (h, f[0].index)->offset;
-
- /* Delete unused bins. */
- for (i = 0; i <= i_last; i++)
- if (g.index != f[i].index)
- {
- ti = get_free_elt (v, elt_at (h, f[i].index), &tb);
- remove_free_block (v, tb, ti);
- elt_delete (h, elt_at (h, f[i].index));
- }
-
- /* Initialize new element. */
- elt_at (h, g.index)->offset = g_offset;
- set_free_elt (v, elt_at (h, g.index), g.bin_index);
-}
-
-uword
-heap_len (void *v, word handle)
-{
- heap_header_t *h = heap_header (v);
-
- if (CLIB_DEBUG > 0)
- ASSERT (clib_bitmap_get (h->used_elt_bitmap, handle));
- return heap_elt_size (v, elt_at (h, handle));
-}
-
-void *
-_heap_free (void *v)
-{
- heap_header_t *h = heap_header (v);
- uword b;
-
- if (!v)
- return v;
-
- clib_bitmap_free (h->used_elt_bitmap);
- for (b = 0; b < vec_len (h->free_lists); b++)
- vec_free (h->free_lists[b]);
- vec_free (h->free_lists);
- vec_free (h->elts);
- vec_free (h->free_elts);
- vec_free (h->small_free_elt_free_index);
- if (!(h->flags & HEAP_IS_STATIC))
- vec_free_h (v, sizeof (h[0]));
- return v;
-}
-
-uword
-heap_bytes (void *v)
-{
- heap_header_t *h = heap_header (v);
- uword bytes, b;
-
- if (!v)
- return 0;
-
- bytes = sizeof (h[0]);
- bytes += vec_len (v) * sizeof (h->elt_bytes);
- for (b = 0; b < vec_len (h->free_lists); b++)
- bytes += vec_capacity (h->free_lists[b], 0);
- bytes += vec_bytes (h->free_lists);
- bytes += vec_capacity (h->elts, 0);
- bytes += vec_capacity (h->free_elts, 0);
- bytes += vec_bytes (h->used_elt_bitmap);
-
- return bytes;
-}
-
-static u8 *
-debug_elt (u8 * s, void *v, word i, word n)
-{
- heap_elt_t *e, *e0, *e1;
- heap_header_t *h = heap_header (v);
- word j;
-
- if (vec_len (h->elts) == 0)
- return s;
-
- if (i < 0)
- e0 = first (h);
- else
- {
- e0 = h->elts + i;
- for (j = 0; j < n / 2; j++)
- e0 = heap_prev (e0);
- }
-
- if (n < 0)
- e1 = h->elts + h->tail;
- else
- {
- e1 = h->elts + i;
- for (j = 0; j < n / 2; j++)
- e1 = heap_next (e1);
- }
-
- i = -n / 2;
- for (e = e0; 1; e = heap_next (e))
- {
- if (heap_is_free (e))
- s = format (s, "index %4d, free\n", e - h->elts);
- else if (h->format_elt)
- s = format (s, "%U", h->format_elt, v, elt_data (v, e));
- else
- s = format (s, "index %4d, used\n", e - h->elts);
- i++;
- if (e == e1)
- break;
- }
-
- return s;
-}
-
-u8 *
-format_heap (u8 * s, va_list * va)
-{
- void *v = va_arg (*va, void *);
- uword verbose = va_arg (*va, uword);
- heap_header_t *h = heap_header (v);
- heap_header_t zero;
-
- memset (&zero, 0, sizeof (zero));
-
- if (!v)
- h = &zero;
-
- {
- f64 elt_bytes = vec_len (v) * h->elt_bytes;
- f64 overhead_bytes = heap_bytes (v);
-
- s = format (s, "heap %p, %6d objects, size %.1fk + overhead %.1fk\n",
- v, h->used_count, elt_bytes / 1024,
- (overhead_bytes - elt_bytes) / 1024);
- }
-
- if (v && verbose)
- s = debug_elt (s, v, -1, -1);
-
- return s;
-}
-
-void
-heap_validate (void *v)
-{
- heap_header_t *h = heap_header (v);
- uword i, o, s;
- u8 *free_map;
- heap_elt_t *e, *n;
-
- uword used_count, total_size;
- uword free_count, free_size;
-
- ASSERT (h->used_count == clib_bitmap_count_set_bits (h->used_elt_bitmap));
-
- ASSERT (first (h)->prev == 0);
- ASSERT (last (h)->next == 0);
-
- /* Validate number of elements and size. */
- free_size = free_count = 0;
- for (i = 0; i < vec_len (h->free_lists); i++)
- {
- free_count += vec_len (h->free_lists[i]);
- for (o = 0; o < vec_len (h->free_lists[i]); o++)
- {
- e = h->elts + h->free_lists[i][o];
- s = heap_elt_size (v, e);
- ASSERT (size_to_bin (s) == i);
- ASSERT (heap_is_free (e));
- free_size += s;
- }
- }
-
- {
- uword elt_free_size, elt_free_count;
-
- used_count = total_size = elt_free_size = elt_free_count = 0;
- for (e = first (h); 1; e = n)
- {
- int is_free = heap_is_free (e);
- used_count++;
- s = heap_elt_size (v, e);
- total_size += s;
- ASSERT (is_free ==
- !clib_bitmap_get (h->used_elt_bitmap, e - h->elts));
- if (is_free)
- {
- elt_free_count++;
- elt_free_size += s;
- }
- n = heap_next (e);
- if (e == n)
- {
- ASSERT (last (h) == n);
- break;
- }
-
- /* We should never have two free adjacent elements. */
- ASSERT (!(heap_is_free (e) && heap_is_free (n)));
- }
-
- ASSERT (free_count == elt_free_count);
- ASSERT (free_size == elt_free_size);
- ASSERT (used_count == h->used_count + free_count);
- ASSERT (total_size == vec_len (v));
- }
-
- free_map = vec_new (u8, used_count);
-
- e = first (h);
- for (i = o = 0; 1; i++)
- {
- ASSERT (heap_offset (e) == o);
- s = heap_elt_size (v, e);
-
- if (heap_is_free (e))
- {
- uword fb, fi;
-
- fi = get_free_elt (v, e, &fb);
-
- ASSERT (fb < vec_len (h->free_lists));
- ASSERT (fi < vec_len (h->free_lists[fb]));
- ASSERT (h->free_lists[fb][fi] == e - h->elts);
-
- ASSERT (!free_map[i]);
- free_map[i] = 1;
- }
-
- n = heap_next (e);
-
- if (e == n)
- break;
-
- ASSERT (heap_prev (n) == e);
-
- o += s;
- e = n;
- }
-
- vec_free (free_map);
-}
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */