diff options
Diffstat (limited to 'app/nginx/src/core/ngx_slab.c')
-rw-r--r-- | app/nginx/src/core/ngx_slab.c | 806 |
1 files changed, 0 insertions, 806 deletions
diff --git a/app/nginx/src/core/ngx_slab.c b/app/nginx/src/core/ngx_slab.c deleted file mode 100644 index 1d4ce2b..0000000 --- a/app/nginx/src/core/ngx_slab.c +++ /dev/null @@ -1,806 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) Nginx, Inc. - */ - -#include <ngx_config.h> -#include <ngx_core.h> - - -#define NGX_SLAB_PAGE_MASK 3 -#define NGX_SLAB_PAGE 0 -#define NGX_SLAB_BIG 1 -#define NGX_SLAB_EXACT 2 -#define NGX_SLAB_SMALL 3 - -#if (NGX_PTR_SIZE == 4) - -#define NGX_SLAB_PAGE_FREE 0 -#define NGX_SLAB_PAGE_BUSY 0xffffffff -#define NGX_SLAB_PAGE_START 0x80000000 - -#define NGX_SLAB_SHIFT_MASK 0x0000000f -#define NGX_SLAB_MAP_MASK 0xffff0000 -#define NGX_SLAB_MAP_SHIFT 16 - -#define NGX_SLAB_BUSY 0xffffffff - -#else /* (NGX_PTR_SIZE == 8) */ - -#define NGX_SLAB_PAGE_FREE 0 -#define NGX_SLAB_PAGE_BUSY 0xffffffffffffffff -#define NGX_SLAB_PAGE_START 0x8000000000000000 - -#define NGX_SLAB_SHIFT_MASK 0x000000000000000f -#define NGX_SLAB_MAP_MASK 0xffffffff00000000 -#define NGX_SLAB_MAP_SHIFT 32 - -#define NGX_SLAB_BUSY 0xffffffffffffffff - -#endif - - -#define ngx_slab_slots(pool) \ - (ngx_slab_page_t *) ((u_char *) (pool) + sizeof(ngx_slab_pool_t)) - -#define ngx_slab_page_type(page) ((page)->prev & NGX_SLAB_PAGE_MASK) - -#define ngx_slab_page_prev(page) \ - (ngx_slab_page_t *) ((page)->prev & ~NGX_SLAB_PAGE_MASK) - -#define ngx_slab_page_addr(pool, page) \ - ((((page) - (pool)->pages) << ngx_pagesize_shift) \ - + (uintptr_t) (pool)->start) - - -#if (NGX_DEBUG_MALLOC) - -#define ngx_slab_junk(p, size) ngx_memset(p, 0xA5, size) - -#elif (NGX_HAVE_DEBUG_MALLOC) - -#define ngx_slab_junk(p, size) \ - if (ngx_debug_malloc) ngx_memset(p, 0xA5, size) - -#else - -#define ngx_slab_junk(p, size) - -#endif - -static ngx_slab_page_t *ngx_slab_alloc_pages(ngx_slab_pool_t *pool, - ngx_uint_t pages); -static void ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page, - ngx_uint_t pages); -static void ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level, - char *text); - - -static ngx_uint_t ngx_slab_max_size; -static ngx_uint_t ngx_slab_exact_size; -static ngx_uint_t ngx_slab_exact_shift; - - -void -ngx_slab_init(ngx_slab_pool_t *pool) -{ - u_char *p; - size_t size; - ngx_int_t m; - ngx_uint_t i, n, pages; - ngx_slab_page_t *slots, *page; - - /* STUB */ - if (ngx_slab_max_size == 0) { - ngx_slab_max_size = ngx_pagesize / 2; - ngx_slab_exact_size = ngx_pagesize / (8 * sizeof(uintptr_t)); - for (n = ngx_slab_exact_size; n >>= 1; ngx_slab_exact_shift++) { - /* void */ - } - } - /**/ - - pool->min_size = (size_t) 1 << pool->min_shift; - - slots = ngx_slab_slots(pool); - - p = (u_char *) slots; - size = pool->end - p; - - ngx_slab_junk(p, size); - - n = ngx_pagesize_shift - pool->min_shift; - - for (i = 0; i < n; i++) { - /* only "next" is used in list head */ - slots[i].slab = 0; - slots[i].next = &slots[i]; - slots[i].prev = 0; - } - - p += n * sizeof(ngx_slab_page_t); - - pool->stats = (ngx_slab_stat_t *) p; - ngx_memzero(pool->stats, n * sizeof(ngx_slab_stat_t)); - - p += n * sizeof(ngx_slab_stat_t); - - size -= n * (sizeof(ngx_slab_page_t) + sizeof(ngx_slab_stat_t)); - - pages = (ngx_uint_t) (size / (ngx_pagesize + sizeof(ngx_slab_page_t))); - - pool->pages = (ngx_slab_page_t *) p; - ngx_memzero(pool->pages, pages * sizeof(ngx_slab_page_t)); - - page = pool->pages; - - /* only "next" is used in list head */ - pool->free.slab = 0; - pool->free.next = page; - pool->free.prev = 0; - - page->slab = pages; - page->next = &pool->free; - page->prev = (uintptr_t) &pool->free; - - pool->start = ngx_align_ptr(p + pages * sizeof(ngx_slab_page_t), - ngx_pagesize); - - m = pages - (pool->end - pool->start) / ngx_pagesize; - if (m > 0) { - pages -= m; - page->slab = pages; - } - - pool->last = pool->pages + pages; - pool->pfree = pages; - - pool->log_nomem = 1; - pool->log_ctx = &pool->zero; - pool->zero = '\0'; -} - - -void * -ngx_slab_alloc(ngx_slab_pool_t *pool, size_t size) -{ - void *p; - - ngx_shmtx_lock(&pool->mutex); - - p = ngx_slab_alloc_locked(pool, size); - - ngx_shmtx_unlock(&pool->mutex); - - return p; -} - - -void * -ngx_slab_alloc_locked(ngx_slab_pool_t *pool, size_t size) -{ - size_t s; - uintptr_t p, n, m, mask, *bitmap; - ngx_uint_t i, slot, shift, map; - ngx_slab_page_t *page, *prev, *slots; - - if (size > ngx_slab_max_size) { - - ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, - "slab alloc: %uz", size); - - page = ngx_slab_alloc_pages(pool, (size >> ngx_pagesize_shift) - + ((size % ngx_pagesize) ? 1 : 0)); - if (page) { - p = ngx_slab_page_addr(pool, page); - - } else { - p = 0; - } - - goto done; - } - - if (size > pool->min_size) { - shift = 1; - for (s = size - 1; s >>= 1; shift++) { /* void */ } - slot = shift - pool->min_shift; - - } else { - shift = pool->min_shift; - slot = 0; - } - - pool->stats[slot].reqs++; - - ngx_log_debug2(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, - "slab alloc: %uz slot: %ui", size, slot); - - slots = ngx_slab_slots(pool); - page = slots[slot].next; - - if (page->next != page) { - - if (shift < ngx_slab_exact_shift) { - - bitmap = (uintptr_t *) ngx_slab_page_addr(pool, page); - - map = (ngx_pagesize >> shift) / (sizeof(uintptr_t) * 8); - - for (n = 0; n < map; n++) { - - if (bitmap[n] != NGX_SLAB_BUSY) { - - for (m = 1, i = 0; m; m <<= 1, i++) { - if (bitmap[n] & m) { - continue; - } - - bitmap[n] |= m; - - i = (n * sizeof(uintptr_t) * 8 + i) << shift; - - p = (uintptr_t) bitmap + i; - - pool->stats[slot].used++; - - if (bitmap[n] == NGX_SLAB_BUSY) { - for (n = n + 1; n < map; n++) { - if (bitmap[n] != NGX_SLAB_BUSY) { - goto done; - } - } - - prev = ngx_slab_page_prev(page); - prev->next = page->next; - page->next->prev = page->prev; - - page->next = NULL; - page->prev = NGX_SLAB_SMALL; - } - - goto done; - } - } - } - - } else if (shift == ngx_slab_exact_shift) { - - for (m = 1, i = 0; m; m <<= 1, i++) { - if (page->slab & m) { - continue; - } - - page->slab |= m; - - if (page->slab == NGX_SLAB_BUSY) { - prev = ngx_slab_page_prev(page); - prev->next = page->next; - page->next->prev = page->prev; - - page->next = NULL; - page->prev = NGX_SLAB_EXACT; - } - - p = ngx_slab_page_addr(pool, page) + (i << shift); - - pool->stats[slot].used++; - - goto done; - } - - } else { /* shift > ngx_slab_exact_shift */ - - mask = ((uintptr_t) 1 << (ngx_pagesize >> shift)) - 1; - mask <<= NGX_SLAB_MAP_SHIFT; - - for (m = (uintptr_t) 1 << NGX_SLAB_MAP_SHIFT, i = 0; - m & mask; - m <<= 1, i++) - { - if (page->slab & m) { - continue; - } - - page->slab |= m; - - if ((page->slab & NGX_SLAB_MAP_MASK) == mask) { - prev = ngx_slab_page_prev(page); - prev->next = page->next; - page->next->prev = page->prev; - - page->next = NULL; - page->prev = NGX_SLAB_BIG; - } - - p = ngx_slab_page_addr(pool, page) + (i << shift); - - pool->stats[slot].used++; - - goto done; - } - } - - ngx_slab_error(pool, NGX_LOG_ALERT, "ngx_slab_alloc(): page is busy"); - ngx_debug_point(); - } - - page = ngx_slab_alloc_pages(pool, 1); - - if (page) { - if (shift < ngx_slab_exact_shift) { - bitmap = (uintptr_t *) ngx_slab_page_addr(pool, page); - - n = (ngx_pagesize >> shift) / ((1 << shift) * 8); - - if (n == 0) { - n = 1; - } - - /* "n" elements for bitmap, plus one requested */ - bitmap[0] = ((uintptr_t) 2 << n) - 1; - - map = (ngx_pagesize >> shift) / (sizeof(uintptr_t) * 8); - - for (i = 1; i < map; i++) { - bitmap[i] = 0; - } - - page->slab = shift; - page->next = &slots[slot]; - page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL; - - slots[slot].next = page; - - pool->stats[slot].total += (ngx_pagesize >> shift) - n; - - p = ngx_slab_page_addr(pool, page) + (n << shift); - - pool->stats[slot].used++; - - goto done; - - } else if (shift == ngx_slab_exact_shift) { - - page->slab = 1; - page->next = &slots[slot]; - page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT; - - slots[slot].next = page; - - pool->stats[slot].total += sizeof(uintptr_t) * 8; - - p = ngx_slab_page_addr(pool, page); - - pool->stats[slot].used++; - - goto done; - - } else { /* shift > ngx_slab_exact_shift */ - - page->slab = ((uintptr_t) 1 << NGX_SLAB_MAP_SHIFT) | shift; - page->next = &slots[slot]; - page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG; - - slots[slot].next = page; - - pool->stats[slot].total += ngx_pagesize >> shift; - - p = ngx_slab_page_addr(pool, page); - - pool->stats[slot].used++; - - goto done; - } - } - - p = 0; - - pool->stats[slot].fails++; - -done: - - ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, - "slab alloc: %p", (void *) p); - - return (void *) p; -} - - -void * -ngx_slab_calloc(ngx_slab_pool_t *pool, size_t size) -{ - void *p; - - ngx_shmtx_lock(&pool->mutex); - - p = ngx_slab_calloc_locked(pool, size); - - ngx_shmtx_unlock(&pool->mutex); - - return p; -} - - -void * -ngx_slab_calloc_locked(ngx_slab_pool_t *pool, size_t size) -{ - void *p; - - p = ngx_slab_alloc_locked(pool, size); - if (p) { - ngx_memzero(p, size); - } - - return p; -} - - -void -ngx_slab_free(ngx_slab_pool_t *pool, void *p) -{ - ngx_shmtx_lock(&pool->mutex); - - ngx_slab_free_locked(pool, p); - - ngx_shmtx_unlock(&pool->mutex); -} - - -void -ngx_slab_free_locked(ngx_slab_pool_t *pool, void *p) -{ - size_t size; - uintptr_t slab, m, *bitmap; - ngx_uint_t i, n, type, slot, shift, map; - ngx_slab_page_t *slots, *page; - - ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, "slab free: %p", p); - - if ((u_char *) p < pool->start || (u_char *) p > pool->end) { - ngx_slab_error(pool, NGX_LOG_ALERT, "ngx_slab_free(): outside of pool"); - goto fail; - } - - n = ((u_char *) p - pool->start) >> ngx_pagesize_shift; - page = &pool->pages[n]; - slab = page->slab; - type = ngx_slab_page_type(page); - - switch (type) { - - case NGX_SLAB_SMALL: - - shift = slab & NGX_SLAB_SHIFT_MASK; - size = (size_t) 1 << shift; - - if ((uintptr_t) p & (size - 1)) { - goto wrong_chunk; - } - - n = ((uintptr_t) p & (ngx_pagesize - 1)) >> shift; - m = (uintptr_t) 1 << (n % (sizeof(uintptr_t) * 8)); - n /= sizeof(uintptr_t) * 8; - bitmap = (uintptr_t *) - ((uintptr_t) p & ~((uintptr_t) ngx_pagesize - 1)); - - if (bitmap[n] & m) { - slot = shift - pool->min_shift; - - if (page->next == NULL) { - slots = ngx_slab_slots(pool); - - page->next = slots[slot].next; - slots[slot].next = page; - - page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL; - page->next->prev = (uintptr_t) page | NGX_SLAB_SMALL; - } - - bitmap[n] &= ~m; - - n = (ngx_pagesize >> shift) / ((1 << shift) * 8); - - if (n == 0) { - n = 1; - } - - if (bitmap[0] & ~(((uintptr_t) 1 << n) - 1)) { - goto done; - } - - map = (ngx_pagesize >> shift) / (sizeof(uintptr_t) * 8); - - for (i = 1; i < map; i++) { - if (bitmap[i]) { - goto done; - } - } - - ngx_slab_free_pages(pool, page, 1); - - pool->stats[slot].total -= (ngx_pagesize >> shift) - n; - - goto done; - } - - goto chunk_already_free; - - case NGX_SLAB_EXACT: - - m = (uintptr_t) 1 << - (((uintptr_t) p & (ngx_pagesize - 1)) >> ngx_slab_exact_shift); - size = ngx_slab_exact_size; - - if ((uintptr_t) p & (size - 1)) { - goto wrong_chunk; - } - - if (slab & m) { - slot = ngx_slab_exact_shift - pool->min_shift; - - if (slab == NGX_SLAB_BUSY) { - slots = ngx_slab_slots(pool); - - page->next = slots[slot].next; - slots[slot].next = page; - - page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT; - page->next->prev = (uintptr_t) page | NGX_SLAB_EXACT; - } - - page->slab &= ~m; - - if (page->slab) { - goto done; - } - - ngx_slab_free_pages(pool, page, 1); - - pool->stats[slot].total -= sizeof(uintptr_t) * 8; - - goto done; - } - - goto chunk_already_free; - - case NGX_SLAB_BIG: - - shift = slab & NGX_SLAB_SHIFT_MASK; - size = (size_t) 1 << shift; - - if ((uintptr_t) p & (size - 1)) { - goto wrong_chunk; - } - - m = (uintptr_t) 1 << ((((uintptr_t) p & (ngx_pagesize - 1)) >> shift) - + NGX_SLAB_MAP_SHIFT); - - if (slab & m) { - slot = shift - pool->min_shift; - - if (page->next == NULL) { - slots = ngx_slab_slots(pool); - - page->next = slots[slot].next; - slots[slot].next = page; - - page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG; - page->next->prev = (uintptr_t) page | NGX_SLAB_BIG; - } - - page->slab &= ~m; - - if (page->slab & NGX_SLAB_MAP_MASK) { - goto done; - } - - ngx_slab_free_pages(pool, page, 1); - - pool->stats[slot].total -= ngx_pagesize >> shift; - - goto done; - } - - goto chunk_already_free; - - case NGX_SLAB_PAGE: - - if ((uintptr_t) p & (ngx_pagesize - 1)) { - goto wrong_chunk; - } - - if (!(slab & NGX_SLAB_PAGE_START)) { - ngx_slab_error(pool, NGX_LOG_ALERT, - "ngx_slab_free(): page is already free"); - goto fail; - } - - if (slab == NGX_SLAB_PAGE_BUSY) { - ngx_slab_error(pool, NGX_LOG_ALERT, - "ngx_slab_free(): pointer to wrong page"); - goto fail; - } - - n = ((u_char *) p - pool->start) >> ngx_pagesize_shift; - size = slab & ~NGX_SLAB_PAGE_START; - - ngx_slab_free_pages(pool, &pool->pages[n], size); - - ngx_slab_junk(p, size << ngx_pagesize_shift); - - return; - } - - /* not reached */ - - return; - -done: - - pool->stats[slot].used--; - - ngx_slab_junk(p, size); - - return; - -wrong_chunk: - - ngx_slab_error(pool, NGX_LOG_ALERT, - "ngx_slab_free(): pointer to wrong chunk"); - - goto fail; - -chunk_already_free: - - ngx_slab_error(pool, NGX_LOG_ALERT, - "ngx_slab_free(): chunk is already free"); - -fail: - - return; -} - - -static ngx_slab_page_t * -ngx_slab_alloc_pages(ngx_slab_pool_t *pool, ngx_uint_t pages) -{ - ngx_slab_page_t *page, *p; - - for (page = pool->free.next; page != &pool->free; page = page->next) { - - if (page->slab >= pages) { - - if (page->slab > pages) { - page[page->slab - 1].prev = (uintptr_t) &page[pages]; - - page[pages].slab = page->slab - pages; - page[pages].next = page->next; - page[pages].prev = page->prev; - - p = (ngx_slab_page_t *) page->prev; - p->next = &page[pages]; - page->next->prev = (uintptr_t) &page[pages]; - - } else { - p = (ngx_slab_page_t *) page->prev; - p->next = page->next; - page->next->prev = page->prev; - } - - page->slab = pages | NGX_SLAB_PAGE_START; - page->next = NULL; - page->prev = NGX_SLAB_PAGE; - - pool->pfree -= pages; - - if (--pages == 0) { - return page; - } - - for (p = page + 1; pages; pages--) { - p->slab = NGX_SLAB_PAGE_BUSY; - p->next = NULL; - p->prev = NGX_SLAB_PAGE; - p++; - } - - return page; - } - } - - if (pool->log_nomem) { - ngx_slab_error(pool, NGX_LOG_CRIT, - "ngx_slab_alloc() failed: no memory"); - } - - return NULL; -} - - -static void -ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page, - ngx_uint_t pages) -{ - ngx_slab_page_t *prev, *join; - - pool->pfree += pages; - - page->slab = pages--; - - if (pages) { - ngx_memzero(&page[1], pages * sizeof(ngx_slab_page_t)); - } - - if (page->next) { - prev = ngx_slab_page_prev(page); - prev->next = page->next; - page->next->prev = page->prev; - } - - join = page + page->slab; - - if (join < pool->last) { - - if (ngx_slab_page_type(join) == NGX_SLAB_PAGE) { - - if (join->next != NULL) { - pages += join->slab; - page->slab += join->slab; - - prev = ngx_slab_page_prev(join); - prev->next = join->next; - join->next->prev = join->prev; - - join->slab = NGX_SLAB_PAGE_FREE; - join->next = NULL; - join->prev = NGX_SLAB_PAGE; - } - } - } - - if (page > pool->pages) { - join = page - 1; - - if (ngx_slab_page_type(join) == NGX_SLAB_PAGE) { - - if (join->slab == NGX_SLAB_PAGE_FREE) { - join = ngx_slab_page_prev(join); - } - - if (join->next != NULL) { - pages += join->slab; - join->slab += page->slab; - - prev = ngx_slab_page_prev(join); - prev->next = join->next; - join->next->prev = join->prev; - - page->slab = NGX_SLAB_PAGE_FREE; - page->next = NULL; - page->prev = NGX_SLAB_PAGE; - - page = join; - } - } - } - - if (pages) { - page[pages].prev = (uintptr_t) page; - } - - page->prev = (uintptr_t) &pool->free; - page->next = pool->free.next; - - page->next->prev = (uintptr_t) page; - - pool->free.next = page; -} - - -static void -ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level, char *text) -{ - ngx_log_error(level, ngx_cycle->log, 0, "%s%s", text, pool->log_ctx); -} |