From 703faabf2d44d245fe1dd0b75f1736bf6114a557 Mon Sep 17 00:00:00 2001 From: Konstantin Ananyev Date: Thu, 27 Jun 2019 19:28:26 +0100 Subject: v6: memtank introduction For analogy with mempool, named this structure memtank. Same a s mempool it allows to alloc/free objects of fixed size in a lightweight manner (not as lightweight as mempool, but hopefully close enough). The whole idea is that alloc/free is used at fast-path and don't allocate/free more than *min_free* objects at one call. So for majority of cases our fast-path alloc/free should be lightweight (LIFO enqueue/dequeue operations). Also user will need to call grow/shrink periodically (ideally from the slow-path) to make sure there is enough free objects in the tank. Internally it is just a simple LIFO for up to *max_free* objects plus a list of memory buffers (memchunk) from where these objects were allocated. v1 -> v2 - Added UT - Fixed few bugs v2 -> v3 - extend UT with more parameters v3 -> v4 - add object alignement as parameter for memtank_create - extend UT with more parameters - added memtank dump routine v4 -> v5 - fixed few bugs inside memtank lib - extend UT with: - new test case - new command-line options: '-s ', '-m ' v5 -> v6 - extend memtank dump to collect/display extra information - make memtank dump routine MT safe - add memtank sanity check function - add proper comments for pubic API Signed-off-by: Konstantin Ananyev Change-Id: I8939772577f5d9e293088eaa9a9fe316c3fe8f87 --- lib/Makefile | 1 + lib/libtle_memtank/Makefile | 40 ++ lib/libtle_memtank/memtank.c | 507 ++++++++++++++++++++++ lib/libtle_memtank/memtank.h | 107 +++++ lib/libtle_memtank/misc.c | 381 +++++++++++++++++ lib/libtle_memtank/tle_memtank.h | 274 ++++++++++++ lib/libtle_memtank/tle_memtank_pub.h | 149 +++++++ test/Makefile | 1 + test/memtank/Makefile | 42 ++ test/memtank/test_memtank.c | 793 +++++++++++++++++++++++++++++++++++ 10 files changed, 2295 insertions(+) create mode 100644 lib/libtle_memtank/Makefile create mode 100644 lib/libtle_memtank/memtank.c create mode 100644 lib/libtle_memtank/memtank.h create mode 100644 lib/libtle_memtank/misc.c create mode 100644 lib/libtle_memtank/tle_memtank.h create mode 100644 lib/libtle_memtank/tle_memtank_pub.h create mode 100644 test/memtank/Makefile create mode 100644 test/memtank/test_memtank.c diff --git a/lib/Makefile b/lib/Makefile index 6317af9..8d61a08 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -24,6 +24,7 @@ include $(RTE_SDK)/mk/rte.vars.mk DIRS-y += libtle_misc DIRS-y += libtle_dring DIRS-y += libtle_timer +DIRS-y += libtle_memtank DIRS-y += libtle_l4p include $(TLDK_ROOT)/mk/tle.subdir.mk diff --git a/lib/libtle_memtank/Makefile b/lib/libtle_memtank/Makefile new file mode 100644 index 0000000..d87e320 --- /dev/null +++ b/lib/libtle_memtank/Makefile @@ -0,0 +1,40 @@ +# Copyright (c) 2016 Intel Corporation. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ifeq ($(RTE_SDK),) +$(error "Please define RTE_SDK environment variable") +endif + +# Default target, can be overwritten by command line or environment +RTE_TARGET ?= x86_64-native-linuxapp-gcc + +include $(RTE_SDK)/mk/rte.vars.mk + +# library name +LIB = libtle_memtank.a + +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) + +EXPORT_MAP := tle_memtank_version.map + +LIBABIVER := 1 + +#source files +SRCS-y += memtank.c +SRCS-y += misc.c + +SYMLINK-y-include += tle_memtank_pub.h +SYMLINK-y-include += tle_memtank.h + +include $(TLDK_ROOT)/mk/tle.lib.mk diff --git a/lib/libtle_memtank/memtank.c b/lib/libtle_memtank/memtank.c new file mode 100644 index 0000000..ceb209c --- /dev/null +++ b/lib/libtle_memtank/memtank.c @@ -0,0 +1,507 @@ +/* + * Copyright (c) 2019 Intel Corporation. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "memtank.h" +#include + +#define ALIGN_MUL_CEIL(v, mul) \ + ((typeof(v))(((uint64_t)(v) + (mul) - 1) / (mul))) + + +static inline size_t +memtank_meta_size(uint32_t nb_free) +{ + size_t sz; + static const struct memtank *mt; + + sz = sizeof(*mt) + nb_free * sizeof(mt->pub.free[0]); + sz = RTE_ALIGN_CEIL(sz, alignof(*mt)); + return sz; +} + +static inline size_t +memchunk_meta_size(uint32_t nb_obj) +{ + size_t sz; + static const struct memchunk *ch; + + sz = sizeof(*ch) + nb_obj * sizeof(ch->free[0]); + sz = RTE_ALIGN_CEIL(sz, alignof(*ch)); + return sz; +} + +static inline size_t +memobj_size(uint32_t obj_size, uint32_t obj_align) +{ + size_t sz; + static const struct memobj *obj; + + sz = sizeof(*obj) + obj_size; + sz = RTE_ALIGN_CEIL(sz, obj_align); + return sz; +} + +static inline size_t +memchunk_size(uint32_t nb_obj, uint32_t obj_size, uint32_t obj_align) +{ + size_t algn, sz; + static const struct memchunk *ch; + + algn = RTE_MAX(alignof(*ch), obj_align); + sz = memchunk_meta_size(nb_obj); + sz += nb_obj * memobj_size(obj_size, obj_align); + sz = RTE_ALIGN_CEIL(sz + algn - 1, algn); + return sz; +} + +static void +init_chunk(struct memtank *mt, struct memchunk *ch) +{ + uint32_t i, n, sz; + uintptr_t p; + struct memobj *obj; + + const struct memobj cobj = { + .red_zone1 = RED_ZONE_V1, + .chunk = ch, + .red_zone2 = RED_ZONE_V2, + }; + + n = mt->prm.nb_obj_chunk; + sz = mt->obj_size; + + /* get start of memobj array */ + p = (uintptr_t)ch + memchunk_meta_size(n); + p = RTE_ALIGN_CEIL(p, mt->prm.obj_align); + + for (i = 0; i != n; i++) { + obj = obj_pub_full(p, sz); + obj[0] = cobj; + ch->free[i] = (void *)p; + p += sz; + } + + ch->nb_total = n; + ch->nb_free = n; + + if (mt->prm.init != NULL) + mt->prm.init(ch->free, n, mt->prm.udata); +} + +static void +put_chunk(struct memtank *mt, struct memchunk *ch, void * const obj[], + uint32_t num) +{ + uint32_t k, n; + struct mchunk_list *ls; + + /* chunk should be in the *used* list */ + k = MC_USED; + ls = &mt->chl[k]; + rte_spinlock_lock(&ls->lock); + + n = ch->nb_free; + RTE_ASSERT(n + num <= ch->nb_total); + + _copy_objs(ch->free + n, obj, num); + ch->nb_free = n + num; + + /* chunk is full now */ + if (ch->nb_free == ch->nb_total) { + TAILQ_REMOVE(&ls->chunk, ch, link); + k = MC_FULL; + /* chunk is not empty anymore, move it to the head */ + } else if (n == 0) { + TAILQ_REMOVE(&ls->chunk, ch, link); + TAILQ_INSERT_HEAD(&ls->chunk, ch, link); + } + + rte_spinlock_unlock(&ls->lock); + + /* insert this chunk into the *full* list */ + if (k == MC_FULL) { + ls = &mt->chl[k]; + rte_spinlock_lock(&ls->lock); + TAILQ_INSERT_HEAD(&ls->chunk, ch, link); + rte_spinlock_unlock(&ls->lock); + } +} + +static uint32_t +shrink_chunk(struct memtank *mt, uint32_t num) +{ + uint32_t i, k; + struct mchunk_list *ls; + struct memchunk *ch[num]; + + ls = &mt->chl[MC_FULL]; + rte_spinlock_lock(&ls->lock); + + for (k = 0; k != num; k++) { + ch[k] = TAILQ_LAST(&ls->chunk, mchunk_head); + if (ch[k] == NULL) + break; + TAILQ_REMOVE(&ls->chunk, ch[k], link); + } + + rte_spinlock_unlock(&ls->lock); + + rte_atomic32_sub(&mt->nb_chunks, k); + + for (i = 0; i != k; i++) + mt->prm.free(ch[i]->raw, mt->prm.udata); + + return k; +} + +static struct memchunk * +alloc_chunk(struct memtank *mt) +{ + void *p; + struct memchunk *ch; + + p = mt->prm.alloc(mt->chunk_size, mt->prm.udata); + if (p == NULL) + return NULL; + ch = RTE_PTR_ALIGN_CEIL(p, alignof(*ch)); + ch->raw = p; + return ch; +} + +/* Determine by how many chunks we can actually grow */ +static inline uint32_t +grow_num(struct memtank *mt, uint32_t num) +{ + uint32_t k, n, max; + + max = mt->max_chunk; + n = rte_atomic32_add_return(&mt->nb_chunks, num); + + if (n <= max) + return num; + + k = n - max; + return (k >= num) ? 0 : num - k; +} + +static uint32_t +grow_chunk(struct memtank *mt, uint32_t num) +{ + uint32_t i, k, n; + struct mchunk_list *fls; + struct mchunk_head ls; + struct memchunk *ch[num]; + + /* check can we grow further */ + k = grow_num(mt, num); + + for (n = 0; n != k; n++) { + ch[n] = alloc_chunk(mt); + if (ch[n] == NULL) + break; + } + + TAILQ_INIT(&ls); + + for (i = 0; i != n; i++) { + init_chunk(mt, ch[i]); + TAILQ_INSERT_HEAD(&ls, ch[i], link); + } + + if (n != 0) { + fls = &mt->chl[MC_FULL]; + rte_spinlock_lock(&fls->lock); + TAILQ_CONCAT(&fls->chunk, &ls, link); + rte_spinlock_unlock(&fls->lock); + } + + if (n != num) + rte_atomic32_sub(&mt->nb_chunks, num - n); + + return n; +} + +static void +obj_dbg_alloc(struct memtank *mt, void * const obj[], uint32_t nb_obj) +{ + uint32_t i, sz; + struct memobj *po; + + sz = mt->obj_size; + for (i = 0; i != nb_obj; i++) { + po = obj_pub_full((uintptr_t)obj[i], sz); + RTE_VERIFY(memobj_verify(po, 0) == 0); + po->dbg.nb_alloc++; + } +} + +static void +obj_dbg_free(struct memtank *mt, void * const obj[], uint32_t nb_obj) +{ + uint32_t i, sz; + struct memobj *po; + + sz = mt->obj_size; + for (i = 0; i != nb_obj; i++) { + po = obj_pub_full((uintptr_t)obj[i], sz); + RTE_VERIFY(memobj_verify(po, 1) == 0); + po->dbg.nb_free++; + } +} + + +void +tle_memtank_chunk_free(struct tle_memtank *t, void * const obj[], + uint32_t nb_obj, uint32_t flags) +{ + uint32_t i, j, k, sz; + struct memtank *mt; + struct memobj *mo; + struct memchunk *ch[nb_obj]; + + mt = tank_pub_full(t); + sz = mt->obj_size; + + if (mt->flags & TLE_MTANK_OBJ_DBG) + obj_dbg_free(mt, obj, nb_obj); + + for (i = 0; i != nb_obj; i++) { + mo = obj_pub_full((uintptr_t)obj[i], sz); + ch[i] = mo->chunk; + } + + k = 0; + for (i = 0; i != nb_obj; i = j) { + + /* find number of consequtive objs from the same chunk */ + for (j = i + 1; j != nb_obj && ch[j] == ch[i]; j++) + ; + + put_chunk(mt, ch[i], obj + i, j - i); + k++; + } + + if (flags & TLE_MTANK_FREE_SHRINK) + shrink_chunk(mt, k); +} + +static uint32_t +get_chunk(struct mchunk_list *ls, struct mchunk_head *els, + struct mchunk_head *uls, void *obj[], uint32_t nb_obj) +{ + uint32_t l, k, n; + struct memchunk *ch, *nch; + + rte_spinlock_lock(&ls->lock); + + n = 0; + for (ch = TAILQ_FIRST(&ls->chunk); + n != nb_obj && ch != NULL && ch->nb_free != 0; + ch = nch, n += k) { + + k = RTE_MIN(nb_obj - n, ch->nb_free); + l = ch->nb_free - k; + _copy_objs(obj + n, ch->free + l, k); + ch->nb_free = l; + + nch = TAILQ_NEXT(ch, link); + + /* chunk is empty now */ + if (l == 0) { + TAILQ_REMOVE(&ls->chunk, ch, link); + TAILQ_INSERT_TAIL(els, ch, link); + } else if (uls != NULL) { + TAILQ_REMOVE(&ls->chunk, ch, link); + TAILQ_INSERT_HEAD(uls, ch, link); + } + } + + rte_spinlock_unlock(&ls->lock); + return n; +} + +uint32_t +tle_memtank_chunk_alloc(struct tle_memtank *t, void *obj[], uint32_t nb_obj, + uint32_t flags) +{ + uint32_t k, n; + struct memtank *mt; + struct mchunk_head els, uls; + + mt = tank_pub_full(t); + + /* walk though the the *used* list first */ + n = get_chunk(&mt->chl[MC_USED], &mt->chl[MC_USED].chunk, NULL, + obj, nb_obj); + + if (n != nb_obj) { + + TAILQ_INIT(&els); + TAILQ_INIT(&uls); + + /* walk though the the *full* list */ + n += get_chunk(&mt->chl[MC_FULL], &els, &uls, + obj + n, nb_obj - n); + + if (n != nb_obj && (flags & TLE_MTANK_ALLOC_GROW) != 0) { + + /* try to allocate extra memchunks */ + k = ALIGN_MUL_CEIL(nb_obj - n, + mt->prm.nb_obj_chunk); + k = grow_chunk(mt, k); + + /* walk through the *full* list again */ + if (k != 0) + n += get_chunk(&mt->chl[MC_FULL], &els, &uls, + obj + n, nb_obj - n); + } + + /* concatenate with *used* list our temporary lists */ + rte_spinlock_lock(&mt->chl[MC_USED].lock); + + /* put new non-emtpy elems at head of the *used* list */ + TAILQ_CONCAT(&uls, &mt->chl[MC_USED].chunk, link); + TAILQ_CONCAT(&mt->chl[MC_USED].chunk, &uls, link); + + /* put new emtpy elems at tail of the *used* list */ + TAILQ_CONCAT(&mt->chl[MC_USED].chunk, &els, link); + + rte_spinlock_unlock(&mt->chl[MC_USED].lock); + } + + if (mt->flags & TLE_MTANK_OBJ_DBG) + obj_dbg_alloc(mt, obj, n); + + return n; +} + +int +tle_memtank_grow(struct tle_memtank *t) +{ + uint32_t k, n, num; + struct memtank *mt; + + mt = tank_pub_full(t); + + /* how many chunks we need to grow */ + k = t->min_free - t->nb_free; + if ((int32_t)k <= 0) + return 0; + + num = ALIGN_MUL_CEIL(k, mt->prm.nb_obj_chunk); + + /* try to grow and refill the *free* */ + n = grow_chunk(mt, num); + if (n != 0) + _fill_free(t, k, 0); + + return n; +} + +int +tle_memtank_shrink(struct tle_memtank *t) +{ + uint32_t n; + struct memtank *mt; + + mt = tank_pub_full(t); + + /* how many chunks we need to shrink */ + if (t->nb_free < t->max_free) + return 0; + + /* how many chunks we need to free */ + n = ALIGN_MUL_CEIL(t->min_free, mt->prm.nb_obj_chunk); + + /* free up to *num* chunks */ + return shrink_chunk(mt, n); +} + +static int +check_param(const struct tle_memtank_prm *prm) +{ + if (prm->alloc == NULL || prm->free == NULL || + prm->min_free > prm->max_free || + rte_is_power_of_2(prm->obj_align) == 0) + return -EINVAL; + return 0; +} + +struct tle_memtank * +tle_memtank_create(const struct tle_memtank_prm *prm) +{ + int32_t rc; + size_t sz; + void *p; + struct memtank *mt; + + rc = check_param(prm); + if (rc != 0) { + rte_errno = -rc; + return NULL; + } + + sz = memtank_meta_size(prm->max_free); + p = prm->alloc(sz, prm->udata); + if (p == NULL) { + rte_errno = ENOMEM; + return NULL; + } + + mt = RTE_PTR_ALIGN_CEIL(p, alignof(*mt)); + + memset(mt, 0, sizeof(*mt)); + mt->prm = *prm; + + mt->raw = p; + mt->chunk_size = memchunk_size(prm->nb_obj_chunk, prm->obj_size, + prm->obj_align); + mt->obj_size = memobj_size(prm->obj_size, prm->obj_align); + mt->max_chunk = ALIGN_MUL_CEIL(prm->max_obj, prm->nb_obj_chunk); + mt->flags = prm->flags; + + mt->pub.min_free = prm->min_free; + mt->pub.max_free = prm->max_free; + + TAILQ_INIT(&mt->chl[MC_FULL].chunk); + TAILQ_INIT(&mt->chl[MC_USED].chunk); + + return &mt->pub; +} + +static void +free_mchunk_list(struct memtank *mt, struct mchunk_list *ls) +{ + struct memchunk *ch; + + for (ch = TAILQ_FIRST(&ls->chunk); ch != NULL; + ch = TAILQ_FIRST(&ls->chunk)) { + TAILQ_REMOVE(&ls->chunk, ch, link); + mt->prm.free(ch->raw, mt->prm.udata); + } +} + +void +tle_memtank_destroy(struct tle_memtank *t) +{ + struct memtank *mt; + + if (t != NULL) { + mt = tank_pub_full(t); + free_mchunk_list(mt, &mt->chl[MC_FULL]); + free_mchunk_list(mt, &mt->chl[MC_USED]); + mt->prm.free(mt->raw, mt->prm.udata); + } +} diff --git a/lib/libtle_memtank/memtank.h b/lib/libtle_memtank/memtank.h new file mode 100644 index 0000000..ba3f160 --- /dev/null +++ b/lib/libtle_memtank/memtank.h @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2019 Intel Corporation. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _MEMTANK_H_ +#define _MEMTANK_H_ + +#include +#include + +struct memobj { + uint64_t red_zone1; + struct memchunk *chunk; /* ptr to the chunk it belongs to */ + struct { + uint32_t nb_alloc; + uint32_t nb_free; + } dbg; + uint64_t red_zone2; +}; + +#define RED_ZONE_V1 UINT64_C(0xBADECAFEBADECAFE) +#define RED_ZONE_V2 UINT64_C(0xDEADBEEFDEADBEEF) + +struct memchunk { + TAILQ_ENTRY(memchunk) link; /* link to the next chunk in the tank */ + void *raw; /* un-aligned ptr returned by alloc() */ + uint32_t nb_total; /* total number of objects in the chunk */ + uint32_t nb_free; /* number of free object in the chunk */ + void *free[]; /* array of free objects */ +} __rte_cache_aligned; + + +TAILQ_HEAD(mchunk_head, memchunk); + +struct mchunk_list { + rte_spinlock_t lock; + struct mchunk_head chunk; /* list of chunks */ +} __rte_cache_aligned; + +enum { + MC_FULL, /* all memchunk objs are free */ + MC_USED, /* some of memchunk objs are allocated */ + MC_NUM, +}; + +struct memtank { + /* user provided data */ + struct tle_memtank_prm prm; + + /*run-time data */ + void *raw; /* un-aligned ptr returned by alloc() */ + size_t chunk_size; /* full size of each memchunk */ + uint32_t obj_size; /* full size of each memobj */ + uint32_t max_chunk; /* max allowed number of chunks */ + uint32_t flags; /* behavior flags */ + rte_atomic32_t nb_chunks; /* number of allocated chunks */ + + struct mchunk_list chl[MC_NUM]; /* lists of memchunks */ + + struct tle_memtank pub; +}; + +/* + * Obtain pointer to interal memtank struct from public one + */ +static inline struct memtank * +tank_pub_full(const void *p) +{ + uintptr_t v; + + v = (uintptr_t)p - offsetof(struct memtank, pub); + return (struct memtank *)v; +} + +/* + * Obtain pointer to interal memobj struct from public one + */ +static inline struct memobj * +obj_pub_full(uintptr_t p, uint32_t obj_sz) +{ + uintptr_t v; + + v = p + obj_sz - sizeof(struct memobj); + return (struct memobj *)v; +} + +static inline int +memobj_verify(const struct memobj *mo, uint32_t finc) +{ + if (mo->red_zone1 != RED_ZONE_V1 || mo->red_zone2 != RED_ZONE_V2 || + mo->dbg.nb_alloc != mo->dbg.nb_free + finc) + return -EINVAL; + return 0; +} + +#endif /* _MEMTANK_H_ */ diff --git a/lib/libtle_memtank/misc.c b/lib/libtle_memtank/misc.c new file mode 100644 index 0000000..07e73db --- /dev/null +++ b/lib/libtle_memtank/misc.c @@ -0,0 +1,381 @@ +/* + * Copyright (c) 2019 Intel Corporation. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "memtank.h" +#include + +#define CHUNK_OBJ_LT_NUM 4 + +struct mchunk_stat { + uint32_t nb_empty; + uint32_t nb_full; + struct { + uint32_t nb_chunk; + uint32_t nb_obj; + struct { + uint32_t val; + uint32_t num; + } chunk_obj_lt[CHUNK_OBJ_LT_NUM]; + } used; +}; + +struct mfree_stat { + uint32_t nb_chunk; + struct mchunk_stat chunk; +}; + +#define MTANK_LOG(lvl, fmt, args...) RTE_LOG(lvl, USER1, fmt, ##args) + + +static void +mchunk_stat_dump(FILE *f, const struct mchunk_stat *st) +{ + uint32_t i; + + fprintf(f, "\t\tstat={\n"); + fprintf(f, "\t\t\tnb_empty=%u,\n", st->nb_empty); + fprintf(f, "\t\t\tnb_full=%u,\n", st->nb_full); + fprintf(f, "\t\t\tused={\n"); + fprintf(f, "\t\t\t\tnb_chunk=%u,\n", st->used.nb_chunk); + fprintf(f, "\t\t\t\tnb_obj=%u,\n", st->used.nb_obj); + + for (i = 0; i != RTE_DIM(st->used.chunk_obj_lt); i++) { + if (st->used.chunk_obj_lt[i].num != 0) + fprintf(f, "\t\t\t\tnb_chunk_obj_lt_%u=%u,\n", + st->used.chunk_obj_lt[i].val, + st->used.chunk_obj_lt[i].num); + } + + fprintf(f, "\t\t\t},\n"); + fprintf(f, "\t\t},\n"); +} + +static void +mchunk_stat_init(struct mchunk_stat *st, uint32_t nb_obj_chunk) +{ + uint32_t i; + + memset(st, 0, sizeof(*st)); + for (i = 0; i != RTE_DIM(st->used.chunk_obj_lt); i++) { + st->used.chunk_obj_lt[i].val = (i + 1) * nb_obj_chunk / + RTE_DIM(st->used.chunk_obj_lt); + } +} + +static void +mchunk_stat_collect(struct mchunk_stat *st, const struct memchunk *ch) +{ + uint32_t i, n; + + n = ch->nb_total - ch->nb_free; + + if (ch->nb_free == 0) + st->nb_empty++; + else if (n == 0) + st->nb_full++; + else { + st->used.nb_chunk++; + st->used.nb_obj += n; + + for (i = 0; i != RTE_DIM(st->used.chunk_obj_lt); i++) { + if (n < st->used.chunk_obj_lt[i].val) { + st->used.chunk_obj_lt[i].num++; + break; + } + } + } +} + +static void +mchunk_list_dump(FILE *f, struct memtank *mt, uint32_t idx, uint32_t flags) +{ + struct mchunk_list *ls; + const struct memchunk *ch; + struct mchunk_stat mcs; + + ls = &mt->chl[idx]; + mchunk_stat_init(&mcs, mt->prm.nb_obj_chunk); + + rte_spinlock_lock(&ls->lock); + + for (ch = TAILQ_FIRST(&ls->chunk); ch != NULL; + ch = TAILQ_NEXT(ch, link)) { + + /* collect chunk stats */ + if (flags & TLE_MTANK_DUMP_CHUNK_STAT) + mchunk_stat_collect(&mcs, ch); + + /* dump chunk metadata */ + if (flags & TLE_MTANK_DUMP_CHUNK) { + fprintf(f, "\t\tmemchunk@%p={\n", ch); + fprintf(f, "\t\t\traw=%p,\n", ch->raw); + fprintf(f, "\t\t\tnb_total=%u,\n", ch->nb_total); + fprintf(f, "\t\t\tnb_free=%u,\n", ch->nb_free); + fprintf(f, "\t\t},\n"); + } + } + + rte_spinlock_unlock(&ls->lock); + + /* print chunk stats */ + if (flags & TLE_MTANK_DUMP_CHUNK_STAT) + mchunk_stat_dump(f, &mcs); +} + +static void +mfree_stat_init(struct mfree_stat *st, uint32_t nb_obj_chunk) +{ + st->nb_chunk = 0; + mchunk_stat_init(&st->chunk, nb_obj_chunk); +} + +static int +ptr_cmp(const void *p1, const void *p2) +{ + const intptr_t *v1, *v2; + + v1 = p1; + v2 = p2; + return v1[0] - v2[0]; +} + +static void +mfree_stat_collect(struct mfree_stat *st, struct memtank *mt) +{ + uint32_t i, j, n, sz; + uintptr_t *p; + const struct memobj *mo; + + sz = mt->obj_size; + + p = malloc(mt->pub.max_free * sizeof(*p)); + if (p == NULL) + return; + + /** + * grab free lock and keep it till we analyze related memchunks, + * to make sure none of these memchunks will be freed untill + * we are finished. + */ + rte_spinlock_lock(&mt->pub.lock); + + /* collect chunks for all objects in free[] */ + n = mt->pub.nb_free; + memcpy(p, mt->pub.free, n * sizeof(*p)); + for (i = 0; i != n; i++) { + mo = obj_pub_full(p[i], sz); + p[i] = (uintptr_t)mo->chunk; + } + + /* sort chunk pointers */ + qsort(p, n, sizeof(*p), ptr_cmp); + + /* for each chunk collect stats */ + for (i = 0; i != n; i = j) { + + st->nb_chunk++; + mchunk_stat_collect(&st->chunk, (const struct memchunk *)p[i]); + for (j = i + 1; j != n && p[i] == p[j]; j++) + ; + } + + rte_spinlock_unlock(&mt->pub.lock); + free(p); +} + +static void +mfree_stat_dump(FILE *f, const struct mfree_stat *st) +{ + fprintf(f, "\tfree_stat={\n"); + fprintf(f, "\t\tnb_chunk=%u,\n", st->nb_chunk); + mchunk_stat_dump(f, &st->chunk); + fprintf(f, "\t},\n"); +} + +void +tle_memtank_dump(FILE *f, const struct tle_memtank *t, uint32_t flags) +{ + struct memtank *mt; + + if (f == NULL || t == NULL) + return; + + mt = tank_pub_full(t); + + fprintf(f, "tle_memtank@%p={\n", t); + fprintf(f, "\tmin_free=%u,\n", t->min_free); + fprintf(f, "\tmax_free=%u,\n", t->max_free); + fprintf(f, "\tnb_free=%u,\n", t->nb_free); + fprintf(f, "\tchunk_size=%zu,\n", mt->chunk_size); + fprintf(f, "\tobj_size=%u,\n", mt->obj_size); + fprintf(f, "\tmax_chunk=%u,\n", mt->max_chunk); + fprintf(f, "\tflags=%#x,\n", mt->flags); + fprintf(f, "\tnb_chunks=%u,\n", rte_atomic32_read(&mt->nb_chunks)); + + if (flags & TLE_MTANK_DUMP_FREE_STAT) { + struct mfree_stat mfs; + mfree_stat_init(&mfs, mt->prm.nb_obj_chunk); + mfree_stat_collect(&mfs, mt); + mfree_stat_dump(f, &mfs); + } + + if (flags & (TLE_MTANK_DUMP_CHUNK | TLE_MTANK_DUMP_CHUNK_STAT)) { + + fprintf(f, "\t[FULL]={\n"); + mchunk_list_dump(f, mt, MC_FULL, flags); + fprintf(f, "\t},\n"); + + fprintf(f, "\t[USED]={,\n"); + mchunk_list_dump(f, mt, MC_USED, flags); + fprintf(f, "\t},\n"); + } + fprintf(f, "};\n"); +} + +static int +mobj_bulk_check(const char *fname, const struct memtank *mt, + const uintptr_t p[], uint32_t num, uint32_t fmsk) +{ + int32_t ret; + uintptr_t align; + uint32_t i, k, sz; + const struct memobj *mo; + + k = ((mt->flags & TLE_MTANK_OBJ_DBG) != 0) & fmsk; + sz = mt->obj_size; + align = mt->prm.obj_align - 1; + + ret = 0; + for (i = 0; i != num; i++) { + + if (p[i] == (uintptr_t)NULL) { + ret--; + MTANK_LOG(ERR, + "%s(mt=%p, %p[%u]): NULL object\n", + fname, mt, p, i); + } else if ((p[i] & align) != 0) { + ret--; + MTANK_LOG(ERR, + "%s(mt=%p, %p[%u]): object %#zx violates " + "expected alignment %#zx\n", + fname, mt, p, i, p[i], align); + } else { + mo = obj_pub_full(p[i], sz); + if (memobj_verify(mo, k) != 0) { + ret--; + MTANK_LOG(ERR, + "%s(mt=%p, %p[%u]): " + "invalid object header @%#zx={" + "red_zone1=%#" PRIx64 "," + "dbg={nb_alloc=%u,nb_free=%u}," + "red_zone2=%#" PRIx64 + "}\n", + fname, mt, p, i, p[i], + mo->red_zone1, + mo->dbg.nb_alloc, mo->dbg.nb_free, + mo->red_zone2); + } + } + } + + return ret; +} + +/* grab free lock and check objects in free[] */ +static int +mfree_check(struct memtank *mt) +{ + int32_t rc; + + rte_spinlock_lock(&mt->pub.lock); + rc = mobj_bulk_check(__func__, mt, (const uintptr_t *)mt->pub.free, + mt->pub.nb_free, 1); + rte_spinlock_unlock(&mt->pub.lock); + return rc; +} + +static int +mchunk_check(const struct memtank *mt, const struct memchunk *mc, uint32_t tc) +{ + int32_t n, rc; + + rc = 0; + n = mc->nb_total - mc->nb_free; + + rc -= (mc->nb_total != mt->prm.nb_obj_chunk); + rc -= (tc == MC_FULL) ? (n != 0) : (n <= 0); + rc -= (RTE_PTR_ALIGN_CEIL(mc->raw, alignof(*mc)) != mc); + + if (rc != 0) + MTANK_LOG(ERR, "%s(mt=%p, tc=%u): invalid memchunk @%p={" + "raw=%p, nb_total=%u, nb_free=%u}\n", + __func__, mt, tc, mc, + mc->raw, mc->nb_total, mc->nb_free); + + rc += mobj_bulk_check(__func__, mt, (const uintptr_t *)mc->free, + mc->nb_free, 0); + return rc; +} + +static int +mchunk_list_check(struct memtank *mt, uint32_t tc, uint32_t *nb_chunk) +{ + int32_t rc; + uint32_t n; + struct mchunk_list *ls; + const struct memchunk *ch; + + ls = &mt->chl[tc]; + rte_spinlock_lock(&ls->lock); + + rc = 0; + for (n = 0, ch = TAILQ_FIRST(&ls->chunk); ch != NULL; + ch = TAILQ_NEXT(ch, link), n++) + rc += mchunk_check(mt, ch, tc); + + rte_spinlock_unlock(&ls->lock); + + *nb_chunk = n; + return rc; +} + +int +tle_memtank_sanity_check(const struct tle_memtank *t, int32_t ct) +{ + int32_t rc; + uint32_t n, nf, nu; + struct memtank *mt; + + mt = tank_pub_full(t); + rc = mfree_check(mt); + + nf = 0, nu = 0; + rc += mchunk_list_check(mt, MC_FULL, &nf); + rc += mchunk_list_check(mt, MC_USED, &nu); + + /* + * if some other threads concurently do alloc/free/grow/shrink + * these numbers can still not match. + */ + n = rte_atomic32_read(&mt->nb_chunks); + if (nf + nu != n && ct == 0) { + MTANK_LOG(ERR, + "%s(mt=%p) nb_chunks: expected=%u, full=%u, used=%u\n", + __func__, mt, n, nf, nu); + rc--; + } + + return rc; +} diff --git a/lib/libtle_memtank/tle_memtank.h b/lib/libtle_memtank/tle_memtank.h new file mode 100644 index 0000000..9f12fe0 --- /dev/null +++ b/lib/libtle_memtank/tle_memtank.h @@ -0,0 +1,274 @@ +/* + * Copyright (c) 2019 Intel Corporation. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _TLE_MEMTANK_H_ +#define _TLE_MEMTANK_H_ + +#include + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @file + * TLE memtank + * + * Same a s mempool it allows to alloc/free objects of fixed size + * in a lightweight manner (probably not as lightweight as mempool, + * but hopefully close enough). + * But in addition it can grow/shrink dynamically plus provides extra + * additional API for higher flexibility: + * - manual grow()/shrink() functions + * - different alloc/free policies + * (can be specified by user via flags parameter). + * Internally it consists of: + * - LIFO queue (fast allocator/deallocator) + * - lists of memchunks (USED, FREE). + * + * For perfomance reasons memtank tries to allocate memory in + * relatively big chunks (memchunks) and then split each memchunk + * in dozens (or hundreds) of objects. + * There are two thresholds: + * - min_free (grow threshold) + * - max_free (shrink threshold) + */ + +struct tle_memtank; + +/** generic memtank behavior flags */ +enum { + TLE_MTANK_OBJ_DBG = 1, +}; + +struct tle_memtank_prm { + /** min number of free objs in the ring (grow threshold). */ + uint32_t min_free; + uint32_t max_free; /**< max number of free objs (empty threshold) */ + uint32_t max_obj; /**< max number of objs (grow limit) */ + uint32_t obj_size; /**< size of each mem object */ + uint32_t obj_align; /**< alignment of each mem object */ + uint32_t nb_obj_chunk; /**< number of objects per chunk */ + uint32_t flags; /**< behavior flags */ + /** user provided function to alloc chunk of memory */ + void * (*alloc)(size_t, void *); + /** user provided function to free chunk of memory */ + void (*free)(void *, void *); + /** user provided function to initialiaze an object */ + void (*init)(void *[], uint32_t, void *); + void *udata; /**< opaque user data for alloc/free/init */ +}; + +/** + * Allocate and intitialize new memtank instance, based on the + * parameters provided. Note that it uses user-provided *alloc()* function + * to allocate space for the memtank metadata. + * @param prm + * Parameters used to create and initialise new memtank. + * @return + * - Pointer to new memtank insteance created, if operation completed + * successfully. + * - NULL on error with rte_errno set appropriately. + */ +struct tle_memtank * +tle_memtank_create(const struct tle_memtank_prm *prm); + +/** + * Destroy the memtank and free all memory referenced by the memtank. + * The objects must not be used by other cores as they will be freed. + * + * @param t + * A pointer to the memtank instance. + */ +void +tle_memtank_destroy(struct tle_memtank *t); + + +/** alloc flags */ +enum { + TLE_MTANK_ALLOC_CHUNK = 1, + TLE_MTANK_ALLOC_GROW = 2, +}; + +/** + * Allocate up to requested number of objects from the memtank. + * Note that depending on *alloc* behavior (flags) some new memory chunks + * can be allocated from the underlying memory subsystem. + * + * @param t + * A pointer to the memtank instance. + * @param obj + * An array of void * pointers (objects) that will be filled. + * @param num + * Number of objects to allocate from the memtank. + * @param flags + * Flags that control allocation behavior. + * @return + * Number of allocated objects. + */ +static inline uint32_t +tle_memtank_alloc(struct tle_memtank *t, void *obj[], uint32_t num, + uint32_t flags); + +/** + * Allocate up to requested number of objects from the memtank. + * Note that this function bypasses *free* cache(s) and tries to allocate + * objects straight from the memory chunks. + * Note that depending on *alloc* behavior (flags) some new memory chunks + * can be allocated from the underlying memory subsystem. + * + * @param t + * A pointer to the memtank instance. + * @param obj + * An array of void * pointers (objects) that will be filled. + * @param nb_obj + * Number of objects to allocate from the memtank. + * @param flags + * Flags that control allocation behavior. + * @return + * Number of allocated objects. + */ +uint32_t +tle_memtank_chunk_alloc(struct tle_memtank *t, void *obj[], uint32_t nb_obj, + uint32_t flags); + +/** free flags */ +enum { + TLE_MTANK_FREE_SHRINK = 1, +}; + +/** + * Free (put) provided objects back to the memtank. + * Note that depending on *free* behavior (flags) some memory chunks can be + * returned (freed) to the underlying memory subsystem. + * + * @param t + * A pointer to the memtank instance. + * @param obj + * An array of object pointers to be freed. + * @param num + * Number of objects to free. + * @param flags + * Flags that control free behavior. + */ +static inline void +tle_memtank_free(struct tle_memtank *t, void * const obj[], uint32_t num, + uint32_t flags); + +/** + * Free (put) provided objects back to the memtank. + * Note that this function bypasses *free* cache(s) and tries to put + * objects straight to the memory chunks. + * Note that depending on *free* behavior (flags) some memory chunks can be + * returned (freed) to the underlying memory subsystem. + * + * @param t + * A pointer to the memtank instance. + * @param obj + * An array of object pointers to be freed. + * @param nb_obj + * Number of objects to allocate from the memtank. + * @param flags + * Flags that control allocation behavior. + */ +void +tle_memtank_chunk_free(struct tle_memtank *t, void * const obj[], + uint32_t nb_obj, uint32_t flags); + +/** + * Check does number of objects in *free* cache is below memtank grow + * threshold (min_free). If yes, then tries to allocate memory for new + * objects from the underlying memory subsystem. + * + * @param t + * A pointer to the memtank instance. + * @return + * Number of newly allocated memory chunks. + */ +int +tle_memtank_grow(struct tle_memtank *t); + +/** + * Check does number of objects in *free* cache have reached memtank shrink + * threshold (max_free). If yes, then tries to return excessive memory to + * the the underlying memory subsystem. + * + * @param t + * A pointer to the memtank instance. + * @return + * Number of freed memory chunks. + */ +int +tle_memtank_shrink(struct tle_memtank *t); + +/* dump flags */ +enum { + TLE_MTANK_DUMP_FREE_STAT = 1, + TLE_MTANK_DUMP_CHUNK_STAT = 2, + TLE_MTANK_DUMP_CHUNK = 4, + /* first not used power of two */ + TLE_MTANK_DUMP_END, + + /* dump all stats */ + TLE_MTANK_DUMP_STAT = + (TLE_MTANK_DUMP_FREE_STAT | TLE_MTANK_DUMP_CHUNK_STAT), + /* dump everything */ + TLE_MTANK_DUMP_ALL = TLE_MTANK_DUMP_END - 1, +}; + +/** + * Dump information about the memtank to the file. + * Note that depending of *flags* value it might cause some internal locks + * grabbing, and might affect perfomance of others threads that + * concurently use same memtank. + * + * @param f + * A pinter to the file. + * @param t + * A pointer to the memtank instance. + * @param flags + * Flags that control dump behavior. + */ +void +tle_memtank_dump(FILE *f, const struct tle_memtank *t, uint32_t flags); + +/** + * Check the consistency of the given memtank instance. + * Dumps error messages to the RTE log subsystem, if some inconsitency + * is detected. + * + * @param t + * A pointer to the memtank instance. + * @param ct + * Value greater then zero, if some other threads do concurently use + * that memtank. + * @return + * Zero on success, or negaive value otherwise. + */ +int +tle_memtank_sanity_check(const struct tle_memtank *t, int32_t ct); + +#ifdef __cplusplus +} +#endif + +#include + +#endif /* _TLE_MEMTANK_H_ */ diff --git a/lib/libtle_memtank/tle_memtank_pub.h b/lib/libtle_memtank/tle_memtank_pub.h new file mode 100644 index 0000000..78e89b2 --- /dev/null +++ b/lib/libtle_memtank/tle_memtank_pub.h @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2019 Intel Corporation. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _TLE_MEMTANK_PUB_H_ +#define _TLE_MEMTANK_PUB_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @file + * TLE memtank public + * It is not recommended to include this file directly, + * include instead. + */ + +struct tle_memtank { + rte_spinlock_t lock; + uint32_t min_free; + uint32_t max_free; + uint32_t nb_free; + void *free[]; +} __rte_cache_aligned; + + +static inline void +_copy_objs(void *dst[], void * const src[], uint32_t num) +{ + uint32_t i, n; + + n = RTE_ALIGN_FLOOR(num, 4); + + for (i = 0; i != n; i += 4) { + dst[i] = src[i]; + dst[i + 1] = src[i + 1]; + dst[i + 2] = src[i + 2]; + dst[i + 3] = src[i + 3]; + } + + switch (num % 4) { + case 3: + dst[i + 2] = src[i + 2]; + /* fallthrough */ + case 2: + dst[i + 1] = src[i + 1]; + /* fallthrough */ + case 1: + dst[i] = src[i]; + /* fallthrough */ + } +} + +static inline uint32_t +_get_free(struct tle_memtank *t, void *obj[], uint32_t num) +{ + uint32_t len, n; + + rte_spinlock_lock(&t->lock); + + len = t->nb_free; + n = RTE_MIN(num, len); + len -= n; + _copy_objs(obj, t->free + len, n); + t->nb_free = len; + + rte_spinlock_unlock(&t->lock); + return n; +} + +static inline uint32_t +_put_free(struct tle_memtank *t, void * const obj[], uint32_t num) +{ + uint32_t len, n; + + rte_spinlock_lock(&t->lock); + + len = t->nb_free; + n = t->max_free - len; + n = RTE_MIN(num, n); + _copy_objs(t->free + len, obj, n); + t->nb_free = len + n; + + rte_spinlock_unlock(&t->lock); + return n; +} + +static inline void +_fill_free(struct tle_memtank *t, uint32_t num, uint32_t flags) +{ + uint32_t k, n; + void *free[num]; + + k = tle_memtank_chunk_alloc(t, free, RTE_DIM(free), flags); + n = _put_free(t, free, k); + if (n != k) + tle_memtank_chunk_free(t, free + n, k - n, 0); +} + +static inline uint32_t +tle_memtank_alloc(struct tle_memtank *t, void *obj[], uint32_t num, + uint32_t flags) +{ + uint32_t n; + + n = _get_free(t, obj, num); + + /* not enough free objects, try to allocate via memchunks */ + if (n != num && flags != 0) { + n += tle_memtank_chunk_alloc(t, obj + n, num - n, flags); + + /* refill *free* tank */ + if (n == num) + _fill_free(t, t->min_free, flags); + } + + return n; +} + +static inline void +tle_memtank_free(struct tle_memtank *t, void * const obj[], uint32_t num, + uint32_t flags) +{ + uint32_t n; + + n = _put_free(t, obj, num); + if (n != num) + tle_memtank_chunk_free(t, obj + n, num - n, flags); +} + +#ifdef __cplusplus +} +#endif + +#endif /* _TLE_MEMTANK_PUB_H_ */ diff --git a/test/Makefile b/test/Makefile index c5cf270..c82b123 100644 --- a/test/Makefile +++ b/test/Makefile @@ -23,6 +23,7 @@ include $(RTE_SDK)/mk/rte.vars.mk DIRS-y += dring DIRS-y += gtest +DIRS-y += memtank DIRS-y += timer include $(TLDK_ROOT)/mk/tle.subdir.mk diff --git a/test/memtank/Makefile b/test/memtank/Makefile new file mode 100644 index 0000000..b8e4483 --- /dev/null +++ b/test/memtank/Makefile @@ -0,0 +1,42 @@ +# Copyright (c) 2019 Intel Corporation. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ifeq ($(RTE_SDK),) +$(error "Please define RTE_SDK environment variable") +endif + +ifeq ($(RTE_TARGET),) +$(error "Please define RTE_TARGET environment variable") +endif + +ifeq ($(TLDK_ROOT),) +$(error "Please define TLDK_ROOT environment variable") +endif + +include $(RTE_SDK)/mk/rte.vars.mk + +# binary name +APP = test_memtank + +# all source are stored in SRCS-y +SRCS-y += test_memtank.c + +CFLAGS += $(WERROR_FLAGS) +CFLAGS += -I$(RTE_OUTPUT)/include + +LDLIBS += -L$(RTE_OUTPUT)/lib +LDLIBS += -ltle_memtank + +EXTRA_CFLAGS += -O3 + +include $(TLDK_ROOT)/mk/tle.app.mk diff --git a/test/memtank/test_memtank.c b/test/memtank/test_memtank.c new file mode 100644 index 0000000..51e86be --- /dev/null +++ b/test/memtank/test_memtank.c @@ -0,0 +1,793 @@ +/* + * Copyright (c) 2016 Intel Corporation. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +struct memstat { + struct { + rte_atomic64_t nb_call; + rte_atomic64_t nb_fail; + rte_atomic64_t sz; + } alloc; + struct { + rte_atomic64_t nb_call; + rte_atomic64_t nb_fail; + } free; + uint64_t nb_alloc_obj; +}; + +struct memtank_stat { + uint64_t nb_cycle; + struct { + uint64_t nb_call; + uint64_t nb_req; + uint64_t nb_alloc; + uint64_t nb_cycle; + } alloc; + struct { + uint64_t nb_call; + uint64_t nb_free; + uint64_t nb_cycle; + } free; + struct { + uint64_t nb_call; + uint64_t nb_chunk; + uint64_t nb_cycle; + } grow; + struct { + uint64_t nb_call; + uint64_t nb_chunk; + uint64_t nb_cycle; + } shrink; +}; + +struct master_args { + uint64_t run_cycles; + uint32_t delay_us; + uint32_t flags; +}; + +struct worker_args { + uint32_t max_obj; + uint32_t obj_size; + uint32_t alloc_flags; + uint32_t free_flags; +}; + +struct memtank_arg { + struct tle_memtank *mt; + union { + struct master_args master; + struct worker_args worker; + }; + struct memtank_stat stats; +}; + +#define BULK_NUM 32 +#define MAX_OBJ 0x100000 + +#define OBJ_SZ_MIN 1 +#define OBJ_SZ_MAX 0x100000 +#define OBJ_SZ_DEF (4 * RTE_CACHE_LINE_SIZE + 1) + +#define TEST_TIME 10 + +#define FREE_THRSH_MIN 0 +#define FREE_THRSH_MAX 100 + +enum { + WRK_CMD_STOP, + WRK_CMD_RUN, +}; + +enum { + MASTER_FLAG_GROW = 1, + MASTER_FLAG_SHRINK = 2, +}; + +enum { + MEM_FUNC_SYS, + MEM_FUNC_RTE, +}; + +static uint32_t wrk_cmd __rte_cache_aligned; + +static struct tle_memtank_prm mtnk_prm = { + .min_free = 4 * BULK_NUM, + .max_free = 32 * BULK_NUM, + .max_obj = MAX_OBJ, + .obj_size = OBJ_SZ_DEF, + .obj_align = RTE_CACHE_LINE_SIZE, + .nb_obj_chunk = BULK_NUM, + .flags = TLE_MTANK_OBJ_DBG, +}; + +static struct { + uint32_t run_time; /* test run-time in seconds */ + uint32_t wrk_max_obj; /* max alloced objects per worker */ + uint32_t wrk_free_thrsh; /* wrk free thresh % (0-100) */ + int32_t mem_func; /* memory subsystem to use for alloc/free */ +} global_cfg = { + .run_time = TEST_TIME, + .wrk_max_obj = 2 * BULK_NUM, + .wrk_free_thrsh = FREE_THRSH_MIN, + .mem_func = MEM_FUNC_SYS, +}; + +static void * +alloc_func(size_t sz) +{ + switch (global_cfg.mem_func) { + case MEM_FUNC_SYS: + return malloc(sz); + case MEM_FUNC_RTE: + return rte_malloc(NULL, sz, 0); + } + + return NULL; +} + +static void +free_func(void *p) +{ + switch (global_cfg.mem_func) { + case MEM_FUNC_SYS: + return free(p); + case MEM_FUNC_RTE: + return rte_free(p); + } +} + +static void * +test_alloc1(size_t sz, void *p) +{ + struct memstat *ms; + void *buf; + + ms = p; + buf = alloc_func(sz); + rte_atomic64_inc(&ms->alloc.nb_call); + if (buf != NULL) { + memset(buf, 0, sz); + rte_atomic64_add(&ms->alloc.sz, sz); + } else + rte_atomic64_inc(&ms->alloc.nb_fail); + + return buf; +} + +static void +test_free1(void *buf, void *p) +{ + struct memstat *ms; + + ms = p; + + free_func(buf); + rte_atomic64_inc(&ms->free.nb_call); + if (buf == NULL) + rte_atomic64_inc(&ms->free.nb_fail); +} + +static void +memstat_dump(FILE *f, struct memstat *ms) +{ + + uint64_t alloc_sz, nb_alloc; + long double muc, mut; + + nb_alloc = rte_atomic64_read(&ms->alloc.nb_call) - + rte_atomic64_read(&ms->alloc.nb_fail); + alloc_sz = rte_atomic64_read(&ms->alloc.sz) / nb_alloc; + nb_alloc -= rte_atomic64_read(&ms->free.nb_call) - + rte_atomic64_read(&ms->free.nb_fail); + alloc_sz *= nb_alloc; + mut = (alloc_sz == 0) ? 1 : + (long double)ms->nb_alloc_obj * mtnk_prm.obj_size / alloc_sz; + muc = (alloc_sz == 0) ? 1 : + (long double)(ms->nb_alloc_obj + mtnk_prm.max_free) * + mtnk_prm.obj_size / alloc_sz; + + fprintf(f, "%s(%p)={\n", __func__, ms); + fprintf(f, "\talloc={\n"); + fprintf(f, "\t\tnb_call=%" PRIu64 ",\n", + rte_atomic64_read(&ms->alloc.nb_call)); + fprintf(f, "\t\tnb_fail=%" PRIu64 ",\n", + rte_atomic64_read(&ms->alloc.nb_fail)); + fprintf(f, "\t\tsz=%" PRIu64 ",\n", + rte_atomic64_read(&ms->alloc.sz)); + fprintf(f, "\t},\n"); + fprintf(f, "\tfree={\n"); + fprintf(f, "\t\tnb_call=%" PRIu64 ",\n", + rte_atomic64_read(&ms->free.nb_call)); + fprintf(f, "\t\tnb_fail=%" PRIu64 ",\n", + rte_atomic64_read(&ms->free.nb_fail)); + fprintf(f, "\t},\n"); + fprintf(f, "\tnb_alloc_obj=%" PRIu64 ",\n", ms->nb_alloc_obj); + fprintf(f, "\tnb_alloc_chunk=%" PRIu64 ",\n", nb_alloc); + fprintf(f, "\talloc_sz=%" PRIu64 ",\n", alloc_sz); + fprintf(f, "\tmem_util(total)=%.2Lf %%,\n", mut * 100); + fprintf(f, "\tmem_util(cached)=%.2Lf %%,\n", muc * 100); + fprintf(f, "};\n"); + +} + +static void +memtank_stat_dump(FILE *f, uint32_t lc, const struct memtank_stat *ms) +{ + uint64_t t; + + fprintf(f, "%s(lc=%u)={\n", __func__, lc); + fprintf(f, "\tnb_cycle=%" PRIu64 ",\n", ms->nb_cycle); + if (ms->alloc.nb_call != 0) { + fprintf(f, "\talloc={\n"); + fprintf(f, "\t\tnb_call=%" PRIu64 ",\n", ms->alloc.nb_call); + fprintf(f, "\t\tnb_req=%" PRIu64 ",\n", ms->alloc.nb_req); + fprintf(f, "\t\tnb_alloc=%" PRIu64 ",\n", ms->alloc.nb_alloc); + fprintf(f, "\t\tnb_cycle=%" PRIu64 ",\n", ms->alloc.nb_cycle); + + t = ms->alloc.nb_req - ms->alloc.nb_alloc; + fprintf(f, "\t\tfailed req: %"PRIu64 "(%.2Lf %%)\n", + t, (long double)t * 100 / ms->alloc.nb_req); + fprintf(f, "\t\tcycles/alloc: %.2Lf\n", + (long double)ms->alloc.nb_cycle / ms->alloc.nb_alloc); + fprintf(f, "\t\tobj/call(avg): %.2Lf\n", + (long double)ms->alloc.nb_alloc / ms->alloc.nb_call); + + fprintf(f, "\t},\n"); + } + if (ms->free.nb_call != 0) { + fprintf(f, "\tfree={\n"); + fprintf(f, "\t\tnb_call=%" PRIu64 ",\n", ms->free.nb_call); + fprintf(f, "\t\tnb_free=%" PRIu64 ",\n", ms->free.nb_free); + fprintf(f, "\t\tnb_cycle=%" PRIu64 ",\n", ms->free.nb_cycle); + + fprintf(f, "\t\tcycles/free: %.2Lf\n", + (long double)ms->free.nb_cycle / ms->free.nb_free); + fprintf(f, "\t\tobj/call(avg): %.2Lf\n", + (long double)ms->free.nb_free / ms->free.nb_call); + + fprintf(f, "\t},\n"); + } + if (ms->grow.nb_call != 0) { + fprintf(f, "\tgrow={\n"); + fprintf(f, "\t\tnb_call=%" PRIu64 ",\n", ms->grow.nb_call); + fprintf(f, "\t\tnb_chunk=%" PRIu64 ",\n", ms->grow.nb_chunk); + fprintf(f, "\t\tnb_cycle=%" PRIu64 ",\n", ms->grow.nb_cycle); + + fprintf(f, "\t\tcycles/chunk: %.2Lf\n", + (long double)ms->grow.nb_cycle / ms->grow.nb_chunk); + fprintf(f, "\t\tobj/call(avg): %.2Lf\n", + (long double)ms->grow.nb_chunk / ms->grow.nb_call); + + fprintf(f, "\t},\n"); + } + if (ms->shrink.nb_call != 0) { + fprintf(f, "\tshrink={\n"); + fprintf(f, "\t\tnb_call=%" PRIu64 ",\n", ms->shrink.nb_call); + fprintf(f, "\t\tnb_chunk=%" PRIu64 ",\n", ms->shrink.nb_chunk); + fprintf(f, "\t\tnb_cycle=%" PRIu64 ",\n", ms->shrink.nb_cycle); + + fprintf(f, "\t\tcycles/chunk: %.2Lf\n", + (long double)ms->shrink.nb_cycle / ms->shrink.nb_chunk); + fprintf(f, "\t\tobj/call(avg): %.2Lf\n", + (long double)ms->shrink.nb_chunk / ms->shrink.nb_call); + + fprintf(f, "\t},\n"); + } + fprintf(f, "};\n"); +} + +static int32_t +check_fill_objs(void *obj[], uint32_t sz, uint32_t num, + uint8_t check, uint8_t fill) +{ + uint32_t i; + uint8_t buf[sz]; + + static rte_spinlock_t dump_lock; + + memset(buf, check, sz); + + for (i = 0; i != num; i++) { + if (memcmp(buf, obj[i], sz) != 0) { + rte_spinlock_lock(&dump_lock); + printf ("%s(%u, %u, %hu, %hu) failed at %u-th iter, " + "offendig object: %p\n", + __func__, sz, num, check, fill, i, obj[i]); + rte_memdump(stdout, "expected", buf, sz); + rte_memdump(stdout, "result", obj[i], sz); + rte_spinlock_unlock(&dump_lock); + return -EINVAL; + } + memset(obj[i], fill, sz); + } + return 0; +} + +static int +test_memtank_worker(void *arg) +{ + int32_t rc; + size_t sz; + uint32_t ft, lc, n, num; + uint64_t cl, tm0, tm1; + struct memtank_arg *ma; + struct rte_ring *ring; + void *obj[BULK_NUM]; + + ma = arg; + lc = rte_lcore_id(); + + sz = rte_ring_get_memsize(ma->worker.max_obj); + ring = alloca(sz); + if (ring == NULL) { + printf("%s(%u): alloca(%zu) for FIFO with %u elems failed", + __func__, lc, sz, ma->worker.max_obj); + return -ENOMEM; + } + rc = rte_ring_init(ring, "", ma->worker.max_obj, + RING_F_SP_ENQ | RING_F_SC_DEQ); + if (rc != 0) { + printf("%s(%u): rte_ring_init(%p, %u) failed, error: %d(%s)\n", + __func__, lc, ring, ma->worker.max_obj, + rc, strerror(-rc)); + return rc; + } + + /* calculate free threshold */ + ft = ma->worker.max_obj * global_cfg.wrk_free_thrsh / FREE_THRSH_MAX; + + while (wrk_cmd != WRK_CMD_RUN) { + rte_smp_rmb(); + rte_pause(); + } + + cl = rte_rdtsc_precise(); + + do { + num = rte_rand() % RTE_DIM(obj); + n = rte_ring_free_count(ring); + num = RTE_MIN(num, n); + + /* perform alloc*/ + if (num != 0) { + tm0 = rte_rdtsc_precise(); + n = tle_memtank_alloc(ma->mt, obj, num, + ma->worker.alloc_flags); + tm1 = rte_rdtsc_precise(); + + /* check and fill contents of allocated objects */ + rc = check_fill_objs(obj, ma->worker.obj_size, n, + 0, lc); + if (rc != 0) + break; + + /* collect alloc stat */ + ma->stats.alloc.nb_call++; + ma->stats.alloc.nb_req += num; + ma->stats.alloc.nb_alloc += n; + ma->stats.alloc.nb_cycle += tm1 - tm0; + + /* store allocated objects */ + rte_ring_enqueue_bulk(ring, obj, n, NULL); + } + + /* get some objects to free */ + num = rte_rand() % RTE_DIM(obj); + n = rte_ring_count(ring); + num = (n >= ft) ? RTE_MIN(num, n) : 0; + + /* perform free*/ + if (num != 0) { + + /* retrieve objects to free */ + rte_ring_dequeue_bulk(ring, obj, num, NULL); + + /* check and fill contents of freeing objects */ + rc = check_fill_objs(obj, ma->worker.obj_size, num, + lc, 0); + if (rc != 0) + break; + + tm0 = rte_rdtsc_precise(); + tle_memtank_free(ma->mt, obj, num, + ma->worker.free_flags); + tm1 = rte_rdtsc_precise(); + + /* collect free stat */ + ma->stats.free.nb_call++; + ma->stats.free.nb_free += num; + ma->stats.free.nb_cycle += tm1 - tm0; + } + + rte_smp_mb(); + } while (wrk_cmd == WRK_CMD_RUN); + + ma->stats.nb_cycle = rte_rdtsc_precise() - cl; + + return rc; +} + +static int +test_memtank_master(void *arg) +{ + struct memtank_arg *ma; + uint64_t cl, tm0, tm1, tm2; + uint32_t i, n; + + ma = (struct memtank_arg *)arg; + + for (cl = 0, i = 0; cl < ma->master.run_cycles; + cl += tm2 - tm0, i++) { + + tm0 = rte_rdtsc_precise(); + + if (ma->master.flags & MASTER_FLAG_SHRINK) { + + n = tle_memtank_shrink(ma->mt); + tm1 = rte_rdtsc_precise(); + ma->stats.shrink.nb_call++; + ma->stats.shrink.nb_chunk += n; + if (n != 0) + ma->stats.shrink.nb_cycle += tm1 - tm0; + } + + if (ma->master.flags & MASTER_FLAG_GROW) { + + tm1 = rte_rdtsc_precise(); + n = tle_memtank_grow(ma->mt); + tm2 = rte_rdtsc_precise(); + ma->stats.grow.nb_call++; + ma->stats.grow.nb_chunk += n; + if (n != 0) + ma->stats.grow.nb_cycle += tm2 - tm1; + } + + wrk_cmd = WRK_CMD_RUN; + rte_smp_mb(); + + rte_delay_us(ma->master.delay_us); + tm2 = rte_rdtsc_precise(); + } + + ma->stats.nb_cycle = cl; + + rte_smp_mb(); + wrk_cmd = WRK_CMD_STOP; + + return 0; +} + +static void +fill_worker_args(struct worker_args *wa, uint32_t alloc_flags, + uint32_t free_flags) +{ + wa->max_obj = global_cfg.wrk_max_obj; + wa->obj_size = mtnk_prm.obj_size; + wa->alloc_flags = alloc_flags; + wa->free_flags = free_flags; +} + +static void +fill_master_args(struct master_args *ma, uint32_t flags) +{ + uint64_t tm; + + tm = global_cfg.run_time * rte_get_timer_hz(); + + ma->run_cycles = tm; + ma->delay_us = US_PER_S / MS_PER_S; + ma->flags = flags; +} + +/* + * alloc/free by workers threads. + * grow/shrink by master + */ +static int +test_memtank_mt1(void) +{ + int32_t rc; + uint32_t lc; + struct tle_memtank *mt; + struct tle_memtank_prm prm; + struct memstat ms; + struct memtank_arg arg[RTE_MAX_LCORE]; + + printf("%s start\n", __func__); + + memset(&prm, 0, sizeof(prm)); + memset(&ms, 0, sizeof(ms)); + + prm = mtnk_prm; + prm.alloc = test_alloc1; + prm.free = test_free1; + prm.udata = &ms; + + mt = tle_memtank_create(&prm); + if (mt == NULL) { + printf("%s: memtank_create() failed\n", __func__); + return -ENOMEM; + } + + memset(arg, 0, sizeof(arg)); + + /* launch on all slaves */ + RTE_LCORE_FOREACH_SLAVE(lc) { + arg[lc].mt = mt; + fill_worker_args(&arg[lc].worker, 0, 0); + rte_eal_remote_launch(test_memtank_worker, &arg[lc], lc); + } + + /* launch on master */ + lc = rte_lcore_id(); + arg[lc].mt = mt; + fill_master_args(&arg[lc].master, + MASTER_FLAG_GROW | MASTER_FLAG_SHRINK); + test_memtank_master(&arg[lc]); + + /* wait for slaves and collect stats. */ + rc = 0; + RTE_LCORE_FOREACH_SLAVE(lc) { + rc |= rte_eal_wait_lcore(lc); + memtank_stat_dump(stdout, lc, &arg[lc].stats); + ms.nb_alloc_obj += arg[lc].stats.alloc.nb_alloc - + arg[lc].stats.free.nb_free; + } + + lc = rte_lcore_id(); + memtank_stat_dump(stdout, lc, &arg[lc].stats); + tle_memtank_dump(stdout, mt, TLE_MTANK_DUMP_STAT); + + memstat_dump(stdout, &ms); + + rc |= tle_memtank_sanity_check(mt, 0); + tle_memtank_destroy(mt); + return rc; +} + +/* + * alloc/free with grow/shrink by worker threads. + * master does nothing + */ +static int +test_memtank_mt2(void) +{ + int32_t rc; + uint32_t lc; + struct tle_memtank *mt; + struct tle_memtank_prm prm; + struct memstat ms; + struct memtank_arg arg[RTE_MAX_LCORE]; + + const uint32_t alloc_flags = TLE_MTANK_ALLOC_CHUNK | + TLE_MTANK_ALLOC_GROW; + const uint32_t free_flags = TLE_MTANK_FREE_SHRINK; + + printf("%s start\n", __func__); + + memset(&prm, 0, sizeof(prm)); + memset(&ms, 0, sizeof(ms)); + + prm = mtnk_prm; + prm.alloc = test_alloc1; + prm.free = test_free1; + prm.udata = &ms; + + mt = tle_memtank_create(&prm); + if (mt == NULL) { + printf("%s: memtank_create() failed\n", __func__); + return -ENOMEM; + } + + memset(arg, 0, sizeof(arg)); + + /* launch on all slaves */ + RTE_LCORE_FOREACH_SLAVE(lc) { + arg[lc].mt = mt; + fill_worker_args(&arg[lc].worker, alloc_flags, free_flags); + rte_eal_remote_launch(test_memtank_worker, &arg[lc], lc); + } + + /* launch on master */ + lc = rte_lcore_id(); + arg[lc].mt = mt; + fill_master_args(&arg[lc].master, 0); + test_memtank_master(&arg[lc]); + + /* wait for slaves and collect stats. */ + rc = 0; + RTE_LCORE_FOREACH_SLAVE(lc) { + rc |= rte_eal_wait_lcore(lc); + memtank_stat_dump(stdout, lc, &arg[lc].stats); + ms.nb_alloc_obj += arg[lc].stats.alloc.nb_alloc - + arg[lc].stats.free.nb_free; + } + + lc = rte_lcore_id(); + memtank_stat_dump(stdout, lc, &arg[lc].stats); + tle_memtank_dump(stdout, mt, TLE_MTANK_DUMP_STAT); + + memstat_dump(stdout, &ms); + + rc |= tle_memtank_sanity_check(mt, 0); + tle_memtank_destroy(mt); + return rc; +} + +static int +parse_uint_val(const char *str, uint32_t *val, uint32_t min, uint32_t max) +{ + unsigned long v; + char *end; + + errno = 0; + v = strtoul(str, &end, 0); + if (errno != 0 || end[0] != 0 || v < min || v > max) + return -EINVAL; + + val[0] = v; + return 0; +} + +static int +parse_mem_str(const char *str) +{ + uint32_t i; + + static const struct { + const char *name; + int32_t val; + } name2val[] = { + { + .name = "sys", + .val = MEM_FUNC_SYS, + }, + { + .name = "rte", + .val = MEM_FUNC_RTE, + }, + }; + + for (i = 0; i != RTE_DIM(name2val); i++) { + if (strcmp(str, name2val[i].name) == 0) + return name2val[i].val; + } + return -EINVAL; +} + +static int +parse_opt(int argc, char * const argv[]) +{ + int32_t opt, rc; + uint32_t v; + + rc = 0; + optind = 0; + optarg = NULL; + + while ((opt = getopt(argc, argv, "f:m:s:t:w:")) != EOF) { + switch (opt) { + case 'f': + rc = parse_uint_val(optarg, &v, FREE_THRSH_MIN, + FREE_THRSH_MAX); + if (rc == 0) + global_cfg.wrk_free_thrsh = v; + break; + case 'm': + rc = parse_mem_str(optarg); + if (rc >= 0) + global_cfg.mem_func = rc; + break; + case 's': + rc = parse_uint_val(optarg, &v, OBJ_SZ_MIN, + OBJ_SZ_MAX); + if (rc == 0) + mtnk_prm.obj_size = v; + break; + case 't': + rc = parse_uint_val(optarg, &v, 0, UINT32_MAX); + if (rc == 0) + global_cfg.run_time = v; + break; + case 'w': + rc = parse_uint_val(optarg, &v, 0, UINT32_MAX); + if (rc == 0) + global_cfg.wrk_max_obj = v; + break; + default: + rc = -EINVAL; + } + } + + if (rc < 0) + printf("%s: invalid value: \"%s\" for option: \'%c\'\n", + __func__, optarg, opt); + return rc; +} + +int +main(int argc, char * argv[]) +{ + int32_t rc; + uint32_t i, k; + + const struct { + const char *name; + int (*func)(void); + } tests[] = { + { + .name = "MT1-WRK_ALLOC_FREE-MST_GROW_SHRINK", + .func = test_memtank_mt1, + }, + { + .name = "MT1-WRK_ALLOC+GROW_FREE+SHRINK", + .func = test_memtank_mt2, + }, + }; + + + rc = rte_eal_init(argc, argv); + if (rc < 0) + rte_exit(EXIT_FAILURE, + "%s: rte_eal_init failed with error code: %d\n", + __func__, rc); + + rc = parse_opt(argc - rc, argv + rc); + if (rc < 0) + rte_exit(EXIT_FAILURE, + "%s: parse_op failed with error code: %d\n", + __func__, rc); + + for (i = 0, k = 0; i != RTE_DIM(tests); i++) { + + printf("TEST %s START\n", tests[i].name); + + rc = tests[i].func(); + k += (rc == 0); + + if (rc != 0) + printf("TEST %s FAILED\n", tests[i].name); + else + printf("TEST %s OK\n", tests[i].name); + } + + printf("Number of tests:\t%u\nSuccess:\t%u\nFailed:\t%u\n", + i, k, i - k); + return (k != i); +} -- cgit 1.2.3-korg