v6: memtank introduction

For analogy with mempool, named this structure memtank.
Same a s mempool it allows to alloc/free objects of fixed size
in a lightweight manner (not as lightweight as mempool,
but hopefully close enough).
The whole idea is that alloc/free is used at fast-path and don't
allocate/free more than *min_free* objects at one call.
So for majority of cases our fast-path alloc/free should be lightweight
(LIFO enqueue/dequeue operations).
Also user will need to call grow/shrink periodically
(ideally from the slow-path) to make sure there is enough
free objects in the tank.
Internally it is just a simple LIFO for up to *max_free* objects plus
a list of memory buffers (memchunk) from where these objects were
allocated.

v1 -> v2
 - Added UT
 - Fixed few bugs

v2 -> v3
 - extend UT with more parameters

v3 -> v4
 - add object alignement as parameter for memtank_create
 - extend UT with more parameters
 - added memtank dump routine

v4 -> v5
 - fixed few bugs inside memtank lib
 - extend UT with:
   - new test case
   - new command-line options: '-s <obj_size>', '-m <mem_func>'

v5 -> v6
 - extend memtank dump to collect/display extra information
 - make memtank dump routine MT safe
 - add memtank sanity check function
 - add proper comments for pubic API

Signed-off-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
Change-Id: I8939772577f5d9e293088eaa9a9fe316c3fe8f87

10 files changed, 2295 insertions, 0 deletions

diff --git a/lib/Makefile b/lib/Makefile
index 6317af9..8d61a08 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -24,6 +24,7 @@ include $(RTE_SDK)/mk/rte.vars.mk
 DIRS-y += libtle_misc
 DIRS-y += libtle_dring
 DIRS-y += libtle_timer
+DIRS-y += libtle_memtank
 DIRS-y += libtle_l4p
 
 include $(TLDK_ROOT)/mk/tle.subdir.mk
diff --git a/lib/libtle_memtank/Makefile b/lib/libtle_memtank/Makefile
new file mode 100644
index 0000000..d87e320
--- /dev/null
+++ b/lib/libtle_memtank/Makefile
@@ -0,0 +1,40 @@
+# Copyright (c) 2016 Intel Corporation.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ifeq ($(RTE_SDK),)
+$(error "Please define RTE_SDK environment variable")
+endif
+
+# Default target, can be overwritten by command line or environment
+RTE_TARGET ?= x86_64-native-linuxapp-gcc
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# library name
+LIB = libtle_memtank.a
+
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR)
+
+EXPORT_MAP := tle_memtank_version.map
+
+LIBABIVER := 1
+
+#source files
+SRCS-y += memtank.c
+SRCS-y += misc.c
+
+SYMLINK-y-include += tle_memtank_pub.h
+SYMLINK-y-include += tle_memtank.h
+
+include $(TLDK_ROOT)/mk/tle.lib.mk
diff --git a/lib/libtle_memtank/memtank.c b/lib/libtle_memtank/memtank.c
new file mode 100644
index 0000000..ceb209c
--- /dev/null
+++ b/lib/libtle_memtank/memtank.c
@@ -0,0 +1,507 @@
+/*
+ * Copyright (c) 2019  Intel Corporation.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "memtank.h"
+#include <rte_errno.h>
+
+#define	ALIGN_MUL_CEIL(v, mul)	\
+	((typeof(v))(((uint64_t)(v) + (mul) - 1) / (mul)))
+
+
+static inline size_t
+memtank_meta_size(uint32_t nb_free)
+{
+	size_t sz;
+	static const struct memtank *mt;
+
+	sz = sizeof(*mt) + nb_free * sizeof(mt->pub.free[0]);
+	sz = RTE_ALIGN_CEIL(sz, alignof(*mt));
+	return sz;
+}
+
+static inline size_t
+memchunk_meta_size(uint32_t nb_obj)
+{
+	size_t sz;
+	static const struct memchunk *ch;
+
+	sz = sizeof(*ch) +  nb_obj * sizeof(ch->free[0]);
+	sz = RTE_ALIGN_CEIL(sz, alignof(*ch));
+	return sz;
+}
+
+static inline size_t
+memobj_size(uint32_t obj_size, uint32_t obj_align)
+{
+	size_t sz;
+	static const struct memobj *obj;
+
+	sz = sizeof(*obj) + obj_size;
+	sz = RTE_ALIGN_CEIL(sz, obj_align);
+	return sz;
+}
+
+static inline size_t
+memchunk_size(uint32_t nb_obj, uint32_t obj_size, uint32_t obj_align)
+{
+	size_t algn, sz;
+	static const struct memchunk *ch;
+
+	algn = RTE_MAX(alignof(*ch), obj_align);
+	sz = memchunk_meta_size(nb_obj);
+	sz += nb_obj * memobj_size(obj_size, obj_align);
+	sz = RTE_ALIGN_CEIL(sz + algn - 1, algn);
+	return sz;
+}
+
+static void
+init_chunk(struct memtank *mt, struct memchunk *ch)
+{
+	uint32_t i, n, sz;
+	uintptr_t p;
+	struct memobj *obj;
+
+	const struct memobj cobj = {
+		.red_zone1 = RED_ZONE_V1,
+		.chunk = ch,
+		.red_zone2 = RED_ZONE_V2,
+	};
+
+	n = mt->prm.nb_obj_chunk;
+	sz = mt->obj_size;
+
+	/* get start of memobj array */
+	p = (uintptr_t)ch + memchunk_meta_size(n);
+	p = RTE_ALIGN_CEIL(p, mt->prm.obj_align);
+
+	for (i = 0; i != n; i++) {
+		obj = obj_pub_full(p, sz);
+		obj[0] = cobj;
+		ch->free[i] = (void *)p;
+		p += sz;
+	}
+
+	ch->nb_total = n;
+	ch->nb_free = n;
+
+	if (mt->prm.init != NULL)
+		mt->prm.init(ch->free, n, mt->prm.udata);
+}
+
+static void
+put_chunk(struct memtank *mt, struct memchunk *ch, void * const obj[],
+	uint32_t num)
+{
+	uint32_t k, n;
+	struct mchunk_list *ls;
+
+	/* chunk should be in the *used* list */
+	k = MC_USED;
+	ls = &mt->chl[k];
+	rte_spinlock_lock(&ls->lock);
+
+	n = ch->nb_free;
+	RTE_ASSERT(n + num <= ch->nb_total);
+
+	_copy_objs(ch->free + n, obj, num);
+	ch->nb_free = n + num;
+
+	/* chunk is full now */
+	if (ch->nb_free == ch->nb_total) {
+		TAILQ_REMOVE(&ls->chunk, ch, link);
+		k = MC_FULL;
+	/* chunk is not empty anymore, move it to the head */
+	} else if (n == 0) {
+		TAILQ_REMOVE(&ls->chunk, ch, link);
+		TAILQ_INSERT_HEAD(&ls->chunk, ch, link);
+	}
+
+	rte_spinlock_unlock(&ls->lock);
+
+	/* insert this chunk into the *full* list */
+	if (k == MC_FULL) {
+		ls = &mt->chl[k];
+		rte_spinlock_lock(&ls->lock);
+		TAILQ_INSERT_HEAD(&ls->chunk, ch, link);
+		rte_spinlock_unlock(&ls->lock);
+	}
+}
+
+static uint32_t
+shrink_chunk(struct memtank *mt, uint32_t num)
+{
+	uint32_t i, k;
+	struct mchunk_list *ls;
+	struct memchunk *ch[num];
+
+	ls = &mt->chl[MC_FULL];
+	rte_spinlock_lock(&ls->lock);
+
+	for (k = 0; k != num; k++) {
+		ch[k] = TAILQ_LAST(&ls->chunk, mchunk_head);
+		if (ch[k] == NULL)
+			break;
+		TAILQ_REMOVE(&ls->chunk, ch[k], link);
+	}
+
+	rte_spinlock_unlock(&ls->lock);
+
+	rte_atomic32_sub(&mt->nb_chunks, k);
+
+	for (i = 0; i != k; i++)
+		mt->prm.free(ch[i]->raw, mt->prm.udata);
+
+	return k;
+}
+
+static struct memchunk *
+alloc_chunk(struct memtank *mt)
+{
+	void *p;
+	struct memchunk *ch;
+
+	p = mt->prm.alloc(mt->chunk_size, mt->prm.udata);
+	if (p == NULL)
+		return NULL;
+	ch = RTE_PTR_ALIGN_CEIL(p, alignof(*ch));
+	ch->raw = p;
+	return ch;
+}
+
+/* Determine by how many chunks we can actually grow */
+static inline uint32_t
+grow_num(struct memtank *mt, uint32_t num)
+{
+	uint32_t k, n, max;
+
+	max = mt->max_chunk;
+	n = rte_atomic32_add_return(&mt->nb_chunks, num);
+
+	if (n <= max)
+		return num;
+
+	k = n - max;
+	return (k >= num) ? 0 : num - k;
+}
+
+static uint32_t
+grow_chunk(struct memtank *mt, uint32_t num)
+{
+	uint32_t i, k, n;
+	struct mchunk_list *fls;
+	struct mchunk_head ls;
+	struct memchunk *ch[num];
+
+	/* check can we grow further */
+	k = grow_num(mt, num);
+
+	for (n = 0; n != k; n++) {
+		ch[n] = alloc_chunk(mt);
+		if (ch[n] == NULL)
+			break;
+	}
+
+	TAILQ_INIT(&ls);
+
+	for (i = 0; i != n; i++) {
+		init_chunk(mt, ch[i]);
+		TAILQ_INSERT_HEAD(&ls, ch[i], link);
+	}
+
+	if (n != 0) {
+		fls = &mt->chl[MC_FULL];
+		rte_spinlock_lock(&fls->lock);
+		TAILQ_CONCAT(&fls->chunk, &ls, link);
+		rte_spinlock_unlock(&fls->lock);
+	}
+
+	if (n != num)
+		rte_atomic32_sub(&mt->nb_chunks, num - n);
+
+	return n;
+}
+
+static void
+obj_dbg_alloc(struct memtank *mt, void * const obj[], uint32_t nb_obj)
+{
+	uint32_t i, sz;
+	struct memobj *po;
+
+	sz = mt->obj_size;
+	for (i = 0; i != nb_obj; i++) {
+		po = obj_pub_full((uintptr_t)obj[i], sz);
+		RTE_VERIFY(memobj_verify(po, 0) == 0);
+		po->dbg.nb_alloc++;
+	}
+}
+
+static void
+obj_dbg_free(struct memtank *mt, void * const obj[], uint32_t nb_obj)
+{
+	uint32_t i, sz;
+	struct memobj *po;
+
+	sz = mt->obj_size;
+	for (i = 0; i != nb_obj; i++) {
+		po = obj_pub_full((uintptr_t)obj[i], sz);
+		RTE_VERIFY(memobj_verify(po, 1) == 0);
+		po->dbg.nb_free++;
+	}
+}
+
+
+void
+tle_memtank_chunk_free(struct tle_memtank *t, void * const obj[],
+	uint32_t nb_obj, uint32_t flags)
+{
+	uint32_t i, j, k, sz;
+	struct memtank *mt;
+	struct memobj *mo;
+	struct memchunk *ch[nb_obj];
+
+	mt = tank_pub_full(t);
+	sz = mt->obj_size;
+
+	if (mt->flags & TLE_MTANK_OBJ_DBG)
+		obj_dbg_free(mt, obj, nb_obj);
+
+	for (i = 0; i != nb_obj; i++) {
+		mo = obj_pub_full((uintptr_t)obj[i], sz);
+		ch[i] = mo->chunk;
+	}
+
+	k = 0;
+	for (i = 0; i != nb_obj; i = j) {
+
+		/* find number of consequtive objs from the same chunk */
+		for (j = i + 1; j != nb_obj && ch[j] == ch[i]; j++)
+			;
+
+		put_chunk(mt, ch[i], obj + i, j - i);
+		k++;
+	}
+
+	if (flags & TLE_MTANK_FREE_SHRINK)
+		shrink_chunk(mt, k);
+}
+
+static uint32_t
+get_chunk(struct mchunk_list *ls, struct mchunk_head *els,
+	struct mchunk_head *uls, void *obj[], uint32_t nb_obj)
+{
+	uint32_t l, k, n;
+	struct memchunk *ch, *nch;
+
+	rte_spinlock_lock(&ls->lock);
+
+	n = 0;
+	for (ch = TAILQ_FIRST(&ls->chunk);
+			n != nb_obj && ch != NULL && ch->nb_free != 0;
+			ch = nch, n += k) {
+
+		k = RTE_MIN(nb_obj - n, ch->nb_free);
+		l = ch->nb_free - k;
+		_copy_objs(obj + n, ch->free + l, k);
+		ch->nb_free = l;
+
+		nch = TAILQ_NEXT(ch, link);
+
+		/* chunk is empty now */
+		if (l == 0) {
+			TAILQ_REMOVE(&ls->chunk, ch, link);
+			TAILQ_INSERT_TAIL(els, ch, link);
+		} else if (uls != NULL) {
+			TAILQ_REMOVE(&ls->chunk, ch, link);
+			TAILQ_INSERT_HEAD(uls, ch, link);
+		}
+	}
+
+	rte_spinlock_unlock(&ls->lock);
+	return n;
+}
+
+uint32_t
+tle_memtank_chunk_alloc(struct tle_memtank *t, void *obj[], uint32_t nb_obj,
+	uint32_t flags)
+{
+	uint32_t k, n;
+	struct memtank *mt;
+	struct mchunk_head els, uls;
+
+	mt = tank_pub_full(t);
+
+	/* walk though the the *used* list first */
+	n = get_chunk(&mt->chl[MC_USED], &mt->chl[MC_USED].chunk, NULL,
+		obj, nb_obj);
+
+	if (n != nb_obj) {
+
+		TAILQ_INIT(&els);
+		TAILQ_INIT(&uls);
+
+		/* walk though the the *full* list */
+		n += get_chunk(&mt->chl[MC_FULL], &els, &uls,
+			obj + n, nb_obj - n);
+
+		if (n != nb_obj && (flags & TLE_MTANK_ALLOC_GROW) != 0) {
+
+			/* try to allocate extra memchunks */
+			k = ALIGN_MUL_CEIL(nb_obj - n,
+				mt->prm.nb_obj_chunk);
+			k = grow_chunk(mt, k);
+
+			/* walk through the *full* list again */
+			if (k != 0)
+				n += get_chunk(&mt->chl[MC_FULL], &els, &uls,
+					obj + n, nb_obj - n);
+		}
+
+		/* concatenate with *used* list our temporary lists */
+		rte_spinlock_lock(&mt->chl[MC_USED].lock);
+
+		/* put new non-emtpy elems at head of the *used* list */
+		TAILQ_CONCAT(&uls, &mt->chl[MC_USED].chunk, link);
+		TAILQ_CONCAT(&mt->chl[MC_USED].chunk, &uls, link);
+
+		/* put new emtpy elems at tail of the *used* list */
+		TAILQ_CONCAT(&mt->chl[MC_USED].chunk, &els, link);
+
+		rte_spinlock_unlock(&mt->chl[MC_USED].lock);
+	}
+
+	if (mt->flags & TLE_MTANK_OBJ_DBG)
+		obj_dbg_alloc(mt, obj, n);
+
+	return n;
+}
+
+int
+tle_memtank_grow(struct tle_memtank *t)
+{
+	uint32_t k, n, num;
+	struct memtank *mt;
+
+	mt = tank_pub_full(t);
+
+	/* how many chunks we need to grow */
+	k = t->min_free - t->nb_free;
+	if ((int32_t)k <= 0)
+		return 0;
+
+	num = ALIGN_MUL_CEIL(k, mt->prm.nb_obj_chunk);
+
+	/* try to grow and refill the *free* */
+	n = grow_chunk(mt, num);
+	if (n != 0)
+		_fill_free(t, k, 0);
+
+	return n;
+}
+
+int
+tle_memtank_shrink(struct tle_memtank *t)
+{
+	uint32_t n;
+	struct memtank *mt;
+
+	mt = tank_pub_full(t);
+
+	/* how many chunks we need to shrink */
+	if (t->nb_free < t->max_free)
+		return 0;
+
+	/* how many chunks we need to free */
+	n = ALIGN_MUL_CEIL(t->min_free, mt->prm.nb_obj_chunk);
+
+	/* free up to *num* chunks */
+	return shrink_chunk(mt, n);
+}
+
+static int
+check_param(const struct tle_memtank_prm *prm)
+{
+	if (prm->alloc == NULL || prm->free == NULL ||
+			prm->min_free > prm->max_free ||
+			rte_is_power_of_2(prm->obj_align) == 0)
+		return -EINVAL;
+	return 0;
+}
+
+struct tle_memtank *
+tle_memtank_create(const struct tle_memtank_prm *prm)
+{
+	int32_t rc;
+	size_t sz;
+	void *p;
+	struct memtank *mt;
+
+	rc = check_param(prm);
+	if (rc != 0) {
+		rte_errno = -rc;
+		return NULL;
+	}
+
+	sz = memtank_meta_size(prm->max_free);
+	p = prm->alloc(sz, prm->udata);
+	if (p == NULL) {
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+
+	mt = RTE_PTR_ALIGN_CEIL(p, alignof(*mt));
+
+	memset(mt, 0, sizeof(*mt));
+	mt->prm = *prm;
+
+	mt->raw = p;
+	mt->chunk_size = memchunk_size(prm->nb_obj_chunk, prm->obj_size,
+		prm->obj_align);
+	mt->obj_size = memobj_size(prm->obj_size, prm->obj_align);
+	mt->max_chunk = ALIGN_MUL_CEIL(prm->max_obj, prm->nb_obj_chunk);
+	mt->flags = prm->flags;
+
+	mt->pub.min_free = prm->min_free;
+	mt->pub.max_free = prm->max_free;
+
+	TAILQ_INIT(&mt->chl[MC_FULL].chunk);
+	TAILQ_INIT(&mt->chl[MC_USED].chunk);
+
+	return &mt->pub;
+}
+
+static void
+free_mchunk_list(struct memtank *mt, struct mchunk_list *ls)
+{
+	struct memchunk *ch;
+
+	for (ch = TAILQ_FIRST(&ls->chunk); ch != NULL;
+			ch = TAILQ_FIRST(&ls->chunk)) {
+		TAILQ_REMOVE(&ls->chunk, ch, link);
+		mt->prm.free(ch->raw, mt->prm.udata);
+	}
+}
+
+void
+tle_memtank_destroy(struct tle_memtank *t)
+{
+	struct memtank *mt;
+
+	if (t != NULL) {
+		mt = tank_pub_full(t);
+		free_mchunk_list(mt, &mt->chl[MC_FULL]);
+		free_mchunk_list(mt, &mt->chl[MC_USED]);
+		mt->prm.free(mt->raw, mt->prm.udata);
+	}
+}
diff --git a/lib/libtle_memtank/memtank.h b/lib/libtle_memtank/memtank.h
new file mode 100644
index 0000000..ba3f160
--- /dev/null
+++ b/lib/libtle_memtank/memtank.h
@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2019  Intel Corporation.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef	_MEMTANK_H_
+#define	_MEMTANK_H_
+
+#include <tle_memtank.h>
+#include <stdalign.h>
+
+struct memobj {
+	uint64_t red_zone1;
+	struct memchunk *chunk; /* ptr to the chunk it belongs to */
+	struct {
+		uint32_t nb_alloc;
+		uint32_t nb_free;
+	} dbg;
+	uint64_t red_zone2;
+};
+
+#define RED_ZONE_V1	UINT64_C(0xBADECAFEBADECAFE)
+#define RED_ZONE_V2	UINT64_C(0xDEADBEEFDEADBEEF)
+
+struct memchunk {
+	TAILQ_ENTRY(memchunk) link;  /* link to the next chunk in the tank */
+	void *raw;		     /* un-aligned ptr returned by alloc() */
+	uint32_t nb_total;           /* total number of objects in the chunk */
+	uint32_t nb_free;            /*  number of free object in the chunk */
+	void *free[];                /* array of free objects */
+} __rte_cache_aligned;
+
+
+TAILQ_HEAD(mchunk_head, memchunk);
+
+struct mchunk_list {
+	rte_spinlock_t lock;
+	struct mchunk_head chunk;  /* list of chunks */
+} __rte_cache_aligned;
+
+enum {
+	MC_FULL,  /* all memchunk objs are free */
+	MC_USED,  /* some of memchunk objs are allocated */
+	MC_NUM,
+};
+
+struct memtank {
+	/* user provided data */
+	struct tle_memtank_prm prm;
+
+	/*run-time data */
+	void *raw;		     /* un-aligned ptr returned by alloc() */
+	size_t chunk_size;           /* full size of each memchunk */
+	uint32_t obj_size;	     /* full size of each memobj */
+	uint32_t max_chunk;          /* max allowed number of chunks */
+	uint32_t flags;              /* behavior flags */
+	rte_atomic32_t nb_chunks;    /* number of allocated chunks */
+
+	struct mchunk_list chl[MC_NUM];  /* lists of memchunks */
+
+	struct tle_memtank pub;
+};
+
+/*
+ * Obtain pointer to interal memtank struct from public one
+ */
+static inline struct memtank *
+tank_pub_full(const void *p)
+{
+	uintptr_t v;
+
+	v = (uintptr_t)p - offsetof(struct memtank, pub);
+	return (struct memtank *)v;
+}
+
+/*
+ * Obtain pointer to interal memobj struct from public one
+ */
+static inline struct memobj *
+obj_pub_full(uintptr_t p, uint32_t obj_sz)
+{
+	uintptr_t v;
+
+	v = p + obj_sz - sizeof(struct memobj);
+	return (struct memobj *)v;
+}
+
+static inline int
+memobj_verify(const struct memobj *mo, uint32_t finc)
+{
+	if (mo->red_zone1 != RED_ZONE_V1 || mo->red_zone2 != RED_ZONE_V2 ||
+			mo->dbg.nb_alloc != mo->dbg.nb_free + finc)
+		return -EINVAL;
+	return 0;
+}
+
+#endif	/* _MEMTANK_H_ */
diff --git a/lib/libtle_memtank/misc.c b/lib/libtle_memtank/misc.c
new file mode 100644
index 0000000..07e73db
--- /dev/null
+++ b/lib/libtle_memtank/misc.c
@@ -0,0 +1,381 @@
+/*
+ * Copyright (c) 2019  Intel Corporation.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "memtank.h"
+#include <inttypes.h>
+
+#define CHUNK_OBJ_LT_NUM	4
+
+struct mchunk_stat {
+	uint32_t nb_empty;
+	uint32_t nb_full;
+	struct {
+		uint32_t nb_chunk;
+		uint32_t nb_obj;
+		struct {
+			uint32_t val;
+			uint32_t num;
+		} chunk_obj_lt[CHUNK_OBJ_LT_NUM];
+	} used;
+};
+
+struct mfree_stat {
+	uint32_t nb_chunk;
+	struct mchunk_stat chunk;
+};
+
+#define	MTANK_LOG(lvl, fmt, args...)	RTE_LOG(lvl, USER1, fmt, ##args)
+
+
+static void
+mchunk_stat_dump(FILE *f, const struct mchunk_stat *st)
+{
+	uint32_t i;
+
+	fprintf(f, "\t\tstat={\n");
+	fprintf(f, "\t\t\tnb_empty=%u,\n", st->nb_empty);
+	fprintf(f, "\t\t\tnb_full=%u,\n", st->nb_full);
+	fprintf(f, "\t\t\tused={\n");
+	fprintf(f, "\t\t\t\tnb_chunk=%u,\n", st->used.nb_chunk);
+	fprintf(f, "\t\t\t\tnb_obj=%u,\n", st->used.nb_obj);
+
+	for (i = 0; i != RTE_DIM(st->used.chunk_obj_lt); i++) {
+		if (st->used.chunk_obj_lt[i].num != 0)
+			fprintf(f, "\t\t\t\tnb_chunk_obj_lt_%u=%u,\n",
+				st->used.chunk_obj_lt[i].val,
+				st->used.chunk_obj_lt[i].num);
+	}
+
+	fprintf(f, "\t\t\t},\n");
+	fprintf(f, "\t\t},\n");
+}
+
+static void
+mchunk_stat_init(struct mchunk_stat *st, uint32_t nb_obj_chunk)
+{
+	uint32_t i;
+
+	memset(st, 0, sizeof(*st));
+	for (i = 0; i != RTE_DIM(st->used.chunk_obj_lt); i++) {
+		st->used.chunk_obj_lt[i].val = (i + 1) * nb_obj_chunk /
+			RTE_DIM(st->used.chunk_obj_lt);
+	}
+}
+
+static void
+mchunk_stat_collect(struct mchunk_stat *st, const struct memchunk *ch)
+{
+	uint32_t i, n;
+
+	n = ch->nb_total - ch->nb_free;
+
+	if (ch->nb_free == 0)
+		st->nb_empty++;
+	else if (n == 0)
+		st->nb_full++;
+	else {
+		st->used.nb_chunk++;
+		st->used.nb_obj += n;
+
+		for (i = 0; i != RTE_DIM(st->used.chunk_obj_lt); i++) {
+			if (n < st->used.chunk_obj_lt[i].val) {
+				st->used.chunk_obj_lt[i].num++;
+				break;
+			}
+		}
+	}
+}
+
+static void
+mchunk_list_dump(FILE *f, struct memtank *mt, uint32_t idx, uint32_t flags)
+{
+	struct mchunk_list *ls;
+	const struct memchunk *ch;
+	struct mchunk_stat mcs;
+
+	ls = &mt->chl[idx];
+	mchunk_stat_init(&mcs, mt->prm.nb_obj_chunk);
+
+	rte_spinlock_lock(&ls->lock);
+
+	for (ch = TAILQ_FIRST(&ls->chunk); ch != NULL;
+			ch = TAILQ_NEXT(ch, link)) {
+
+		/* collect chunk stats */
+		if (flags & TLE_MTANK_DUMP_CHUNK_STAT)
+			mchunk_stat_collect(&mcs, ch);
+
+		/* dump chunk metadata */
+		if (flags & TLE_MTANK_DUMP_CHUNK) {
+			fprintf(f, "\t\tmemchunk@%p={\n", ch);
+			fprintf(f, "\t\t\traw=%p,\n", ch->raw);
+			fprintf(f, "\t\t\tnb_total=%u,\n", ch->nb_total);
+			fprintf(f, "\t\t\tnb_free=%u,\n", ch->nb_free);
+			fprintf(f, "\t\t},\n");
+		}
+	}
+
+	rte_spinlock_unlock(&ls->lock);
+
+	/* print chunk stats */
+	if (flags & TLE_MTANK_DUMP_CHUNK_STAT)
+		mchunk_stat_dump(f, &mcs);
+}
+
+static void
+mfree_stat_init(struct mfree_stat *st, uint32_t nb_obj_chunk)
+{
+	st->nb_chunk = 0;
+	mchunk_stat_init(&st->chunk, nb_obj_chunk);
+}
+
+static int
+ptr_cmp(const void *p1, const void *p2)
+{
+	const intptr_t *v1, *v2;
+
+	v1 = p1;
+	v2 = p2;
+	return v1[0] - v2[0];
+}
+
+static void
+mfree_stat_collect(struct mfree_stat *st, struct memtank *mt)
+{
+	uint32_t i, j, n, sz;
+	uintptr_t *p;
+	const struct memobj *mo;
+
+	sz = mt->obj_size;
+
+	p = malloc(mt->pub.max_free * sizeof(*p));
+	if (p == NULL)
+		return;
+
+	/**
+	 * grab free lock and keep it till we analyze related memchunks,
+	 * to make sure none of these memchunks will be freed untill
+	 * we are finished.
+	 */
+	rte_spinlock_lock(&mt->pub.lock);
+
+	/* collect chunks for all objects in free[] */
+	n = mt->pub.nb_free;
+	memcpy(p, mt->pub.free, n * sizeof(*p));
+	for (i = 0; i != n; i++) {
+		mo = obj_pub_full(p[i], sz);
+		p[i] = (uintptr_t)mo->chunk;
+	}
+
+	/* sort chunk pointers */
+	qsort(p, n, sizeof(*p), ptr_cmp);
+
+	/* for each chunk collect stats */
+	for (i = 0; i != n; i = j) {
+
+		st->nb_chunk++;
+		mchunk_stat_collect(&st->chunk, (const struct memchunk *)p[i]);
+		for (j = i + 1; j != n && p[i] == p[j]; j++)
+			;
+	}
+
+	rte_spinlock_unlock(&mt->pub.lock);
+	free(p);
+}
+
+static void
+mfree_stat_dump(FILE *f, const struct mfree_stat *st)
+{
+	fprintf(f, "\tfree_stat={\n");
+	fprintf(f, "\t\tnb_chunk=%u,\n", st->nb_chunk);
+	mchunk_stat_dump(f, &st->chunk);
+	fprintf(f, "\t},\n");
+}
+
+void
+tle_memtank_dump(FILE *f, const struct tle_memtank *t, uint32_t flags)
+{
+	struct memtank *mt;
+
+	if (f == NULL || t == NULL)
+		return;
+
+	mt = tank_pub_full(t);
+
+	fprintf(f, "tle_memtank@%p={\n", t);
+	fprintf(f, "\tmin_free=%u,\n", t->min_free);
+	fprintf(f, "\tmax_free=%u,\n", t->max_free);
+	fprintf(f, "\tnb_free=%u,\n", t->nb_free);
+	fprintf(f, "\tchunk_size=%zu,\n", mt->chunk_size);
+	fprintf(f, "\tobj_size=%u,\n", mt->obj_size);
+	fprintf(f, "\tmax_chunk=%u,\n", mt->max_chunk);
+	fprintf(f, "\tflags=%#x,\n", mt->flags);
+	fprintf(f, "\tnb_chunks=%u,\n", rte_atomic32_read(&mt->nb_chunks));
+
+	if (flags & TLE_MTANK_DUMP_FREE_STAT) {
+		struct mfree_stat mfs;
+		mfree_stat_init(&mfs, mt->prm.nb_obj_chunk);
+		mfree_stat_collect(&mfs, mt);
+		mfree_stat_dump(f, &mfs);
+	}
+
+	if (flags & (TLE_MTANK_DUMP_CHUNK | TLE_MTANK_DUMP_CHUNK_STAT)) {
+
+		fprintf(f, "\t[FULL]={\n");
+		mchunk_list_dump(f, mt, MC_FULL, flags);
+		fprintf(f, "\t},\n");
+
+		fprintf(f, "\t[USED]={,\n");
+		mchunk_list_dump(f, mt, MC_USED, flags);
+		fprintf(f, "\t},\n");
+	}
+	fprintf(f, "};\n");
+}
+
+static int
+mobj_bulk_check(const char *fname, const struct memtank *mt,
+	const uintptr_t p[], uint32_t num, uint32_t fmsk)
+{
+	int32_t ret;
+	uintptr_t align;
+	uint32_t i, k, sz;
+	const struct memobj *mo;
+
+	k = ((mt->flags & TLE_MTANK_OBJ_DBG) != 0) & fmsk;
+	sz = mt->obj_size;
+	align = mt->prm.obj_align - 1;
+
+	ret = 0;
+	for (i = 0; i != num; i++) {
+
+		if (p[i] == (uintptr_t)NULL) {
+			ret--;
+			MTANK_LOG(ERR,
+				"%s(mt=%p, %p[%u]): NULL object\n",
+				fname, mt, p, i);
+		} else if ((p[i] & align) != 0) {
+			ret--;
+			MTANK_LOG(ERR,
+				"%s(mt=%p, %p[%u]): object %#zx violates "
+				"expected alignment %#zx\n",
+				fname, mt, p, i, p[i], align);
+		} else {
+			mo = obj_pub_full(p[i], sz);
+			if (memobj_verify(mo, k) != 0) {
+				ret--;
+				MTANK_LOG(ERR,
+					"%s(mt=%p, %p[%u]): "
+					"invalid object header @%#zx={"
+					"red_zone1=%#" PRIx64 ","
+					"dbg={nb_alloc=%u,nb_free=%u},"
+					"red_zone2=%#" PRIx64
+					"}\n",
+					fname, mt, p, i, p[i],
+					mo->red_zone1,
+					mo->dbg.nb_alloc, mo->dbg.nb_free,
+					mo->red_zone2);
+			}
+		}
+	}
+
+	return ret;
+}
+
+/* grab free lock and check objects in free[] */
+static int
+mfree_check(struct memtank *mt)
+{
+	int32_t rc;
+
+	rte_spinlock_lock(&mt->pub.lock);
+	rc = mobj_bulk_check(__func__, mt, (const uintptr_t *)mt->pub.free,
+		mt->pub.nb_free, 1);
+	rte_spinlock_unlock(&mt->pub.lock);
+	return rc;
+}
+
+static int
+mchunk_check(const struct memtank *mt, const struct memchunk *mc, uint32_t tc)
+{
+	int32_t n, rc;
+
+	rc = 0;
+	n = mc->nb_total - mc->nb_free;
+
+	rc -= (mc->nb_total != mt->prm.nb_obj_chunk);
+	rc -= (tc == MC_FULL) ? (n != 0) : (n <= 0);
+	rc -= (RTE_PTR_ALIGN_CEIL(mc->raw, alignof(*mc)) != mc);
+
+	if (rc != 0)
+		MTANK_LOG(ERR, "%s(mt=%p, tc=%u): invalid memchunk @%p={"
+			"raw=%p, nb_total=%u, nb_free=%u}\n",
+			__func__, mt, tc, mc,
+			mc->raw, mc->nb_total, mc->nb_free);
+
+	rc += mobj_bulk_check(__func__, mt, (const uintptr_t *)mc->free,
+		mc->nb_free, 0);
+	return rc;
+}
+
+static int
+mchunk_list_check(struct memtank *mt, uint32_t tc, uint32_t *nb_chunk)
+{
+	int32_t rc;
+	uint32_t n;
+	struct mchunk_list *ls;
+	const struct memchunk *ch;
+
+	ls = &mt->chl[tc];
+	rte_spinlock_lock(&ls->lock);
+
+	rc = 0;
+	for (n = 0, ch = TAILQ_FIRST(&ls->chunk); ch != NULL;
+			ch = TAILQ_NEXT(ch, link), n++)
+		rc += mchunk_check(mt, ch, tc);
+
+	rte_spinlock_unlock(&ls->lock);
+
+	*nb_chunk = n;
+	return rc;
+}
+
+int
+tle_memtank_sanity_check(const struct tle_memtank *t, int32_t ct)
+{
+	int32_t rc;
+	uint32_t n, nf, nu;
+	struct memtank *mt;
+
+	mt = tank_pub_full(t);
+	rc = mfree_check(mt);
+
+	nf = 0, nu = 0;
+	rc += mchunk_list_check(mt, MC_FULL, &nf);
+	rc += mchunk_list_check(mt, MC_USED, &nu);
+
+	/*
+	 * if some other threads concurently do alloc/free/grow/shrink
+	 * these numbers can still not match.
+	 */
+	n = rte_atomic32_read(&mt->nb_chunks);
+	if (nf + nu != n && ct == 0) {
+		MTANK_LOG(ERR,
+			"%s(mt=%p) nb_chunks: expected=%u, full=%u, used=%u\n",
+			__func__, mt, n, nf, nu);
+		rc--;
+	}
+
+	return rc;
+}
diff --git a/lib/libtle_memtank/tle_memtank.h b/lib/libtle_memtank/tle_memtank.h
new file mode 100644
index 0000000..9f12fe0
--- /dev/null
+++ b/lib/libtle_memtank/tle_memtank.h
@@ -0,0 +1,274 @@
+/*
+ * Copyright (c) 2019  Intel Corporation.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _TLE_MEMTANK_H_
+#define _TLE_MEMTANK_H_
+
+#include <string.h>
+
+#include <rte_common.h>
+#include <rte_memory.h>
+#include <rte_atomic.h>
+#include <rte_spinlock.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @file
+ * TLE memtank
+ *
+ * Same a s mempool it allows to alloc/free objects of fixed size
+ * in a lightweight manner (probably not as lightweight as mempool,
+ * but hopefully close enough).
+ * But in addition it can grow/shrink dynamically plus provides extra
+ * additional API for higher flexibility:
+ *	- manual grow()/shrink() functions
+ *	- different alloc/free policies
+ *        (can be specified by user via flags parameter).
+ * Internally it consists of:
+ *	- LIFO queue (fast allocator/deallocator)
+ *	- lists of memchunks (USED, FREE).
+ *
+ * For perfomance reasons memtank tries to allocate memory in
+ * relatively big chunks (memchunks) and then split each memchunk
+ * in dozens (or hundreds) of objects.
+ * There are two thresholds:
+ *	- min_free (grow threshold)
+ *	- max_free (shrink threshold)
+ */
+
+struct tle_memtank;
+
+/** generic memtank behavior flags */
+enum {
+	TLE_MTANK_OBJ_DBG = 1,
+};
+
+struct tle_memtank_prm {
+	/** min number of free objs in the ring (grow threshold). */
+	uint32_t min_free;
+	uint32_t max_free;  /**< max number of free objs (empty threshold) */
+	uint32_t max_obj; /**< max number of objs (grow limit) */
+	uint32_t obj_size;  /**< size of each mem object */
+	uint32_t obj_align;  /**< alignment of each mem object */
+	uint32_t nb_obj_chunk; /**< number of objects per chunk */
+	uint32_t flags; /**< behavior flags */
+	/** user provided function to alloc chunk of memory */
+	void * (*alloc)(size_t, void *);
+	/** user provided function to free chunk of memory */
+	void (*free)(void *, void *);
+	/** user provided function to initialiaze an object */
+	void (*init)(void *[], uint32_t, void *);
+	void *udata;        /**< opaque user data for alloc/free/init */
+};
+
+/**
+ * Allocate and intitialize new memtank instance, based on the
+ * parameters provided. Note that it uses user-provided *alloc()* function
+ * to allocate space for the memtank metadata.
+ * @param prm
+ *   Parameters used to create and initialise new memtank.
+ * @return
+ *   - Pointer to new memtank insteance created, if operation completed
+ *     successfully.
+ *   - NULL on error with rte_errno set appropriately.
+ */
+struct tle_memtank *
+tle_memtank_create(const struct tle_memtank_prm *prm);
+
+/**
+ * Destroy the memtank and free all memory referenced by the memtank.
+ * The objects must not be used by other cores as they will be freed.
+ *
+ * @param t
+ *   A pointer to the memtank instance.
+ */
+void
+tle_memtank_destroy(struct tle_memtank *t);
+
+
+/** alloc flags */
+enum {
+	TLE_MTANK_ALLOC_CHUNK = 1,
+	TLE_MTANK_ALLOC_GROW = 2,
+};
+
+/**
+ * Allocate up to requested number of objects from the memtank.
+ * Note that depending on *alloc* behavior (flags) some new memory chunks
+ * can be allocated from the underlying memory subsystem.
+ *
+ * @param t
+ *   A pointer to the memtank instance.
+ * @param obj
+ *   An array of void * pointers (objects) that will be filled.
+ * @param num
+ *   Number of objects to allocate from the memtank.
+ * @param flags
+ *   Flags that control allocation behavior.
+ * @return
+ *   Number of allocated objects.
+ */
+static inline uint32_t
+tle_memtank_alloc(struct tle_memtank *t, void *obj[], uint32_t num,
+		uint32_t flags);
+
+/**
+ * Allocate up to requested number of objects from the memtank.
+ * Note that this function bypasses *free* cache(s) and tries to allocate
+ * objects straight from the memory chunks.
+ * Note that depending on *alloc* behavior (flags) some new memory chunks
+ * can be allocated from the underlying memory subsystem.
+ *
+ * @param t
+ *   A pointer to the memtank instance.
+ * @param obj
+ *   An array of void * pointers (objects) that will be filled.
+ * @param nb_obj
+ *   Number of objects to allocate from the memtank.
+ * @param flags
+ *   Flags that control allocation behavior.
+ * @return
+ *   Number of allocated objects.
+ */
+uint32_t
+tle_memtank_chunk_alloc(struct tle_memtank *t, void *obj[], uint32_t nb_obj,
+		uint32_t flags);
+
+/** free flags */
+enum {
+	TLE_MTANK_FREE_SHRINK = 1,
+};
+
+/**
+ * Free (put) provided objects back to the memtank.
+ * Note that depending on *free* behavior (flags) some memory chunks can be
+ * returned (freed) to the underlying memory subsystem.
+ *
+ * @param t
+ *   A pointer to the memtank instance.
+ * @param obj
+ *   An array of object pointers to be freed.
+ * @param num
+ *   Number of objects to free.
+ * @param flags
+ *   Flags that control free behavior.
+ */
+static inline void
+tle_memtank_free(struct tle_memtank *t, void * const obj[],  uint32_t num,
+		uint32_t flags);
+
+/**
+ * Free (put) provided objects back to the memtank.
+ * Note that this function bypasses *free* cache(s) and tries to put
+ * objects straight to the memory chunks.
+ * Note that depending on *free* behavior (flags) some memory chunks can be
+ * returned (freed) to the underlying memory subsystem.
+ *
+ * @param t
+ *   A pointer to the memtank instance.
+ * @param obj
+ *   An array of object pointers to be freed.
+ * @param nb_obj
+ *   Number of objects to allocate from the memtank.
+ * @param flags
+ *   Flags that control allocation behavior.
+ */
+void
+tle_memtank_chunk_free(struct tle_memtank *t, void * const obj[],
+		uint32_t nb_obj, uint32_t flags);
+
+/**
+ * Check does number of objects in *free* cache is below memtank grow
+ * threshold (min_free). If yes, then tries to allocate memory for new
+ * objects from the underlying memory subsystem.
+ *
+ * @param t
+ *   A pointer to the memtank instance.
+ * @return
+ *   Number of newly allocated memory chunks.
+ */
+int
+tle_memtank_grow(struct tle_memtank *t);
+
+/**
+ * Check does number of objects in *free* cache have reached memtank shrink
+ * threshold (max_free). If yes, then tries to return excessive memory to
+ * the the underlying memory subsystem.
+ *
+ * @param t
+ *   A pointer to the memtank instance.
+ * @return
+ *   Number of freed memory chunks.
+ */
+int
+tle_memtank_shrink(struct tle_memtank *t);
+
+/* dump flags */
+enum {
+	TLE_MTANK_DUMP_FREE_STAT = 1,
+	TLE_MTANK_DUMP_CHUNK_STAT = 2,
+	TLE_MTANK_DUMP_CHUNK = 4,
+	/* first not used power of two */
+	TLE_MTANK_DUMP_END,
+
+	/* dump all stats */
+	TLE_MTANK_DUMP_STAT =
+		(TLE_MTANK_DUMP_FREE_STAT | TLE_MTANK_DUMP_CHUNK_STAT),
+	/* dump everything */
+	TLE_MTANK_DUMP_ALL = TLE_MTANK_DUMP_END - 1,
+};
+
+/**
+ * Dump information about the memtank to the file.
+ * Note that depending of *flags* value it might cause some internal locks
+ * grabbing, and might affect perfomance of others threads that
+ * concurently use same memtank.
+ *
+ * @param f
+ *   A pinter to the file.
+ * @param t
+ *   A pointer to the memtank instance.
+ * @param flags
+ *   Flags that control dump behavior.
+ */
+void
+tle_memtank_dump(FILE *f, const struct tle_memtank *t, uint32_t flags);
+
+/**
+ * Check the consistency of the given memtank instance.
+ * Dumps error messages to the RTE log subsystem, if some inconsitency
+ * is detected.
+ *
+ * @param t
+ *   A pointer to the memtank instance.
+ * @param ct
+ *   Value greater then zero, if some other threads do concurently use
+ *   that memtank.
+ * @return
+ *   Zero on success, or negaive value otherwise.
+ */
+int
+tle_memtank_sanity_check(const struct tle_memtank *t, int32_t ct);
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <tle_memtank_pub.h>
+
+#endif /* _TLE_MEMTANK_H_ */
diff --git a/lib/libtle_memtank/tle_memtank_pub.h b/lib/libtle_memtank/tle_memtank_pub.h
new file mode 100644
index 0000000..78e89b2
--- /dev/null
+++ b/lib/libtle_memtank/tle_memtank_pub.h
@@ -0,0 +1,149 @@
+/*
+ * Copyright (c) 2019  Intel Corporation.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _TLE_MEMTANK_PUB_H_
+#define _TLE_MEMTANK_PUB_H_
+
+#include <tle_memtank.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @file
+ * TLE memtank public
+ * It is not recommended to include this file directly,
+ * include <tle_memtank.h> instead.
+ */
+
+struct tle_memtank {
+	rte_spinlock_t lock;
+	uint32_t min_free;
+	uint32_t max_free;
+	uint32_t nb_free;
+	void *free[];
+} __rte_cache_aligned;
+
+
+static inline void
+_copy_objs(void *dst[], void * const src[], uint32_t num)
+{
+	uint32_t i, n;
+
+	n = RTE_ALIGN_FLOOR(num, 4);
+
+	for (i = 0; i != n; i += 4) {
+		dst[i] = src[i];
+		dst[i + 1] = src[i + 1];
+		dst[i + 2] = src[i + 2];
+		dst[i + 3] = src[i + 3];
+	}
+
+	switch (num % 4) {
+	case 3:
+		dst[i + 2] = src[i + 2];
+		/* fallthrough */
+	case 2:
+		dst[i + 1] = src[i + 1];
+		/* fallthrough */
+	case 1:
+		dst[i] = src[i];
+		/* fallthrough */
+	}
+}
+
+static inline uint32_t
+_get_free(struct tle_memtank *t, void *obj[], uint32_t num)
+{
+	uint32_t len, n;
+
+	rte_spinlock_lock(&t->lock);
+
+	len = t->nb_free;
+	n = RTE_MIN(num, len);
+	len -= n;
+	_copy_objs(obj, t->free + len, n);
+	t->nb_free = len;
+
+	rte_spinlock_unlock(&t->lock);
+	return n;
+}
+
+static inline uint32_t
+_put_free(struct tle_memtank *t, void * const obj[], uint32_t num)
+{
+	uint32_t len, n;
+
+	rte_spinlock_lock(&t->lock);
+
+	len = t->nb_free;
+	n = t->max_free - len;
+	n = RTE_MIN(num, n);
+	_copy_objs(t->free + len, obj, n);
+	t->nb_free = len + n;
+
+	rte_spinlock_unlock(&t->lock);
+	return n;
+}
+
+static inline void
+_fill_free(struct tle_memtank *t, uint32_t num, uint32_t flags)
+{
+	uint32_t k, n;
+	void *free[num];
+
+	k = tle_memtank_chunk_alloc(t, free, RTE_DIM(free), flags);
+	n = _put_free(t, free, k);
+	if (n != k)
+		tle_memtank_chunk_free(t, free + n, k - n, 0);
+}
+
+static inline uint32_t
+tle_memtank_alloc(struct tle_memtank *t, void *obj[], uint32_t num,
+	uint32_t flags)
+{
+	uint32_t n;
+
+	n = _get_free(t, obj, num);
+
+	/* not enough free objects, try to allocate via memchunks */
+	if (n != num && flags != 0) {
+		n += tle_memtank_chunk_alloc(t, obj + n, num - n, flags);
+
+		/* refill *free* tank */
+		if (n == num)
+			_fill_free(t, t->min_free, flags);
+	}
+
+	return n;
+}
+
+static inline void
+tle_memtank_free(struct tle_memtank *t, void * const obj[], uint32_t num,
+	uint32_t flags)
+{
+	uint32_t n;
+
+	n = _put_free(t, obj, num);
+	if (n != num)
+		tle_memtank_chunk_free(t, obj + n, num - n, flags);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _TLE_MEMTANK_PUB_H_ */
diff --git a/test/Makefile b/test/Makefile
index c5cf270..c82b123 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -23,6 +23,7 @@ include $(RTE_SDK)/mk/rte.vars.mk
 
 DIRS-y += dring
 DIRS-y += gtest
+DIRS-y += memtank
 DIRS-y += timer
 
 include $(TLDK_ROOT)/mk/tle.subdir.mk
diff --git a/test/memtank/Makefile b/test/memtank/Makefile
new file mode 100644
index 0000000..b8e4483
--- /dev/null
+++ b/test/memtank/Makefile
@@ -0,0 +1,42 @@
+# Copyright (c) 2019 Intel Corporation.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ifeq ($(RTE_SDK),)
+$(error "Please define RTE_SDK environment variable")
+endif
+
+ifeq ($(RTE_TARGET),)
+$(error "Please define RTE_TARGET environment variable")
+endif
+
+ifeq ($(TLDK_ROOT),)
+$(error "Please define TLDK_ROOT environment variable")
+endif
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# binary name
+APP = test_memtank
+
+# all source are stored in SRCS-y
+SRCS-y += test_memtank.c
+
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -I$(RTE_OUTPUT)/include
+
+LDLIBS += -L$(RTE_OUTPUT)/lib
+LDLIBS += -ltle_memtank
+
+EXTRA_CFLAGS += -O3
+
+include $(TLDK_ROOT)/mk/tle.app.mk
diff --git a/test/memtank/test_memtank.c b/test/memtank/test_memtank.c
new file mode 100644
index 0000000..51e86be
--- /dev/null
+++ b/test/memtank/test_memtank.c
@@ -0,0 +1,793 @@
+/*
+ * Copyright (c) 2016  Intel Corporation.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <inttypes.h>
+#include <errno.h>
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_log.h>
+#include <rte_errno.h>
+#include <rte_launch.h>
+#include <rte_cycles.h>
+#include <rte_eal.h>
+#include <rte_ring.h>
+#include <rte_per_lcore.h>
+#include <rte_lcore.h>
+#include <rte_random.h>
+#include <rte_hexdump.h>
+#include <rte_malloc.h>
+
+#include <tle_memtank.h>
+
+struct memstat {
+	struct {
+		rte_atomic64_t nb_call;
+		rte_atomic64_t nb_fail;
+		rte_atomic64_t sz;
+	} alloc;
+	struct {
+		rte_atomic64_t nb_call;
+		rte_atomic64_t nb_fail;
+	} free;
+	uint64_t nb_alloc_obj;
+};
+
+struct memtank_stat {
+	uint64_t nb_cycle;
+	struct {
+		uint64_t nb_call;
+		uint64_t nb_req;
+		uint64_t nb_alloc;
+		uint64_t nb_cycle;
+	} alloc;
+	struct {
+		uint64_t nb_call;
+		uint64_t nb_free;
+		uint64_t nb_cycle;
+	} free;
+	struct {
+		uint64_t nb_call;
+		uint64_t nb_chunk;
+		uint64_t nb_cycle;
+	} grow;
+	struct {
+		uint64_t nb_call;
+		uint64_t nb_chunk;
+		uint64_t nb_cycle;
+	} shrink;
+};
+
+struct master_args {
+	uint64_t run_cycles;
+	uint32_t delay_us;
+	uint32_t flags;
+};
+
+struct worker_args {
+	uint32_t max_obj;
+	uint32_t obj_size;
+	uint32_t alloc_flags;
+	uint32_t free_flags;
+};
+
+struct memtank_arg {
+	struct tle_memtank *mt;
+	union {
+		struct master_args master;
+		struct worker_args worker;
+	};
+	struct memtank_stat stats;
+};
+
+#define BULK_NUM	32
+#define	MAX_OBJ		0x100000
+
+#define	OBJ_SZ_MIN	1
+#define	OBJ_SZ_MAX	0x100000
+#define	OBJ_SZ_DEF	(4 * RTE_CACHE_LINE_SIZE + 1)
+
+#define TEST_TIME	10
+
+#define FREE_THRSH_MIN	0
+#define FREE_THRSH_MAX	100
+
+enum {
+	WRK_CMD_STOP,
+	WRK_CMD_RUN,
+};
+
+enum {
+	MASTER_FLAG_GROW = 1,
+	MASTER_FLAG_SHRINK = 2,
+};
+
+enum {
+	MEM_FUNC_SYS,
+	MEM_FUNC_RTE,
+};
+
+static uint32_t wrk_cmd __rte_cache_aligned;
+
+static struct tle_memtank_prm mtnk_prm = {
+	.min_free = 4 * BULK_NUM,
+	.max_free = 32 * BULK_NUM,
+	.max_obj = MAX_OBJ,
+	.obj_size = OBJ_SZ_DEF,
+	.obj_align = RTE_CACHE_LINE_SIZE,
+	.nb_obj_chunk = BULK_NUM,
+	.flags = TLE_MTANK_OBJ_DBG,
+};
+
+static struct {
+	uint32_t run_time;       /* test run-time in seconds */
+	uint32_t wrk_max_obj;    /* max alloced objects per worker */
+	uint32_t wrk_free_thrsh; /* wrk free thresh % (0-100) */
+	int32_t mem_func;        /* memory subsystem to use for alloc/free */
+} global_cfg = {
+	.run_time = TEST_TIME,
+	.wrk_max_obj = 2 * BULK_NUM,
+	.wrk_free_thrsh = FREE_THRSH_MIN,
+	.mem_func = MEM_FUNC_SYS,
+};
+
+static void *
+alloc_func(size_t sz)
+{
+	switch (global_cfg.mem_func) {
+	case MEM_FUNC_SYS:
+		return malloc(sz);
+	case MEM_FUNC_RTE:
+		return rte_malloc(NULL, sz, 0);
+	}
+
+	return NULL;
+}
+
+static void
+free_func(void *p)
+{
+	switch (global_cfg.mem_func) {
+	case MEM_FUNC_SYS:
+		return free(p);
+	case MEM_FUNC_RTE:
+		return rte_free(p);
+	}
+}
+
+static void *
+test_alloc1(size_t sz, void *p)
+{
+	struct memstat *ms;
+	void *buf;
+
+	ms = p;
+	buf = alloc_func(sz);
+	rte_atomic64_inc(&ms->alloc.nb_call);
+	if (buf != NULL) {
+		memset(buf, 0, sz);
+		rte_atomic64_add(&ms->alloc.sz, sz);
+	} else
+		rte_atomic64_inc(&ms->alloc.nb_fail);
+
+	return buf;
+}
+
+static void
+test_free1(void *buf, void *p)
+{
+	struct memstat *ms;
+
+	ms = p;
+
+	free_func(buf);
+	rte_atomic64_inc(&ms->free.nb_call);
+	if (buf == NULL)
+		rte_atomic64_inc(&ms->free.nb_fail);
+}
+
+static void
+memstat_dump(FILE *f, struct memstat *ms)
+{
+
+	uint64_t alloc_sz, nb_alloc;
+	long double muc, mut;
+
+	nb_alloc = rte_atomic64_read(&ms->alloc.nb_call) -
+		rte_atomic64_read(&ms->alloc.nb_fail);
+	alloc_sz = rte_atomic64_read(&ms->alloc.sz) / nb_alloc;
+	nb_alloc -= rte_atomic64_read(&ms->free.nb_call) -
+		rte_atomic64_read(&ms->free.nb_fail);
+	alloc_sz *= nb_alloc;
+	mut = (alloc_sz == 0) ? 1 :
+		(long double)ms->nb_alloc_obj * mtnk_prm.obj_size / alloc_sz;
+	muc = (alloc_sz == 0) ? 1 :
+		(long double)(ms->nb_alloc_obj + mtnk_prm.max_free) *
+		mtnk_prm.obj_size / alloc_sz;
+
+	fprintf(f, "%s(%p)={\n", __func__, ms);
+	fprintf(f, "\talloc={\n");
+	fprintf(f, "\t\tnb_call=%" PRIu64 ",\n",
+		rte_atomic64_read(&ms->alloc.nb_call));
+	fprintf(f, "\t\tnb_fail=%" PRIu64 ",\n",
+		rte_atomic64_read(&ms->alloc.nb_fail));
+	fprintf(f, "\t\tsz=%" PRIu64 ",\n",
+		rte_atomic64_read(&ms->alloc.sz));
+	fprintf(f, "\t},\n");
+	fprintf(f, "\tfree={\n");
+	fprintf(f, "\t\tnb_call=%" PRIu64 ",\n",
+		rte_atomic64_read(&ms->free.nb_call));
+	fprintf(f, "\t\tnb_fail=%" PRIu64 ",\n",
+		rte_atomic64_read(&ms->free.nb_fail));
+	fprintf(f, "\t},\n");
+	fprintf(f, "\tnb_alloc_obj=%" PRIu64 ",\n", ms->nb_alloc_obj);
+	fprintf(f, "\tnb_alloc_chunk=%" PRIu64 ",\n", nb_alloc);
+	fprintf(f, "\talloc_sz=%" PRIu64 ",\n", alloc_sz);
+	fprintf(f, "\tmem_util(total)=%.2Lf %%,\n", mut * 100);
+	fprintf(f, "\tmem_util(cached)=%.2Lf %%,\n", muc * 100);
+	fprintf(f, "};\n");
+
+}
+
+static void
+memtank_stat_dump(FILE *f, uint32_t lc, const struct memtank_stat *ms)
+{
+	uint64_t t;
+
+	fprintf(f, "%s(lc=%u)={\n", __func__, lc);
+	fprintf(f, "\tnb_cycle=%" PRIu64 ",\n", ms->nb_cycle);
+	if (ms->alloc.nb_call != 0) {
+		fprintf(f, "\talloc={\n");
+		fprintf(f, "\t\tnb_call=%" PRIu64 ",\n", ms->alloc.nb_call);
+		fprintf(f, "\t\tnb_req=%" PRIu64 ",\n", ms->alloc.nb_req);
+		fprintf(f, "\t\tnb_alloc=%" PRIu64 ",\n", ms->alloc.nb_alloc);
+		fprintf(f, "\t\tnb_cycle=%" PRIu64 ",\n", ms->alloc.nb_cycle);
+
+		t = ms->alloc.nb_req - ms->alloc.nb_alloc;
+		fprintf(f, "\t\tfailed req: %"PRIu64 "(%.2Lf %%)\n",
+			t, (long double)t * 100 /  ms->alloc.nb_req);
+		fprintf(f, "\t\tcycles/alloc: %.2Lf\n",
+			(long double)ms->alloc.nb_cycle / ms->alloc.nb_alloc);
+		fprintf(f, "\t\tobj/call(avg): %.2Lf\n",
+			(long double)ms->alloc.nb_alloc /  ms->alloc.nb_call);
+
+		fprintf(f, "\t},\n");
+	}
+	if (ms->free.nb_call != 0) {
+		fprintf(f, "\tfree={\n");
+		fprintf(f, "\t\tnb_call=%" PRIu64 ",\n", ms->free.nb_call);
+		fprintf(f, "\t\tnb_free=%" PRIu64 ",\n", ms->free.nb_free);
+		fprintf(f, "\t\tnb_cycle=%" PRIu64 ",\n", ms->free.nb_cycle);
+
+		fprintf(f, "\t\tcycles/free: %.2Lf\n",
+			(long double)ms->free.nb_cycle / ms->free.nb_free);
+		fprintf(f, "\t\tobj/call(avg): %.2Lf\n",
+			(long double)ms->free.nb_free /  ms->free.nb_call);
+
+		fprintf(f, "\t},\n");
+	}
+	if (ms->grow.nb_call != 0) {
+		fprintf(f, "\tgrow={\n");
+		fprintf(f, "\t\tnb_call=%" PRIu64 ",\n", ms->grow.nb_call);
+		fprintf(f, "\t\tnb_chunk=%" PRIu64 ",\n", ms->grow.nb_chunk);
+		fprintf(f, "\t\tnb_cycle=%" PRIu64 ",\n", ms->grow.nb_cycle);
+
+		fprintf(f, "\t\tcycles/chunk: %.2Lf\n",
+			(long double)ms->grow.nb_cycle / ms->grow.nb_chunk);
+		fprintf(f, "\t\tobj/call(avg): %.2Lf\n",
+			(long double)ms->grow.nb_chunk /  ms->grow.nb_call);
+
+		fprintf(f, "\t},\n");
+	}
+	if (ms->shrink.nb_call != 0) {
+		fprintf(f, "\tshrink={\n");
+		fprintf(f, "\t\tnb_call=%" PRIu64 ",\n", ms->shrink.nb_call);
+		fprintf(f, "\t\tnb_chunk=%" PRIu64 ",\n", ms->shrink.nb_chunk);
+		fprintf(f, "\t\tnb_cycle=%" PRIu64 ",\n", ms->shrink.nb_cycle);
+
+		fprintf(f, "\t\tcycles/chunk: %.2Lf\n",
+			(long double)ms->shrink.nb_cycle / ms->shrink.nb_chunk);
+		fprintf(f, "\t\tobj/call(avg): %.2Lf\n",
+			(long double)ms->shrink.nb_chunk /  ms->shrink.nb_call);
+
+		fprintf(f, "\t},\n");
+	}
+	fprintf(f, "};\n");
+}
+
+static int32_t
+check_fill_objs(void *obj[], uint32_t sz, uint32_t num,
+	uint8_t check, uint8_t fill)
+{
+	uint32_t i;
+	uint8_t buf[sz];
+
+	static rte_spinlock_t dump_lock;
+
+	memset(buf, check, sz);
+
+	for (i = 0; i != num; i++) {
+		if (memcmp(buf, obj[i], sz) != 0) {
+			rte_spinlock_lock(&dump_lock);
+			printf ("%s(%u, %u, %hu, %hu) failed at %u-th iter, "
+				"offendig object: %p\n",
+				__func__, sz, num, check, fill, i, obj[i]);
+			rte_memdump(stdout, "expected", buf, sz);
+			rte_memdump(stdout, "result", obj[i], sz);
+			rte_spinlock_unlock(&dump_lock);
+			return -EINVAL;
+		}
+		memset(obj[i], fill, sz);
+	}
+	return 0;
+}
+
+static int
+test_memtank_worker(void *arg)
+{
+	int32_t rc;
+	size_t sz;
+	uint32_t ft, lc, n, num;
+	uint64_t cl, tm0, tm1;
+	struct memtank_arg *ma;
+	struct rte_ring *ring;
+	void *obj[BULK_NUM];
+
+	ma = arg;
+	lc = rte_lcore_id();
+
+	sz = rte_ring_get_memsize(ma->worker.max_obj);
+	ring = alloca(sz);
+	if (ring == NULL) {
+		printf("%s(%u): alloca(%zu) for FIFO with %u elems failed",
+			__func__, lc, sz, ma->worker.max_obj);
+		return -ENOMEM;
+	}
+	rc = rte_ring_init(ring, "", ma->worker.max_obj,
+		RING_F_SP_ENQ | RING_F_SC_DEQ);
+	if (rc != 0) {
+		printf("%s(%u): rte_ring_init(%p, %u) failed, error: %d(%s)\n",
+			__func__, lc, ring, ma->worker.max_obj,
+			rc, strerror(-rc));
+		return rc;
+	}
+
+	/* calculate free threshold */
+	ft = ma->worker.max_obj * global_cfg.wrk_free_thrsh / FREE_THRSH_MAX;
+
+	while (wrk_cmd != WRK_CMD_RUN) {
+		rte_smp_rmb();
+		rte_pause();
+	}
+
+	cl = rte_rdtsc_precise();
+
+	do {
+		num = rte_rand() % RTE_DIM(obj);
+		n = rte_ring_free_count(ring);
+		num = RTE_MIN(num, n);
+
+		/* perform alloc*/
+		if (num != 0) {
+			tm0 = rte_rdtsc_precise();
+			n = tle_memtank_alloc(ma->mt, obj, num,
+				ma->worker.alloc_flags);
+			tm1 = rte_rdtsc_precise();
+
+			/* check and fill contents of allocated objects */
+			rc = check_fill_objs(obj, ma->worker.obj_size, n,
+				0, lc);
+			if (rc != 0)
+				break;
+
+			/* collect alloc stat */
+			ma->stats.alloc.nb_call++;
+			ma->stats.alloc.nb_req += num;
+			ma->stats.alloc.nb_alloc += n;
+			ma->stats.alloc.nb_cycle += tm1 - tm0;
+
+			/* store allocated objects */
+			rte_ring_enqueue_bulk(ring, obj, n, NULL);
+		}
+
+		/* get some objects to free */
+		num = rte_rand() % RTE_DIM(obj);
+		n = rte_ring_count(ring);
+		num = (n >= ft) ? RTE_MIN(num, n) : 0;
+
+		/* perform free*/
+		if (num != 0) {
+
+			/* retrieve objects to free */
+			rte_ring_dequeue_bulk(ring, obj, num, NULL);
+
+			/* check and fill contents of freeing objects */
+			rc = check_fill_objs(obj, ma->worker.obj_size, num,
+				lc, 0);
+			if (rc != 0)
+				break;
+
+			tm0 = rte_rdtsc_precise();
+			tle_memtank_free(ma->mt, obj, num,
+				ma->worker.free_flags);
+			tm1 = rte_rdtsc_precise();
+
+			/* collect free stat */
+			ma->stats.free.nb_call++;
+			ma->stats.free.nb_free += num;
+			ma->stats.free.nb_cycle += tm1 - tm0;
+		}
+
+		rte_smp_mb();
+	} while (wrk_cmd == WRK_CMD_RUN);
+
+	ma->stats.nb_cycle = rte_rdtsc_precise() - cl;
+
+	return rc;
+}
+
+static int
+test_memtank_master(void *arg)
+{
+	struct memtank_arg *ma;
+	uint64_t cl, tm0, tm1, tm2;
+	uint32_t i, n;
+
+	ma = (struct memtank_arg *)arg;
+
+	for (cl = 0, i = 0; cl < ma->master.run_cycles;
+			cl += tm2 - tm0, i++)  {
+
+		tm0 = rte_rdtsc_precise();
+
+		if (ma->master.flags & MASTER_FLAG_SHRINK) {
+
+			n = tle_memtank_shrink(ma->mt);
+			tm1 = rte_rdtsc_precise();
+			ma->stats.shrink.nb_call++;
+			ma->stats.shrink.nb_chunk += n;
+			if (n != 0)
+				ma->stats.shrink.nb_cycle += tm1 - tm0;
+		}
+
+		if (ma->master.flags & MASTER_FLAG_GROW) {
+
+			tm1 = rte_rdtsc_precise();
+			n = tle_memtank_grow(ma->mt);
+			tm2 = rte_rdtsc_precise();
+			ma->stats.grow.nb_call++;
+			ma->stats.grow.nb_chunk += n;
+			if (n != 0)
+				ma->stats.grow.nb_cycle += tm2 - tm1;
+		}
+
+		wrk_cmd = WRK_CMD_RUN;
+		rte_smp_mb();
+
+		rte_delay_us(ma->master.delay_us);
+		tm2 = rte_rdtsc_precise();
+	}
+
+	ma->stats.nb_cycle = cl;
+
+	rte_smp_mb();
+	wrk_cmd = WRK_CMD_STOP;
+
+	return 0;
+}
+
+static void
+fill_worker_args(struct worker_args *wa, uint32_t alloc_flags,
+	uint32_t free_flags)
+{
+	wa->max_obj = global_cfg.wrk_max_obj;
+	wa->obj_size = mtnk_prm.obj_size;
+	wa->alloc_flags = alloc_flags;
+	wa->free_flags = free_flags;
+}
+
+static void
+fill_master_args(struct master_args *ma, uint32_t flags)
+{
+	uint64_t tm;
+
+	tm = global_cfg.run_time * rte_get_timer_hz();
+
+	ma->run_cycles = tm;
+	ma->delay_us = US_PER_S / MS_PER_S;
+	ma->flags = flags;
+}
+
+/*
+ * alloc/free by workers threads.
+ * grow/shrink by master
+ */
+static int
+test_memtank_mt1(void)
+{
+	int32_t rc;
+	uint32_t lc;
+	struct tle_memtank *mt;
+	struct tle_memtank_prm prm;
+	struct memstat ms;
+	struct memtank_arg arg[RTE_MAX_LCORE];
+
+	printf("%s start\n", __func__);
+
+	memset(&prm, 0, sizeof(prm));
+	memset(&ms, 0, sizeof(ms));
+
+	prm = mtnk_prm;
+	prm.alloc = test_alloc1;
+	prm.free = test_free1;
+	prm.udata = &ms;
+
+	mt = tle_memtank_create(&prm);
+	if (mt == NULL) {
+		printf("%s: memtank_create() failed\n", __func__);
+		return -ENOMEM;
+	}
+
+	memset(arg, 0, sizeof(arg));
+
+	/* launch on all slaves */
+	RTE_LCORE_FOREACH_SLAVE(lc) {
+		arg[lc].mt = mt;
+		fill_worker_args(&arg[lc].worker, 0, 0);
+		rte_eal_remote_launch(test_memtank_worker, &arg[lc], lc);
+	}
+
+	/* launch on master */
+	lc = rte_lcore_id();
+	arg[lc].mt = mt;
+	fill_master_args(&arg[lc].master,
+		MASTER_FLAG_GROW | MASTER_FLAG_SHRINK);
+	test_memtank_master(&arg[lc]);
+
+	/* wait for slaves and collect stats. */
+	rc = 0;
+	RTE_LCORE_FOREACH_SLAVE(lc) {
+		rc |= rte_eal_wait_lcore(lc);
+		memtank_stat_dump(stdout, lc, &arg[lc].stats);
+		ms.nb_alloc_obj += arg[lc].stats.alloc.nb_alloc -
+			arg[lc].stats.free.nb_free;
+	}
+
+	lc = rte_lcore_id();
+	memtank_stat_dump(stdout, lc, &arg[lc].stats);
+	tle_memtank_dump(stdout, mt, TLE_MTANK_DUMP_STAT);
+
+	memstat_dump(stdout, &ms);
+
+	rc |= tle_memtank_sanity_check(mt, 0);
+	tle_memtank_destroy(mt);
+	return rc;
+}
+
+/*
+ * alloc/free with grow/shrink by worker threads.
+ * master does nothing
+ */
+static int
+test_memtank_mt2(void)
+{
+	int32_t rc;
+	uint32_t lc;
+	struct tle_memtank *mt;
+	struct tle_memtank_prm prm;
+	struct memstat ms;
+	struct memtank_arg arg[RTE_MAX_LCORE];
+
+	const uint32_t alloc_flags = TLE_MTANK_ALLOC_CHUNK |
+				TLE_MTANK_ALLOC_GROW;
+	const uint32_t free_flags = TLE_MTANK_FREE_SHRINK;
+
+	printf("%s start\n", __func__);
+
+	memset(&prm, 0, sizeof(prm));
+	memset(&ms, 0, sizeof(ms));
+
+	prm = mtnk_prm;
+	prm.alloc = test_alloc1;
+	prm.free = test_free1;
+	prm.udata = &ms;
+
+	mt = tle_memtank_create(&prm);
+	if (mt == NULL) {
+		printf("%s: memtank_create() failed\n", __func__);
+		return -ENOMEM;
+	}
+
+	memset(arg, 0, sizeof(arg));
+
+	/* launch on all slaves */
+	RTE_LCORE_FOREACH_SLAVE(lc) {
+		arg[lc].mt = mt;
+		fill_worker_args(&arg[lc].worker, alloc_flags, free_flags);
+		rte_eal_remote_launch(test_memtank_worker, &arg[lc], lc);
+	}
+
+	/* launch on master */
+	lc = rte_lcore_id();
+	arg[lc].mt = mt;
+	fill_master_args(&arg[lc].master, 0);
+	test_memtank_master(&arg[lc]);
+
+	/* wait for slaves and collect stats. */
+	rc = 0;
+	RTE_LCORE_FOREACH_SLAVE(lc) {
+		rc |= rte_eal_wait_lcore(lc);
+		memtank_stat_dump(stdout, lc, &arg[lc].stats);
+		ms.nb_alloc_obj += arg[lc].stats.alloc.nb_alloc -
+			arg[lc].stats.free.nb_free;
+	}
+
+	lc = rte_lcore_id();
+	memtank_stat_dump(stdout, lc, &arg[lc].stats);
+	tle_memtank_dump(stdout, mt, TLE_MTANK_DUMP_STAT);
+
+	memstat_dump(stdout, &ms);
+
+	rc |= tle_memtank_sanity_check(mt, 0);
+	tle_memtank_destroy(mt);
+	return rc;
+}
+
+static int
+parse_uint_val(const char *str, uint32_t *val, uint32_t min, uint32_t max)
+{
+	unsigned long v;
+	char *end;
+
+	errno = 0;
+	v = strtoul(str, &end, 0);
+	if (errno != 0 || end[0] != 0 || v < min || v > max)
+		return -EINVAL;
+
+	val[0] = v;
+	return 0;
+}
+
+static int
+parse_mem_str(const char *str)
+{
+	uint32_t i;
+
+	static const struct {
+		const char *name;
+		int32_t val;
+	} name2val[] = {
+		{
+			.name = "sys",
+			.val = MEM_FUNC_SYS,
+		},
+		{
+			.name = "rte",
+			.val = MEM_FUNC_RTE,
+		},
+	};
+
+	for (i = 0; i != RTE_DIM(name2val); i++) {
+		if (strcmp(str, name2val[i].name) == 0)
+			return name2val[i].val;
+	}
+	return -EINVAL;
+}
+
+static int
+parse_opt(int argc, char * const argv[])
+{
+	int32_t opt, rc;
+	uint32_t v;
+
+	rc = 0;
+	optind = 0;
+	optarg = NULL;
+
+	while ((opt = getopt(argc, argv, "f:m:s:t:w:")) != EOF) {
+		switch (opt) {
+		case 'f':
+			rc = parse_uint_val(optarg, &v, FREE_THRSH_MIN,
+				FREE_THRSH_MAX);
+			if (rc == 0)
+				global_cfg.wrk_free_thrsh = v;
+			break;
+		case 'm':
+			rc = parse_mem_str(optarg);
+			if (rc >= 0)
+				global_cfg.mem_func = rc;
+			break;
+		case 's':
+			rc = parse_uint_val(optarg, &v, OBJ_SZ_MIN,
+				OBJ_SZ_MAX);
+			if (rc == 0)
+				mtnk_prm.obj_size = v;
+			break;
+		case 't':
+			rc = parse_uint_val(optarg, &v, 0, UINT32_MAX);
+			if (rc == 0)
+				global_cfg.run_time = v;
+			break;
+		case 'w':
+			rc = parse_uint_val(optarg, &v, 0, UINT32_MAX);
+			if (rc == 0)
+				global_cfg.wrk_max_obj = v;
+			break;
+		default:
+			rc = -EINVAL;
+		}
+	}
+
+	if (rc < 0)
+		printf("%s: invalid value: \"%s\" for option: \'%c\'\n",
+			__func__, optarg, opt);
+	return  rc;
+}
+
+int
+main(int argc, char * argv[])
+{
+	int32_t rc;
+	uint32_t i, k;
+
+	const struct {
+		const char *name;
+		int (*func)(void);
+	} tests[] = {
+		{
+			.name = "MT1-WRK_ALLOC_FREE-MST_GROW_SHRINK",
+			.func = test_memtank_mt1,
+		},
+		{
+			.name = "MT1-WRK_ALLOC+GROW_FREE+SHRINK",
+			.func = test_memtank_mt2,
+		},
+	};
+		
+
+	rc = rte_eal_init(argc, argv);
+	if (rc < 0)
+		rte_exit(EXIT_FAILURE,
+			"%s: rte_eal_init failed with error code: %d\n",
+			__func__, rc);
+
+	rc = parse_opt(argc - rc, argv + rc);
+	if (rc < 0)
+		rte_exit(EXIT_FAILURE,
+			"%s: parse_op failed with error code: %d\n",
+			__func__, rc);
+
+	for (i = 0, k = 0; i != RTE_DIM(tests); i++) {
+
+		printf("TEST %s START\n", tests[i].name);
+
+		rc = tests[i].func();
+		k += (rc == 0);
+
+		if (rc != 0)
+			printf("TEST %s FAILED\n", tests[i].name);
+		else
+			printf("TEST %s OK\n", tests[i].name);
+	}
+
+	printf("Number of tests:\t%u\nSuccess:\t%u\nFailed:\t%u\n",
+		i, k, i - k);
+	return (k != i);
+}