diff options
Diffstat (limited to 'lib/librte_mempool')
| -rw-r--r-- | lib/librte_mempool/Makefile | 3 | ||||
| -rw-r--r-- | lib/librte_mempool/rte_mempool.c | 177 | ||||
| -rw-r--r-- | lib/librte_mempool/rte_mempool.h | 194 | ||||
| -rw-r--r-- | lib/librte_mempool/rte_mempool_ops.c | 29 | ||||
| -rw-r--r-- | lib/librte_mempool/rte_mempool_version.map | 10 |
5 files changed, 296 insertions, 117 deletions
diff --git a/lib/librte_mempool/Makefile b/lib/librte_mempool/Makefile index 7b5bdfee..46654e32 100644 --- a/lib/librte_mempool/Makefile +++ b/lib/librte_mempool/Makefile @@ -35,10 +35,11 @@ include $(RTE_SDK)/mk/rte.vars.mk LIB = librte_mempool.a CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 +LDLIBS += -lrte_eal -lrte_ring EXPORT_MAP := rte_mempool_version.map -LIBABIVER := 2 +LIBABIVER := 3 # all source are stored in SRCS-y SRCS-$(CONFIG_RTE_LIBRTE_MEMPOOL) += rte_mempool.c diff --git a/lib/librte_mempool/rte_mempool.c b/lib/librte_mempool/rte_mempool.c index 6fc3c9c7..d50dba49 100644 --- a/lib/librte_mempool/rte_mempool.c +++ b/lib/librte_mempool/rte_mempool.c @@ -128,7 +128,7 @@ static unsigned optimize_object_size(unsigned obj_size) } static void -mempool_add_elem(struct rte_mempool *mp, void *obj, phys_addr_t physaddr) +mempool_add_elem(struct rte_mempool *mp, void *obj, rte_iova_t iova) { struct rte_mempool_objhdr *hdr; struct rte_mempool_objtlr *tlr __rte_unused; @@ -136,7 +136,7 @@ mempool_add_elem(struct rte_mempool *mp, void *obj, phys_addr_t physaddr) /* set mempool ptr in header */ hdr = RTE_PTR_SUB(obj, sizeof(*hdr)); hdr->mp = mp; - hdr->physaddr = physaddr; + hdr->iova = iova; STAILQ_INSERT_TAIL(&mp->elt_list, hdr, next); mp->populated_size++; @@ -238,9 +238,16 @@ rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags, * Calculate maximum amount of memory required to store given number of objects. */ size_t -rte_mempool_xmem_size(uint32_t elt_num, size_t total_elt_sz, uint32_t pg_shift) +rte_mempool_xmem_size(uint32_t elt_num, size_t total_elt_sz, uint32_t pg_shift, + unsigned int flags) { size_t obj_per_page, pg_num, pg_sz; + unsigned int mask; + + mask = MEMPOOL_F_CAPA_BLK_ALIGNED_OBJECTS | MEMPOOL_F_CAPA_PHYS_CONTIG; + if ((flags & mask) == mask) + /* alignment need one additional object */ + elt_num += 1; if (total_elt_sz == 0) return 0; @@ -263,23 +270,29 @@ rte_mempool_xmem_size(uint32_t elt_num, size_t total_elt_sz, uint32_t pg_shift) */ ssize_t rte_mempool_xmem_usage(__rte_unused void *vaddr, uint32_t elt_num, - size_t total_elt_sz, const phys_addr_t paddr[], uint32_t pg_num, - uint32_t pg_shift) + size_t total_elt_sz, const rte_iova_t iova[], uint32_t pg_num, + uint32_t pg_shift, unsigned int flags) { uint32_t elt_cnt = 0; - phys_addr_t start, end; - uint32_t paddr_idx; + rte_iova_t start, end; + uint32_t iova_idx; size_t pg_sz = (size_t)1 << pg_shift; + unsigned int mask; + + mask = MEMPOOL_F_CAPA_BLK_ALIGNED_OBJECTS | MEMPOOL_F_CAPA_PHYS_CONTIG; + if ((flags & mask) == mask) + /* alignment need one additional object */ + elt_num += 1; - /* if paddr is NULL, assume contiguous memory */ - if (paddr == NULL) { + /* if iova is NULL, assume contiguous memory */ + if (iova == NULL) { start = 0; end = pg_sz * pg_num; - paddr_idx = pg_num; + iova_idx = pg_num; } else { - start = paddr[0]; - end = paddr[0] + pg_sz; - paddr_idx = 1; + start = iova[0]; + end = iova[0] + pg_sz; + iova_idx = 1; } while (elt_cnt < elt_num) { @@ -287,15 +300,15 @@ rte_mempool_xmem_usage(__rte_unused void *vaddr, uint32_t elt_num, /* enough contiguous memory, add an object */ start += total_elt_sz; elt_cnt++; - } else if (paddr_idx < pg_num) { + } else if (iova_idx < pg_num) { /* no room to store one obj, add a page */ - if (end == paddr[paddr_idx]) { + if (end == iova[iova_idx]) { end += pg_sz; } else { - start = paddr[paddr_idx]; - end = paddr[paddr_idx] + pg_sz; + start = iova[iova_idx]; + end = iova[iova_idx] + pg_sz; } - paddr_idx++; + iova_idx++; } else { /* no more page, return how many elements fit */ @@ -303,7 +316,7 @@ rte_mempool_xmem_usage(__rte_unused void *vaddr, uint32_t elt_num, } } - return (size_t)paddr_idx << pg_shift; + return (size_t)iova_idx << pg_shift; } /* free a memchunk allocated with rte_memzone_reserve() */ @@ -344,8 +357,8 @@ rte_mempool_free_memchunks(struct rte_mempool *mp) * on error. */ int -rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, - phys_addr_t paddr, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, +rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr, + rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque) { unsigned total_elt_sz; @@ -354,6 +367,11 @@ rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, struct rte_mempool_memhdr *memhdr; int ret; + /* Notify memory area to mempool */ + ret = rte_mempool_ops_register_memory_area(mp, vaddr, iova, len); + if (ret != -ENOTSUP && ret < 0) + return ret; + /* create the internal ring if not already done */ if ((mp->flags & MEMPOOL_F_POOL_CREATED) == 0) { ret = rte_mempool_ops_alloc(mp); @@ -368,29 +386,42 @@ rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size; + /* Detect pool area has sufficient space for elements */ + if (mp->flags & MEMPOOL_F_CAPA_PHYS_CONTIG) { + if (len < total_elt_sz * mp->size) { + RTE_LOG(ERR, MEMPOOL, + "pool area %" PRIx64 " not enough\n", + (uint64_t)len); + return -ENOSPC; + } + } + memhdr = rte_zmalloc("MEMPOOL_MEMHDR", sizeof(*memhdr), 0); if (memhdr == NULL) return -ENOMEM; memhdr->mp = mp; memhdr->addr = vaddr; - memhdr->phys_addr = paddr; + memhdr->iova = iova; memhdr->len = len; memhdr->free_cb = free_cb; memhdr->opaque = opaque; - if (mp->flags & MEMPOOL_F_NO_CACHE_ALIGN) + if (mp->flags & MEMPOOL_F_CAPA_BLK_ALIGNED_OBJECTS) + /* align object start address to a multiple of total_elt_sz */ + off = total_elt_sz - ((uintptr_t)vaddr % total_elt_sz); + else if (mp->flags & MEMPOOL_F_NO_CACHE_ALIGN) off = RTE_PTR_ALIGN_CEIL(vaddr, 8) - vaddr; else off = RTE_PTR_ALIGN_CEIL(vaddr, RTE_CACHE_LINE_SIZE) - vaddr; while (off + total_elt_sz <= len && mp->populated_size < mp->size) { off += mp->header_size; - if (paddr == RTE_BAD_PHYS_ADDR) + if (iova == RTE_BAD_IOVA) mempool_add_elem(mp, (char *)vaddr + off, - RTE_BAD_PHYS_ADDR); + RTE_BAD_IOVA); else - mempool_add_elem(mp, (char *)vaddr + off, paddr + off); + mempool_add_elem(mp, (char *)vaddr + off, iova + off); off += mp->elt_size + mp->trailer_size; i++; } @@ -404,12 +435,20 @@ rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, return i; } +int +rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, + phys_addr_t paddr, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, + void *opaque) +{ + return rte_mempool_populate_iova(mp, vaddr, paddr, len, free_cb, opaque); +} + /* Add objects in the pool, using a table of physical pages. Return the * number of objects added, or a negative value on error. */ int -rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr, - const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift, +rte_mempool_populate_iova_tab(struct rte_mempool *mp, char *vaddr, + const rte_iova_t iova[], uint32_t pg_num, uint32_t pg_shift, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque) { uint32_t i, n; @@ -421,18 +460,18 @@ rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr, return -EEXIST; if (mp->flags & MEMPOOL_F_NO_PHYS_CONTIG) - return rte_mempool_populate_phys(mp, vaddr, RTE_BAD_PHYS_ADDR, + return rte_mempool_populate_iova(mp, vaddr, RTE_BAD_IOVA, pg_num * pg_sz, free_cb, opaque); for (i = 0; i < pg_num && mp->populated_size < mp->size; i += n) { /* populate with the largest group of contiguous pages */ for (n = 1; (i + n) < pg_num && - paddr[i + n - 1] + pg_sz == paddr[i + n]; n++) + iova[i + n - 1] + pg_sz == iova[i + n]; n++) ; - ret = rte_mempool_populate_phys(mp, vaddr + i * pg_sz, - paddr[i], n * pg_sz, free_cb, opaque); + ret = rte_mempool_populate_iova(mp, vaddr + i * pg_sz, + iova[i], n * pg_sz, free_cb, opaque); if (ret < 0) { rte_mempool_free_memchunks(mp); return ret; @@ -444,6 +483,15 @@ rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr, return cnt; } +int +rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr, + const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift, + rte_mempool_memchunk_free_cb_t *free_cb, void *opaque) +{ + return rte_mempool_populate_iova_tab(mp, vaddr, paddr, pg_num, pg_shift, + free_cb, opaque); +} + /* Populate the mempool with a virtual area. Return the number of * objects added, or a negative value on error. */ @@ -452,7 +500,7 @@ rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque) { - phys_addr_t paddr; + rte_iova_t iova; size_t off, phys_len; int ret, cnt = 0; @@ -466,33 +514,30 @@ rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, return -EINVAL; if (mp->flags & MEMPOOL_F_NO_PHYS_CONTIG) - return rte_mempool_populate_phys(mp, addr, RTE_BAD_PHYS_ADDR, + return rte_mempool_populate_iova(mp, addr, RTE_BAD_IOVA, len, free_cb, opaque); for (off = 0; off + pg_sz <= len && mp->populated_size < mp->size; off += phys_len) { - paddr = rte_mem_virt2phy(addr + off); - /* required for xen_dom0 to get the machine address */ - paddr = rte_mem_phy2mch(-1, paddr); + iova = rte_mem_virt2iova(addr + off); - if (paddr == RTE_BAD_PHYS_ADDR && rte_eal_has_hugepages()) { + if (iova == RTE_BAD_IOVA && rte_eal_has_hugepages()) { ret = -EINVAL; goto fail; } /* populate with the largest group of contiguous pages */ for (phys_len = pg_sz; off + phys_len < len; phys_len += pg_sz) { - phys_addr_t paddr_tmp; + rte_iova_t iova_tmp; - paddr_tmp = rte_mem_virt2phy(addr + off + phys_len); - paddr_tmp = rte_mem_phy2mch(-1, paddr_tmp); + iova_tmp = rte_mem_virt2iova(addr + off + phys_len); - if (paddr_tmp != paddr + phys_len) + if (iova_tmp != iova + phys_len) break; } - ret = rte_mempool_populate_phys(mp, addr + off, paddr, + ret = rte_mempool_populate_iova(mp, addr + off, iova, phys_len, free_cb, opaque); if (ret < 0) goto fail; @@ -515,23 +560,29 @@ rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, int rte_mempool_populate_default(struct rte_mempool *mp) { - int mz_flags = RTE_MEMZONE_1GB|RTE_MEMZONE_SIZE_HINT_ONLY; + unsigned int mz_flags = RTE_MEMZONE_1GB|RTE_MEMZONE_SIZE_HINT_ONLY; char mz_name[RTE_MEMZONE_NAMESIZE]; const struct rte_memzone *mz; size_t size, total_elt_sz, align, pg_sz, pg_shift; - phys_addr_t paddr; + rte_iova_t iova; unsigned mz_id, n; + unsigned int mp_flags; int ret; /* mempool must not be populated */ if (mp->nb_mem_chunks != 0) return -EEXIST; - if (rte_xen_dom0_supported()) { - pg_sz = RTE_PGSIZE_2M; - pg_shift = rte_bsf32(pg_sz); - align = pg_sz; - } else if (rte_eal_has_hugepages()) { + /* Get mempool capabilities */ + mp_flags = 0; + ret = rte_mempool_ops_get_capabilities(mp, &mp_flags); + if ((ret < 0) && (ret != -ENOTSUP)) + return ret; + + /* update mempool capabilities */ + mp->flags |= mp_flags; + + if (rte_eal_has_hugepages()) { pg_shift = 0; /* not needed, zone is physically contiguous */ pg_sz = 0; align = RTE_CACHE_LINE_SIZE; @@ -543,7 +594,8 @@ rte_mempool_populate_default(struct rte_mempool *mp) total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size; for (mz_id = 0, n = mp->size; n > 0; mz_id++, n -= ret) { - size = rte_mempool_xmem_size(n, total_elt_sz, pg_shift); + size = rte_mempool_xmem_size(n, total_elt_sz, pg_shift, + mp->flags); ret = snprintf(mz_name, sizeof(mz_name), RTE_MEMPOOL_MZ_FORMAT "_%d", mp->name, mz_id); @@ -564,13 +616,13 @@ rte_mempool_populate_default(struct rte_mempool *mp) } if (mp->flags & MEMPOOL_F_NO_PHYS_CONTIG) - paddr = RTE_BAD_PHYS_ADDR; + iova = RTE_BAD_IOVA; else - paddr = mz->phys_addr; + iova = mz->iova; - if (rte_eal_has_hugepages() && !rte_xen_dom0_supported()) - ret = rte_mempool_populate_phys(mp, mz->addr, - paddr, mz->len, + if (rte_eal_has_hugepages()) + ret = rte_mempool_populate_iova(mp, mz->addr, + iova, mz->len, rte_mempool_memchunk_mz_free, (void *)(uintptr_t)mz); else @@ -600,7 +652,8 @@ get_anon_size(const struct rte_mempool *mp) pg_sz = getpagesize(); pg_shift = rte_bsf32(pg_sz); total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size; - size = rte_mempool_xmem_size(mp->size, total_elt_sz, pg_shift); + size = rte_mempool_xmem_size(mp->size, total_elt_sz, pg_shift, + mp->flags); return size; } @@ -742,7 +795,7 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, struct rte_tailq_entry *te = NULL; const struct rte_memzone *mz = NULL; size_t mempool_size; - int mz_flags = RTE_MEMZONE_1GB|RTE_MEMZONE_SIZE_HINT_ONLY; + unsigned int mz_flags = RTE_MEMZONE_1GB|RTE_MEMZONE_SIZE_HINT_ONLY; struct rte_mempool_objsz objsz; unsigned lcore_id; int ret; @@ -922,7 +975,7 @@ rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size, rte_mempool_ctor_t *mp_init, void *mp_init_arg, rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, int socket_id, unsigned flags, void *vaddr, - const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift) + const rte_iova_t iova[], uint32_t pg_num, uint32_t pg_shift) { struct rte_mempool *mp = NULL; int ret; @@ -934,7 +987,7 @@ rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size, obj_init, obj_init_arg, socket_id, flags); /* check that we have both VA and PA */ - if (paddr == NULL) { + if (iova == NULL) { rte_errno = EINVAL; return NULL; } @@ -954,7 +1007,7 @@ rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size, if (mp_init) mp_init(mp, mp_init_arg); - ret = rte_mempool_populate_phys_tab(mp, vaddr, paddr, pg_num, pg_shift, + ret = rte_mempool_populate_iova_tab(mp, vaddr, iova, pg_num, pg_shift, NULL, NULL); if (ret < 0 || ret != (int)mp->size) goto fail; @@ -1177,7 +1230,7 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp) fprintf(f, "mempool <%s>@%p\n", mp->name, mp); fprintf(f, " flags=%x\n", mp->flags); fprintf(f, " pool=%p\n", mp->pool_data); - fprintf(f, " phys_addr=0x%" PRIx64 "\n", mp->mz->phys_addr); + fprintf(f, " iova=0x%" PRIx64 "\n", mp->mz->iova); fprintf(f, " nb_mem_chunks=%u\n", mp->nb_mem_chunks); fprintf(f, " size=%"PRIu32"\n", mp->size); fprintf(f, " populated_size=%"PRIu32"\n", mp->populated_size); diff --git a/lib/librte_mempool/rte_mempool.h b/lib/librte_mempool/rte_mempool.h index 76b5b3b1..721227f6 100644 --- a/lib/librte_mempool/rte_mempool.h +++ b/lib/librte_mempool/rte_mempool.h @@ -157,7 +157,11 @@ struct rte_mempool_objsz { struct rte_mempool_objhdr { STAILQ_ENTRY(rte_mempool_objhdr) next; /**< Next in list. */ struct rte_mempool *mp; /**< The mempool owning the object. */ - phys_addr_t physaddr; /**< Physical address of the object. */ + RTE_STD_C11 + union { + rte_iova_t iova; /**< IO address of the object. */ + phys_addr_t physaddr; /**< deprecated - Physical address of the object. */ + }; #ifdef RTE_LIBRTE_MEMPOOL_DEBUG uint64_t cookie; /**< Debug cookie. */ #endif @@ -203,7 +207,11 @@ struct rte_mempool_memhdr { STAILQ_ENTRY(rte_mempool_memhdr) next; /**< Next in list. */ struct rte_mempool *mp; /**< The mempool owning the chunk */ void *addr; /**< Virtual address of the chunk */ - phys_addr_t phys_addr; /**< Physical address of the chunk */ + RTE_STD_C11 + union { + rte_iova_t iova; /**< IO address of the chunk */ + phys_addr_t phys_addr; /**< Physical address of the chunk */ + }; size_t len; /**< length of the chunk */ rte_mempool_memchunk_free_cb_t *free_cb; /**< Free callback */ void *opaque; /**< Argument passed to the free callback */ @@ -226,7 +234,7 @@ struct rte_mempool { }; void *pool_config; /**< optional args for ops alloc. */ const struct rte_memzone *mz; /**< Memzone where pool is alloc'd. */ - int flags; /**< Flags of the mempool. */ + unsigned int flags; /**< Flags of the mempool. */ int socket_id; /**< Socket id passed at create. */ uint32_t size; /**< Max size of the mempool. */ uint32_t cache_size; @@ -265,6 +273,24 @@ struct rte_mempool { #define MEMPOOL_F_SC_GET 0x0008 /**< Default get is "single-consumer".*/ #define MEMPOOL_F_POOL_CREATED 0x0010 /**< Internal: pool is created. */ #define MEMPOOL_F_NO_PHYS_CONTIG 0x0020 /**< Don't need physically contiguous objs. */ +/** + * This capability flag is advertised by a mempool handler, if the whole + * memory area containing the objects must be physically contiguous. + * Note: This flag should not be passed by application. + */ +#define MEMPOOL_F_CAPA_PHYS_CONTIG 0x0040 +/** + * This capability flag is advertised by a mempool handler. Used for a case + * where mempool driver wants object start address(vaddr) aligned to block + * size(/ total element size). + * + * Note: + * - This flag should not be passed by application. + * Flag used for mempool driver only. + * - Mempool driver must also set MEMPOOL_F_CAPA_PHYS_CONTIG flag along with + * MEMPOOL_F_CAPA_BLK_ALIGNED_OBJECTS. + */ +#define MEMPOOL_F_CAPA_BLK_ALIGNED_OBJECTS 0x0080 /** * @internal When debug is enabled, store some statistics. @@ -389,6 +415,18 @@ typedef int (*rte_mempool_dequeue_t)(struct rte_mempool *mp, */ typedef unsigned (*rte_mempool_get_count)(const struct rte_mempool *mp); +/** + * Get the mempool capabilities. + */ +typedef int (*rte_mempool_get_capabilities_t)(const struct rte_mempool *mp, + unsigned int *flags); + +/** + * Notify new memory area to mempool. + */ +typedef int (*rte_mempool_ops_register_memory_area_t) +(const struct rte_mempool *mp, char *vaddr, rte_iova_t iova, size_t len); + /** Structure defining mempool operations structure */ struct rte_mempool_ops { char name[RTE_MEMPOOL_OPS_NAMESIZE]; /**< Name of mempool ops struct. */ @@ -397,6 +435,14 @@ struct rte_mempool_ops { rte_mempool_enqueue_t enqueue; /**< Enqueue an object. */ rte_mempool_dequeue_t dequeue; /**< Dequeue an object. */ rte_mempool_get_count get_count; /**< Get qty of available objs. */ + /** + * Get the mempool capabilities + */ + rte_mempool_get_capabilities_t get_capabilities; + /** + * Notify new memory area to mempool + */ + rte_mempool_ops_register_memory_area_t register_memory_area; } __rte_cache_aligned; #define RTE_MEMPOOL_MAX_OPS_IDX 16 /**< Max registered ops structs */ @@ -509,6 +555,43 @@ unsigned rte_mempool_ops_get_count(const struct rte_mempool *mp); /** + * @internal wrapper for mempool_ops get_capabilities callback. + * + * @param mp [in] + * Pointer to the memory pool. + * @param flags [out] + * Pointer to the mempool flags. + * @return + * - 0: Success; The mempool driver has advertised his pool capabilities in + * flags param. + * - -ENOTSUP - doesn't support get_capabilities ops (valid case). + * - Otherwise, pool create fails. + */ +int +rte_mempool_ops_get_capabilities(const struct rte_mempool *mp, + unsigned int *flags); +/** + * @internal wrapper for mempool_ops register_memory_area callback. + * API to notify the mempool handler when a new memory area is added to pool. + * + * @param mp + * Pointer to the memory pool. + * @param vaddr + * Pointer to the buffer virtual address. + * @param iova + * Pointer to the buffer IO address. + * @param len + * Pool size. + * @return + * - 0: Success; + * - -ENOTSUP - doesn't support register_memory_area ops (valid error case). + * - Otherwise, rte_mempool_populate_phys fails thus pool create fails. + */ +int +rte_mempool_ops_register_memory_area(const struct rte_mempool *mp, + char *vaddr, rte_iova_t iova, size_t len); + +/** * @internal wrapper for mempool_ops free callback. * * @param mp @@ -722,11 +805,10 @@ rte_mempool_create(const char *name, unsigned n, unsigned elt_size, * @param vaddr * Virtual address of the externally allocated memory buffer. * Will be used to store mempool objects. - * @param paddr - * Array of physical addresses of the pages that comprises given memory - * buffer. + * @param iova + * Array of IO addresses of the pages that comprises given memory buffer. * @param pg_num - * Number of elements in the paddr array. + * Number of elements in the iova array. * @param pg_shift * LOG2 of the physical pages size. * @return @@ -739,7 +821,7 @@ rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size, rte_mempool_ctor_t *mp_init, void *mp_init_arg, rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, int socket_id, unsigned flags, void *vaddr, - const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift); + const rte_iova_t iova[], uint32_t pg_num, uint32_t pg_shift); /** * Create an empty mempool @@ -798,7 +880,7 @@ rte_mempool_free(struct rte_mempool *mp); * Add a virtually and physically contiguous memory chunk in the pool * where objects can be instantiated. * - * If the given physical address is unknown (paddr = RTE_BAD_PHYS_ADDR), + * If the given IO address is unknown (iova = RTE_BAD_IOVA), * the chunk doesn't need to be physically contiguous (only virtually), * and allocated objects may span two pages. * @@ -806,8 +888,8 @@ rte_mempool_free(struct rte_mempool *mp); * A pointer to the mempool structure. * @param vaddr * The virtual address of memory that should be used to store objects. - * @param paddr - * The physical address + * @param iova + * The IO address * @param len * The length of memory in bytes. * @param free_cb @@ -819,6 +901,11 @@ rte_mempool_free(struct rte_mempool *mp); * On error, the chunk is not added in the memory list of the * mempool and a negative errno is returned. */ +int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr, + rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, + void *opaque); + +__rte_deprecated int rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, phys_addr_t paddr, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque); @@ -827,18 +914,17 @@ int rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, * Add physical memory for objects in the pool at init * * Add a virtually contiguous memory chunk in the pool where objects can - * be instantiated. The physical addresses corresponding to the virtual - * area are described in paddr[], pg_num, pg_shift. + * be instantiated. The IO addresses corresponding to the virtual + * area are described in iova[], pg_num, pg_shift. * * @param mp * A pointer to the mempool structure. * @param vaddr * The virtual address of memory that should be used to store objects. - * @param paddr - * An array of physical addresses of each page composing the virtual - * area. + * @param iova + * An array of IO addresses of each page composing the virtual area. * @param pg_num - * Number of elements in the paddr array. + * Number of elements in the iova array. * @param pg_shift * LOG2 of the physical pages size. * @param free_cb @@ -850,6 +936,11 @@ int rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, * On error, the chunks are not added in the memory list of the * mempool and a negative errno is returned. */ +int rte_mempool_populate_iova_tab(struct rte_mempool *mp, char *vaddr, + const rte_iova_t iova[], uint32_t pg_num, uint32_t pg_shift, + rte_mempool_memchunk_free_cb_t *free_cb, void *opaque); + +__rte_deprecated int rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr, const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque); @@ -1034,13 +1125,10 @@ rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id) * positive. * @param cache * A pointer to a mempool cache structure. May be NULL if not needed. - * @param flags - * The flags used for the mempool creation. - * Single-producer (MEMPOOL_F_SP_PUT flag) or multi-producers. */ static __rte_always_inline void __mempool_generic_put(struct rte_mempool *mp, void * const *obj_table, - unsigned n, struct rte_mempool_cache *cache) + unsigned int n, struct rte_mempool_cache *cache) { void **cache_objs; @@ -1096,14 +1184,10 @@ ring_enqueue: * The number of objects to add in the mempool from the obj_table. * @param cache * A pointer to a mempool cache structure. May be NULL if not needed. - * @param flags - * The flags used for the mempool creation. - * Single-producer (MEMPOOL_F_SP_PUT flag) or multi-producers. */ static __rte_always_inline void rte_mempool_generic_put(struct rte_mempool *mp, void * const *obj_table, - unsigned n, struct rte_mempool_cache *cache, - __rte_unused int flags) + unsigned int n, struct rte_mempool_cache *cache) { __mempool_check_cookies(mp, obj_table, n, 0); __mempool_generic_put(mp, obj_table, n, cache); @@ -1125,11 +1209,11 @@ rte_mempool_generic_put(struct rte_mempool *mp, void * const *obj_table, */ static __rte_always_inline void rte_mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table, - unsigned n) + unsigned int n) { struct rte_mempool_cache *cache; cache = rte_mempool_default_cache(mp, rte_lcore_id()); - rte_mempool_generic_put(mp, obj_table, n, cache, mp->flags); + rte_mempool_generic_put(mp, obj_table, n, cache); } /** @@ -1160,16 +1244,13 @@ rte_mempool_put(struct rte_mempool *mp, void *obj) * The number of objects to get, must be strictly positive. * @param cache * A pointer to a mempool cache structure. May be NULL if not needed. - * @param flags - * The flags used for the mempool creation. - * Single-consumer (MEMPOOL_F_SC_GET flag) or multi-consumers. * @return * - >=0: Success; number of objects supplied. * - <0: Error; code of ring dequeue function. */ static __rte_always_inline int __mempool_generic_get(struct rte_mempool *mp, void **obj_table, - unsigned n, struct rte_mempool_cache *cache) + unsigned int n, struct rte_mempool_cache *cache) { int ret; uint32_t index, len; @@ -1241,16 +1322,13 @@ ring_dequeue: * The number of objects to get from mempool to obj_table. * @param cache * A pointer to a mempool cache structure. May be NULL if not needed. - * @param flags - * The flags used for the mempool creation. - * Single-consumer (MEMPOOL_F_SC_GET flag) or multi-consumers. * @return * - 0: Success; objects taken. * - -ENOENT: Not enough entries in the mempool; no object is retrieved. */ static __rte_always_inline int -rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table, unsigned n, - struct rte_mempool_cache *cache, __rte_unused int flags) +rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table, + unsigned int n, struct rte_mempool_cache *cache) { int ret; ret = __mempool_generic_get(mp, obj_table, n, cache); @@ -1282,11 +1360,11 @@ rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table, unsigned n, * - -ENOENT: Not enough entries in the mempool; no object is retrieved. */ static __rte_always_inline int -rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned n) +rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n) { struct rte_mempool_cache *cache; cache = rte_mempool_default_cache(mp, rte_lcore_id()); - return rte_mempool_generic_get(mp, obj_table, n, cache, mp->flags); + return rte_mempool_generic_get(mp, obj_table, n, cache); } /** @@ -1383,24 +1461,29 @@ rte_mempool_empty(const struct rte_mempool *mp) } /** - * Return the physical address of elt, which is an element of the pool mp. + * Return the IO address of elt, which is an element of the pool mp. * - * @param mp - * A pointer to the mempool structure. * @param elt * A pointer (virtual address) to the element of the pool. * @return - * The physical address of the elt element. + * The IO address of the elt element. * If the mempool was created with MEMPOOL_F_NO_PHYS_CONTIG, the - * returned value is RTE_BAD_PHYS_ADDR. + * returned value is RTE_BAD_IOVA. */ -static inline phys_addr_t -rte_mempool_virt2phy(__rte_unused const struct rte_mempool *mp, const void *elt) +static inline rte_iova_t +rte_mempool_virt2iova(const void *elt) { const struct rte_mempool_objhdr *hdr; hdr = (const struct rte_mempool_objhdr *)RTE_PTR_SUB(elt, sizeof(*hdr)); - return hdr->physaddr; + return hdr->iova; +} + +__rte_deprecated +static inline phys_addr_t +rte_mempool_virt2phy(__rte_unused const struct rte_mempool *mp, const void *elt) +{ + return rte_mempool_virt2iova(elt); } /** @@ -1489,11 +1572,13 @@ uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags, * by rte_mempool_calc_obj_size(). * @param pg_shift * LOG2 of the physical pages size. If set to 0, ignore page boundaries. + * @param flags + * The mempool flags. * @return * Required memory size aligned at page boundary. */ size_t rte_mempool_xmem_size(uint32_t elt_num, size_t total_elt_sz, - uint32_t pg_shift); + uint32_t pg_shift, unsigned int flags); /** * Get the size of memory required to store mempool elements. @@ -1509,13 +1594,14 @@ size_t rte_mempool_xmem_size(uint32_t elt_num, size_t total_elt_sz, * @param total_elt_sz * The size of each element, including header and trailer, as returned * by rte_mempool_calc_obj_size(). - * @param paddr - * Array of physical addresses of the pages that comprises given memory - * buffer. + * @param iova + * Array of IO addresses of the pages that comprises given memory buffer. * @param pg_num - * Number of elements in the paddr array. + * Number of elements in the iova array. * @param pg_shift * LOG2 of the physical pages size. + * @param flags + * The mempool flags. * @return * On success, the number of bytes needed to store given number of * objects, aligned to the given page size. If the provided memory @@ -1523,8 +1609,8 @@ size_t rte_mempool_xmem_size(uint32_t elt_num, size_t total_elt_sz, * is the actual number of elements that can be stored in that buffer. */ ssize_t rte_mempool_xmem_usage(void *vaddr, uint32_t elt_num, - size_t total_elt_sz, const phys_addr_t paddr[], uint32_t pg_num, - uint32_t pg_shift); + size_t total_elt_sz, const rte_iova_t iova[], uint32_t pg_num, + uint32_t pg_shift, unsigned int flags); /** * Walk list of all memory pools diff --git a/lib/librte_mempool/rte_mempool_ops.c b/lib/librte_mempool/rte_mempool_ops.c index 5f24de25..92b9f90c 100644 --- a/lib/librte_mempool/rte_mempool_ops.c +++ b/lib/librte_mempool/rte_mempool_ops.c @@ -37,6 +37,7 @@ #include <rte_mempool.h> #include <rte_errno.h> +#include <rte_dev.h> /* indirect jump table to support external memory pools. */ struct rte_mempool_ops_table rte_mempool_ops_table = { @@ -85,6 +86,8 @@ rte_mempool_register_ops(const struct rte_mempool_ops *h) ops->enqueue = h->enqueue; ops->dequeue = h->dequeue; ops->get_count = h->get_count; + ops->get_capabilities = h->get_capabilities; + ops->register_memory_area = h->register_memory_area; rte_spinlock_unlock(&rte_mempool_ops_table.sl); @@ -123,6 +126,32 @@ rte_mempool_ops_get_count(const struct rte_mempool *mp) return ops->get_count(mp); } +/* wrapper to get external mempool capabilities. */ +int +rte_mempool_ops_get_capabilities(const struct rte_mempool *mp, + unsigned int *flags) +{ + struct rte_mempool_ops *ops; + + ops = rte_mempool_get_ops(mp->ops_index); + + RTE_FUNC_PTR_OR_ERR_RET(ops->get_capabilities, -ENOTSUP); + return ops->get_capabilities(mp, flags); +} + +/* wrapper to notify new memory area to external mempool */ +int +rte_mempool_ops_register_memory_area(const struct rte_mempool *mp, char *vaddr, + rte_iova_t iova, size_t len) +{ + struct rte_mempool_ops *ops; + + ops = rte_mempool_get_ops(mp->ops_index); + + RTE_FUNC_PTR_OR_ERR_RET(ops->register_memory_area, -ENOTSUP); + return ops->register_memory_area(mp, vaddr, iova, len); +} + /* sets mempool ops previously registered by rte_mempool_register_ops. */ int rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name, diff --git a/lib/librte_mempool/rte_mempool_version.map b/lib/librte_mempool/rte_mempool_version.map index f9c07944..62b76f91 100644 --- a/lib/librte_mempool/rte_mempool_version.map +++ b/lib/librte_mempool/rte_mempool_version.map @@ -41,3 +41,13 @@ DPDK_16.07 { rte_mempool_set_ops_byname; } DPDK_2.0; + +DPDK_17.11 { + global: + + rte_mempool_ops_get_capabilities; + rte_mempool_ops_register_memory_area; + rte_mempool_populate_iova; + rte_mempool_populate_iova_tab; + +} DPDK_16.07; |