diff options
Diffstat (limited to 'lib/librte_eal/common/include')
19 files changed, 642 insertions, 128 deletions
diff --git a/lib/librte_eal/common/include/arch/arm/rte_cycles_32.h b/lib/librte_eal/common/include/arch/arm/rte_cycles_32.h index c4f974fe..859b0974 100644 --- a/lib/librte_eal/common/include/arch/arm/rte_cycles_32.h +++ b/lib/librte_eal/common/include/arch/arm/rte_cycles_32.h @@ -29,8 +29,8 @@ extern "C" { #ifndef RTE_ARM_EAL_RDTSC_USE_PMU /** - * This call is easily portable to any ARM architecture, however, - * it may be damn slow and inprecise for some tasks. + * This call is easily portable to any architecture, however, + * it may require a system call and inprecise for some tasks. */ static inline uint64_t __rte_rdtsc_syscall(void) diff --git a/lib/librte_eal/common/include/arch/ppc_64/meson.build b/lib/librte_eal/common/include/arch/ppc_64/meson.build new file mode 100644 index 00000000..00f96117 --- /dev/null +++ b/lib/librte_eal/common/include/arch/ppc_64/meson.build @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Luca Boccassi <bluca@debian.org> + +install_headers( + 'rte_atomic.h', + 'rte_byteorder.h', + 'rte_cpuflags.h', + 'rte_cycles.h', + 'rte_io.h', + 'rte_memcpy.h', + 'rte_pause.h', + 'rte_prefetch.h', + 'rte_rwlock.h', + 'rte_spinlock.h', + 'rte_vect.h', + subdir: get_option('include_subdir_arch')) diff --git a/lib/librte_eal/common/include/arch/ppc_64/rte_pause.h b/lib/librte_eal/common/include/arch/ppc_64/rte_pause.h index 8bd83576..16e47ce2 100644 --- a/lib/librte_eal/common/include/arch/ppc_64/rte_pause.h +++ b/lib/librte_eal/common/include/arch/ppc_64/rte_pause.h @@ -9,10 +9,17 @@ extern "C" { #endif +#include "rte_atomic.h" + #include "generic/rte_pause.h" static inline void rte_pause(void) { + /* Set hardware multi-threading low priority */ + asm volatile("or 1,1,1"); + /* Set hardware multi-threading medium priority */ + asm volatile("or 2,2,2"); + rte_compiler_barrier(); } #ifdef __cplusplus diff --git a/lib/librte_eal/common/include/generic/rte_cycles.h b/lib/librte_eal/common/include/generic/rte_cycles.h index 0ff1af50..ac379e87 100644 --- a/lib/librte_eal/common/include/generic/rte_cycles.h +++ b/lib/librte_eal/common/include/generic/rte_cycles.h @@ -13,6 +13,7 @@ */ #include <stdint.h> +#include <rte_compat.h> #include <rte_debug.h> #include <rte_atomic.h> @@ -158,6 +159,16 @@ rte_delay_ms(unsigned ms) void rte_delay_us_block(unsigned int us); /** + * Delay function that uses system sleep. + * Does not block the CPU core. + * + * @param us + * Number of microseconds to wait. + */ +void __rte_experimental +rte_delay_us_sleep(unsigned int us); + +/** * Replace rte_delay_us with user defined function. * * @param userfunc diff --git a/lib/librte_eal/common/include/rte_bitmap.h b/lib/librte_eal/common/include/rte_bitmap.h index d9facc64..7a36ce73 100644 --- a/lib/librte_eal/common/include/rte_bitmap.h +++ b/lib/librte_eal/common/include/rte_bitmap.h @@ -88,7 +88,7 @@ __rte_bitmap_index1_inc(struct rte_bitmap *bmp) static inline uint64_t __rte_bitmap_mask1_get(struct rte_bitmap *bmp) { - return (~1lu) << bmp->offset1; + return (~1llu) << bmp->offset1; } static inline void @@ -317,7 +317,7 @@ rte_bitmap_get(struct rte_bitmap *bmp, uint32_t pos) index2 = pos >> RTE_BITMAP_SLAB_BIT_SIZE_LOG2; offset2 = pos & RTE_BITMAP_SLAB_BIT_MASK; slab2 = bmp->array2 + index2; - return (*slab2) & (1lu << offset2); + return (*slab2) & (1llu << offset2); } /** @@ -342,8 +342,8 @@ rte_bitmap_set(struct rte_bitmap *bmp, uint32_t pos) slab2 = bmp->array2 + index2; slab1 = bmp->array1 + index1; - *slab2 |= 1lu << offset2; - *slab1 |= 1lu << offset1; + *slab2 |= 1llu << offset2; + *slab1 |= 1llu << offset1; } /** @@ -370,7 +370,7 @@ rte_bitmap_set_slab(struct rte_bitmap *bmp, uint32_t pos, uint64_t slab) slab1 = bmp->array1 + index1; *slab2 |= slab; - *slab1 |= 1lu << offset1; + *slab1 |= 1llu << offset1; } static inline uint64_t @@ -408,7 +408,7 @@ rte_bitmap_clear(struct rte_bitmap *bmp, uint32_t pos) slab2 = bmp->array2 + index2; /* Return if array2 slab is not all-zeros */ - *slab2 &= ~(1lu << offset2); + *slab2 &= ~(1llu << offset2); if (*slab2){ return; } @@ -424,7 +424,7 @@ rte_bitmap_clear(struct rte_bitmap *bmp, uint32_t pos) index1 = pos >> (RTE_BITMAP_SLAB_BIT_SIZE_LOG2 + RTE_BITMAP_CL_BIT_SIZE_LOG2); offset1 = (pos >> RTE_BITMAP_CL_BIT_SIZE_LOG2) & RTE_BITMAP_SLAB_BIT_MASK; slab1 = bmp->array1 + index1; - *slab1 &= ~(1lu << offset1); + *slab1 &= ~(1llu << offset1); return; } diff --git a/lib/librte_eal/common/include/rte_bus.h b/lib/librte_eal/common/include/rte_bus.h index b7b5b084..6be4b5ca 100644 --- a/lib/librte_eal/common/include/rte_bus.h +++ b/lib/librte_eal/common/include/rte_bus.h @@ -168,6 +168,35 @@ typedef int (*rte_bus_unplug_t)(struct rte_device *dev); typedef int (*rte_bus_parse_t)(const char *name, void *addr); /** + * Implement a specific hot-unplug handler, which is responsible for + * handle the failure when device be hot-unplugged. When the event of + * hot-unplug be detected, it could call this function to handle + * the hot-unplug failure and avoid app crash. + * @param dev + * Pointer of the device structure. + * + * @return + * 0 on success. + * !0 on error. + */ +typedef int (*rte_bus_hot_unplug_handler_t)(struct rte_device *dev); + +/** + * Implement a specific sigbus handler, which is responsible for handling + * the sigbus error which is either original memory error, or specific memory + * error that caused of device be hot-unplugged. When sigbus error be captured, + * it could call this function to handle sigbus error. + * @param failure_addr + * Pointer of the fault address of the sigbus error. + * + * @return + * 0 for success handle the sigbus for hot-unplug. + * 1 for not process it, because it is a generic sigbus error. + * -1 for failed to handle the sigbus for hot-unplug. + */ +typedef int (*rte_bus_sigbus_handler_t)(const void *failure_addr); + +/** * Bus scan policies */ enum rte_bus_scan_mode { @@ -212,6 +241,11 @@ struct rte_bus { struct rte_bus_conf conf; /**< Bus configuration */ rte_bus_get_iommu_class_t get_iommu_class; /**< Get iommu class */ rte_dev_iterate_t dev_iterate; /**< Device iterator. */ + rte_bus_hot_unplug_handler_t hot_unplug_handler; + /**< handle hot-unplug failure on the bus */ + rte_bus_sigbus_handler_t sigbus_handler; + /**< handle sigbus error on the bus */ + }; /** diff --git a/lib/librte_eal/common/include/rte_common.h b/lib/librte_eal/common/include/rte_common.h index 069c13ec..cba7bbc1 100644 --- a/lib/librte_eal/common/include/rte_common.h +++ b/lib/librte_eal/common/include/rte_common.h @@ -68,6 +68,11 @@ typedef uint16_t unaligned_uint16_t; /******* Macro to mark functions and fields scheduled for removal *****/ #define __rte_deprecated __attribute__((__deprecated__)) +/** + * Mark a function or variable to a weak reference. + */ +#define __rte_weak __attribute__((__weak__)) + /*********** Macros to eliminate unused variable warnings ********/ /** @@ -164,6 +169,12 @@ static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void) */ #define RTE_PTR_DIFF(ptr1, ptr2) ((uintptr_t)(ptr1) - (uintptr_t)(ptr2)) +/** + * Workaround to cast a const field of a structure to non-const type. + */ +#define RTE_CAST_FIELD(var, field, type) \ + (*(type *)((uintptr_t)(var) + offsetof(typeof(*(var)), field))) + /*********** Macros/static functions for doing alignment ********/ diff --git a/lib/librte_eal/common/include/rte_dev.h b/lib/librte_eal/common/include/rte_dev.h index b80a8059..cd6c187c 100644 --- a/lib/librte_eal/common/include/rte_dev.h +++ b/lib/librte_eal/common/include/rte_dev.h @@ -39,7 +39,7 @@ struct rte_dev_event { char *devname; /**< device name */ }; -typedef void (*rte_dev_event_cb_fn)(char *device_name, +typedef void (*rte_dev_event_cb_fn)(const char *device_name, enum rte_dev_event_type event, void *cb_arg); @@ -156,63 +156,67 @@ struct rte_driver { struct rte_device { TAILQ_ENTRY(rte_device) next; /**< Next device */ const char *name; /**< Device name */ - const struct rte_driver *driver;/**< Associated driver */ + const struct rte_driver *driver; /**< Driver assigned after probing */ + const struct rte_bus *bus; /**< Bus handle assigned on scan */ int numa_node; /**< NUMA node connection */ - struct rte_devargs *devargs; /**< Device user arguments */ + struct rte_devargs *devargs; /**< Arguments for latest probing */ }; /** - * Attach a device to a registered driver. + * @warning + * @b EXPERIMENTAL: this API may change without prior notice * - * @param name - * The device name, that refers to a pci device (or some private - * way of designating a vdev device). Based on this device name, eal - * will identify a driver capable of handling it and pass it to the - * driver probing function. - * @param devargs - * Device arguments to be passed to the driver. - * @return - * 0 on success, negative on error. - */ -__rte_deprecated -int rte_eal_dev_attach(const char *name, const char *devargs); - -/** - * Detach a device from its driver. + * Query status of a device. * * @param dev - * A pointer to a rte_device structure. + * Generic device pointer. * @return - * 0 on success, negative on error. + * (int)true if already probed successfully, 0 otherwise. */ -__rte_deprecated -int rte_eal_dev_detach(struct rte_device *dev); +__rte_experimental +int rte_dev_is_probed(const struct rte_device *dev); /** - * @warning - * @b EXPERIMENTAL: this API may change without prior notice - * * Hotplug add a given device to a specific bus. * + * In multi-process, it will request other processes to add the same device. + * A failure, in any process, will rollback the action + * * @param busname * The bus name the device is added to. * @param devname * The device name. Based on this device name, eal will identify a driver * capable of handling it and pass it to the driver probing function. - * @param devargs + * @param drvargs * Device arguments to be passed to the driver. * @return * 0 on success, negative on error. */ -int __rte_experimental rte_eal_hotplug_add(const char *busname, const char *devname, - const char *devargs); +int rte_eal_hotplug_add(const char *busname, const char *devname, + const char *drvargs); /** * @warning * @b EXPERIMENTAL: this API may change without prior notice * + * Add matching devices. + * + * In multi-process, it will request other processes to add the same device. + * A failure, in any process, will rollback the action + * + * @param devargs + * Device arguments including bus, class and driver properties. + * @return + * 0 on success, negative on error. + */ +int __rte_experimental rte_dev_probe(const char *devargs); + +/** * Hotplug remove a given device from a specific bus. * + * In multi-process, it will request other processes to remove the same device. + * A failure, in any process, will rollback the action + * * @param busname * The bus name the device is removed from. * @param devname @@ -220,8 +224,23 @@ int __rte_experimental rte_eal_hotplug_add(const char *busname, const char *devn * @return * 0 on success, negative on error. */ -int __rte_experimental rte_eal_hotplug_remove(const char *busname, - const char *devname); +int rte_eal_hotplug_remove(const char *busname, const char *devname); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Remove one device. + * + * In multi-process, it will request other processes to remove the same device. + * A failure, in any process, will rollback the action + * + * @param dev + * Data structure of the device to remove. + * @return + * 0 on success, negative on error. + */ +int __rte_experimental rte_dev_remove(struct rte_device *dev); /** * Device comparison function. @@ -438,6 +457,22 @@ rte_dev_event_callback_unregister(const char *device_name, * @warning * @b EXPERIMENTAL: this API may change without prior notice * + * Executes all the user application registered callbacks for + * the specific device. + * + * @param device_name + * The device name. + * @param event + * the device event type. + */ +void __rte_experimental +rte_dev_event_callback_process(const char *device_name, + enum rte_dev_event_type event); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * * Start the device event monitoring. * * @return @@ -460,4 +495,30 @@ rte_dev_event_monitor_start(void); int __rte_experimental rte_dev_event_monitor_stop(void); +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Enable hotplug handling for devices. + * + * @return + * - On success, zero. + * - On failure, a negative value. + */ +int __rte_experimental +rte_dev_hotplug_handle_enable(void); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Disable hotplug handling for devices. + * + * @return + * - On success, zero. + * - On failure, a negative value. + */ +int __rte_experimental +rte_dev_hotplug_handle_disable(void); + #endif /* _RTE_DEV_H_ */ diff --git a/lib/librte_eal/common/include/rte_devargs.h b/lib/librte_eal/common/include/rte_devargs.h index 097a4ce7..b1f121f8 100644 --- a/lib/librte_eal/common/include/rte_devargs.h +++ b/lib/librte_eal/common/include/rte_devargs.h @@ -67,36 +67,6 @@ struct rte_devargs { }; /** - * @deprecated - * Parse a devargs string. - * - * For PCI devices, the format of arguments string is "PCI_ADDR" or - * "PCI_ADDR,key=val,key2=val2,...". Examples: "08:00.1", "0000:5:00.0", - * "04:00.0,arg=val". - * - * For virtual devices, the format of arguments string is "DRIVER_NAME*" - * or "DRIVER_NAME*,key=val,key2=val2,...". Examples: "net_ring", - * "net_ring0", "net_pmdAnything,arg=0:arg2=1". - * - * The function parses the arguments string to get driver name and driver - * arguments. - * - * @param devargs_str - * The arguments as given by the user. - * @param drvname - * The pointer to the string to store parsed driver name. - * @param drvargs - * The pointer to the string to store parsed driver arguments. - * - * @return - * - 0 on success - * - A negative value on error - */ -__rte_deprecated -int rte_eal_parse_devargs_str(const char *devargs_str, - char **drvname, char **drvargs); - -/** * Parse a device string. * * Verify that a bus is capable of handling the device passed @@ -202,32 +172,12 @@ __rte_experimental int rte_devargs_add(enum rte_devtype devtype, const char *devargs_str); /** - * @deprecated - * Add a device to the user device list - * See rte_devargs_parse() for details. - * - * @param devtype - * The type of the device. - * @param devargs_str - * The arguments as given by the user. - * - * @return - * - 0 on success - * - A negative value on error - */ -__rte_deprecated -int rte_eal_devargs_add(enum rte_devtype devtype, const char *devargs_str); - -/** * Remove a device from the user device list. * Its resources are freed. * If the devargs cannot be found, nothing happens. * - * @param busname - * bus name of the devargs to remove. - * - * @param devname - * device name of the devargs to remove. + * @param devargs + * The instance or a copy of devargs to remove. * * @return * 0 on success. @@ -235,8 +185,7 @@ int rte_eal_devargs_add(enum rte_devtype devtype, const char *devargs_str); * >0 if the devargs was not within the user device list. */ __rte_experimental -int rte_devargs_remove(const char *busname, - const char *devname); +int rte_devargs_remove(struct rte_devargs *devargs); /** * Count the number of user devices of a specified type @@ -252,20 +201,6 @@ unsigned int rte_devargs_type_count(enum rte_devtype devtype); /** - * @deprecated - * Count the number of user devices of a specified type - * - * @param devtype - * The type of the devices to counted. - * - * @return - * The number of devices. - */ -__rte_deprecated -unsigned int -rte_eal_devargs_type_count(enum rte_devtype devtype); - -/** * This function dumps the list of user device and their arguments. * * @param f @@ -275,16 +210,6 @@ __rte_experimental void rte_devargs_dump(FILE *f); /** - * @deprecated - * This function dumps the list of user device and their arguments. - * - * @param f - * A pointer to a file for output - */ -__rte_deprecated -void rte_eal_devargs_dump(FILE *f); - -/** * Find next rte_devargs matching the provided bus name. * * @param busname diff --git a/lib/librte_eal/common/include/rte_eal.h b/lib/librte_eal/common/include/rte_eal.h index e114dcbd..a0cedd57 100644 --- a/lib/librte_eal/common/include/rte_eal.h +++ b/lib/librte_eal/common/include/rte_eal.h @@ -316,7 +316,7 @@ rte_mp_sendmsg(struct rte_mp_msg *msg); * * @param reply * The reply argument will be for storing all the replied messages; - * the caller is responsible for free reply->replies. + * the caller is responsible for free reply->msgs. * * @param ts * The ts argument specifies how long we can wait for the peer(s) to reply. @@ -378,6 +378,15 @@ int __rte_experimental rte_mp_reply(struct rte_mp_msg *msg, const char *peer); /** + * Register all mp action callbacks for hotplug. + * + * @return + * 0 on success, negative on error. + */ +int __rte_experimental +rte_mp_dev_hotplug_init(void); + +/** * Usage function typedef used by the application usage function. * * Use this function typedef to define and call rte_set_application_usage_hook() @@ -498,6 +507,15 @@ enum rte_iova_mode rte_eal_iova_mode(void); const char * rte_eal_mbuf_user_pool_ops(void); +/** + * Get the runtime directory of DPDK + * + * @return + * The runtime directory path of DPDK + */ +const char * +rte_eal_get_runtime_dir(void); + #ifdef __cplusplus } #endif diff --git a/lib/librte_eal/common/include/rte_eal_interrupts.h b/lib/librte_eal/common/include/rte_eal_interrupts.h index 6eb49327..9d302f41 100644 --- a/lib/librte_eal/common/include/rte_eal_interrupts.h +++ b/lib/librte_eal/common/include/rte_eal_interrupts.h @@ -35,6 +35,7 @@ enum rte_intr_handle_type { RTE_INTR_HANDLE_EXT, /**< external handler */ RTE_INTR_HANDLE_VDEV, /**< virtual device */ RTE_INTR_HANDLE_DEV_EVENT, /**< device event handle */ + RTE_INTR_HANDLE_VFIO_REQ, /**< VFIO request handle */ RTE_INTR_HANDLE_MAX /**< count of elements */ }; diff --git a/lib/librte_eal/common/include/rte_eal_memconfig.h b/lib/librte_eal/common/include/rte_eal_memconfig.h index aff0688d..84aabe36 100644 --- a/lib/librte_eal/common/include/rte_eal_memconfig.h +++ b/lib/librte_eal/common/include/rte_eal_memconfig.h @@ -30,9 +30,11 @@ struct rte_memseg_list { uint64_t addr_64; /**< Makes sure addr is always 64-bits */ }; - int socket_id; /**< Socket ID for all memsegs in this list. */ uint64_t page_sz; /**< Page size for all memsegs in this list. */ + int socket_id; /**< Socket ID for all memsegs in this list. */ volatile uint32_t version; /**< version number for multiprocess sync. */ + size_t len; /**< Length of memory area covered by this memseg list. */ + unsigned int external; /**< 1 if this list points to external memory */ struct rte_fbarray memseg_arr; }; @@ -70,13 +72,23 @@ struct rte_mem_config { struct rte_tailq_head tailq_head[RTE_MAX_TAILQ]; /**< Tailqs for objects */ - /* Heaps of Malloc per socket */ - struct malloc_heap malloc_heaps[RTE_MAX_NUMA_NODES]; + /* Heaps of Malloc */ + struct malloc_heap malloc_heaps[RTE_MAX_HEAPS]; + + /* next socket ID for external malloc heap */ + int next_socket_id; /* address of mem_config in primary process. used to map shared config into * exact same address the primary process maps it. */ uint64_t mem_cfg_addr; + + /* legacy mem and single file segments options are shared */ + uint32_t legacy_mem; + uint32_t single_file_segments; + + /* keeps the more restricted dma mask */ + uint8_t dma_maskbits; } __attribute__((__packed__)); diff --git a/lib/librte_eal/common/include/rte_malloc.h b/lib/librte_eal/common/include/rte_malloc.h index a9fb7e45..7249e6aa 100644 --- a/lib/librte_eal/common/include/rte_malloc.h +++ b/lib/librte_eal/common/include/rte_malloc.h @@ -264,6 +264,198 @@ rte_malloc_get_socket_stats(int socket, struct rte_malloc_socket_stats *socket_stats); /** + * Add memory chunk to a heap with specified name. + * + * @note Multiple memory chunks can be added to the same heap + * + * @note Before accessing this memory in other processes, it needs to be + * attached in each of those processes by calling + * ``rte_malloc_heap_memory_attach`` in each other process. + * + * @note Memory must be previously allocated for DPDK to be able to use it as a + * malloc heap. Failing to do so will result in undefined behavior, up to and + * including segmentation faults. + * + * @note Calling this function will erase any contents already present at the + * supplied memory address. + * + * @param heap_name + * Name of the heap to add memory chunk to + * @param va_addr + * Start of virtual area to add to the heap + * @param len + * Length of virtual area to add to the heap + * @param iova_addrs + * Array of page IOVA addresses corresponding to each page in this memory + * area. Can be NULL, in which case page IOVA addresses will be set to + * RTE_BAD_IOVA. + * @param n_pages + * Number of elements in the iova_addrs array. Ignored if ``iova_addrs`` + * is NULL. + * @param page_sz + * Page size of the underlying memory + * + * @return + * - 0 on success + * - -1 in case of error, with rte_errno set to one of the following: + * EINVAL - one of the parameters was invalid + * EPERM - attempted to add memory to a reserved heap + * ENOSPC - no more space in internal config to store a new memory chunk + */ +int __rte_experimental +rte_malloc_heap_memory_add(const char *heap_name, void *va_addr, size_t len, + rte_iova_t iova_addrs[], unsigned int n_pages, size_t page_sz); + +/** + * Remove memory chunk from heap with specified name. + * + * @note Memory chunk being removed must be the same as one that was added; + * partially removing memory chunks is not supported + * + * @note Memory area must not contain any allocated elements to allow its + * removal from the heap + * + * @note All other processes must detach from the memory chunk prior to it being + * removed from the heap. + * + * @param heap_name + * Name of the heap to remove memory from + * @param va_addr + * Virtual address to remove from the heap + * @param len + * Length of virtual area to remove from the heap + * + * @return + * - 0 on success + * - -1 in case of error, with rte_errno set to one of the following: + * EINVAL - one of the parameters was invalid + * EPERM - attempted to remove memory from a reserved heap + * ENOENT - heap or memory chunk was not found + * EBUSY - memory chunk still contains data + */ +int __rte_experimental +rte_malloc_heap_memory_remove(const char *heap_name, void *va_addr, size_t len); + +/** + * Attach to an already existing chunk of external memory in another process. + * + * @note This function must be called before any attempt is made to use an + * already existing external memory chunk. This function does *not* need to + * be called if a call to ``rte_malloc_heap_memory_add`` was made in the + * current process. + * + * @param heap_name + * Heap name to which this chunk of memory belongs + * @param va_addr + * Start address of memory chunk to attach to + * @param len + * Length of memory chunk to attach to + * @return + * 0 on successful attach + * -1 on unsuccessful attach, with rte_errno set to indicate cause for error: + * EINVAL - one of the parameters was invalid + * EPERM - attempted to attach memory to a reserved heap + * ENOENT - heap or memory chunk was not found + */ +int __rte_experimental +rte_malloc_heap_memory_attach(const char *heap_name, void *va_addr, size_t len); + +/** + * Detach from a chunk of external memory in secondary process. + * + * @note This function must be called in before any attempt is made to remove + * external memory from the heap in another process. This function does *not* + * need to be called if a call to ``rte_malloc_heap_memory_remove`` will be + * called in current process. + * + * @param heap_name + * Heap name to which this chunk of memory belongs + * @param va_addr + * Start address of memory chunk to attach to + * @param len + * Length of memory chunk to attach to + * @return + * 0 on successful detach + * -1 on unsuccessful detach, with rte_errno set to indicate cause for error: + * EINVAL - one of the parameters was invalid + * EPERM - attempted to detach memory from a reserved heap + * ENOENT - heap or memory chunk was not found + */ +int __rte_experimental +rte_malloc_heap_memory_detach(const char *heap_name, void *va_addr, size_t len); + +/** + * Creates a new empty malloc heap with a specified name. + * + * @note Heaps created via this call will automatically get assigned a unique + * socket ID, which can be found using ``rte_malloc_heap_get_socket()`` + * + * @param heap_name + * Name of the heap to create. + * + * @return + * - 0 on successful creation + * - -1 in case of error, with rte_errno set to one of the following: + * EINVAL - ``heap_name`` was NULL, empty or too long + * EEXIST - heap by name of ``heap_name`` already exists + * ENOSPC - no more space in internal config to store a new heap + */ +int __rte_experimental +rte_malloc_heap_create(const char *heap_name); + +/** + * Destroys a previously created malloc heap with specified name. + * + * @note This function will return a failure result if not all memory allocated + * from the heap has been freed back to the heap + * + * @note This function will return a failure result if not all memory segments + * were removed from the heap prior to its destruction + * + * @param heap_name + * Name of the heap to create. + * + * @return + * - 0 on success + * - -1 in case of error, with rte_errno set to one of the following: + * EINVAL - ``heap_name`` was NULL, empty or too long + * ENOENT - heap by the name of ``heap_name`` was not found + * EPERM - attempting to destroy reserved heap + * EBUSY - heap still contains data + */ +int __rte_experimental +rte_malloc_heap_destroy(const char *heap_name); + +/** + * Find socket ID corresponding to a named heap. + * + * @param name + * Heap name to find socket ID for + * @return + * Socket ID in case of success (a non-negative number) + * -1 in case of error, with rte_errno set to one of the following: + * EINVAL - ``name`` was NULL + * ENOENT - heap identified by the name ``name`` was not found + */ +int __rte_experimental +rte_malloc_heap_get_socket(const char *name); + +/** + * Check if a given socket ID refers to externally allocated memory. + * + * @note Passing SOCKET_ID_ANY will return 0. + * + * @param socket_id + * Socket ID to check + * @return + * 1 if socket ID refers to externally allocated memory + * 0 if socket ID refers to internal DPDK memory + * -1 if socket ID is invalid + */ +int __rte_experimental +rte_malloc_heap_socket_is_external(int socket_id); + +/** * Dump statistics. * * Dump for the specified type to a file. If the type argument is diff --git a/lib/librte_eal/common/include/rte_malloc_heap.h b/lib/librte_eal/common/include/rte_malloc_heap.h index d43fa909..4a7e0eb1 100644 --- a/lib/librte_eal/common/include/rte_malloc_heap.h +++ b/lib/librte_eal/common/include/rte_malloc_heap.h @@ -12,6 +12,7 @@ /* Number of free lists per heap, grouped by size. */ #define RTE_HEAP_NUM_FREELISTS 13 +#define RTE_HEAP_NAME_MAX_LEN 32 /* dummy definition, for pointers */ struct malloc_elem; @@ -26,7 +27,9 @@ struct malloc_heap { struct malloc_elem *volatile last; unsigned alloc_count; + unsigned int socket_id; size_t total_size; + char name[RTE_HEAP_NAME_MAX_LEN]; } __rte_cache_aligned; #endif /* _RTE_MALLOC_HEAP_H_ */ diff --git a/lib/librte_eal/common/include/rte_memory.h b/lib/librte_eal/common/include/rte_memory.h index c4b7f4cf..ce937058 100644 --- a/lib/librte_eal/common/include/rte_memory.h +++ b/lib/librte_eal/common/include/rte_memory.h @@ -215,6 +215,9 @@ typedef int (*rte_memseg_list_walk_t)(const struct rte_memseg_list *msl, * @note This function read-locks the memory hotplug subsystem, and thus cannot * be used within memory-related callback functions. * + * @note This function will also walk through externally allocated segments. It + * is up to the user to decide whether to skip through these segments. + * * @param func * Iterator function * @param arg @@ -233,6 +236,9 @@ rte_memseg_walk(rte_memseg_walk_t func, void *arg); * @note This function read-locks the memory hotplug subsystem, and thus cannot * be used within memory-related callback functions. * + * @note This function will also walk through externally allocated segments. It + * is up to the user to decide whether to skip through these segments. + * * @param func * Iterator function * @param arg @@ -251,6 +257,9 @@ rte_memseg_contig_walk(rte_memseg_contig_walk_t func, void *arg); * @note This function read-locks the memory hotplug subsystem, and thus cannot * be used within memory-related callback functions. * + * @note This function will also walk through externally allocated segments. It + * is up to the user to decide whether to skip through these segments. + * * @param func * Iterator function * @param arg @@ -318,6 +327,103 @@ int __rte_experimental rte_memseg_list_walk_thread_unsafe(rte_memseg_list_walk_t func, void *arg); /** + * Return file descriptor associated with a particular memseg (if available). + * + * @note This function read-locks the memory hotplug subsystem, and thus cannot + * be used within memory-related callback functions. + * + * @note This returns an internal file descriptor. Performing any operations on + * this file descriptor is inherently dangerous, so it should be treated + * as read-only for all intents and purposes. + * + * @param ms + * A pointer to memseg for which to get file descriptor. + * + * @return + * Valid file descriptor in case of success. + * -1 in case of error, with ``rte_errno`` set to the following values: + * - EINVAL - ``ms`` pointer was NULL or did not point to a valid memseg + * - ENODEV - ``ms`` fd is not available + * - ENOENT - ``ms`` is an unused segment + * - ENOTSUP - segment fd's are not supported + */ +int __rte_experimental +rte_memseg_get_fd(const struct rte_memseg *ms); + +/** + * Return file descriptor associated with a particular memseg (if available). + * + * @note This function does not perform any locking, and is only safe to call + * from within memory-related callback functions. + * + * @note This returns an internal file descriptor. Performing any operations on + * this file descriptor is inherently dangerous, so it should be treated + * as read-only for all intents and purposes. + * + * @param ms + * A pointer to memseg for which to get file descriptor. + * + * @return + * Valid file descriptor in case of success. + * -1 in case of error, with ``rte_errno`` set to the following values: + * - EINVAL - ``ms`` pointer was NULL or did not point to a valid memseg + * - ENODEV - ``ms`` fd is not available + * - ENOENT - ``ms`` is an unused segment + * - ENOTSUP - segment fd's are not supported + */ +int __rte_experimental +rte_memseg_get_fd_thread_unsafe(const struct rte_memseg *ms); + +/** + * Get offset into segment file descriptor associated with a particular memseg + * (if available). + * + * @note This function read-locks the memory hotplug subsystem, and thus cannot + * be used within memory-related callback functions. + * + * @param ms + * A pointer to memseg for which to get file descriptor. + * @param offset + * A pointer to offset value where the result will be stored. + * + * @return + * Valid file descriptor in case of success. + * -1 in case of error, with ``rte_errno`` set to the following values: + * - EINVAL - ``ms`` pointer was NULL or did not point to a valid memseg + * - EINVAL - ``offset`` pointer was NULL + * - ENODEV - ``ms`` fd is not available + * - ENOENT - ``ms`` is an unused segment + * - ENOTSUP - segment fd's are not supported + */ +int __rte_experimental +rte_memseg_get_fd_offset(const struct rte_memseg *ms, size_t *offset); + +/** + * Get offset into segment file descriptor associated with a particular memseg + * (if available). + * + * @note This function does not perform any locking, and is only safe to call + * from within memory-related callback functions. + * + * @param ms + * A pointer to memseg for which to get file descriptor. + * @param offset + * A pointer to offset value where the result will be stored. + * + * @return + * Valid file descriptor in case of success. + * -1 in case of error, with ``rte_errno`` set to the following values: + * - EINVAL - ``ms`` pointer was NULL or did not point to a valid memseg + * - EINVAL - ``offset`` pointer was NULL + * - ENODEV - ``ms`` fd is not available + * - ENOENT - ``ms`` is an unused segment + * - ENOTSUP - segment fd's are not supported + */ +int __rte_experimental +rte_memseg_get_fd_offset_thread_unsafe(const struct rte_memseg *ms, + size_t *offset); + +/** * Dump the physical memory layout to a file. * * @note This function read-locks the memory hotplug subsystem, and thus cannot @@ -357,6 +463,9 @@ unsigned rte_memory_get_nchannel(void); */ unsigned rte_memory_get_nrank(void); +/* check memsegs iovas are within a range based on dma mask */ +int __rte_experimental rte_eal_check_dma_mask(uint8_t maskbits); + /** * Drivers based on uio will not load unless physical * addresses are obtainable. It is only possible to get diff --git a/lib/librte_eal/common/include/rte_option.h b/lib/librte_eal/common/include/rte_option.h new file mode 100644 index 00000000..8957b970 --- /dev/null +++ b/lib/librte_eal/common/include/rte_option.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation. + */ + +#ifndef __INCLUDE_RTE_OPTION_H__ +#define __INCLUDE_RTE_OPTION_H__ + +/** + * @file + * + * This API offers the ability to register options to the EAL command line and + * map those options to functions that will be executed at the end of EAL + * initialization. These options will be available as part of the EAL command + * line of applications and are dynamically managed. + * + * This is used primarily by DPDK libraries offering command line options. + * Currently, this API is limited to registering options without argument. + * + * The register API can be used to resolve circular dependency issues + * between EAL and the library. The library uses EAL, but is also initialized + * by EAL. Hence, EAL depends on the init function of the library. The API + * introduced in rte_option allows us to register the library init with EAL + * (passing a function pointer) and avoid the circular dependency. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef int (*rte_option_cb)(void); + +/* + * Structure describing the EAL command line option being registered. + */ +struct rte_option { + TAILQ_ENTRY(rte_option) next; /**< Next entry in the list. */ + char *opt_str; /**< The option name. */ + rte_option_cb cb; /**< Function called when option is used. */ + int enabled; /**< Set when the option is used. */ +}; + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Register an option to the EAL command line. + * When recognized, the associated function will be executed at the end of EAL + * initialization. + * + * The associated structure must be available the whole time this option is + * registered (i.e. not stack memory). + * + * @param opt + * Structure describing the option to parse. + */ +void __rte_experimental +rte_option_register(struct rte_option *opt); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/lib/librte_eal/common/include/rte_string_fns.h b/lib/librte_eal/common/include/rte_string_fns.h index 97597a14..9a2a1ff9 100644 --- a/lib/librte_eal/common/include/rte_string_fns.h +++ b/lib/librte_eal/common/include/rte_string_fns.h @@ -16,6 +16,7 @@ extern "C" { #endif #include <stdio.h> +#include <string.h> /** * Takes string "string" parameter and splits it at character "delim" @@ -60,12 +61,10 @@ rte_strlcpy(char *dst, const char *src, size_t size) /* pull in a strlcpy function */ #ifdef RTE_EXEC_ENV_BSDAPP -#include <string.h> #ifndef __BSD_VISIBLE /* non-standard functions are hidden */ #define strlcpy(dst, src, size) rte_strlcpy(dst, src, size) #endif - #else /* non-BSD platforms */ #ifdef RTE_USE_LIBBSD #include <bsd/string.h> @@ -76,6 +75,29 @@ rte_strlcpy(char *dst, const char *src, size_t size) #endif /* RTE_USE_LIBBSD */ #endif /* BSDAPP */ +/** + * Copy string src to buffer dst of size dsize. + * At most dsize-1 chars will be copied. + * Always NUL-terminates, unless (dsize == 0). + * Returns number of bytes copied (terminating NUL-byte excluded) on success ; + * negative errno on error. + * + * @param dst + * The destination string. + * + * @param src + * The input string to be copied. + * + * @param dsize + * Length in bytes of the destination buffer. + * + * @return + * The number of bytes copied on success + * -E2BIG if the destination buffer is too small. + */ +ssize_t +rte_strscpy(char *dst, const char *src, size_t dsize); + #ifdef __cplusplus } #endif diff --git a/lib/librte_eal/common/include/rte_version.h b/lib/librte_eal/common/include/rte_version.h index 7c6714a2..412ed2db 100644 --- a/lib/librte_eal/common/include/rte_version.h +++ b/lib/librte_eal/common/include/rte_version.h @@ -32,7 +32,7 @@ extern "C" { /** * Minor version/month number i.e. the mm in yy.mm.z */ -#define RTE_VER_MONTH 8 +#define RTE_VER_MONTH 11 /** * Patch level number i.e. the z in yy.mm.z @@ -42,14 +42,14 @@ extern "C" { /** * Extra string to be appended to version number */ -#define RTE_VER_SUFFIX "" +#define RTE_VER_SUFFIX "-rc" /** * Patch release number * 0-15 = release candidates * 16 = release */ -#define RTE_VER_RELEASE 16 +#define RTE_VER_RELEASE 1 /** * Macro to compute a version number usable for comparisons diff --git a/lib/librte_eal/common/include/rte_vfio.h b/lib/librte_eal/common/include/rte_vfio.h index 5ca13fcc..cae96fab 100644 --- a/lib/librte_eal/common/include/rte_vfio.h +++ b/lib/librte_eal/common/include/rte_vfio.h @@ -14,6 +14,8 @@ extern "C" { #endif +#include <stdint.h> + /* * determine if VFIO is present on the system */ @@ -22,6 +24,9 @@ extern "C" { #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0) #define VFIO_PRESENT #endif /* kernel version >= 3.6.0 */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0) +#define HAVE_VFIO_DEV_REQ_INTERFACE +#endif /* kernel version >= 4.0.0 */ #endif /* RTE_EAL_VFIO */ #ifdef VFIO_PRESENT @@ -44,6 +49,30 @@ extern "C" { #define RTE_VFIO_NOIOMMU 8 #endif +/* + * capabilities are only supported on kernel 4.6+. there were also some API + * changes as well, so add a macro to get cap offset. + */ +#ifdef VFIO_REGION_INFO_FLAG_CAPS +#define RTE_VFIO_INFO_FLAG_CAPS VFIO_REGION_INFO_FLAG_CAPS +#define VFIO_CAP_OFFSET(x) (x->cap_offset) +#else +#define RTE_VFIO_INFO_FLAG_CAPS (1 << 3) +#define VFIO_CAP_OFFSET(x) (x->resv) +struct vfio_info_cap_header { + uint16_t id; + uint16_t version; + uint32_t next; +}; +#endif + +/* kernels 4.16+ can map BAR containing MSI-X table */ +#ifdef VFIO_REGION_INFO_CAP_MSIX_MAPPABLE +#define RTE_VFIO_CAP_MSIX_MAPPABLE VFIO_REGION_INFO_CAP_MSIX_MAPPABLE +#else +#define RTE_VFIO_CAP_MSIX_MAPPABLE 3 +#endif + #else /* not VFIO_PRESENT */ /* we don't need an actual definition, only pointer is used */ @@ -227,7 +256,7 @@ rte_vfio_get_group_num(const char *sysfs_base, const char *dev_addr, int *iommu_group_num); /** - * Open VFIO container fd or get an existing one + * Open a new VFIO container fd * * This function is only relevant to linux and will return * an error on BSD. |