diff options
Diffstat (limited to 'lib/librte_eal')
87 files changed, 3520 insertions, 624 deletions
diff --git a/lib/librte_eal/bsdapp/contigmem/contigmem.c b/lib/librte_eal/bsdapp/contigmem/contigmem.c index da971deb..e8fb9087 100644 --- a/lib/librte_eal/bsdapp/contigmem/contigmem.c +++ b/lib/librte_eal/bsdapp/contigmem/contigmem.c @@ -50,24 +50,37 @@ __FBSDID("$FreeBSD$"); #include <vm/vm.h> #include <vm/pmap.h> +#include <vm/vm_param.h> #include <vm/vm_object.h> #include <vm/vm_page.h> #include <vm/vm_pager.h> +#include <vm/vm_phys.h> + +struct contigmem_buffer { + void *addr; + int refcnt; + struct mtx mtx; +}; + +struct contigmem_vm_handle { + int buffer_index; +}; static int contigmem_load(void); static int contigmem_unload(void); static int contigmem_physaddr(SYSCTL_HANDLER_ARGS); -static d_mmap_t contigmem_mmap; static d_mmap_single_t contigmem_mmap_single; static d_open_t contigmem_open; +static d_close_t contigmem_close; static int contigmem_num_buffers = RTE_CONTIGMEM_DEFAULT_NUM_BUFS; static int64_t contigmem_buffer_size = RTE_CONTIGMEM_DEFAULT_BUF_SIZE; static eventhandler_tag contigmem_eh_tag; -static void *contigmem_buffers[RTE_CONTIGMEM_MAX_NUM_BUFS]; +static struct contigmem_buffer contigmem_buffers[RTE_CONTIGMEM_MAX_NUM_BUFS]; static struct cdev *contigmem_cdev = NULL; +static int contigmem_refcnt; TUNABLE_INT("hw.contigmem.num_buffers", &contigmem_num_buffers); TUNABLE_QUAD("hw.contigmem.buffer_size", &contigmem_buffer_size); @@ -78,6 +91,8 @@ SYSCTL_INT(_hw_contigmem, OID_AUTO, num_buffers, CTLFLAG_RD, &contigmem_num_buffers, 0, "Number of contigmem buffers allocated"); SYSCTL_QUAD(_hw_contigmem, OID_AUTO, buffer_size, CTLFLAG_RD, &contigmem_buffer_size, 0, "Size of each contiguous buffer"); +SYSCTL_INT(_hw_contigmem, OID_AUTO, num_references, CTLFLAG_RD, + &contigmem_refcnt, 0, "Number of references to contigmem"); static SYSCTL_NODE(_hw_contigmem, OID_AUTO, physaddr, CTLFLAG_RD, 0, "physaddr"); @@ -114,42 +129,49 @@ MODULE_VERSION(contigmem, 1); static struct cdevsw contigmem_ops = { .d_name = "contigmem", .d_version = D_VERSION, - .d_mmap = contigmem_mmap, + .d_flags = D_TRACKCLOSE, .d_mmap_single = contigmem_mmap_single, .d_open = contigmem_open, + .d_close = contigmem_close, }; static int contigmem_load() { char index_string[8], description[32]; - int i; + int i, error = 0; + void *addr; if (contigmem_num_buffers > RTE_CONTIGMEM_MAX_NUM_BUFS) { printf("%d buffers requested is greater than %d allowed\n", contigmem_num_buffers, RTE_CONTIGMEM_MAX_NUM_BUFS); - return EINVAL; + error = EINVAL; + goto error; } if (contigmem_buffer_size < PAGE_SIZE || (contigmem_buffer_size & (contigmem_buffer_size - 1)) != 0) { printf("buffer size 0x%lx is not greater than PAGE_SIZE and " "power of two\n", contigmem_buffer_size); - return EINVAL; + error = EINVAL; + goto error; } for (i = 0; i < contigmem_num_buffers; i++) { - contigmem_buffers[i] = - contigmalloc(contigmem_buffer_size, M_CONTIGMEM, M_ZERO, 0, - BUS_SPACE_MAXADDR, contigmem_buffer_size, 0); - - if (contigmem_buffers[i] == NULL) { + addr = contigmalloc(contigmem_buffer_size, M_CONTIGMEM, M_ZERO, + 0, BUS_SPACE_MAXADDR, contigmem_buffer_size, 0); + if (addr == NULL) { printf("contigmalloc failed for buffer %d\n", i); - return ENOMEM; + error = ENOMEM; + goto error; } - printf("%2u: virt=%p phys=%p\n", i, contigmem_buffers[i], - (void *)pmap_kextract((vm_offset_t)contigmem_buffers[i])); + printf("%2u: virt=%p phys=%p\n", i, addr, + (void *)pmap_kextract((vm_offset_t)addr)); + + mtx_init(&contigmem_buffers[i].mtx, "contigmem", NULL, MTX_DEF); + contigmem_buffers[i].addr = addr; + contigmem_buffers[i].refcnt = 0; snprintf(index_string, sizeof(index_string), "%d", i); snprintf(description, sizeof(description), @@ -165,6 +187,17 @@ contigmem_load() GID_WHEEL, 0600, "contigmem"); return 0; + +error: + for (i = 0; i < contigmem_num_buffers; i++) { + if (contigmem_buffers[i].addr != NULL) + contigfree(contigmem_buffers[i].addr, + contigmem_buffer_size, M_CONTIGMEM); + if (mtx_initialized(&contigmem_buffers[i].mtx)) + mtx_destroy(&contigmem_buffers[i].mtx); + } + + return error; } static int @@ -172,16 +205,22 @@ contigmem_unload() { int i; + if (contigmem_refcnt > 0) + return EBUSY; + if (contigmem_cdev != NULL) destroy_dev(contigmem_cdev); if (contigmem_eh_tag != NULL) EVENTHANDLER_DEREGISTER(process_exit, contigmem_eh_tag); - for (i = 0; i < RTE_CONTIGMEM_MAX_NUM_BUFS; i++) - if (contigmem_buffers[i] != NULL) - contigfree(contigmem_buffers[i], contigmem_buffer_size, - M_CONTIGMEM); + for (i = 0; i < RTE_CONTIGMEM_MAX_NUM_BUFS; i++) { + if (contigmem_buffers[i].addr != NULL) + contigfree(contigmem_buffers[i].addr, + contigmem_buffer_size, M_CONTIGMEM); + if (mtx_initialized(&contigmem_buffers[i].mtx)) + mtx_destroy(&contigmem_buffers[i].mtx); + } return 0; } @@ -192,7 +231,7 @@ contigmem_physaddr(SYSCTL_HANDLER_ARGS) uint64_t physaddr; int index = (int)(uintptr_t)arg1; - physaddr = (uint64_t)vtophys(contigmem_buffers[index]); + physaddr = (uint64_t)vtophys(contigmem_buffers[index].addr); return sysctl_handle_64(oidp, &physaddr, 0, req); } @@ -200,22 +239,121 @@ static int contigmem_open(struct cdev *cdev, int fflags, int devtype, struct thread *td) { + + atomic_add_int(&contigmem_refcnt, 1); + return 0; } static int -contigmem_mmap(struct cdev *cdev, vm_ooffset_t offset, vm_paddr_t *paddr, - int prot, vm_memattr_t *memattr) +contigmem_close(struct cdev *cdev, int fflags, int devtype, + struct thread *td) { - *paddr = offset; + atomic_subtract_int(&contigmem_refcnt, 1); + return 0; } static int +contigmem_cdev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, + vm_ooffset_t foff, struct ucred *cred, u_short *color) +{ + struct contigmem_vm_handle *vmh = handle; + struct contigmem_buffer *buf; + + buf = &contigmem_buffers[vmh->buffer_index]; + + atomic_add_int(&contigmem_refcnt, 1); + + mtx_lock(&buf->mtx); + if (buf->refcnt == 0) + memset(buf->addr, 0, contigmem_buffer_size); + buf->refcnt++; + mtx_unlock(&buf->mtx); + + return 0; +} + +static void +contigmem_cdev_pager_dtor(void *handle) +{ + struct contigmem_vm_handle *vmh = handle; + struct contigmem_buffer *buf; + + buf = &contigmem_buffers[vmh->buffer_index]; + + mtx_lock(&buf->mtx); + buf->refcnt--; + mtx_unlock(&buf->mtx); + + free(vmh, M_CONTIGMEM); + + atomic_subtract_int(&contigmem_refcnt, 1); +} + +static int +contigmem_cdev_pager_fault(vm_object_t object, vm_ooffset_t offset, int prot, + vm_page_t *mres) +{ + vm_paddr_t paddr; + vm_page_t m_paddr, page; + vm_memattr_t memattr, memattr1; + + memattr = object->memattr; + + VM_OBJECT_WUNLOCK(object); + + paddr = offset; + + m_paddr = vm_phys_paddr_to_vm_page(paddr); + if (m_paddr != NULL) { + memattr1 = pmap_page_get_memattr(m_paddr); + if (memattr1 != memattr) + memattr = memattr1; + } + + if (((*mres)->flags & PG_FICTITIOUS) != 0) { + /* + * If the passed in result page is a fake page, update it with + * the new physical address. + */ + page = *mres; + VM_OBJECT_WLOCK(object); + vm_page_updatefake(page, paddr, memattr); + } else { + vm_page_t mret; + /* + * Replace the passed in reqpage page with our own fake page and + * free up the original page. + */ + page = vm_page_getfake(paddr, memattr); + VM_OBJECT_WLOCK(object); + mret = vm_page_replace(page, object, (*mres)->pindex); + KASSERT(mret == *mres, + ("invalid page replacement, old=%p, ret=%p", *mres, mret)); + vm_page_lock(mret); + vm_page_free(mret); + vm_page_unlock(mret); + *mres = page; + } + + page->valid = VM_PAGE_BITS_ALL; + + return VM_PAGER_OK; +} + +static struct cdev_pager_ops contigmem_cdev_pager_ops = { + .cdev_pg_ctor = contigmem_cdev_pager_ctor, + .cdev_pg_dtor = contigmem_cdev_pager_dtor, + .cdev_pg_fault = contigmem_cdev_pager_fault, +}; + +static int contigmem_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, vm_size_t size, struct vm_object **obj, int nprot) { + struct contigmem_vm_handle *vmh; uint64_t buffer_index; /* @@ -227,10 +365,17 @@ contigmem_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, vm_size_t size, if (buffer_index >= contigmem_num_buffers) return EINVAL; - memset(contigmem_buffers[buffer_index], 0, contigmem_buffer_size); - *offset = (vm_ooffset_t)vtophys(contigmem_buffers[buffer_index]); - *obj = vm_pager_allocate(OBJT_DEVICE, cdev, size, nprot, *offset, - curthread->td_ucred); + if (size > contigmem_buffer_size) + return EINVAL; + + vmh = malloc(sizeof(*vmh), M_CONTIGMEM, M_NOWAIT | M_ZERO); + if (vmh == NULL) + return ENOMEM; + vmh->buffer_index = buffer_index; + + *offset = (vm_ooffset_t)vtophys(contigmem_buffers[buffer_index].addr); + *obj = cdev_pager_allocate(vmh, OBJT_DEVICE, &contigmem_cdev_pager_ops, + size, nprot, *offset, curthread->td_ucred); return 0; } diff --git a/lib/librte_eal/bsdapp/eal/Makefile b/lib/librte_eal/bsdapp/eal/Makefile index a0f99502..005019ed 100644 --- a/lib/librte_eal/bsdapp/eal/Makefile +++ b/lib/librte_eal/bsdapp/eal/Makefile @@ -48,7 +48,7 @@ LDLIBS += -lgcc_s EXPORT_MAP := rte_eal_version.map -LIBABIVER := 4 +LIBABIVER := 5 # specific to bsdapp exec-env SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) := eal.c @@ -87,6 +87,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_malloc.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += malloc_elem.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += malloc_heap.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_keepalive.c +SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_service.c # from arch dir SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_cpuflags.c diff --git a/lib/librte_eal/bsdapp/eal/eal.c b/lib/librte_eal/bsdapp/eal/eal.c index 05f0c1f9..5fa59884 100644 --- a/lib/librte_eal/bsdapp/eal/eal.c +++ b/lib/librte_eal/bsdapp/eal/eal.c @@ -45,7 +45,6 @@ #include <stddef.h> #include <errno.h> #include <limits.h> -#include <errno.h> #include <sys/mman.h> #include <sys/queue.h> @@ -59,6 +58,7 @@ #include <rte_errno.h> #include <rte_per_lcore.h> #include <rte_lcore.h> +#include <rte_service_component.h> #include <rte_log.h> #include <rte_random.h> #include <rte_cycles.h> @@ -69,7 +69,6 @@ #include <rte_pci.h> #include <rte_dev.h> #include <rte_devargs.h> -#include <rte_common.h> #include <rte_version.h> #include <rte_atomic.h> #include <malloc_heap.h> @@ -615,6 +614,11 @@ rte_eal_init(int argc, char **argv) rte_config.master_lcore, thread_id, cpuset, ret == 0 ? "" : "..."); + if (eal_option_device_parse()) { + rte_errno = ENODEV; + return -1; + } + if (rte_bus_scan()) { rte_eal_init_alert("Cannot scan the buses for devices\n"); rte_errno = ENODEV; @@ -653,6 +657,14 @@ rte_eal_init(int argc, char **argv) rte_eal_mp_remote_launch(sync_func, NULL, SKIP_MASTER); rte_eal_mp_wait_lcore(); + /* initialize services so vdevs register service during bus_probe. */ + ret = rte_service_init(); + if (ret) { + rte_eal_init_alert("rte_service_init() failed\n"); + rte_errno = ENOEXEC; + return -1; + } + /* Probe all the buses and devices/drivers on them */ if (rte_bus_probe()) { rte_eal_init_alert("Cannot probe devices\n"); @@ -660,6 +672,15 @@ rte_eal_init(int argc, char **argv) return -1; } + /* initialize default service/lcore mappings and start running. Ignore + * -ENOTSUP, as it indicates no service coremask passed to EAL. + */ + ret = rte_service_start_with_defaults(); + if (ret < 0 && ret != -ENOTSUP) { + rte_errno = ENOEXEC; + return -1; + } + rte_eal_mcfg_complete(); return fctret; diff --git a/lib/librte_eal/bsdapp/eal/eal_pci.c b/lib/librte_eal/bsdapp/eal/eal_pci.c index e321461d..04eacdcc 100644 --- a/lib/librte_eal/bsdapp/eal/eal_pci.c +++ b/lib/librte_eal/bsdapp/eal/eal_pci.c @@ -41,7 +41,6 @@ #include <sys/types.h> #include <sys/stat.h> #include <fcntl.h> -#include <stdarg.h> #include <errno.h> #include <dirent.h> #include <limits.h> @@ -52,7 +51,6 @@ #include <dev/pci/pcireg.h> #if defined(RTE_ARCH_X86) -#include <sys/types.h> #include <machine/cpufunc.h> #endif @@ -282,8 +280,7 @@ pci_scan_one(int dev_pci_fd, struct pci_conf *conf) /* FreeBSD has no NUMA support (yet) */ dev->device.numa_node = 0; - rte_pci_device_name(&dev->addr, dev->name, sizeof(dev->name)); - dev->device.name = dev->name; + pci_name_set(dev); /* FreeBSD has only one pass through driver */ dev->kdrv = RTE_KDRV_NIC_UIO; @@ -334,6 +331,7 @@ pci_scan_one(int dev_pci_fd, struct pci_conf *conf) } else { /* already registered */ dev2->kdrv = dev->kdrv; dev2->max_vfs = dev->max_vfs; + pci_name_set(dev2); memmove(dev2->mem_resource, dev->mem_resource, sizeof(dev->mem_resource)); @@ -396,7 +394,7 @@ rte_pci_scan(void) close(fd); - RTE_LOG(ERR, EAL, "PCI scan found %u devices\n", dev_count); + RTE_LOG(DEBUG, EAL, "PCI scan found %u devices\n", dev_count); return 0; error: diff --git a/lib/librte_eal/bsdapp/eal/eal_thread.c b/lib/librte_eal/bsdapp/eal/eal_thread.c index 1b8cd8a6..783d68c5 100644 --- a/lib/librte_eal/bsdapp/eal/eal_thread.c +++ b/lib/librte_eal/bsdapp/eal/eal_thread.c @@ -49,7 +49,6 @@ #include <rte_memzone.h> #include <rte_per_lcore.h> #include <rte_eal.h> -#include <rte_per_lcore.h> #include <rte_lcore.h> #include "eal_private.h" diff --git a/lib/librte_eal/bsdapp/eal/rte_eal_version.map b/lib/librte_eal/bsdapp/eal/rte_eal_version.map index 2e48a736..aac6fd77 100644 --- a/lib/librte_eal/bsdapp/eal/rte_eal_version.map +++ b/lib/librte_eal/bsdapp/eal/rte_eal_version.map @@ -193,3 +193,47 @@ DPDK_17.05 { vfio_get_group_no; } DPDK_17.02; + +DPDK_17.08 { + global: + + rte_bus_find; + rte_bus_find_by_device; + rte_bus_find_by_name; + rte_log_get_level; + +} DPDK_17.05; + +EXPERIMENTAL { + global: + + rte_eal_devargs_insert; + rte_eal_devargs_parse; + rte_eal_devargs_remove; + rte_eal_hotplug_add; + rte_eal_hotplug_remove; + rte_service_disable_on_lcore; + rte_service_dump; + rte_service_enable_on_lcore; + rte_service_get_by_id; + rte_service_get_by_name; + rte_service_get_count; + rte_service_get_enabled_on_lcore; + rte_service_is_running; + rte_service_lcore_add; + rte_service_lcore_count; + rte_service_lcore_del; + rte_service_lcore_list; + rte_service_lcore_reset_all; + rte_service_lcore_start; + rte_service_lcore_stop; + rte_service_probe_capability; + rte_service_register; + rte_service_reset; + rte_service_set_stats_enable; + rte_service_start; + rte_service_start_with_defaults; + rte_service_stop; + rte_service_unregister; + +} DPDK_17.08; diff --git a/lib/librte_eal/common/Makefile b/lib/librte_eal/common/Makefile index a5bd1089..e8fd67a2 100644 --- a/lib/librte_eal/common/Makefile +++ b/lib/librte_eal/common/Makefile @@ -41,10 +41,11 @@ INC += rte_eal_memconfig.h rte_malloc_heap.h INC += rte_hexdump.h rte_devargs.h rte_bus.h rte_dev.h rte_vdev.h INC += rte_pci_dev_feature_defs.h rte_pci_dev_features.h INC += rte_malloc.h rte_keepalive.h rte_time.h +INC += rte_service.h rte_service_component.h GENERIC_INC := rte_atomic.h rte_byteorder.h rte_cycles.h rte_prefetch.h GENERIC_INC += rte_spinlock.h rte_memcpy.h rte_cpuflags.h rte_rwlock.h -GENERIC_INC += rte_vect.h rte_io.h +GENERIC_INC += rte_vect.h rte_pause.h rte_io.h # defined in mk/arch/$(RTE_ARCH)/rte.vars.mk ARCH_DIR ?= $(RTE_ARCH) diff --git a/lib/librte_eal/common/arch/arm/rte_cpuflags.c b/lib/librte_eal/common/arch/arm/rte_cpuflags.c index 79160a60..5636e9c1 100644 --- a/lib/librte_eal/common/arch/arm/rte_cpuflags.c +++ b/lib/librte_eal/common/arch/arm/rte_cpuflags.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2015. + * Copyright (C) Cavium, Inc. 2015. * Copyright(c) 2015 RehiveTech. All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -14,7 +14,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/lib/librte_eal/common/eal_common_bus.c b/lib/librte_eal/common/eal_common_bus.c index 8f9baf8b..08bec2d9 100644 --- a/lib/librte_eal/common/eal_common_bus.c +++ b/lib/librte_eal/common/eal_common_bus.c @@ -1,8 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 NXP - * All rights reserved. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -50,6 +49,9 @@ rte_bus_register(struct rte_bus *bus) /* A bus should mandatorily have the scan implemented */ RTE_VERIFY(bus->scan); RTE_VERIFY(bus->probe); + RTE_VERIFY(bus->find_device); + /* Buses supporting driver plug also require unplug. */ + RTE_VERIFY(!bus->plug || bus->unplug); TAILQ_INSERT_TAIL(&rte_bus_list, bus, next); RTE_LOG(DEBUG, EAL, "Registered [%s] bus.\n", bus->name); @@ -89,7 +91,7 @@ rte_bus_probe(void) struct rte_bus *bus, *vbus = NULL; TAILQ_FOREACH(bus, &rte_bus_list, next) { - if (!strcmp(bus->name, "virtual")) { + if (!strcmp(bus->name, "vdev")) { vbus = bus; continue; } @@ -145,3 +147,78 @@ rte_bus_dump(FILE *f) } } } + +struct rte_bus * +rte_bus_find(const struct rte_bus *start, rte_bus_cmp_t cmp, + const void *data) +{ + struct rte_bus *bus = NULL; + + TAILQ_FOREACH(bus, &rte_bus_list, next) { + if (start && bus == start) { + start = NULL; /* starting point found */ + continue; + } + if (cmp(bus, data) == 0) + break; + } + return bus; +} + +static int +cmp_rte_device(const struct rte_device *dev1, const void *_dev2) +{ + const struct rte_device *dev2 = _dev2; + + return dev1 != dev2; +} + +static int +bus_find_device(const struct rte_bus *bus, const void *_dev) +{ + struct rte_device *dev; + + dev = bus->find_device(NULL, cmp_rte_device, _dev); + return dev == NULL; +} + +struct rte_bus * +rte_bus_find_by_device(const struct rte_device *dev) +{ + return rte_bus_find(NULL, bus_find_device, (const void *)dev); +} + +static int +cmp_bus_name(const struct rte_bus *bus, const void *_name) +{ + const char *name = _name; + + return strcmp(bus->name, name); +} + +struct rte_bus * +rte_bus_find_by_name(const char *busname) +{ + return rte_bus_find(NULL, cmp_bus_name, (const void *)busname); +} + +static int +bus_can_parse(const struct rte_bus *bus, const void *_name) +{ + const char *name = _name; + + return !(bus->parse && bus->parse(name, NULL) == 0); +} + +struct rte_bus * +rte_bus_find_by_device_name(const char *str) +{ + char name[RTE_DEV_NAME_MAX_LEN]; + char *c; + + snprintf(name, sizeof(name), "%s", str); + c = strchr(name, ','); + if (c != NULL) + c[0] = '\0'; + return rte_bus_find(NULL, bus_can_parse, name); +} diff --git a/lib/librte_eal/common/eal_common_dev.c b/lib/librte_eal/common/eal_common_dev.c index a400ddd0..e2512755 100644 --- a/lib/librte_eal/common/eal_common_dev.c +++ b/lib/librte_eal/common/eal_common_dev.c @@ -37,58 +37,210 @@ #include <inttypes.h> #include <sys/queue.h> +#include <rte_bus.h> #include <rte_dev.h> #include <rte_devargs.h> #include <rte_debug.h> -#include <rte_devargs.h> #include <rte_log.h> #include "eal_private.h" +static int cmp_detached_dev_name(const struct rte_device *dev, + const void *_name) +{ + const char *name = _name; + + /* skip attached devices */ + if (dev->driver != NULL) + return 1; + + return strcmp(dev->name, name); +} + +static int cmp_dev_name(const struct rte_device *dev, const void *_name) +{ + const char *name = _name; + + return strcmp(dev->name, name); +} + int rte_eal_dev_attach(const char *name, const char *devargs) { - struct rte_pci_addr addr; + struct rte_bus *bus; + int ret; if (name == NULL || devargs == NULL) { RTE_LOG(ERR, EAL, "Invalid device or arguments provided\n"); return -EINVAL; } - if (eal_parse_pci_DomBDF(name, &addr) == 0) { - if (rte_pci_probe_one(&addr) < 0) - goto err; + bus = rte_bus_find_by_device_name(name); + if (bus == NULL) { + RTE_LOG(ERR, EAL, "Unable to find a bus for the device '%s'\n", + name); + return -EINVAL; + } + if (strcmp(bus->name, "pci") == 0) + return rte_eal_hotplug_add("pci", name, devargs); + if (strcmp(bus->name, "vdev") != 0) { + RTE_LOG(ERR, EAL, "Device attach is only supported for PCI and vdev devices.\n"); + return -ENOTSUP; + } - } else { - if (rte_vdev_init(name, devargs)) - goto err; + /* + * If we haven't found a bus device the user meant to "hotplug" a + * virtual device instead. + */ + ret = rte_vdev_init(name, devargs); + if (ret) + RTE_LOG(ERR, EAL, "Driver cannot attach the device (%s)\n", + name); + return ret; +} + +int rte_eal_dev_detach(struct rte_device *dev) +{ + struct rte_bus *bus; + int ret; + + if (dev == NULL) { + RTE_LOG(ERR, EAL, "Invalid device provided.\n"); + return -EINVAL; } - return 0; + bus = rte_bus_find_by_device(dev); + if (bus == NULL) { + RTE_LOG(ERR, EAL, "Cannot find bus for device (%s)\n", + dev->name); + return -EINVAL; + } + + if (bus->unplug == NULL) { + RTE_LOG(ERR, EAL, "Bus function not supported\n"); + return -ENOTSUP; + } -err: - RTE_LOG(ERR, EAL, "Driver cannot attach the device (%s)\n", name); - return -EINVAL; + ret = bus->unplug(dev); + if (ret) + RTE_LOG(ERR, EAL, "Driver cannot detach the device (%s)\n", + dev->name); + return ret; } -int rte_eal_dev_detach(const char *name) +static char * +full_dev_name(const char *bus, const char *dev, const char *args) { - struct rte_pci_addr addr; + char *name; + size_t len; + len = snprintf(NULL, 0, "%s:%s,%s", bus, dev, args) + 1; + name = calloc(1, len); if (name == NULL) { - RTE_LOG(ERR, EAL, "Invalid device provided.\n"); - return -EINVAL; + RTE_LOG(ERR, EAL, "Could not allocate full device name\n"); + return NULL; } + snprintf(name, len, "%s:%s,%s", bus, dev, args); + return name; +} - if (eal_parse_pci_DomBDF(name, &addr) == 0) { - if (rte_pci_detach(&addr) < 0) - goto err; - } else { - if (rte_vdev_uninit(name)) - goto err; +int rte_eal_hotplug_add(const char *busname, const char *devname, + const char *devargs) +{ + struct rte_bus *bus; + struct rte_device *dev; + struct rte_devargs *da; + char *name; + int ret; + + bus = rte_bus_find_by_name(busname); + if (bus == NULL) { + RTE_LOG(ERR, EAL, "Cannot find bus (%s)\n", busname); + return -ENOENT; + } + + if (bus->plug == NULL) { + RTE_LOG(ERR, EAL, "Function plug not supported by bus (%s)\n", + bus->name); + return -ENOTSUP; + } + + name = full_dev_name(busname, devname, devargs); + if (name == NULL) + return -ENOMEM; + + da = calloc(1, sizeof(*da)); + if (da == NULL) { + ret = -ENOMEM; + goto err_name; + } + + ret = rte_eal_devargs_parse(name, da); + if (ret) + goto err_devarg; + + ret = rte_eal_devargs_insert(da); + if (ret) + goto err_devarg; + + ret = bus->scan(); + if (ret) + goto err_devarg; + + dev = bus->find_device(NULL, cmp_detached_dev_name, devname); + if (dev == NULL) { + RTE_LOG(ERR, EAL, "Cannot find unplugged device (%s)\n", + devname); + ret = -ENODEV; + goto err_devarg; + } + + ret = bus->plug(dev); + if (ret) { + RTE_LOG(ERR, EAL, "Driver cannot attach the device (%s)\n", + dev->name); + goto err_devarg; } + free(name); return 0; -err: - RTE_LOG(ERR, EAL, "Driver cannot detach the device (%s)\n", name); - return -EINVAL; +err_devarg: + if (rte_eal_devargs_remove(busname, devname)) { + free(da->args); + free(da); + } +err_name: + free(name); + return ret; +} + +int rte_eal_hotplug_remove(const char *busname, const char *devname) +{ + struct rte_bus *bus; + struct rte_device *dev; + int ret; + + bus = rte_bus_find_by_name(busname); + if (bus == NULL) { + RTE_LOG(ERR, EAL, "Cannot find bus (%s)\n", busname); + return -ENOENT; + } + + if (bus->unplug == NULL) { + RTE_LOG(ERR, EAL, "Function unplug not supported by bus (%s)\n", + bus->name); + return -ENOTSUP; + } + + dev = bus->find_device(NULL, cmp_dev_name, devname); + if (dev == NULL) { + RTE_LOG(ERR, EAL, "Cannot find plugged device (%s)\n", devname); + return -EINVAL; + } + + ret = bus->unplug(dev); + if (ret) + RTE_LOG(ERR, EAL, "Driver cannot detach the device (%s)\n", + dev->name); + rte_eal_devargs_remove(busname, devname); + return ret; } diff --git a/lib/librte_eal/common/eal_common_devargs.c b/lib/librte_eal/common/eal_common_devargs.c index ffa8ad96..6ac88d6a 100644 --- a/lib/librte_eal/common/eal_common_devargs.c +++ b/lib/librte_eal/common/eal_common_devargs.c @@ -40,8 +40,9 @@ #include <stdio.h> #include <string.h> -#include <rte_pci.h> +#include <rte_dev.h> #include <rte_devargs.h> +#include <rte_tailq.h> #include "eal_private.h" /** Global list of user devices */ @@ -78,50 +79,107 @@ rte_eal_parse_devargs_str(const char *devargs_str, return 0; } +static int +bus_name_cmp(const struct rte_bus *bus, const void *name) +{ + return strncmp(bus->name, name, strlen(bus->name)); +} + +int +rte_eal_devargs_parse(const char *dev, struct rte_devargs *da) +{ + struct rte_bus *bus = NULL; + const char *devname; + const size_t maxlen = sizeof(da->name); + size_t i; + + if (dev == NULL || da == NULL) + return -EINVAL; + /* Retrieve eventual bus info */ + do { + devname = dev; + bus = rte_bus_find(bus, bus_name_cmp, dev); + if (bus == NULL) + break; + devname = dev + strlen(bus->name) + 1; + if (rte_bus_find_by_device_name(devname) == bus) + break; + } while (1); + /* Store device name */ + i = 0; + while (devname[i] != '\0' && devname[i] != ',') { + da->name[i] = devname[i]; + i++; + if (i == maxlen) { + fprintf(stderr, "WARNING: Parsing \"%s\": device name should be shorter than %zu\n", + dev, maxlen); + da->name[i - 1] = '\0'; + return -EINVAL; + } + } + da->name[i] = '\0'; + if (bus == NULL) { + bus = rte_bus_find_by_device_name(da->name); + if (bus == NULL) { + fprintf(stderr, "ERROR: failed to parse device \"%s\"\n", + da->name); + return -EFAULT; + } + } + da->bus = bus; + /* Parse eventual device arguments */ + if (devname[i] == ',') + da->args = strdup(&devname[i + 1]); + else + da->args = strdup(""); + if (da->args == NULL) { + fprintf(stderr, "ERROR: not enough memory to parse arguments\n"); + return -ENOMEM; + } + return 0; +} + +int +rte_eal_devargs_insert(struct rte_devargs *da) +{ + int ret; + + ret = rte_eal_devargs_remove(da->bus->name, da->name); + if (ret < 0) + return ret; + TAILQ_INSERT_TAIL(&devargs_list, da, next); + return 0; +} + /* store a whitelist parameter for later parsing */ int rte_eal_devargs_add(enum rte_devtype devtype, const char *devargs_str) { struct rte_devargs *devargs = NULL; - char *buf = NULL; - int ret; + struct rte_bus *bus = NULL; + const char *dev = devargs_str; - /* use malloc instead of rte_malloc as it's called early at init */ - devargs = malloc(sizeof(*devargs)); + /* use calloc instead of rte_zmalloc as it's called early at init */ + devargs = calloc(1, sizeof(*devargs)); if (devargs == NULL) goto fail; - memset(devargs, 0, sizeof(*devargs)); - devargs->type = devtype; - - if (rte_eal_parse_devargs_str(devargs_str, &buf, &devargs->args)) + if (rte_eal_devargs_parse(dev, devargs)) goto fail; - - switch (devargs->type) { - case RTE_DEVTYPE_WHITELISTED_PCI: - case RTE_DEVTYPE_BLACKLISTED_PCI: - /* try to parse pci identifier */ - if (eal_parse_pci_BDF(buf, &devargs->pci.addr) != 0 && - eal_parse_pci_DomBDF(buf, &devargs->pci.addr) != 0) - goto fail; - - break; - case RTE_DEVTYPE_VIRTUAL: - /* save driver name */ - ret = snprintf(devargs->virt.drv_name, - sizeof(devargs->virt.drv_name), "%s", buf); - if (ret < 0 || ret >= (int)sizeof(devargs->virt.drv_name)) - goto fail; - - break; + devargs->type = devtype; + bus = devargs->bus; + if (devargs->type == RTE_DEVTYPE_BLACKLISTED_PCI) + devargs->policy = RTE_DEV_BLACKLISTED; + if (bus->conf.scan_mode == RTE_BUS_SCAN_UNDEFINED) { + if (devargs->policy == RTE_DEV_WHITELISTED) + bus->conf.scan_mode = RTE_BUS_SCAN_WHITELIST; + else if (devargs->policy == RTE_DEV_BLACKLISTED) + bus->conf.scan_mode = RTE_BUS_SCAN_BLACKLIST; } - - free(buf); TAILQ_INSERT_TAIL(&devargs_list, devargs, next); return 0; fail: - free(buf); if (devargs) { free(devargs->args); free(devargs); @@ -130,6 +188,24 @@ fail: return -1; } +int +rte_eal_devargs_remove(const char *busname, const char *devname) +{ + struct rte_devargs *d; + void *tmp; + + TAILQ_FOREACH_SAFE(d, &devargs_list, next, tmp) { + if (strcmp(d->bus->name, busname) == 0 && + strcmp(d->name, devname) == 0) { + TAILQ_REMOVE(&devargs_list, d, next); + free(d->args); + free(d); + return 0; + } + } + return 1; +} + /* count the number of devices of a specified type */ unsigned int rte_eal_devargs_type_count(enum rte_devtype devtype) @@ -151,27 +227,10 @@ rte_eal_devargs_dump(FILE *f) { struct rte_devargs *devargs; - fprintf(f, "User device white list:\n"); + fprintf(f, "User device list:\n"); TAILQ_FOREACH(devargs, &devargs_list, next) { - if (devargs->type == RTE_DEVTYPE_WHITELISTED_PCI) - fprintf(f, " PCI whitelist " PCI_PRI_FMT " %s\n", - devargs->pci.addr.domain, - devargs->pci.addr.bus, - devargs->pci.addr.devid, - devargs->pci.addr.function, - devargs->args); - else if (devargs->type == RTE_DEVTYPE_BLACKLISTED_PCI) - fprintf(f, " PCI blacklist " PCI_PRI_FMT " %s\n", - devargs->pci.addr.domain, - devargs->pci.addr.bus, - devargs->pci.addr.devid, - devargs->pci.addr.function, - devargs->args); - else if (devargs->type == RTE_DEVTYPE_VIRTUAL) - fprintf(f, " VIRTUAL %s %s\n", - devargs->virt.drv_name, - devargs->args); - else - fprintf(f, " UNKNOWN %s\n", devargs->args); + fprintf(f, " [%s]: %s %s\n", + (devargs->bus ? devargs->bus->name : "??"), + devargs->name, devargs->args); } } diff --git a/lib/librte_eal/common/eal_common_launch.c b/lib/librte_eal/common/eal_common_launch.c index 229c3a03..137c191d 100644 --- a/lib/librte_eal/common/eal_common_launch.c +++ b/lib/librte_eal/common/eal_common_launch.c @@ -41,6 +41,7 @@ #include <rte_memzone.h> #include <rte_eal.h> #include <rte_atomic.h> +#include <rte_pause.h> #include <rte_per_lcore.h> #include <rte_lcore.h> @@ -54,7 +55,8 @@ rte_eal_wait_lcore(unsigned slave_id) return 0; while (lcore_config[slave_id].state != WAIT && - lcore_config[slave_id].state != FINISHED); + lcore_config[slave_id].state != FINISHED) + rte_pause(); rte_rmb(); diff --git a/lib/librte_eal/common/eal_common_lcore.c b/lib/librte_eal/common/eal_common_lcore.c index 84fa0cb5..0db1555b 100644 --- a/lib/librte_eal/common/eal_common_lcore.c +++ b/lib/librte_eal/common/eal_common_lcore.c @@ -81,6 +81,7 @@ rte_eal_cpu_init(void) /* By default, each detected core is enabled */ config->lcore_role[lcore_id] = ROLE_RTE; + lcore_config[lcore_id].core_role = ROLE_RTE; lcore_config[lcore_id].core_id = eal_cpu_core_id(lcore_id); lcore_config[lcore_id].socket_id = eal_cpu_socket_id(lcore_id); if (lcore_config[lcore_id].socket_id >= RTE_MAX_NUMA_NODES) { diff --git a/lib/librte_eal/common/eal_common_log.c b/lib/librte_eal/common/eal_common_log.c index ddf65b7f..0e3b9320 100644 --- a/lib/librte_eal/common/eal_common_log.c +++ b/lib/librte_eal/common/eal_common_log.c @@ -112,6 +112,15 @@ rte_get_log_level(void) return rte_log_get_global_level(); } +int +rte_log_get_level(uint32_t type) +{ + if (type >= rte_logs.dynamic_types_len) + return -1; + + return rte_logs.dynamic_types[type].loglevel; +} + /* Set global log type */ __rte_deprecated void rte_set_log_type(uint32_t type, int enable) @@ -173,13 +182,13 @@ rte_log_set_level_regexp(const char *pattern, uint32_t level) return 0; } -/* get the current loglevel for the message beeing processed */ +/* get the current loglevel for the message being processed */ int rte_log_cur_msg_loglevel(void) { return RTE_PER_LCORE(log_cur_msg).loglevel; } -/* get the current logtype for the message beeing processed */ +/* get the current logtype for the message being processed */ int rte_log_cur_msg_logtype(void) { return RTE_PER_LCORE(log_cur_msg).logtype; diff --git a/lib/librte_eal/common/eal_common_memory.c b/lib/librte_eal/common/eal_common_memory.c index 6155752e..996877ef 100644 --- a/lib/librte_eal/common/eal_common_memory.c +++ b/lib/librte_eal/common/eal_common_memory.c @@ -35,7 +35,9 @@ #include <stdint.h> #include <stdlib.h> #include <stdarg.h> +#include <unistd.h> #include <inttypes.h> +#include <sys/mman.h> #include <sys/queue.h> #include <rte_memory.h> @@ -135,6 +137,16 @@ rte_eal_memdevice_init(void) return 0; } +/* Lock page in physical memory and prevent from swapping. */ +int +rte_mem_lock_page(const void *virt) +{ + unsigned long virtual = (unsigned long)virt; + int page_size = getpagesize(); + unsigned long aligned = (virtual & ~(page_size - 1)); + return mlock((void *)aligned, page_size); +} + /* init memory subsystem */ int rte_eal_memory_init(void) diff --git a/lib/librte_eal/common/eal_common_memzone.c b/lib/librte_eal/common/eal_common_memzone.c index 64f4e0ad..3026e36b 100644 --- a/lib/librte_eal/common/eal_common_memzone.c +++ b/lib/librte_eal/common/eal_common_memzone.c @@ -189,7 +189,8 @@ memzone_reserve_aligned_thread_unsafe(const char *name, size_t len, return NULL; } - if ((socket_id != SOCKET_ID_ANY) && (socket_id >= RTE_MAX_NUMA_NODES)) { + if ((socket_id != SOCKET_ID_ANY) && + (socket_id >= RTE_MAX_NUMA_NODES || socket_id < 0)) { rte_errno = EINVAL; return NULL; } diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c index f470195f..1da185e5 100644 --- a/lib/librte_eal/common/eal_common_options.c +++ b/lib/librte_eal/common/eal_common_options.c @@ -47,6 +47,7 @@ #include <rte_eal.h> #include <rte_log.h> #include <rte_lcore.h> +#include <rte_tailq.h> #include <rte_version.h> #include <rte_devargs.h> #include <rte_memcpy.h> @@ -61,9 +62,11 @@ const char eal_short_options[] = "b:" /* pci-blacklist */ "c:" /* coremask */ + "s:" /* service coremask */ "d:" /* driver */ "h" /* help */ "l:" /* corelist */ + "S:" /* service corelist */ "m:" /* memory size */ "n:" /* memory channels */ "r:" /* memory ranks */ @@ -123,11 +126,67 @@ static const char *default_solib_dir = RTE_EAL_PMD_PATH; static const char dpdk_solib_path[] __attribute__((used)) = "DPDK_PLUGIN_PATH=" RTE_EAL_PMD_PATH; +TAILQ_HEAD(device_option_list, device_option); + +struct device_option { + TAILQ_ENTRY(device_option) next; + + enum rte_devtype type; + char arg[]; +}; + +static struct device_option_list devopt_list = +TAILQ_HEAD_INITIALIZER(devopt_list); static int master_lcore_parsed; static int mem_parsed; static int core_parsed; +static int +eal_option_device_add(enum rte_devtype type, const char *optarg) +{ + struct device_option *devopt; + size_t optlen; + int ret; + + optlen = strlen(optarg) + 1; + devopt = calloc(1, sizeof(*devopt) + optlen); + if (devopt == NULL) { + RTE_LOG(ERR, EAL, "Unable to allocate device option\n"); + return -ENOMEM; + } + + devopt->type = type; + ret = snprintf(devopt->arg, optlen, "%s", optarg); + if (ret < 0) { + RTE_LOG(ERR, EAL, "Unable to copy device option\n"); + free(devopt); + return -EINVAL; + } + TAILQ_INSERT_TAIL(&devopt_list, devopt, next); + return 0; +} + +int +eal_option_device_parse(void) +{ + struct device_option *devopt; + void *tmp; + int ret = 0; + + TAILQ_FOREACH_SAFE(devopt, &devopt_list, next, tmp) { + if (ret == 0) { + ret = rte_eal_devargs_add(devopt->type, devopt->arg); + if (ret) + RTE_LOG(ERR, EAL, "Unable to parse device '%s'\n", + devopt->arg); + } + TAILQ_REMOVE(&devopt_list, devopt, next); + free(devopt); + } + return ret; +} + void eal_reset_internal_config(struct internal_config *internal_cfg) { @@ -267,6 +326,77 @@ static int xdigit2val(unsigned char c) } static int +eal_parse_service_coremask(const char *coremask) +{ + struct rte_config *cfg = rte_eal_get_configuration(); + int i, j, idx = 0; + unsigned int count = 0; + char c; + int val; + + if (coremask == NULL) + return -1; + /* Remove all blank characters ahead and after . + * Remove 0x/0X if exists. + */ + while (isblank(*coremask)) + coremask++; + if (coremask[0] == '0' && ((coremask[1] == 'x') + || (coremask[1] == 'X'))) + coremask += 2; + i = strlen(coremask); + while ((i > 0) && isblank(coremask[i - 1])) + i--; + + if (i == 0) + return -1; + + for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) { + c = coremask[i]; + if (isxdigit(c) == 0) { + /* invalid characters */ + return -1; + } + val = xdigit2val(c); + for (j = 0; j < BITS_PER_HEX && idx < RTE_MAX_LCORE; + j++, idx++) { + if ((1 << j) & val) { + /* handle master lcore already parsed */ + uint32_t lcore = idx; + if (master_lcore_parsed && + cfg->master_lcore == lcore) { + RTE_LOG(ERR, EAL, + "Error: lcore %u is master lcore, cannot use as service core\n", + idx); + return -1; + } + + if (!lcore_config[idx].detected) { + RTE_LOG(ERR, EAL, + "lcore %u unavailable\n", idx); + return -1; + } + lcore_config[idx].core_role = ROLE_SERVICE; + count++; + } + } + } + + for (; i >= 0; i--) + if (coremask[i] != '0') + return -1; + + for (; idx < RTE_MAX_LCORE; idx++) + lcore_config[idx].core_index = -1; + + if (count == 0) + return -1; + + cfg->service_lcore_count = count; + return 0; +} + +static int eal_parse_coremask(const char *coremask) { struct rte_config *cfg = rte_eal_get_configuration(); @@ -330,6 +460,72 @@ eal_parse_coremask(const char *coremask) } static int +eal_parse_service_corelist(const char *corelist) +{ + struct rte_config *cfg = rte_eal_get_configuration(); + int i, idx = 0; + unsigned count = 0; + char *end = NULL; + int min, max; + + if (corelist == NULL) + return -1; + + /* Remove all blank characters ahead and after */ + while (isblank(*corelist)) + corelist++; + i = strlen(corelist); + while ((i > 0) && isblank(corelist[i - 1])) + i--; + + /* Get list of cores */ + min = RTE_MAX_LCORE; + do { + while (isblank(*corelist)) + corelist++; + if (*corelist == '\0') + return -1; + errno = 0; + idx = strtoul(corelist, &end, 10); + if (errno || end == NULL) + return -1; + while (isblank(*end)) + end++; + if (*end == '-') { + min = idx; + } else if ((*end == ',') || (*end == '\0')) { + max = idx; + if (min == RTE_MAX_LCORE) + min = idx; + for (idx = min; idx <= max; idx++) { + if (cfg->lcore_role[idx] != ROLE_SERVICE) { + /* handle master lcore already parsed */ + uint32_t lcore = idx; + if (cfg->master_lcore == lcore && + master_lcore_parsed) { + RTE_LOG(ERR, EAL, + "Error: lcore %u is master lcore, cannot use as service core\n", + idx); + return -1; + } + lcore_config[idx].core_role = + ROLE_SERVICE; + count++; + } + } + min = RTE_MAX_LCORE; + } else + return -1; + corelist = end + 1; + } while (*end != '\0'); + + if (count == 0) + return -1; + + return 0; +} + +static int eal_parse_corelist(const char *corelist) { struct rte_config *cfg = rte_eal_get_configuration(); @@ -409,6 +605,13 @@ eal_parse_master_lcore(const char *arg) if (cfg->master_lcore >= RTE_MAX_LCORE) return -1; master_lcore_parsed = 1; + + /* ensure master core is not used as service core */ + if (lcore_config[cfg->master_lcore].core_role == ROLE_SERVICE) { + RTE_LOG(ERR, EAL, "Error: Master lcore is used as a service core.\n"); + return -1; + } + return 0; } @@ -795,20 +998,29 @@ int eal_parse_common_option(int opt, const char *optarg, struct internal_config *conf) { + static int b_used; + static int w_used; + switch (opt) { /* blacklist */ case 'b': - if (rte_eal_devargs_add(RTE_DEVTYPE_BLACKLISTED_PCI, + if (w_used) + goto bw_used; + if (eal_option_device_add(RTE_DEVTYPE_BLACKLISTED_PCI, optarg) < 0) { return -1; } + b_used = 1; break; /* whitelist */ case 'w': - if (rte_eal_devargs_add(RTE_DEVTYPE_WHITELISTED_PCI, + if (b_used) + goto bw_used; + if (eal_option_device_add(RTE_DEVTYPE_WHITELISTED_PCI, optarg) < 0) { return -1; } + w_used = 1; break; /* coremask */ case 'c': @@ -826,6 +1038,20 @@ eal_parse_common_option(int opt, const char *optarg, } core_parsed = 1; break; + /* service coremask */ + case 's': + if (eal_parse_service_coremask(optarg) < 0) { + RTE_LOG(ERR, EAL, "invalid service coremask\n"); + return -1; + } + break; + /* service corelist */ + case 'S': + if (eal_parse_service_corelist(optarg) < 0) { + RTE_LOG(ERR, EAL, "invalid service core list\n"); + return -1; + } + break; /* size of memory */ case 'm': conf->memory = atoi(optarg); @@ -901,7 +1127,7 @@ eal_parse_common_option(int opt, const char *optarg, break; case OPT_VDEV_NUM: - if (rte_eal_devargs_add(RTE_DEVTYPE_VIRTUAL, + if (eal_option_device_add(RTE_DEVTYPE_VIRTUAL, optarg) < 0) { return -1; } @@ -940,6 +1166,10 @@ eal_parse_common_option(int opt, const char *optarg, } return 0; +bw_used: + RTE_LOG(ERR, EAL, "Options blacklist (-b) and whitelist (-w) " + "cannot be used at the same time\n"); + return -1; } static void @@ -978,8 +1208,10 @@ eal_adjust_config(struct internal_config *internal_cfg) internal_config.process_type = eal_proc_type_detect(); /* default master lcore is the first one */ - if (!master_lcore_parsed) + if (!master_lcore_parsed) { cfg->master_lcore = rte_get_next_lcore(-1, 0, 0); + lcore_config[cfg->master_lcore].core_role = ROLE_RTE; + } /* if no memory amounts were requested, this will result in 0 and * will be overridden later, right after eal_hugepage_info_init() */ @@ -1025,13 +1257,6 @@ eal_check_common_options(struct internal_config *internal_cfg) return -1; } - if (rte_eal_devargs_type_count(RTE_DEVTYPE_WHITELISTED_PCI) != 0 && - rte_eal_devargs_type_count(RTE_DEVTYPE_BLACKLISTED_PCI) != 0) { - RTE_LOG(ERR, EAL, "Options blacklist (-b) and whitelist (-w) " - "cannot be used at the same time\n"); - return -1; - } - return 0; } @@ -1052,6 +1277,7 @@ eal_common_usage(void) " ',' is used for single number separator.\n" " '( )' can be omitted for single element group,\n" " '@' can be omitted if cpus and lcores have the same value\n" + " -s SERVICE COREMASK Hexadecimal bitmask of cores to be used as service cores\n" " --"OPT_MASTER_LCORE" ID Core ID that is used as master\n" " -n CHANNELS Number of memory channels\n" " -m MB Memory to allocate (see also --"OPT_SOCKET_MEM")\n" diff --git a/lib/librte_eal/common/eal_common_pci.c b/lib/librte_eal/common/eal_common_pci.c index b7499913..52fd38cd 100644 --- a/lib/librte_eal/common/eal_common_pci.c +++ b/lib/librte_eal/common/eal_common_pci.c @@ -2,6 +2,7 @@ * BSD LICENSE * * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * Copyright 2013-2014 6WIND S.A. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -30,36 +31,6 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -/* BSD LICENSE - * - * Copyright 2013-2014 6WIND S.A. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ #include <string.h> #include <inttypes.h> @@ -102,17 +73,43 @@ const char *pci_get_sysfs_path(void) static struct rte_devargs *pci_devargs_lookup(struct rte_pci_device *dev) { struct rte_devargs *devargs; + struct rte_pci_addr addr; + struct rte_bus *pbus; + pbus = rte_bus_find_by_name("pci"); TAILQ_FOREACH(devargs, &devargs_list, next) { - if (devargs->type != RTE_DEVTYPE_BLACKLISTED_PCI && - devargs->type != RTE_DEVTYPE_WHITELISTED_PCI) + if (devargs->bus != pbus) continue; - if (!rte_eal_compare_pci_addr(&dev->addr, &devargs->pci.addr)) + devargs->bus->parse(devargs->name, &addr); + if (!rte_eal_compare_pci_addr(&dev->addr, &addr)) return devargs; } return NULL; } +void +pci_name_set(struct rte_pci_device *dev) +{ + struct rte_devargs *devargs; + + /* Each device has its internal, canonical name set. */ + rte_pci_device_name(&dev->addr, + dev->name, sizeof(dev->name)); + devargs = pci_devargs_lookup(dev); + dev->device.devargs = devargs; + /* In blacklist mode, if the device is not blacklisted, no + * rte_devargs exists for it. + */ + if (devargs != NULL) + /* If an rte_devargs exists, the generic rte_device uses the + * given name as its namea + */ + dev->device.name = dev->device.devargs->name; + else + /* Otherwise, it uses the internal, canonical form. */ + dev->device.name = dev->name; +} + /* map a particular resource from a file */ void * pci_map_resource(void *requested_addr, int fd, off_t offset, size_t size, @@ -212,12 +209,9 @@ rte_pci_probe_one_driver(struct rte_pci_driver *dr, loc = &dev->addr; /* The device is not blacklisted; Check if driver supports it */ - if (!rte_pci_match(dr, dev)) { + if (!rte_pci_match(dr, dev)) /* Match of device and driver failed */ - RTE_LOG(DEBUG, EAL, "Driver (%s) doesn't match the device\n", - dr->driver.name); return 1; - } RTE_LOG(INFO, EAL, "PCI device "PCI_PRI_FMT" on NUMA socket %i\n", loc->domain, loc->bus, loc->devid, loc->function, @@ -225,13 +219,18 @@ rte_pci_probe_one_driver(struct rte_pci_driver *dr, /* no initialization when blacklisted, return without error */ if (dev->device.devargs != NULL && - dev->device.devargs->type == - RTE_DEVTYPE_BLACKLISTED_PCI) { + dev->device.devargs->policy == + RTE_DEV_BLACKLISTED) { RTE_LOG(INFO, EAL, " Device is blacklisted, not" " initializing\n"); return 1; } + if (dev->device.numa_node < 0) { + RTE_LOG(WARNING, EAL, " Invalid NUMA socket, default to 0\n"); + dev->device.numa_node = 0; + } + RTE_LOG(INFO, EAL, " probe driver: %x:%x %s\n", dev->id.vendor_id, dev->id.device_id, dr->driver.name); @@ -250,7 +249,13 @@ rte_pci_probe_one_driver(struct rte_pci_driver *dr, ret = dr->probe(dr, dev); if (ret) { dev->driver = NULL; - if (dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING) + dev->device.driver = NULL; + if ((dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING) && + /* Don't unmap if device is unsupported and + * driver needs mapped resources. + */ + !(ret > 0 && + (dr->drv_flags & RTE_PCI_DRV_KEEP_MAPPED_RES))) rte_pci_unmap_device(dev); } @@ -326,7 +331,7 @@ pci_probe_all_drivers(struct rte_pci_device *dev) /* * Find the pci device specified by pci address, then invoke probe function of - * the driver of the devive. + * the driver of the device. */ int rte_pci_probe_one(const struct rte_pci_addr *addr) @@ -413,22 +418,18 @@ rte_pci_probe(void) int probe_all = 0; int ret = 0; - if (rte_eal_devargs_type_count(RTE_DEVTYPE_WHITELISTED_PCI) == 0) + if (rte_pci_bus.bus.conf.scan_mode != RTE_BUS_SCAN_WHITELIST) probe_all = 1; FOREACH_DEVICE_ON_PCIBUS(dev) { probed++; - /* set devargs in PCI structure */ - devargs = pci_devargs_lookup(dev); - if (devargs != NULL) - dev->device.devargs = devargs; - + devargs = dev->device.devargs; /* probe all or only whitelisted devices */ if (probe_all) ret = pci_probe_all_drivers(dev); else if (devargs != NULL && - devargs->type == RTE_DEVTYPE_WHITELISTED_PCI) + devargs->policy == RTE_DEV_WHITELISTED) ret = pci_probe_all_drivers(dev); if (ret < 0) { RTE_LOG(ERR, EAL, "Requested device " PCI_PRI_FMT @@ -474,6 +475,20 @@ rte_pci_dump(FILE *f) } } +static int +pci_parse(const char *name, void *addr) +{ + struct rte_pci_addr *out = addr; + struct rte_pci_addr pci_addr; + bool parse; + + parse = (eal_parse_pci_BDF(name, &pci_addr) == 0 || + eal_parse_pci_DomBDF(name, &pci_addr) == 0); + if (parse && addr != NULL) + *out = pci_addr; + return parse == false; +} + /* register a driver */ void rte_pci_register(struct rte_pci_driver *driver) @@ -512,13 +527,54 @@ rte_pci_remove_device(struct rte_pci_device *pci_dev) TAILQ_REMOVE(&rte_pci_bus.device_list, pci_dev, next); } +static struct rte_device * +pci_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, + const void *data) +{ + struct rte_pci_device *dev; + + FOREACH_DEVICE_ON_PCIBUS(dev) { + if (start && &dev->device == start) { + start = NULL; /* starting point found */ + continue; + } + if (cmp(&dev->device, data) == 0) + return &dev->device; + } + + return NULL; +} + +static int +pci_plug(struct rte_device *dev) +{ + return pci_probe_all_drivers(RTE_DEV_TO_PCI(dev)); +} + +static int +pci_unplug(struct rte_device *dev) +{ + struct rte_pci_device *pdev; + int ret; + + pdev = RTE_DEV_TO_PCI(dev); + ret = rte_pci_detach_dev(pdev); + rte_pci_remove_device(pdev); + free(pdev); + return ret; +} + struct rte_pci_bus rte_pci_bus = { .bus = { .scan = rte_pci_scan, .probe = rte_pci_probe, + .find_device = pci_find_device, + .plug = pci_plug, + .unplug = pci_unplug, + .parse = pci_parse, }, .device_list = TAILQ_HEAD_INITIALIZER(rte_pci_bus.device_list), .driver_list = TAILQ_HEAD_INITIALIZER(rte_pci_bus.driver_list), }; -RTE_REGISTER_BUS(PCI_BUS_NAME, rte_pci_bus.bus); +RTE_REGISTER_BUS(pci, rte_pci_bus.bus); diff --git a/lib/librte_eal/common/eal_common_proc.c b/lib/librte_eal/common/eal_common_proc.c index 12e0fcac..60526cad 100644 --- a/lib/librte_eal/common/eal_common_proc.c +++ b/lib/librte_eal/common/eal_common_proc.c @@ -46,10 +46,10 @@ rte_eal_primary_proc_alive(const char *config_file_path) if (config_file_path) config_fd = open(config_file_path, O_RDONLY); else { - char default_path[PATH_MAX+1]; - snprintf(default_path, PATH_MAX, RUNTIME_CONFIG_FMT, - default_config_dir, "rte"); - config_fd = open(default_path, O_RDONLY); + const char *path; + + path = eal_runtime_config_path(); + config_fd = open(path, O_RDONLY); } if (config_fd < 0) return 0; diff --git a/lib/librte_eal/common/eal_common_tailqs.c b/lib/librte_eal/common/eal_common_tailqs.c index 4f698288..55955f9e 100644 --- a/lib/librte_eal/common/eal_common_tailqs.c +++ b/lib/librte_eal/common/eal_common_tailqs.c @@ -46,7 +46,6 @@ #include <rte_eal_memconfig.h> #include <rte_per_lcore.h> #include <rte_lcore.h> -#include <rte_memory.h> #include <rte_atomic.h> #include <rte_branch_prediction.h> #include <rte_log.h> diff --git a/lib/librte_eal/common/eal_common_timer.c b/lib/librte_eal/common/eal_common_timer.c index 72656176..ed0b16d0 100644 --- a/lib/librte_eal/common/eal_common_timer.c +++ b/lib/librte_eal/common/eal_common_timer.c @@ -41,6 +41,7 @@ #include <rte_common.h> #include <rte_log.h> #include <rte_cycles.h> +#include <rte_pause.h> #include "eal_private.h" diff --git a/lib/librte_eal/common/eal_common_vdev.c b/lib/librte_eal/common/eal_common_vdev.c index 0037a641..f7e547a6 100644 --- a/lib/librte_eal/common/eal_common_vdev.c +++ b/lib/librte_eal/common/eal_common_vdev.c @@ -35,14 +35,20 @@ #include <stdio.h> #include <stdlib.h> #include <stdint.h> +#include <stdbool.h> #include <sys/queue.h> #include <rte_eal.h> +#include <rte_dev.h> #include <rte_bus.h> #include <rte_vdev.h> #include <rte_common.h> #include <rte_devargs.h> #include <rte_memory.h> +#include <rte_errno.h> + +/* Forward declare to access virtual bus name */ +static struct rte_bus rte_vdev_bus; /** Double linked list of virtual device drivers. */ TAILQ_HEAD(vdev_device_list, rte_vdev_device); @@ -52,14 +58,10 @@ static struct vdev_device_list vdev_device_list = struct vdev_driver_list vdev_driver_list = TAILQ_HEAD_INITIALIZER(vdev_driver_list); -static void rte_vdev_bus_register(void); - /* register a driver */ void rte_vdev_register(struct rte_vdev_driver *driver) { - rte_vdev_bus_register(); - TAILQ_INSERT_TAIL(&vdev_driver_list, driver, next); } @@ -70,84 +72,45 @@ rte_vdev_unregister(struct rte_vdev_driver *driver) TAILQ_REMOVE(&vdev_driver_list, driver, next); } -/* - * Parse "driver" devargs without adding a dependency on rte_kvargs.h - */ -static char *parse_driver_arg(const char *args) +static int +vdev_parse(const char *name, void *addr) { - const char *c; - char *str; - - if (!args || args[0] == '\0') - return NULL; + struct rte_vdev_driver **out = addr; + struct rte_vdev_driver *driver = NULL; - c = args; - - do { - if (strncmp(c, "driver=", 7) == 0) { - c += 7; + TAILQ_FOREACH(driver, &vdev_driver_list, next) { + if (strncmp(driver->driver.name, name, + strlen(driver->driver.name)) == 0) break; - } - - c = strchr(c, ','); - if (c) - c++; - } while (c); - - if (c) - str = strdup(c); - else - str = NULL; - - return str; + if (driver->driver.alias && + strncmp(driver->driver.alias, name, + strlen(driver->driver.alias)) == 0) + break; + } + if (driver != NULL && + addr != NULL) + *out = driver; + return driver == NULL; } static int vdev_probe_all_drivers(struct rte_vdev_device *dev) { const char *name; - char *drv_name; struct rte_vdev_driver *driver; - int ret = 1; + int ret; - drv_name = parse_driver_arg(rte_vdev_device_args(dev)); - name = drv_name ? drv_name : rte_vdev_device_name(dev); + name = rte_vdev_device_name(dev); RTE_LOG(DEBUG, EAL, "Search driver %s to probe device %s\n", name, rte_vdev_device_name(dev)); - TAILQ_FOREACH(driver, &vdev_driver_list, next) { - /* - * search a driver prefix in virtual device name. - * For example, if the driver is pcap PMD, driver->name - * will be "net_pcap", but "name" will be "net_pcapN". - * So use strncmp to compare. - */ - if (!strncmp(driver->driver.name, name, - strlen(driver->driver.name))) { - dev->device.driver = &driver->driver; - ret = driver->probe(dev); - if (ret) - dev->device.driver = NULL; - goto out; - } - } - - /* Give new names precedence over aliases. */ - TAILQ_FOREACH(driver, &vdev_driver_list, next) { - if (driver->driver.alias && - !strncmp(driver->driver.alias, name, - strlen(driver->driver.alias))) { - dev->device.driver = &driver->driver; - ret = driver->probe(dev); - if (ret) - dev->device.driver = NULL; - break; - } - } - -out: - free(drv_name); + if (vdev_parse(name, &driver)) + return -1; + dev->device.driver = &driver->driver; + ret = driver->probe(dev); + if (ret) + dev->device.driver = NULL; return ret; } @@ -178,13 +141,14 @@ alloc_devargs(const char *name, const char *args) if (!devargs) return NULL; - devargs->type = RTE_DEVTYPE_VIRTUAL; + devargs->bus = &rte_vdev_bus; if (args) devargs->args = strdup(args); + else + devargs->args = strdup(""); - ret = snprintf(devargs->virt.drv_name, - sizeof(devargs->virt.drv_name), "%s", name); - if (ret < 0 || ret >= (int)sizeof(devargs->virt.drv_name)) { + ret = snprintf(devargs->name, sizeof(devargs->name), "%s", name); + if (ret < 0 || ret >= (int)sizeof(devargs->name)) { free(devargs->args); free(devargs); return NULL; @@ -219,7 +183,7 @@ rte_vdev_init(const char *name, const char *args) dev->device.devargs = devargs; dev->device.numa_node = SOCKET_ID_ANY; - dev->device.name = devargs->virt.drv_name; + dev->device.name = devargs->name; ret = vdev_probe_all_drivers(dev); if (ret) { @@ -293,13 +257,12 @@ vdev_scan(void) struct rte_devargs *devargs; /* for virtual devices we scan the devargs_list populated via cmdline */ - TAILQ_FOREACH(devargs, &devargs_list, next) { - if (devargs->type != RTE_DEVTYPE_VIRTUAL) + if (devargs->bus != &rte_vdev_bus) continue; - dev = find_vdev(devargs->virt.drv_name); + dev = find_vdev(devargs->name); if (dev) continue; @@ -309,7 +272,7 @@ vdev_scan(void) dev->device.devargs = devargs; dev->device.numa_node = SOCKET_ID_ANY; - dev->device.name = devargs->virt.drv_name; + dev->device.name = devargs->name; TAILQ_INSERT_TAIL(&vdev_device_list, dev, next); } @@ -338,21 +301,42 @@ vdev_probe(void) return 0; } -static struct rte_bus rte_vdev_bus = { - .scan = vdev_scan, - .probe = vdev_probe, -}; +static struct rte_device * +vdev_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, + const void *data) +{ + struct rte_vdev_device *dev; -RTE_INIT(rte_vdev_bus_register); + TAILQ_FOREACH(dev, &vdev_device_list, next) { + if (start && &dev->device == start) { + start = NULL; + continue; + } + if (cmp(&dev->device, data) == 0) + return &dev->device; + } + return NULL; +} -static void rte_vdev_bus_register(void) +static int +vdev_plug(struct rte_device *dev) { - static int registered; - - if (registered) - return; + return vdev_probe_all_drivers(RTE_DEV_TO_VDEV(dev)); +} - registered = 1; - rte_vdev_bus.name = RTE_STR(virtual); - rte_bus_register(&rte_vdev_bus); +static int +vdev_unplug(struct rte_device *dev) +{ + return rte_vdev_uninit(dev->name); } + +static struct rte_bus rte_vdev_bus = { + .scan = vdev_scan, + .probe = vdev_probe, + .find_device = vdev_find_device, + .plug = vdev_plug, + .unplug = vdev_unplug, + .parse = vdev_parse, +}; + +RTE_REGISTER_BUS(vdev, rte_vdev_bus); diff --git a/lib/librte_eal/common/eal_options.h b/lib/librte_eal/common/eal_options.h index a881c62e..439a2610 100644 --- a/lib/librte_eal/common/eal_options.h +++ b/lib/librte_eal/common/eal_options.h @@ -91,6 +91,7 @@ extern const struct option eal_long_options[]; int eal_parse_common_option(int opt, const char *argv, struct internal_config *conf); +int eal_option_device_parse(void); int eal_adjust_config(struct internal_config *internal_cfg); int eal_check_common_options(struct internal_config *internal_cfg); void eal_common_usage(void); diff --git a/lib/librte_eal/common/eal_private.h b/lib/librte_eal/common/eal_private.h index 6cacce07..597d82e4 100644 --- a/lib/librte_eal/common/eal_private.h +++ b/lib/librte_eal/common/eal_private.h @@ -113,6 +113,11 @@ struct rte_pci_driver; struct rte_pci_device; /** + * Find the name of a PCI device. + */ +void pci_name_set(struct rte_pci_device *dev); + +/** * Add a PCI device to the PCI Bus (append to PCI Device list). This function * also updates the bus references of the PCI Device (and the generic device * object embedded within. @@ -338,4 +343,16 @@ int rte_eal_hugepage_attach(void); */ bool rte_eal_using_phys_addrs(void); +/** + * Find a bus capable of identifying a device. + * + * @param str + * A device identifier (PCI address, virtual PMD name, ...). + * + * @return + * A valid bus handle if found. + * NULL if no bus is able to parse this device. + */ +struct rte_bus *rte_bus_find_by_device_name(const char *str); + #endif /* _EAL_PRIVATE_H_ */ diff --git a/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h b/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h index dc3a0f3b..0b70d620 100644 --- a/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +++ b/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2015. + * Copyright (C) Cavium, Inc. 2015. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/lib/librte_eal/common/include/arch/arm/rte_byteorder.h b/lib/librte_eal/common/include/arch/arm/rte_byteorder.h index 1b312b30..0a29f4bb 100644 --- a/lib/librte_eal/common/include/arch/arm/rte_byteorder.h +++ b/lib/librte_eal/common/include/arch/arm/rte_byteorder.h @@ -52,7 +52,7 @@ static inline uint16_t rte_arch_bswap16(uint16_t _x) { register uint16_t x = _x; - asm volatile ("rev16 %0,%1" + asm volatile ("rev16 %w0,%w1" : "=r" (x) : "r" (x) ); diff --git a/lib/librte_eal/common/include/arch/arm/rte_cpuflags_64.h b/lib/librte_eal/common/include/arch/arm/rte_cpuflags_64.h index 49aead92..5425f4c7 100644 --- a/lib/librte_eal/common/include/arch/arm/rte_cpuflags_64.h +++ b/lib/librte_eal/common/include/arch/arm/rte_cpuflags_64.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2015. + * Copyright (C) Cavium, Inc. 2015. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/lib/librte_eal/common/include/arch/arm/rte_cycles_64.h b/lib/librte_eal/common/include/arch/arm/rte_cycles_64.h index 867a9468..15457691 100644 --- a/lib/librte_eal/common/include/arch/arm/rte_cycles_64.h +++ b/lib/librte_eal/common/include/arch/arm/rte_cycles_64.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2015. + * Copyright (C) Cavium, Inc. 2015. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/lib/librte_eal/common/include/arch/arm/rte_io.h b/lib/librte_eal/common/include/arch/arm/rte_io.h index 9593b424..3b63ec85 100644 --- a/lib/librte_eal/common/include/arch/arm/rte_io.h +++ b/lib/librte_eal/common/include/arch/arm/rte_io.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright(c) 2016 Cavium networks. All rights reserved. + * Copyright(c) 2016 Cavium, Inc. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -14,7 +14,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/lib/librte_eal/common/include/arch/arm/rte_io_64.h b/lib/librte_eal/common/include/arch/arm/rte_io_64.h index 0402125b..ee9b8d55 100644 --- a/lib/librte_eal/common/include/arch/arm/rte_io_64.h +++ b/lib/librte_eal/common/include/arch/arm/rte_io_64.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -44,7 +44,7 @@ extern "C" { #include "generic/rte_io.h" #include "rte_atomic_64.h" -static inline uint8_t __attribute__((always_inline)) +static __rte_always_inline uint8_t rte_read8_relaxed(const volatile void *addr) { uint8_t val; @@ -56,7 +56,7 @@ rte_read8_relaxed(const volatile void *addr) return val; } -static inline uint16_t __attribute__((always_inline)) +static __rte_always_inline uint16_t rte_read16_relaxed(const volatile void *addr) { uint16_t val; @@ -68,7 +68,7 @@ rte_read16_relaxed(const volatile void *addr) return val; } -static inline uint32_t __attribute__((always_inline)) +static __rte_always_inline uint32_t rte_read32_relaxed(const volatile void *addr) { uint32_t val; @@ -80,7 +80,7 @@ rte_read32_relaxed(const volatile void *addr) return val; } -static inline uint64_t __attribute__((always_inline)) +static __rte_always_inline uint64_t rte_read64_relaxed(const volatile void *addr) { uint64_t val; @@ -92,7 +92,7 @@ rte_read64_relaxed(const volatile void *addr) return val; } -static inline void __attribute__((always_inline)) +static __rte_always_inline void rte_write8_relaxed(uint8_t val, volatile void *addr) { asm volatile( @@ -101,7 +101,7 @@ rte_write8_relaxed(uint8_t val, volatile void *addr) : [val] "r" (val), [addr] "r" (addr)); } -static inline void __attribute__((always_inline)) +static __rte_always_inline void rte_write16_relaxed(uint16_t val, volatile void *addr) { asm volatile( @@ -110,7 +110,7 @@ rte_write16_relaxed(uint16_t val, volatile void *addr) : [val] "r" (val), [addr] "r" (addr)); } -static inline void __attribute__((always_inline)) +static __rte_always_inline void rte_write32_relaxed(uint32_t val, volatile void *addr) { asm volatile( @@ -119,7 +119,7 @@ rte_write32_relaxed(uint32_t val, volatile void *addr) : [val] "r" (val), [addr] "r" (addr)); } -static inline void __attribute__((always_inline)) +static __rte_always_inline void rte_write64_relaxed(uint64_t val, volatile void *addr) { asm volatile( @@ -128,7 +128,7 @@ rte_write64_relaxed(uint64_t val, volatile void *addr) : [val] "r" (val), [addr] "r" (addr)); } -static inline uint8_t __attribute__((always_inline)) +static __rte_always_inline uint8_t rte_read8(const volatile void *addr) { uint8_t val; @@ -137,7 +137,7 @@ rte_read8(const volatile void *addr) return val; } -static inline uint16_t __attribute__((always_inline)) +static __rte_always_inline uint16_t rte_read16(const volatile void *addr) { uint16_t val; @@ -146,7 +146,7 @@ rte_read16(const volatile void *addr) return val; } -static inline uint32_t __attribute__((always_inline)) +static __rte_always_inline uint32_t rte_read32(const volatile void *addr) { uint32_t val; @@ -155,7 +155,7 @@ rte_read32(const volatile void *addr) return val; } -static inline uint64_t __attribute__((always_inline)) +static __rte_always_inline uint64_t rte_read64(const volatile void *addr) { uint64_t val; @@ -164,28 +164,28 @@ rte_read64(const volatile void *addr) return val; } -static inline void __attribute__((always_inline)) +static __rte_always_inline void rte_write8(uint8_t value, volatile void *addr) { rte_io_wmb(); rte_write8_relaxed(value, addr); } -static inline void __attribute__((always_inline)) +static __rte_always_inline void rte_write16(uint16_t value, volatile void *addr) { rte_io_wmb(); rte_write16_relaxed(value, addr); } -static inline void __attribute__((always_inline)) +static __rte_always_inline void rte_write32(uint32_t value, volatile void *addr) { rte_io_wmb(); rte_write32_relaxed(value, addr); } -static inline void __attribute__((always_inline)) +static __rte_always_inline void rte_write64(uint64_t value, volatile void *addr) { rte_io_wmb(); diff --git a/lib/librte_eal/common/include/arch/arm/rte_memcpy_64.h b/lib/librte_eal/common/include/arch/arm/rte_memcpy_64.h index 5db66b63..b80d8ba4 100644 --- a/lib/librte_eal/common/include/arch/arm/rte_memcpy_64.h +++ b/lib/librte_eal/common/include/arch/arm/rte_memcpy_64.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2015. + * Copyright (C) Cavium, Inc. 2015. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/lib/librte_eal/common/include/arch/arm/rte_pause.h b/lib/librte_eal/common/include/arch/arm/rte_pause.h new file mode 100644 index 00000000..b772ca07 --- /dev/null +++ b/lib/librte_eal/common/include/arch/arm/rte_pause.h @@ -0,0 +1,50 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_PAUSE_ARM_H_ +#define _RTE_PAUSE_ARM_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef RTE_ARCH_64 +#include <rte_pause_64.h> +#else +#include <rte_pause_32.h> +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_PAUSE_ARM_H_ */ diff --git a/lib/librte_eal/common/include/arch/arm/rte_pause_32.h b/lib/librte_eal/common/include/arch/arm/rte_pause_32.h new file mode 100644 index 00000000..ec680b5c --- /dev/null +++ b/lib/librte_eal/common/include/arch/arm/rte_pause_32.h @@ -0,0 +1,51 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_PAUSE_ARM32_H_ +#define _RTE_PAUSE_ARM32_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <rte_common.h> +#include "generic/rte_pause.h" + +static inline void rte_pause(void) +{ +} + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_PAUSE_ARM32_H_ */ diff --git a/lib/librte_eal/common/include/arch/arm/rte_pause_64.h b/lib/librte_eal/common/include/arch/arm/rte_pause_64.h new file mode 100644 index 00000000..2da46326 --- /dev/null +++ b/lib/librte_eal/common/include/arch/arm/rte_pause_64.h @@ -0,0 +1,52 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_PAUSE_ARM64_H_ +#define _RTE_PAUSE_ARM64_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <rte_common.h> +#include "generic/rte_pause.h" + +static inline void rte_pause(void) +{ + asm volatile("yield" ::: "memory"); +} + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_PAUSE_ARM64_H_ */ diff --git a/lib/librte_eal/common/include/arch/arm/rte_prefetch_64.h b/lib/librte_eal/common/include/arch/arm/rte_prefetch_64.h index 0d077ea6..ff59509f 100644 --- a/lib/librte_eal/common/include/arch/arm/rte_prefetch_64.h +++ b/lib/librte_eal/common/include/arch/arm/rte_prefetch_64.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2015. + * Copyright (C) Cavium, Inc. 2015. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/lib/librte_eal/common/include/arch/arm/rte_vect.h b/lib/librte_eal/common/include/arch/arm/rte_vect.h index 4107c998..782350d1 100644 --- a/lib/librte_eal/common/include/arch/arm/rte_vect.h +++ b/lib/librte_eal/common/include/arch/arm/rte_vect.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2015 Cavium Networks. All rights reserved. + * Copyright(c) 2015 Cavium, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium Networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -35,6 +35,7 @@ #include <stdint.h> #include "generic/rte_vect.h" +#include "rte_debug.h" #include "arm_neon.h" #ifdef __cplusplus @@ -76,8 +77,122 @@ vqtbl1q_u8(uint8x16_t a, uint8x16_t b) return vld1q_u8(rte_ret.u8); } + +static inline uint16_t +vaddvq_u16(uint16x8_t a) +{ + uint32x4_t m = vpaddlq_u16(a); + uint64x2_t n = vpaddlq_u32(m); + uint64x1_t o = vget_low_u64(n) + vget_high_u64(n); + + return vget_lane_u32((uint32x2_t)o, 0); +} + #endif +#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70000) +static inline uint32x4_t +vcopyq_laneq_u32(uint32x4_t a, const int lane_a, + uint32x4_t b, const int lane_b) +{ + return vsetq_lane_u32(vgetq_lane_u32(b, lane_b), a, lane_a); +} +#endif + +#if defined(RTE_ARCH_ARM64) +#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70000) + +#if (GCC_VERSION < 40900) +typedef uint64_t poly64_t; +typedef uint64x2_t poly64x2_t; +typedef uint8_t poly128_t __attribute__((vector_size(16), aligned(16))); +#endif + +/* NEON intrinsic vreinterpretq_u64_p128() is supported since GCC version 7 */ +static inline uint64x2_t +vreinterpretq_u64_p128(poly128_t x) +{ + return (uint64x2_t)x; +} + +/* NEON intrinsic vreinterpretq_p64_u64() is supported since GCC version 7 */ +static inline poly64x2_t +vreinterpretq_p64_u64(uint64x2_t x) +{ + return (poly64x2_t)x; +} + +/* NEON intrinsic vgetq_lane_p64() is supported since GCC version 7 */ +static inline poly64_t +vgetq_lane_p64(poly64x2_t x, const int lane) +{ + RTE_ASSERT(lane >= 0 && lane <= 1); + + poly64_t *p = (poly64_t *)&x; + + return p[lane]; +} +#endif +#endif + +/* + * If (0 <= index <= 15), then call the ASIMD ext intruction on the + * 128 bit regs v0 and v1 with the appropriate index. + * + * Else returns a zero vector. + */ +static inline uint8x16_t +vextract(uint8x16_t v0, uint8x16_t v1, const int index) +{ + switch (index) { + case 0: return vextq_u8(v0, v1, 0); + case 1: return vextq_u8(v0, v1, 1); + case 2: return vextq_u8(v0, v1, 2); + case 3: return vextq_u8(v0, v1, 3); + case 4: return vextq_u8(v0, v1, 4); + case 5: return vextq_u8(v0, v1, 5); + case 6: return vextq_u8(v0, v1, 6); + case 7: return vextq_u8(v0, v1, 7); + case 8: return vextq_u8(v0, v1, 8); + case 9: return vextq_u8(v0, v1, 9); + case 10: return vextq_u8(v0, v1, 10); + case 11: return vextq_u8(v0, v1, 11); + case 12: return vextq_u8(v0, v1, 12); + case 13: return vextq_u8(v0, v1, 13); + case 14: return vextq_u8(v0, v1, 14); + case 15: return vextq_u8(v0, v1, 15); + } + return vdupq_n_u8(0); +} + +/** + * Shifts right 128 bit register by specified number of bytes + * + * Value of shift parameter must be in range 0 - 16 + */ +static inline uint64x2_t +vshift_bytes_right(uint64x2_t reg, const unsigned int shift) +{ + return vreinterpretq_u64_u8(vextract( + vreinterpretq_u8_u64(reg), + vdupq_n_u8(0), + shift)); +} + +/** + * Shifts left 128 bit register by specified number of bytes + * + * Value of shift parameter must be in range 0 - 16 + */ +static inline uint64x2_t +vshift_bytes_left(uint64x2_t reg, const unsigned int shift) +{ + return vreinterpretq_u64_u8(vextract( + vdupq_n_u8(0), + vreinterpretq_u8_u64(reg), + 16 - shift)); +} + #ifdef __cplusplus } #endif diff --git a/lib/librte_eal/common/include/arch/ppc_64/rte_io.h b/lib/librte_eal/common/include/arch/ppc_64/rte_io.h index be192da7..1f42ced5 100644 --- a/lib/librte_eal/common/include/arch/ppc_64/rte_io.h +++ b/lib/librte_eal/common/include/arch/ppc_64/rte_io.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright(c) 2016 Cavium networks. All rights reserved. + * Copyright(c) 2016 Cavium, Inc. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -14,7 +14,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/lib/librte_eal/common/include/arch/ppc_64/rte_pause.h b/lib/librte_eal/common/include/arch/ppc_64/rte_pause.h new file mode 100644 index 00000000..389682ca --- /dev/null +++ b/lib/librte_eal/common/include/arch/ppc_64/rte_pause.h @@ -0,0 +1,51 @@ +/*- + * BSD LICENSE + * + * Copyright(c) Cavium, Inc. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_PAUSE_PPC64_H_ +#define _RTE_PAUSE_PPC64_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "generic/rte_pause.h" + +static inline void rte_pause(void) +{ +} + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_PAUSE_PPC64_H_ */ diff --git a/lib/librte_eal/common/include/arch/ppc_64/rte_spinlock.h b/lib/librte_eal/common/include/arch/ppc_64/rte_spinlock.h index af139c9d..39815d9e 100644 --- a/lib/librte_eal/common/include/arch/ppc_64/rte_spinlock.h +++ b/lib/librte_eal/common/include/arch/ppc_64/rte_spinlock.h @@ -38,6 +38,7 @@ extern "C" { #endif #include <rte_common.h> +#include <rte_pause.h> #include "generic/rte_spinlock.h" /* Fixme: Use intrinsics to implement the spinlock on Power architecture */ diff --git a/lib/librte_eal/common/include/arch/x86/rte_cycles.h b/lib/librte_eal/common/include/arch/x86/rte_cycles.h index 5eb6ce96..1bb3e1db 100644 --- a/lib/librte_eal/common/include/arch/x86/rte_cycles.h +++ b/lib/librte_eal/common/include/arch/x86/rte_cycles.h @@ -2,6 +2,7 @@ * BSD LICENSE * * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * Copyright(c) 2013 6WIND. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -30,36 +31,6 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -/* BSD LICENSE - * - * Copyright(c) 2013 6WIND. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ #ifndef _RTE_CYCLES_X86_64_H_ #define _RTE_CYCLES_X86_64_H_ diff --git a/lib/librte_eal/common/include/arch/x86/rte_io.h b/lib/librte_eal/common/include/arch/x86/rte_io.h index c8d14043..130022d0 100644 --- a/lib/librte_eal/common/include/arch/x86/rte_io.h +++ b/lib/librte_eal/common/include/arch/x86/rte_io.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright(c) 2016 Cavium networks. All rights reserved. + * Copyright(c) 2016 Cavium, Inc. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -14,7 +14,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/lib/librte_eal/common/include/arch/x86/rte_memcpy.h b/lib/librte_eal/common/include/arch/x86/rte_memcpy.h index b9785e85..74c280c2 100644 --- a/lib/librte_eal/common/include/arch/x86/rte_memcpy.h +++ b/lib/librte_eal/common/include/arch/x86/rte_memcpy.h @@ -44,6 +44,7 @@ #include <stdint.h> #include <string.h> #include <rte_vect.h> +#include <rte_common.h> #ifdef __cplusplus extern "C" { @@ -64,8 +65,8 @@ extern "C" { * @return * Pointer to the destination data. */ -static inline void * -rte_memcpy(void *dst, const void *src, size_t n) __attribute__((always_inline)); +static __rte_always_inline void * +rte_memcpy(void *dst, const void *src, size_t n); #ifdef RTE_MACHINE_CPUFLAG_AVX512F diff --git a/lib/librte_eal/common/include/arch/x86/rte_pause.h b/lib/librte_eal/common/include/arch/x86/rte_pause.h new file mode 100644 index 00000000..29130c4b --- /dev/null +++ b/lib/librte_eal/common/include/arch/x86/rte_pause.h @@ -0,0 +1,53 @@ +/*- + * BSD LICENSE + * + * Copyright(c) Cavium, Inc. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_PAUSE_X86_H_ +#define _RTE_PAUSE_X86_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "generic/rte_pause.h" + +#include <emmintrin.h> +static inline void rte_pause(void) +{ + _mm_pause(); +} + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_PAUSE_X86_H_ */ diff --git a/lib/librte_eal/common/include/arch/x86/rte_spinlock.h b/lib/librte_eal/common/include/arch/x86/rte_spinlock.h index 8e630c21..5675c2b4 100644 --- a/lib/librte_eal/common/include/arch/x86/rte_spinlock.h +++ b/lib/librte_eal/common/include/arch/x86/rte_spinlock.h @@ -43,6 +43,7 @@ extern "C" { #include "rte_cpuflags.h" #include "rte_branch_prediction.h" #include "rte_common.h" +#include "rte_pause.h" #define RTE_RTM_MAX_RETRIES (10) #define RTE_XABORT_LOCK_BUSY (0xff) diff --git a/lib/librte_eal/common/include/arch/x86/rte_vect.h b/lib/librte_eal/common/include/arch/x86/rte_vect.h index 1b4b85dd..03fc991e 100644 --- a/lib/librte_eal/common/include/arch/x86/rte_vect.h +++ b/lib/librte_eal/common/include/arch/x86/rte_vect.h @@ -45,21 +45,7 @@ #if (defined(__ICC) || (__GNUC__ == 4 && __GNUC_MINOR__ < 4)) -#ifdef __SSE__ -#include <xmmintrin.h> -#endif - -#ifdef __SSE2__ -#include <emmintrin.h> -#endif - -#ifdef __SSE3__ -#include <tmmintrin.h> -#endif - -#if defined(__SSE4_2__) || defined(__SSE4_1__) -#include <smmintrin.h> -#endif +#include <smmintrin.h> /* SSE4 */ #if defined(__AVX__) #include <immintrin.h> diff --git a/lib/librte_eal/common/include/generic/rte_byteorder.h b/lib/librte_eal/common/include/generic/rte_byteorder.h index e00bccbc..e5e820d3 100644 --- a/lib/librte_eal/common/include/generic/rte_byteorder.h +++ b/lib/librte_eal/common/include/generic/rte_byteorder.h @@ -74,6 +74,73 @@ #elif defined __LITTLE_ENDIAN__ #define RTE_BYTE_ORDER RTE_LITTLE_ENDIAN #endif +#if !defined(RTE_BYTE_ORDER) +#error Unknown endianness. +#endif + +#define RTE_STATIC_BSWAP16(v) \ + ((((uint16_t)(v) & UINT16_C(0x00ff)) << 8) | \ + (((uint16_t)(v) & UINT16_C(0xff00)) >> 8)) + +#define RTE_STATIC_BSWAP32(v) \ + ((((uint32_t)(v) & UINT32_C(0x000000ff)) << 24) | \ + (((uint32_t)(v) & UINT32_C(0x0000ff00)) << 8) | \ + (((uint32_t)(v) & UINT32_C(0x00ff0000)) >> 8) | \ + (((uint32_t)(v) & UINT32_C(0xff000000)) >> 24)) + +#define RTE_STATIC_BSWAP64(v) \ + ((((uint64_t)(v) & UINT64_C(0x00000000000000ff)) << 56) | \ + (((uint64_t)(v) & UINT64_C(0x000000000000ff00)) << 40) | \ + (((uint64_t)(v) & UINT64_C(0x0000000000ff0000)) << 24) | \ + (((uint64_t)(v) & UINT64_C(0x00000000ff000000)) << 8) | \ + (((uint64_t)(v) & UINT64_C(0x000000ff00000000)) >> 8) | \ + (((uint64_t)(v) & UINT64_C(0x0000ff0000000000)) >> 24) | \ + (((uint64_t)(v) & UINT64_C(0x00ff000000000000)) >> 40) | \ + (((uint64_t)(v) & UINT64_C(0xff00000000000000)) >> 56)) + +/* + * These macros are functionally similar to rte_cpu_to_(be|le)(16|32|64)(), + * they take values in host CPU order and return them converted to the + * intended endianness. + * + * They resolve at compilation time to integer constants which can safely be + * used with static initializers, since those cannot involve function calls. + * + * On the other hand, they are not as optimized as their rte_cpu_to_*() + * counterparts, therefore applications should refrain from using them on + * variable values, particularly inside performance-sensitive code. + */ +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN +#define RTE_BE16(v) (rte_be16_t)(v) +#define RTE_BE32(v) (rte_be32_t)(v) +#define RTE_BE64(v) (rte_be64_t)(v) +#define RTE_LE16(v) (rte_le16_t)(RTE_STATIC_BSWAP16(v)) +#define RTE_LE32(v) (rte_le32_t)(RTE_STATIC_BSWAP32(v)) +#define RTE_LE64(v) (rte_le64_t)(RTE_STATIC_BSWAP64(v)) +#elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN +#define RTE_BE16(v) (rte_be16_t)(RTE_STATIC_BSWAP16(v)) +#define RTE_BE32(v) (rte_be32_t)(RTE_STATIC_BSWAP32(v)) +#define RTE_BE64(v) (rte_be64_t)(RTE_STATIC_BSWAP64(v)) +#define RTE_LE16(v) (rte_be16_t)(v) +#define RTE_LE32(v) (rte_be32_t)(v) +#define RTE_LE64(v) (rte_be64_t)(v) +#else +#error Unsupported endianness. +#endif + +/* + * The following types should be used when handling values according to a + * specific byte ordering, which may differ from that of the host CPU. + * + * Libraries, public APIs and applications are encouraged to use them for + * documentation purposes. + */ +typedef uint16_t rte_be16_t; /**< 16-bit big-endian value. */ +typedef uint32_t rte_be32_t; /**< 32-bit big-endian value. */ +typedef uint64_t rte_be64_t; /**< 64-bit big-endian value. */ +typedef uint16_t rte_le16_t; /**< 16-bit little-endian value. */ +typedef uint32_t rte_le32_t; /**< 32-bit little-endian value. */ +typedef uint64_t rte_le64_t; /**< 64-bit little-endian value. */ /* * An internal function to swap bytes in a 16-bit value. @@ -84,8 +151,7 @@ static inline uint16_t rte_constant_bswap16(uint16_t x) { - return (uint16_t)(((x & 0x00ffU) << 8) | - ((x & 0xff00U) >> 8)); + return RTE_STATIC_BSWAP16(x); } /* @@ -97,10 +163,7 @@ rte_constant_bswap16(uint16_t x) static inline uint32_t rte_constant_bswap32(uint32_t x) { - return ((x & 0x000000ffUL) << 24) | - ((x & 0x0000ff00UL) << 8) | - ((x & 0x00ff0000UL) >> 8) | - ((x & 0xff000000UL) >> 24); + return RTE_STATIC_BSWAP32(x); } /* @@ -112,14 +175,7 @@ rte_constant_bswap32(uint32_t x) static inline uint64_t rte_constant_bswap64(uint64_t x) { - return ((x & 0x00000000000000ffULL) << 56) | - ((x & 0x000000000000ff00ULL) << 40) | - ((x & 0x0000000000ff0000ULL) << 24) | - ((x & 0x00000000ff000000ULL) << 8) | - ((x & 0x000000ff00000000ULL) >> 8) | - ((x & 0x0000ff0000000000ULL) >> 24) | - ((x & 0x00ff000000000000ULL) >> 40) | - ((x & 0xff00000000000000ULL) >> 56); + return RTE_STATIC_BSWAP64(x); } @@ -143,65 +199,65 @@ static uint64_t rte_bswap64(uint64_t x); /** * Convert a 16-bit value from CPU order to little endian. */ -static uint16_t rte_cpu_to_le_16(uint16_t x); +static rte_le16_t rte_cpu_to_le_16(uint16_t x); /** * Convert a 32-bit value from CPU order to little endian. */ -static uint32_t rte_cpu_to_le_32(uint32_t x); +static rte_le32_t rte_cpu_to_le_32(uint32_t x); /** * Convert a 64-bit value from CPU order to little endian. */ -static uint64_t rte_cpu_to_le_64(uint64_t x); +static rte_le64_t rte_cpu_to_le_64(uint64_t x); /** * Convert a 16-bit value from CPU order to big endian. */ -static uint16_t rte_cpu_to_be_16(uint16_t x); +static rte_be16_t rte_cpu_to_be_16(uint16_t x); /** * Convert a 32-bit value from CPU order to big endian. */ -static uint32_t rte_cpu_to_be_32(uint32_t x); +static rte_be32_t rte_cpu_to_be_32(uint32_t x); /** * Convert a 64-bit value from CPU order to big endian. */ -static uint64_t rte_cpu_to_be_64(uint64_t x); +static rte_be64_t rte_cpu_to_be_64(uint64_t x); /** * Convert a 16-bit value from little endian to CPU order. */ -static uint16_t rte_le_to_cpu_16(uint16_t x); +static uint16_t rte_le_to_cpu_16(rte_le16_t x); /** * Convert a 32-bit value from little endian to CPU order. */ -static uint32_t rte_le_to_cpu_32(uint32_t x); +static uint32_t rte_le_to_cpu_32(rte_le32_t x); /** * Convert a 64-bit value from little endian to CPU order. */ -static uint64_t rte_le_to_cpu_64(uint64_t x); +static uint64_t rte_le_to_cpu_64(rte_le64_t x); /** * Convert a 16-bit value from big endian to CPU order. */ -static uint16_t rte_be_to_cpu_16(uint16_t x); +static uint16_t rte_be_to_cpu_16(rte_be16_t x); /** * Convert a 32-bit value from big endian to CPU order. */ -static uint32_t rte_be_to_cpu_32(uint32_t x); +static uint32_t rte_be_to_cpu_32(rte_be32_t x); /** * Convert a 64-bit value from big endian to CPU order. */ -static uint64_t rte_be_to_cpu_64(uint64_t x); +static uint64_t rte_be_to_cpu_64(rte_be64_t x); #endif /* __DOXYGEN__ */ diff --git a/lib/librte_eal/common/include/generic/rte_cycles.h b/lib/librte_eal/common/include/generic/rte_cycles.h index 0e645c2c..0df90474 100644 --- a/lib/librte_eal/common/include/generic/rte_cycles.h +++ b/lib/librte_eal/common/include/generic/rte_cycles.h @@ -2,6 +2,7 @@ * BSD LICENSE * * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * Copyright(c) 2013 6WIND. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -30,36 +31,6 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -/* BSD LICENSE - * - * Copyright(c) 2013 6WIND. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ #ifndef _RTE_CYCLES_H_ #define _RTE_CYCLES_H_ diff --git a/lib/librte_eal/common/include/generic/rte_io.h b/lib/librte_eal/common/include/generic/rte_io.h index d82ee695..0b88c341 100644 --- a/lib/librte_eal/common/include/generic/rte_io.h +++ b/lib/librte_eal/common/include/generic/rte_io.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright(c) 2016 Cavium networks. All rights reserved. + * Copyright(c) 2016 Cavium, Inc. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -14,7 +14,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -34,8 +34,6 @@ #ifndef _RTE_IO_H_ #define _RTE_IO_H_ -#include <rte_atomic.h> - /** * @file * I/O device memory operations @@ -264,55 +262,55 @@ rte_write64(uint64_t value, volatile void *addr); #ifndef RTE_OVERRIDE_IO_H -static inline uint8_t __attribute__((always_inline)) +static __rte_always_inline uint8_t rte_read8_relaxed(const volatile void *addr) { return *(const volatile uint8_t *)addr; } -static inline uint16_t __attribute__((always_inline)) +static __rte_always_inline uint16_t rte_read16_relaxed(const volatile void *addr) { return *(const volatile uint16_t *)addr; } -static inline uint32_t __attribute__((always_inline)) +static __rte_always_inline uint32_t rte_read32_relaxed(const volatile void *addr) { return *(const volatile uint32_t *)addr; } -static inline uint64_t __attribute__((always_inline)) +static __rte_always_inline uint64_t rte_read64_relaxed(const volatile void *addr) { return *(const volatile uint64_t *)addr; } -static inline void __attribute__((always_inline)) +static __rte_always_inline void rte_write8_relaxed(uint8_t value, volatile void *addr) { *(volatile uint8_t *)addr = value; } -static inline void __attribute__((always_inline)) +static __rte_always_inline void rte_write16_relaxed(uint16_t value, volatile void *addr) { *(volatile uint16_t *)addr = value; } -static inline void __attribute__((always_inline)) +static __rte_always_inline void rte_write32_relaxed(uint32_t value, volatile void *addr) { *(volatile uint32_t *)addr = value; } -static inline void __attribute__((always_inline)) +static __rte_always_inline void rte_write64_relaxed(uint64_t value, volatile void *addr) { *(volatile uint64_t *)addr = value; } -static inline uint8_t __attribute__((always_inline)) +static __rte_always_inline uint8_t rte_read8(const volatile void *addr) { uint8_t val; @@ -321,7 +319,7 @@ rte_read8(const volatile void *addr) return val; } -static inline uint16_t __attribute__((always_inline)) +static __rte_always_inline uint16_t rte_read16(const volatile void *addr) { uint16_t val; @@ -330,7 +328,7 @@ rte_read16(const volatile void *addr) return val; } -static inline uint32_t __attribute__((always_inline)) +static __rte_always_inline uint32_t rte_read32(const volatile void *addr) { uint32_t val; @@ -339,7 +337,7 @@ rte_read32(const volatile void *addr) return val; } -static inline uint64_t __attribute__((always_inline)) +static __rte_always_inline uint64_t rte_read64(const volatile void *addr) { uint64_t val; @@ -348,28 +346,28 @@ rte_read64(const volatile void *addr) return val; } -static inline void __attribute__((always_inline)) +static __rte_always_inline void rte_write8(uint8_t value, volatile void *addr) { rte_io_wmb(); rte_write8_relaxed(value, addr); } -static inline void __attribute__((always_inline)) +static __rte_always_inline void rte_write16(uint16_t value, volatile void *addr) { rte_io_wmb(); rte_write16_relaxed(value, addr); } -static inline void __attribute__((always_inline)) +static __rte_always_inline void rte_write32(uint32_t value, volatile void *addr) { rte_io_wmb(); rte_write32_relaxed(value, addr); } -static inline void __attribute__((always_inline)) +static __rte_always_inline void rte_write64(uint64_t value, volatile void *addr) { rte_io_wmb(); diff --git a/lib/librte_eal/common/include/generic/rte_pause.h b/lib/librte_eal/common/include/generic/rte_pause.h new file mode 100644 index 00000000..a8374321 --- /dev/null +++ b/lib/librte_eal/common/include/generic/rte_pause.h @@ -0,0 +1,52 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_PAUSE_H_ +#define _RTE_PAUSE_H_ + +/** + * @file + * + * CPU pause operation. + * + */ + +/** + * Pause CPU execution for a short while + * + * This call is intended for tight loops which poll a shared resource or wait + * for an event. A short pause within the loop may reduce the power consumption. + */ +static inline void rte_pause(void); + +#endif /* _RTE_PAUSE_H_ */ diff --git a/lib/librte_eal/common/include/generic/rte_rwlock.h b/lib/librte_eal/common/include/generic/rte_rwlock.h index 7a0fdc55..fdb3113d 100644 --- a/lib/librte_eal/common/include/generic/rte_rwlock.h +++ b/lib/librte_eal/common/include/generic/rte_rwlock.h @@ -52,6 +52,7 @@ extern "C" { #include <rte_common.h> #include <rte_atomic.h> +#include <rte_pause.h> /** * The rte_rwlock_t type. diff --git a/lib/librte_eal/common/include/generic/rte_spinlock.h b/lib/librte_eal/common/include/generic/rte_spinlock.h index e51fc56b..54f83a4c 100644 --- a/lib/librte_eal/common/include/generic/rte_spinlock.h +++ b/lib/librte_eal/common/include/generic/rte_spinlock.h @@ -51,6 +51,7 @@ #ifdef RTE_FORCE_INTRINSICS #include <rte_common.h> #endif +#include <rte_pause.h> /** * The rte_spinlock_t type. diff --git a/lib/librte_eal/common/include/rte_alarm.h b/lib/librte_eal/common/include/rte_alarm.h index 4012cd67..c275be18 100644 --- a/lib/librte_eal/common/include/rte_alarm.h +++ b/lib/librte_eal/common/include/rte_alarm.h @@ -91,7 +91,7 @@ int rte_eal_alarm_set(uint64_t us, rte_eal_alarm_callback cb, void *cb_arg); * the number of canceled alarm callback functions * - value greater or equal 0 and rte_errno set to EINPROGRESS, at least one * alarm could not be canceled because cancellation was requested from alarm - * callback context. Returned value is the number of succesfuly canceled + * callback context. Returned value is the number of successfully canceled * alarm callbacks * - 0 and rte_errno set to ENOENT - no alarm found * - -1 and rte_errno set to EINVAL - invalid parameter (NULL callback) diff --git a/lib/librte_eal/common/include/rte_bus.h b/lib/librte_eal/common/include/rte_bus.h index 7c369692..c79368d3 100644 --- a/lib/librte_eal/common/include/rte_bus.h +++ b/lib/librte_eal/common/include/rte_bus.h @@ -1,8 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 NXP - * All rights reserved. + * Copyright 2016 NXP * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -58,7 +57,7 @@ TAILQ_HEAD(rte_bus_list, rte_bus); /** * Bus specific scan for devices attached on the bus. - * For each bus object, the scan would be reponsible for finding devices and + * For each bus object, the scan would be responsible for finding devices and * adding them to its private device list. * * A bus should mandatorily implement this method. @@ -82,6 +81,94 @@ typedef int (*rte_bus_scan_t)(void); typedef int (*rte_bus_probe_t)(void); /** + * Device iterator to find a device on a bus. + * + * This function returns an rte_device if one of those held by the bus + * matches the data passed as parameter. + * + * If the comparison function returns zero this function should stop iterating + * over any more devices. To continue a search the device of a previous search + * can be passed via the start parameter. + * + * @param cmp + * Comparison function. + * + * @param data + * Data to compare each device against. + * + * @param start + * starting point for the iteration + * + * @return + * The first device matching the data, NULL if none exists. + */ +typedef struct rte_device * +(*rte_bus_find_device_t)(const struct rte_device *start, rte_dev_cmp_t cmp, + const void *data); + +/** + * Implementation specific probe function which is responsible for linking + * devices on that bus with applicable drivers. + * + * @param dev + * Device pointer that was returned by a previous call to find_device. + * + * @return + * 0 on success. + * !0 on error. + */ +typedef int (*rte_bus_plug_t)(struct rte_device *dev); + +/** + * Implementation specific remove function which is responsible for unlinking + * devices on that bus from assigned driver. + * + * @param dev + * Device pointer that was returned by a previous call to find_device. + * + * @return + * 0 on success. + * !0 on error. + */ +typedef int (*rte_bus_unplug_t)(struct rte_device *dev); + +/** + * Bus specific parsing function. + * Validates the syntax used in the textual representation of a device, + * If the syntax is valid and ``addr`` is not NULL, writes the bus-specific + * device representation to ``addr``. + * + * @param[in] name + * device textual description + * + * @param[out] addr + * device information location address, into which parsed info + * should be written. If NULL, nothing should be written, which + * is not an error. + * + * @return + * 0 if parsing was successful. + * !0 for any error. + */ +typedef int (*rte_bus_parse_t)(const char *name, void *addr); + +/** + * Bus scan policies + */ +enum rte_bus_scan_mode { + RTE_BUS_SCAN_UNDEFINED, + RTE_BUS_SCAN_WHITELIST, + RTE_BUS_SCAN_BLACKLIST, +}; + +/** + * A structure used to configure bus operations. + */ +struct rte_bus_conf { + enum rte_bus_scan_mode scan_mode; /**< Scan policy. */ +}; + +/** * A structure describing a generic bus. */ struct rte_bus { @@ -89,6 +176,11 @@ struct rte_bus { const char *name; /**< Name of the bus */ rte_bus_scan_t scan; /**< Scan for devices attached to bus */ rte_bus_probe_t probe; /**< Probe devices on bus */ + rte_bus_find_device_t find_device; /**< Find a device on the bus */ + rte_bus_plug_t plug; /**< Probe single device for drivers */ + rte_bus_unplug_t unplug; /**< Remove single device from driver */ + rte_bus_parse_t parse; /**< Parse a device name */ + struct rte_bus_conf conf; /**< Bus configuration */ }; /** @@ -133,19 +225,68 @@ int rte_bus_probe(void); * * @param f * A valid and open output stream handle + */ +void rte_bus_dump(FILE *f); + +/** + * Bus comparison function. + * + * @param bus + * Bus under test. + * + * @param data + * Data to compare against. * * @return - * 0 in case of success - * !0 in case there is error in opening the output stream + * 0 if the bus matches the data. + * !0 if the bus does not match. + * <0 if ordering is possible and the bus is lower than the data. + * >0 if ordering is possible and the bus is greater than the data. */ -void rte_bus_dump(FILE *f); +typedef int (*rte_bus_cmp_t)(const struct rte_bus *bus, const void *data); + +/** + * Bus iterator to find a particular bus. + * + * This function compares each registered bus to find one that matches + * the data passed as parameter. + * + * If the comparison function returns zero this function will stop iterating + * over any more buses. To continue a search the bus of a previous search can + * be passed via the start parameter. + * + * @param start + * Starting point for the iteration. + * + * @param cmp + * Comparison function. + * + * @param data + * Data to pass to comparison function. + * + * @return + * A pointer to a rte_bus structure or NULL in case no bus matches + */ +struct rte_bus *rte_bus_find(const struct rte_bus *start, rte_bus_cmp_t cmp, + const void *data); + +/** + * Find the registered bus for a particular device. + */ +struct rte_bus *rte_bus_find_by_device(const struct rte_device *dev); + +/** + * Find the registered bus for a given name. + */ +struct rte_bus *rte_bus_find_by_name(const char *busname); /** * Helper for Bus registration. * The constructor has higher priority than PMD constructors. */ #define RTE_REGISTER_BUS(nm, bus) \ -static void __attribute__((constructor(101), used)) businitfn_ ##nm(void) \ +RTE_INIT_PRIO(businitfn_ ##nm, 101); \ +static void businitfn_ ##nm(void) \ {\ (bus).name = RTE_STR(nm);\ rte_bus_register(&bus); \ diff --git a/lib/librte_eal/common/include/rte_common.h b/lib/librte_eal/common/include/rte_common.h index e057f6e2..1afc66e3 100644 --- a/lib/librte_eal/common/include/rte_common.h +++ b/lib/librte_eal/common/include/rte_common.h @@ -66,6 +66,12 @@ extern "C" { #define RTE_STD_C11 #endif +/** Define GCC_VERSION **/ +#ifdef RTE_TOOLCHAIN_GCC +#define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + \ + __GNUC_PATCHLEVEL__) +#endif + #ifdef RTE_ARCH_STRICT_ALIGN typedef uint64_t unaligned_uint64_t __attribute__ ((aligned(1))); typedef uint32_t unaligned_uint32_t __attribute__ ((aligned(1))); @@ -102,6 +108,16 @@ typedef uint16_t unaligned_uint16_t; */ #define RTE_SET_USED(x) (void)(x) +/** + * Force a function to be inlined + */ +#define __rte_always_inline inline __attribute__((always_inline)) + +/** + * Force a function to be noinlined + */ +#define __rte_noinline __attribute__((noinline)) + /*********** Macros for pointer arithmetic ********/ /** @@ -294,21 +310,6 @@ rte_align64pow2(uint64_t v) /*********** Other general functions / macros ********/ -#ifdef __SSE2__ -#include <emmintrin.h> -/** - * PAUSE instruction for tight loops (avoid busy waiting) - */ -static inline void -rte_pause (void) -{ - _mm_pause(); -} -#else -static inline void -rte_pause(void) {} -#endif - /** * Searches the input parameter for the least significant set bit * (starting from zero). @@ -326,6 +327,23 @@ rte_bsf32(uint32_t v) return __builtin_ctz(v); } +/** + * Return the rounded-up log2 of a integer. + * + * @param v + * The input parameter. + * @return + * The rounded-up log2 of the input, or 0 if the input is 0. + */ +static inline uint32_t +rte_log2_u32(uint32_t v) +{ + if (v == 0) + return 0; + v = rte_align32pow2(v); + return rte_bsf32(v); +} + #ifndef offsetof /** Return the offset of a field in a structure. */ #define offsetof(TYPE, MEMBER) __builtin_offsetof (TYPE, MEMBER) diff --git a/lib/librte_eal/common/include/rte_dev.h b/lib/librte_eal/common/include/rte_dev.h index de20c063..5386d3a2 100644 --- a/lib/librte_eal/common/include/rte_dev.h +++ b/lib/librte_eal/common/include/rte_dev.h @@ -115,6 +115,26 @@ rte_pmd_debug_trace(const char *func_name, const char *fmt, ...) } while (0) /** + * Device driver. + */ +enum rte_kernel_driver { + RTE_KDRV_UNKNOWN = 0, + RTE_KDRV_IGB_UIO, + RTE_KDRV_VFIO, + RTE_KDRV_UIO_GENERIC, + RTE_KDRV_NIC_UIO, + RTE_KDRV_NONE, +}; + +/** + * Device policies. + */ +enum rte_dev_policy { + RTE_DEV_WHITELISTED, + RTE_DEV_BLACKLISTED, +}; + +/** * A generic memory resource representation. */ struct rte_mem_resource { @@ -132,6 +152,8 @@ struct rte_driver { const char *alias; /**< Driver alias. */ }; +#define RTE_DEV_NAME_MAX_LEN (32) + /** * A structure describing a generic device. */ @@ -183,13 +205,67 @@ int rte_eal_dev_attach(const char *name, const char *devargs); /** * Detach a device from its driver. * - * @param name - * Same description as for rte_eal_dev_attach(). - * Here, eal will call the driver detaching function. + * @param dev + * A pointer to a rte_device structure. + * @return + * 0 on success, negative on error. + */ +int rte_eal_dev_detach(struct rte_device *dev); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Hotplug add a given device to a specific bus. + * + * @param busname + * The bus name the device is added to. + * @param devname + * The device name. Based on this device name, eal will identify a driver + * capable of handling it and pass it to the driver probing function. + * @param devargs + * Device arguments to be passed to the driver. + * @return + * 0 on success, negative on error. + */ +int rte_eal_hotplug_add(const char *busname, const char *devname, + const char *devargs); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Hotplug remove a given device from a specific bus. + * + * @param busname + * The bus name the device is removed from. + * @param devname + * The device name being removed. * @return * 0 on success, negative on error. */ -int rte_eal_dev_detach(const char *name); +int rte_eal_hotplug_remove(const char *busname, const char *devname); + +/** + * Device comparison function. + * + * This type of function is used to compare an rte_device with arbitrary + * data. + * + * @param dev + * Device handle. + * + * @param data + * Data to compare against. The type of this parameter is determined by + * the kind of comparison performed by the function. + * + * @return + * 0 if the device matches the data. + * !0 if the device does not match. + * <0 if ordering is possible and the device is lower than the data. + * >0 if ordering is possible and the device is greater than the data. + */ +typedef int (*rte_dev_cmp_t)(const struct rte_device *dev, const void *data); #define RTE_PMD_EXPORT_NAME_ARRAY(n, idx) n##idx[] diff --git a/lib/librte_eal/common/include/rte_devargs.h b/lib/librte_eal/common/include/rte_devargs.h index 88120a1c..58d585df 100644 --- a/lib/librte_eal/common/include/rte_devargs.h +++ b/lib/librte_eal/common/include/rte_devargs.h @@ -50,7 +50,7 @@ extern "C" { #include <stdio.h> #include <sys/queue.h> -#include <rte_pci.h> +#include <rte_bus.h> /** * Type of generic device @@ -76,19 +76,12 @@ struct rte_devargs { TAILQ_ENTRY(rte_devargs) next; /** Type of device. */ enum rte_devtype type; - RTE_STD_C11 - union { - /** Used if type is RTE_DEVTYPE_*_PCI. */ - struct { - /** PCI location. */ - struct rte_pci_addr addr; - } pci; - /** Used if type is RTE_DEVTYPE_VIRTUAL. */ - struct { - /** Driver name. */ - char drv_name[32]; - } virt; - }; + /** Device policy. */ + enum rte_dev_policy policy; + /** Bus handle for the device. */ + struct rte_bus *bus; + /** Name of the device. */ + char name[RTE_DEV_NAME_MAX_LEN]; /** Arguments string as given by user or "" for no argument. */ char *args; }; @@ -128,6 +121,39 @@ int rte_eal_parse_devargs_str(const char *devargs_str, char **drvname, char **drvargs); /** + * Parse a device string. + * + * Verify that a bus is capable of handling the device passed + * in argument. Store which bus will handle the device, its name + * and the eventual device parameters. + * + * @param dev + * The device declaration string. + * @param da + * The devargs structure holding the device information. + * + * @return + * - 0 on success. + * - Negative errno on error. + */ +int +rte_eal_devargs_parse(const char *dev, + struct rte_devargs *da); + +/** + * Insert an rte_devargs in the global list. + * + * @param da + * The devargs structure to insert. + * + * @return + * - 0 on success + * - Negative on error. + */ +int +rte_eal_devargs_insert(struct rte_devargs *da); + +/** * Add a device to the user device list * * For PCI devices, the format of arguments string is "PCI_ADDR" or @@ -152,6 +178,24 @@ int rte_eal_parse_devargs_str(const char *devargs_str, int rte_eal_devargs_add(enum rte_devtype devtype, const char *devargs_str); /** + * Remove a device from the user device list. + * Its resources are freed. + * If the devargs cannot be found, nothing happens. + * + * @param busname + * bus name of the devargs to remove. + * + * @param devname + * device name of the devargs to remove. + * + * @return + * 0 on success. + * <0 on error. + * >0 if the devargs was not within the user device list. + */ +int rte_eal_devargs_remove(const char *busname, const char *devname); + +/** * Count the number of user devices of a specified type * * @param devtype diff --git a/lib/librte_eal/common/include/rte_eal.h b/lib/librte_eal/common/include/rte_eal.h index abf020bf..0e7363d7 100644 --- a/lib/librte_eal/common/include/rte_eal.h +++ b/lib/librte_eal/common/include/rte_eal.h @@ -61,6 +61,7 @@ extern "C" { enum rte_lcore_role_t { ROLE_RTE, ROLE_OFF, + ROLE_SERVICE, }; /** @@ -80,6 +81,7 @@ enum rte_proc_type_t { struct rte_config { uint32_t master_lcore; /**< Id of the master lcore */ uint32_t lcore_count; /**< Number of available logical cores. */ + uint32_t service_lcore_count;/**< Number of available service cores. */ enum rte_lcore_role_t lcore_role[RTE_MAX_LCORE]; /**< State of cores. */ /** Primary or secondary configuration */ @@ -185,6 +187,8 @@ int rte_eal_iopl_init(void); * * EPROTO indicates that the PCI bus is either not present, or is not * readable by the eal. + * + * ENOEXEC indicates that a service core failed to launch successfully. */ int rte_eal_init(int argc, char **argv); @@ -286,6 +290,9 @@ static inline int rte_gettid(void) #define RTE_INIT(func) \ static void __attribute__((constructor, used)) func(void) +#define RTE_INIT_PRIO(func, prio) \ +static void __attribute__((constructor(prio), used)) func(void) + #ifdef __cplusplus } #endif diff --git a/lib/librte_eal/common/include/rte_eal_memconfig.h b/lib/librte_eal/common/include/rte_eal_memconfig.h index 2b5e0b17..b9eee702 100644 --- a/lib/librte_eal/common/include/rte_eal_memconfig.h +++ b/lib/librte_eal/common/include/rte_eal_memconfig.h @@ -39,6 +39,7 @@ #include <rte_memzone.h> #include <rte_malloc_heap.h> #include <rte_rwlock.h> +#include <rte_pause.h> #ifdef __cplusplus extern "C" { diff --git a/lib/librte_eal/common/include/rte_lcore.h b/lib/librte_eal/common/include/rte_lcore.h index fe7b5865..50e0d0fe 100644 --- a/lib/librte_eal/common/include/rte_lcore.h +++ b/lib/librte_eal/common/include/rte_lcore.h @@ -73,6 +73,7 @@ struct lcore_config { unsigned core_id; /**< core number on socket for this lcore */ int core_index; /**< relative index, starting from 0 */ rte_cpuset_t cpuset; /**< cpu set which the lcore affinity to */ + uint8_t core_role; /**< role of core eg: OFF, RTE, SERVICE */ }; /** @@ -175,7 +176,7 @@ rte_lcore_is_enabled(unsigned lcore_id) struct rte_config *cfg = rte_eal_get_configuration(); if (lcore_id >= RTE_MAX_LCORE) return 0; - return cfg->lcore_role[lcore_id] != ROLE_OFF; + return cfg->lcore_role[lcore_id] == ROLE_RTE; } /** diff --git a/lib/librte_eal/common/include/rte_log.h b/lib/librte_eal/common/include/rte_log.h index 34191385..ec8dba79 100644 --- a/lib/librte_eal/common/include/rte_log.h +++ b/lib/librte_eal/common/include/rte_log.h @@ -175,6 +175,16 @@ __rte_deprecated uint32_t rte_get_log_type(void); /** + * Get the log level for a given type. + * + * @param logtype + * The log type identifier. + * @return + * 0 on success, a negative value if logtype is invalid. + */ +int rte_log_get_level(uint32_t logtype); + +/** * Set the log level for a given type. * * @param pattern diff --git a/lib/librte_eal/common/include/rte_malloc.h b/lib/librte_eal/common/include/rte_malloc.h index 008ce134..3d37f79b 100644 --- a/lib/librte_eal/common/include/rte_malloc.h +++ b/lib/librte_eal/common/include/rte_malloc.h @@ -327,9 +327,9 @@ rte_malloc_set_limit(const char *type, size_t max); * rte_malloc * * @param addr - * Adress obtained from a previous rte_malloc call + * Address obtained from a previous rte_malloc call * @return - * NULL on error + * RTE_BAD_PHYS_ADDR on error * otherwise return physical address of the buffer */ phys_addr_t diff --git a/lib/librte_eal/common/include/rte_pci.h b/lib/librte_eal/common/include/rte_pci.h index ab64c63c..8b123391 100644 --- a/lib/librte_eal/common/include/rte_pci.h +++ b/lib/librte_eal/common/include/rte_pci.h @@ -2,6 +2,7 @@ * BSD LICENSE * * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * Copyright 2013-2014 6WIND S.A. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -30,36 +31,6 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -/* BSD LICENSE - * - * Copyright 2013-2014 6WIND S.A. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ #ifndef _RTE_PCI_H_ #define _RTE_PCI_H_ @@ -92,7 +63,7 @@ const char *pci_get_sysfs_path(void); /** Formatting string for PCI device identifier: Ex: 0000:00:01.0 */ #define PCI_PRI_FMT "%.4" PRIx16 ":%.2" PRIx8 ":%.2" PRIx8 ".%" PRIx8 -#define PCI_PRI_STR_SIZE sizeof("XXXX:XX:XX.X") +#define PCI_PRI_STR_SIZE sizeof("XXXXXXXX:XX:XX.X") /** Short formatting string, without domain, for PCI device: Ex: 00:01.0 */ #define PCI_SHORT_PRI_FMT "%.2" PRIx8 ":%.2" PRIx8 ".%" PRIx8 @@ -106,9 +77,6 @@ const char *pci_get_sysfs_path(void); /** Maximum number of PCI resources. */ #define PCI_MAX_RESOURCE 6 -/** Name of PCI Bus */ -#define PCI_BUS_NAME "PCI" - /* Forward declarations */ struct rte_pci_device; struct rte_pci_driver; @@ -141,7 +109,7 @@ struct rte_pci_id { * A structure describing the location of a PCI device. */ struct rte_pci_addr { - uint16_t domain; /**< Device domain */ + uint32_t domain; /**< Device domain */ uint8_t bus; /**< Device bus */ uint8_t devid; /**< Device ID */ uint8_t function; /**< Device function. */ @@ -149,15 +117,6 @@ struct rte_pci_addr { struct rte_devargs; -enum rte_kernel_driver { - RTE_KDRV_UNKNOWN = 0, - RTE_KDRV_IGB_UIO, - RTE_KDRV_VFIO, - RTE_KDRV_UIO_GENERIC, - RTE_KDRV_NIC_UIO, - RTE_KDRV_NONE, -}; - /** * A structure describing a PCI device. */ @@ -241,6 +200,8 @@ struct rte_pci_bus { #define RTE_PCI_DRV_INTR_LSC 0x0008 /** Device driver supports device removal interrupt */ #define RTE_PCI_DRV_INTR_RMV 0x0010 +/** Device driver needs to keep mapped resources if unsupported dev detected */ +#define RTE_PCI_DRV_KEEP_MAPPED_RES 0x0020 /** * A structure describing a PCI mapping. @@ -373,10 +334,10 @@ rte_eal_compare_pci_addr(const struct rte_pci_addr *addr, if ((addr == NULL) || (addr2 == NULL)) return -1; - dev_addr = (addr->domain << 24) | (addr->bus << 16) | - (addr->devid << 8) | addr->function; - dev_addr2 = (addr2->domain << 24) | (addr2->bus << 16) | - (addr2->devid << 8) | addr2->function; + dev_addr = ((uint64_t)addr->domain << 24) | + (addr->bus << 16) | (addr->devid << 8) | addr->function; + dev_addr2 = ((uint64_t)addr2->domain << 24) | + (addr2->bus << 16) | (addr2->devid << 8) | addr2->function; if (dev_addr > dev_addr2) return 1; diff --git a/lib/librte_eal/common/include/rte_service.h b/lib/librte_eal/common/include/rte_service.h new file mode 100644 index 00000000..7c6f7383 --- /dev/null +++ b/lib/librte_eal/common/include/rte_service.h @@ -0,0 +1,387 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_SERVICE_H_ +#define _RTE_SERVICE_H_ + +/** + * @file + * + * Service functions + * + * The service functionality provided by this header allows a DPDK component + * to indicate that it requires a function call in order for it to perform + * its processing. + * + * An example usage of this functionality would be a component that registers + * a service to perform a particular packet processing duty: for example the + * eventdev software PMD. At startup the application requests all services + * that have been registered, and the cores in the service-coremask run the + * required services. The EAL removes these number of cores from the available + * runtime cores, and dedicates them to performing service-core workloads. The + * application has access to the remaining lcores as normal. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include<stdio.h> +#include <stdint.h> +#include <sys/queue.h> + +#include <rte_lcore.h> + +/* forward declaration only. Definition in rte_service_private.h */ +struct rte_service_spec; + +#define RTE_SERVICE_NAME_MAX 32 + +/* Capabilities of a service. + * + * Use the *rte_service_probe_capability* function to check if a service is + * capable of a specific capability. + */ +/** When set, the service is capable of having multiple threads run it at the + * same time. + */ +#define RTE_SERVICE_CAP_MT_SAFE (1 << 0) + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Return the number of services registered. + * + * The number of services registered can be passed to *rte_service_get_by_id*, + * enabling the application to retrieve the specification of each service. + * + * @return The number of services registered. + */ +uint32_t rte_service_get_count(void); + + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Return the specification of a service by integer id. + * + * This function provides the specification of a service. This can be used by + * the application to understand what the service represents. The service + * must not be modified by the application directly, only passed to the various + * rte_service_* functions. + * + * @param id The integer id of the service to retrieve + * @retval non-zero A valid pointer to the service_spec + * @retval NULL Invalid *id* provided. + */ +struct rte_service_spec *rte_service_get_by_id(uint32_t id); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Return the specification of a service by name. + * + * This function provides the specification of a service using the service name + * as lookup key. This can be used by the application to understand what the + * service represents. The service must not be modified by the application + * directly, only passed to the various rte_service_* functions. + * + * @param name The name of the service to retrieve + * @retval non-zero A valid pointer to the service_spec + * @retval NULL Invalid *name* provided. + */ +struct rte_service_spec *rte_service_get_by_name(const char *name); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Return the name of the service. + * + * @return A pointer to the name of the service. The returned pointer remains + * in ownership of the service, and the application must not free it. + */ +const char *rte_service_get_name(const struct rte_service_spec *service); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Check if a service has a specific capability. + * + * This function returns if *service* has implements *capability*. + * See RTE_SERVICE_CAP_* defines for a list of valid capabilities. + * @retval 1 Capability supported by this service instance + * @retval 0 Capability not supported by this service instance + */ +int32_t rte_service_probe_capability(const struct rte_service_spec *service, + uint32_t capability); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Enable a core to run a service. + * + * Each core can be added or removed from running specific services. This + * functions adds *lcore* to the set of cores that will run *service*. + * + * If multiple cores are enabled on a service, an atomic is used to ensure that + * only one cores runs the service at a time. The exception to this is when + * a service indicates that it is multi-thread safe by setting the capability + * called RTE_SERVICE_CAP_MT_SAFE. With the multi-thread safe capability set, + * the service function can be run on multiple threads at the same time. + * + * @retval 0 lcore added successfully + * @retval -EINVAL An invalid service or lcore was provided. + */ +int32_t rte_service_enable_on_lcore(struct rte_service_spec *service, + uint32_t lcore); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Disable a core to run a service. + * + * Each core can be added or removed from running specific services. This + * functions removes *lcore* to the set of cores that will run *service*. + * + * @retval 0 Lcore removed successfully + * @retval -EINVAL An invalid service or lcore was provided. + */ +int32_t rte_service_disable_on_lcore(struct rte_service_spec *service, + uint32_t lcore); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Return if an lcore is enabled for the service. + * + * This function allows the application to query if *lcore* is currently set to + * run *service*. + * + * @retval 1 Lcore enabled on this lcore + * @retval 0 Lcore disabled on this lcore + * @retval -EINVAL An invalid service or lcore was provided. + */ +int32_t rte_service_get_enabled_on_lcore(struct rte_service_spec *service, + uint32_t lcore); + + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Enable *service* to run. + * + * This function switches on a service during runtime. + * @retval 0 The service was successfully started + */ +int32_t rte_service_start(struct rte_service_spec *service); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Disable *service*. + * + * Switch off a service, so it is not run until it is *rte_service_start* is + * called on it. + * @retval 0 Service successfully switched off + */ +int32_t rte_service_stop(struct rte_service_spec *service); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Returns if *service* is currently running. + * + * This function returns true if the service has been started using + * *rte_service_start*, AND a service core is mapped to the service. This + * function can be used to ensure that the service will be run. + * + * @retval 1 Service is currently running, and has a service lcore mapped + * @retval 0 Service is currently stopped, or no service lcore is mapped + * @retval -EINVAL Invalid service pointer provided + */ +int32_t rte_service_is_running(const struct rte_service_spec *service); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Start a service core. + * + * Starting a core makes the core begin polling. Any services assigned to it + * will be run as fast as possible. + * + * @retval 0 Success + * @retval -EINVAL Failed to start core. The *lcore_id* passed in is not + * currently assigned to be a service core. + */ +int32_t rte_service_lcore_start(uint32_t lcore_id); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Stop a service core. + * + * Stopping a core makes the core become idle, but remains assigned as a + * service core. + * + * @retval 0 Success + * @retval -EINVAL Invalid *lcore_id* provided + * @retval -EALREADY Already stopped core + * @retval -EBUSY Failed to stop core, as it would cause a service to not + * be run, as this is the only core currently running the service. + * The application must stop the service first, and then stop the + * lcore. + */ +int32_t rte_service_lcore_stop(uint32_t lcore_id); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Adds lcore to the list of service cores. + * + * This functions can be used at runtime in order to modify the service core + * mask. + * + * @retval 0 Success + * @retval -EBUSY lcore is busy, and not available for service core duty + * @retval -EALREADY lcore is already added to the service core list + * @retval -EINVAL Invalid lcore provided + */ +int32_t rte_service_lcore_add(uint32_t lcore); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Removes lcore from the list of service cores. + * + * This can fail if the core is not stopped, see *rte_service_core_stop*. + * + * @retval 0 Success + * @retval -EBUSY Lcore is not stopped, stop service core before removing. + * @retval -EINVAL failed to add lcore to service core mask. + */ +int32_t rte_service_lcore_del(uint32_t lcore); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Retrieve the number of service cores currently available. + * + * This function returns the integer count of service cores available. The + * service core count can be used in mapping logic when creating mappings + * from service cores to services. + * + * See *rte_service_lcore_list* for details on retrieving the lcore_id of each + * service core. + * + * @return The number of service cores currently configured. + */ +int32_t rte_service_lcore_count(void); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Resets all service core mappings. This does not remove the service cores + * from duty, just unmaps all services / cores, and stops() the service cores. + * The runstate of services is not modified. + * + * @retval 0 Success + */ +int32_t rte_service_lcore_reset_all(void); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Enable or disable statistics collection for *service*. + * + * This function enables per core, per-service cycle count collection. + * @param service The service to enable statistics gathering on. + * @param enable Zero to disable statistics, non-zero to enable. + * @retval 0 Success + * @retval -EINVAL Invalid service pointer passed + */ +int32_t rte_service_set_stats_enable(struct rte_service_spec *service, + int32_t enable); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Retrieve the list of currently enabled service cores. + * + * This function fills in an application supplied array, with each element + * indicating the lcore_id of a service core. + * + * Adding and removing service cores can be performed using + * *rte_service_lcore_add* and *rte_service_lcore_del*. + * @param [out] array An array of at least *rte_service_lcore_count* items. + * If statically allocating the buffer, use RTE_MAX_LCORE. + * @param [out] n The size of *array*. + * @retval >=0 Number of service cores that have been populated in the array + * @retval -ENOMEM The provided array is not large enough to fill in the + * service core list. No items have been populated, call this function + * with a size of at least *rte_service_core_count* items. + */ +int32_t rte_service_lcore_list(uint32_t array[], uint32_t n); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Dumps any information available about the service. If service is NULL, + * dumps info for all services. + */ +int32_t rte_service_dump(FILE *f, struct rte_service_spec *service); + +#ifdef __cplusplus +} +#endif + + +#endif /* _RTE_SERVICE_H_ */ diff --git a/lib/librte_eal/common/include/rte_service_component.h b/lib/librte_eal/common/include/rte_service_component.h new file mode 100644 index 00000000..7a946a1e --- /dev/null +++ b/lib/librte_eal/common/include/rte_service_component.h @@ -0,0 +1,144 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_SERVICE_PRIVATE_H_ +#define _RTE_SERVICE_PRIVATE_H_ + +/* This file specifies the internal service specification. + * Include this file if you are writing a component that requires CPU cycles to + * operate, and you wish to run the component using service cores + */ + +#include <rte_service.h> + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Signature of callback function to run a service. + */ +typedef int32_t (*rte_service_func)(void *args); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * The specification of a service. + * + * This struct contains metadata about the service itself, the callback + * function to run one iteration of the service, a userdata pointer, flags etc. + */ +struct rte_service_spec { + /** The name of the service. This should be used by the application to + * understand what purpose this service provides. + */ + char name[RTE_SERVICE_NAME_MAX]; + /** The callback to invoke to run one iteration of the service. */ + rte_service_func callback; + /** The userdata pointer provided to the service callback. */ + void *callback_userdata; + /** Flags to indicate the capabilities of this service. See defines in + * the public header file for values of RTE_SERVICE_CAP_* + */ + uint32_t capabilities; + /** NUMA socket ID that this service is affinitized to */ + int socket_id; +}; + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Register a new service. + * + * A service represents a component that the requires CPU time periodically to + * achieve its purpose. + * + * For example the eventdev SW PMD requires CPU cycles to perform its + * scheduling. This can be achieved by registering it as a service, and the + * application can then assign CPU resources to it using + * *rte_service_set_coremask*. + * + * @param spec The specification of the service to register + * @retval 0 Successfully registered the service. + * -EINVAL Attempted to register an invalid service (eg, no callback + * set) + */ +int32_t rte_service_register(const struct rte_service_spec *spec); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Unregister a service. + * + * The service being removed must be stopped before calling this function. + * + * @retval 0 The service was successfully unregistered. + * @retval -EBUSY The service is currently running, stop the service before + * calling unregister. No action has been taken. + */ +int32_t rte_service_unregister(struct rte_service_spec *service); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Private function to allow EAL to initialized default mappings. + * + * This function iterates all the services, and maps then to the available + * cores. Based on the capabilities of the services, they are set to run on the + * available cores in a round-robin manner. + * + * @retval 0 Success + * @retval -ENOTSUP No service lcores in use + * @retval -EINVAL Error while iterating over services + * @retval -ENODEV Error in enabling service lcore on a service + * @retval -ENOEXEC Error when starting services + */ +int32_t rte_service_start_with_defaults(void); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Initialize the service library. + * + * In order to use the service library, it must be initialized. EAL initializes + * the library at startup. + * + * @retval 0 Success + * @retval -EALREADY Service library is already initialized + */ +int32_t rte_service_init(void); + +#endif /* _RTE_SERVICE_PRIVATE_H_ */ diff --git a/lib/librte_eal/common/include/rte_time.h b/lib/librte_eal/common/include/rte_time.h index 28c6274c..373c41ac 100644 --- a/lib/librte_eal/common/include/rte_time.h +++ b/lib/librte_eal/common/include/rte_time.h @@ -52,7 +52,7 @@ struct rte_timecounter { uint64_t nsec_mask; /** Sub-nanoseconds count. */ uint64_t nsec_frac; - /** Bitmask for two's complement substraction of non-64 bit counters. */ + /** Bitmask for two's complement subtraction of non-64 bit counters. */ uint64_t cc_mask; /** Cycle to nanosecond divisor (power of two). */ uint32_t cc_shift; diff --git a/lib/librte_eal/common/include/rte_vdev.h b/lib/librte_eal/common/include/rte_vdev.h index e6b678ea..29f5a523 100644 --- a/lib/librte_eal/common/include/rte_vdev.h +++ b/lib/librte_eal/common/include/rte_vdev.h @@ -46,11 +46,18 @@ struct rte_vdev_device { struct rte_device device; /**< Inherit core device */ }; +/** + * @internal + * Helper macro for drivers that need to convert to struct rte_vdev_device. + */ +#define RTE_DEV_TO_VDEV(ptr) \ + container_of(ptr, struct rte_vdev_device, device) + static inline const char * rte_vdev_device_name(const struct rte_vdev_device *dev) { - if (dev && dev->device.devargs) - return dev->device.devargs->virt.drv_name; + if (dev && dev->device.name) + return dev->device.name; return NULL; } diff --git a/lib/librte_eal/common/include/rte_version.h b/lib/librte_eal/common/include/rte_version.h index c36d8526..a69a7075 100644 --- a/lib/librte_eal/common/include/rte_version.h +++ b/lib/librte_eal/common/include/rte_version.h @@ -61,12 +61,12 @@ extern "C" { /** * Minor version/month number i.e. the mm in yy.mm.z */ -#define RTE_VER_MONTH 5 +#define RTE_VER_MONTH 8 /** * Patch level number i.e. the z in yy.mm.z */ -#define RTE_VER_MINOR 1 +#define RTE_VER_MINOR 0 /** * Extra string to be appended to version number diff --git a/lib/librte_eal/common/malloc_elem.c b/lib/librte_eal/common/malloc_elem.c index 42568e1d..15076905 100644 --- a/lib/librte_eal/common/malloc_elem.c +++ b/lib/librte_eal/common/malloc_elem.c @@ -51,7 +51,7 @@ #define MIN_DATA_SIZE (RTE_CACHE_LINE_SIZE) /* - * initialise a general malloc_elem header structure + * Initialize a general malloc_elem header structure */ void malloc_elem_init(struct malloc_elem *elem, @@ -69,7 +69,7 @@ malloc_elem_init(struct malloc_elem *elem, } /* - * initialise a dummy malloc_elem header for the end-of-memseg marker + * Initialize a dummy malloc_elem header for the end-of-memseg marker */ void malloc_elem_mkend(struct malloc_elem *elem, struct malloc_elem *prev) @@ -228,7 +228,7 @@ malloc_elem_alloc(struct malloc_elem *elem, size_t size, unsigned align, elem->pad = old_elem_size; /* put a dummy header in padding, to point to real element header */ - if (elem->pad > 0){ /* pad will be at least 64-bytes, as everything + if (elem->pad > 0) { /* pad will be at least 64-bytes, as everything * is cache-line aligned */ new_elem->pad = elem->pad; new_elem->state = ELEM_PAD; @@ -314,17 +314,16 @@ malloc_elem_free(struct malloc_elem *elem) int malloc_elem_resize(struct malloc_elem *elem, size_t size) { - const size_t new_size = size + MALLOC_ELEM_OVERHEAD; + const size_t new_size = size + elem->pad + MALLOC_ELEM_OVERHEAD; /* if we request a smaller size, then always return ok */ - const size_t current_size = elem->size - elem->pad; - if (current_size >= new_size) + if (elem->size >= new_size) return 0; struct malloc_elem *next = RTE_PTR_ADD(elem, elem->size); rte_spinlock_lock(&elem->heap->lock); if (next ->state != ELEM_FREE) goto err_return; - if (current_size + next->size < new_size) + if (elem->size + next->size < new_size) goto err_return; /* we now know the element fits, so remove from free list, @@ -333,7 +332,7 @@ malloc_elem_resize(struct malloc_elem *elem, size_t size) elem_free_list_remove(next); join_elem(elem, next); - if (elem->size - new_size >= MIN_DATA_SIZE + MALLOC_ELEM_OVERHEAD){ + if (elem->size - new_size >= MIN_DATA_SIZE + MALLOC_ELEM_OVERHEAD) { /* now we have a big block together. Lets cut it down a bit, by splitting */ struct malloc_elem *split_pt = RTE_PTR_ADD(elem, new_size); split_pt = RTE_PTR_ALIGN_CEIL(split_pt, RTE_CACHE_LINE_SIZE); diff --git a/lib/librte_eal/common/rte_keepalive.c b/lib/librte_eal/common/rte_keepalive.c index 9765d1bd..cdd69560 100644 --- a/lib/librte_eal/common/rte_keepalive.c +++ b/lib/librte_eal/common/rte_keepalive.c @@ -38,7 +38,6 @@ #include <rte_log.h> #include <rte_keepalive.h> #include <rte_malloc.h> -#include <rte_cycles.h> struct rte_keepalive { /** Core Liveness. */ diff --git a/lib/librte_eal/common/rte_malloc.c b/lib/librte_eal/common/rte_malloc.c index f4a88352..5c0627bf 100644 --- a/lib/librte_eal/common/rte_malloc.c +++ b/lib/librte_eal/common/rte_malloc.c @@ -253,6 +253,8 @@ rte_malloc_virt2phy(const void *addr) { const struct malloc_elem *elem = malloc_elem_from_data(addr); if (elem == NULL) - return 0; + return RTE_BAD_PHYS_ADDR; + if (elem->ms->phys_addr == RTE_BAD_PHYS_ADDR) + return RTE_BAD_PHYS_ADDR; return elem->ms->phys_addr + ((uintptr_t)addr - (uintptr_t)elem->ms->addr); } diff --git a/lib/librte_eal/common/rte_service.c b/lib/librte_eal/common/rte_service.c new file mode 100644 index 00000000..7efb76dc --- /dev/null +++ b/lib/librte_eal/common/rte_service.c @@ -0,0 +1,706 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <unistd.h> +#include <inttypes.h> +#include <limits.h> +#include <string.h> +#include <dirent.h> + +#include <rte_service.h> +#include "include/rte_service_component.h" + +#include <rte_eal.h> +#include <rte_lcore.h> +#include <rte_common.h> +#include <rte_debug.h> +#include <rte_cycles.h> +#include <rte_atomic.h> +#include <rte_memory.h> +#include <rte_malloc.h> + +#define RTE_SERVICE_NUM_MAX 64 + +#define SERVICE_F_REGISTERED (1 << 0) +#define SERVICE_F_STATS_ENABLED (1 << 1) + +/* runstates for services and lcores, denoting if they are active or not */ +#define RUNSTATE_STOPPED 0 +#define RUNSTATE_RUNNING 1 + +/* internal representation of a service */ +struct rte_service_spec_impl { + /* public part of the struct */ + struct rte_service_spec spec; + + /* atomic lock that when set indicates a service core is currently + * running this service callback. When not set, a core may take the + * lock and then run the service callback. + */ + rte_atomic32_t execute_lock; + + /* API set/get-able variables */ + int32_t runstate; + uint8_t internal_flags; + + /* per service statistics */ + uint32_t num_mapped_cores; + uint64_t calls; + uint64_t cycles_spent; +} __rte_cache_aligned; + +/* the internal values of a service core */ +struct core_state { + /* map of services IDs are run on this core */ + uint64_t service_mask; + uint8_t runstate; /* running or stopped */ + uint8_t is_service_core; /* set if core is currently a service core */ + + /* extreme statistics */ + uint64_t calls_per_service[RTE_SERVICE_NUM_MAX]; +} __rte_cache_aligned; + +static uint32_t rte_service_count; +static struct rte_service_spec_impl *rte_services; +static struct core_state *lcore_states; +static uint32_t rte_service_library_initialized; + +int32_t rte_service_init(void) +{ + if (rte_service_library_initialized) { + printf("service library init() called, init flag %d\n", + rte_service_library_initialized); + return -EALREADY; + } + + rte_services = rte_calloc("rte_services", RTE_SERVICE_NUM_MAX, + sizeof(struct rte_service_spec_impl), + RTE_CACHE_LINE_SIZE); + if (!rte_services) { + printf("error allocating rte services array\n"); + return -ENOMEM; + } + + lcore_states = rte_calloc("rte_service_core_states", RTE_MAX_LCORE, + sizeof(struct core_state), RTE_CACHE_LINE_SIZE); + if (!lcore_states) { + printf("error allocating core states array\n"); + return -ENOMEM; + } + + int i; + int count = 0; + struct rte_config *cfg = rte_eal_get_configuration(); + for (i = 0; i < RTE_MAX_LCORE; i++) { + if (lcore_config[i].core_role == ROLE_SERVICE) { + if ((unsigned int)i == cfg->master_lcore) + continue; + rte_service_lcore_add(i); + count++; + } + } + + rte_service_library_initialized = 1; + return 0; +} + +/* returns 1 if service is registered and has not been unregistered + * Returns 0 if service never registered, or has been unregistered + */ +static inline int +service_valid(uint32_t id) +{ + return !!(rte_services[id].internal_flags & SERVICE_F_REGISTERED); +} + +/* returns 1 if statistics should be colleced for service + * Returns 0 if statistics should not be collected for service + */ +static inline int +service_stats_enabled(struct rte_service_spec_impl *impl) +{ + return !!(impl->internal_flags & SERVICE_F_STATS_ENABLED); +} + +static inline int +service_mt_safe(struct rte_service_spec_impl *s) +{ + return s->spec.capabilities & RTE_SERVICE_CAP_MT_SAFE; +} + +int32_t rte_service_set_stats_enable(struct rte_service_spec *service, + int32_t enabled) +{ + struct rte_service_spec_impl *impl = + (struct rte_service_spec_impl *)service; + if (!impl) + return -EINVAL; + + if (enabled) + impl->internal_flags |= SERVICE_F_STATS_ENABLED; + else + impl->internal_flags &= ~(SERVICE_F_STATS_ENABLED); + + return 0; +} + +uint32_t +rte_service_get_count(void) +{ + return rte_service_count; +} + +struct rte_service_spec * +rte_service_get_by_id(uint32_t id) +{ + struct rte_service_spec *service = NULL; + if (id < rte_service_count) + service = (struct rte_service_spec *)&rte_services[id]; + + return service; +} + +struct rte_service_spec *rte_service_get_by_name(const char *name) +{ + struct rte_service_spec *service = NULL; + int i; + for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) { + if (service_valid(i) && + strcmp(name, rte_services[i].spec.name) == 0) { + service = (struct rte_service_spec *)&rte_services[i]; + break; + } + } + + return service; +} + +const char * +rte_service_get_name(const struct rte_service_spec *service) +{ + return service->name; +} + +int32_t +rte_service_probe_capability(const struct rte_service_spec *service, + uint32_t capability) +{ + return service->capabilities & capability; +} + +int32_t +rte_service_is_running(const struct rte_service_spec *spec) +{ + const struct rte_service_spec_impl *impl = + (const struct rte_service_spec_impl *)spec; + if (!impl) + return -EINVAL; + + return (impl->runstate == RUNSTATE_RUNNING) && + (impl->num_mapped_cores > 0); +} + +int32_t +rte_service_register(const struct rte_service_spec *spec) +{ + uint32_t i; + int32_t free_slot = -1; + + if (spec->callback == NULL || strlen(spec->name) == 0) + return -EINVAL; + + for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) { + if (!service_valid(i)) { + free_slot = i; + break; + } + } + + if ((free_slot < 0) || (i == RTE_SERVICE_NUM_MAX)) + return -ENOSPC; + + struct rte_service_spec_impl *s = &rte_services[free_slot]; + s->spec = *spec; + s->internal_flags |= SERVICE_F_REGISTERED; + + rte_smp_wmb(); + rte_service_count++; + + return 0; +} + +int32_t +rte_service_unregister(struct rte_service_spec *spec) +{ + struct rte_service_spec_impl *s = NULL; + struct rte_service_spec_impl *spec_impl = + (struct rte_service_spec_impl *)spec; + + uint32_t i; + uint32_t service_id; + for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) { + if (&rte_services[i] == spec_impl) { + s = spec_impl; + service_id = i; + break; + } + } + + if (!s) + return -EINVAL; + + rte_service_count--; + rte_smp_wmb(); + + s->internal_flags &= ~(SERVICE_F_REGISTERED); + + for (i = 0; i < RTE_MAX_LCORE; i++) + lcore_states[i].service_mask &= ~(UINT64_C(1) << service_id); + + memset(&rte_services[service_id], 0, + sizeof(struct rte_service_spec_impl)); + + return 0; +} + +int32_t +rte_service_start(struct rte_service_spec *service) +{ + struct rte_service_spec_impl *s = + (struct rte_service_spec_impl *)service; + s->runstate = RUNSTATE_RUNNING; + rte_smp_wmb(); + return 0; +} + +int32_t +rte_service_stop(struct rte_service_spec *service) +{ + struct rte_service_spec_impl *s = + (struct rte_service_spec_impl *)service; + s->runstate = RUNSTATE_STOPPED; + rte_smp_wmb(); + return 0; +} + +static int32_t +rte_service_runner_func(void *arg) +{ + RTE_SET_USED(arg); + uint32_t i; + const int lcore = rte_lcore_id(); + struct core_state *cs = &lcore_states[lcore]; + + while (lcore_states[lcore].runstate == RUNSTATE_RUNNING) { + const uint64_t service_mask = cs->service_mask; + for (i = 0; i < rte_service_count; i++) { + struct rte_service_spec_impl *s = &rte_services[i]; + if (s->runstate != RUNSTATE_RUNNING || + !(service_mask & (UINT64_C(1) << i))) + continue; + + /* check do we need cmpset, if MT safe or <= 1 core + * mapped, atomic ops are not required. + */ + const int need_cmpset = !((service_mt_safe(s) == 0) && + (s->num_mapped_cores > 1)); + uint32_t *lock = (uint32_t *)&s->execute_lock; + + if (need_cmpset || rte_atomic32_cmpset(lock, 0, 1)) { + void *userdata = s->spec.callback_userdata; + + if (service_stats_enabled(s)) { + uint64_t start = rte_rdtsc(); + s->spec.callback(userdata); + uint64_t end = rte_rdtsc(); + s->cycles_spent += end - start; + cs->calls_per_service[i]++; + s->calls++; + } else + s->spec.callback(userdata); + + if (need_cmpset) + rte_atomic32_clear(&s->execute_lock); + } + } + + rte_smp_rmb(); + } + + lcore_config[lcore].state = WAIT; + + return 0; +} + +int32_t +rte_service_lcore_count(void) +{ + int32_t count = 0; + uint32_t i; + for (i = 0; i < RTE_MAX_LCORE; i++) + count += lcore_states[i].is_service_core; + return count; +} + +int32_t +rte_service_lcore_list(uint32_t array[], uint32_t n) +{ + uint32_t count = rte_service_lcore_count(); + if (count > n) + return -ENOMEM; + + if (!array) + return -EINVAL; + + uint32_t i; + uint32_t idx = 0; + for (i = 0; i < RTE_MAX_LCORE; i++) { + struct core_state *cs = &lcore_states[i]; + if (cs->is_service_core) { + array[idx] = i; + idx++; + } + } + + return count; +} + +int32_t +rte_service_start_with_defaults(void) +{ + /* create a default mapping from cores to services, then start the + * services to make them transparent to unaware applications. + */ + uint32_t i; + int ret; + uint32_t count = rte_service_get_count(); + + int32_t lcore_iter = 0; + uint32_t ids[RTE_MAX_LCORE]; + int32_t lcore_count = rte_service_lcore_list(ids, RTE_MAX_LCORE); + + if (lcore_count == 0) + return -ENOTSUP; + + for (i = 0; (int)i < lcore_count; i++) + rte_service_lcore_start(ids[i]); + + for (i = 0; i < count; i++) { + struct rte_service_spec *s = rte_service_get_by_id(i); + if (!s) + return -EINVAL; + + /* do 1:1 core mapping here, with each service getting + * assigned a single core by default. Adding multiple services + * should multiplex to a single core, or 1:1 if there are the + * same amount of services as service-cores + */ + ret = rte_service_enable_on_lcore(s, ids[lcore_iter]); + if (ret) + return -ENODEV; + + lcore_iter++; + if (lcore_iter >= lcore_count) + lcore_iter = 0; + + ret = rte_service_start(s); + if (ret) + return -ENOEXEC; + } + + return 0; +} + +static int32_t +service_update(struct rte_service_spec *service, uint32_t lcore, + uint32_t *set, uint32_t *enabled) +{ + uint32_t i; + int32_t sid = -1; + + for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) { + if ((struct rte_service_spec *)&rte_services[i] == service && + service_valid(i)) { + sid = i; + break; + } + } + + if (sid == -1 || lcore >= RTE_MAX_LCORE) + return -EINVAL; + + if (!lcore_states[lcore].is_service_core) + return -EINVAL; + + uint64_t sid_mask = UINT64_C(1) << sid; + if (set) { + if (*set) { + lcore_states[lcore].service_mask |= sid_mask; + rte_services[sid].num_mapped_cores++; + } else { + lcore_states[lcore].service_mask &= ~(sid_mask); + rte_services[sid].num_mapped_cores--; + } + } + + if (enabled) + *enabled = (lcore_states[lcore].service_mask & (sid_mask)); + + rte_smp_wmb(); + + return 0; +} + +int32_t rte_service_get_enabled_on_lcore(struct rte_service_spec *service, + uint32_t lcore) +{ + uint32_t enabled; + int ret = service_update(service, lcore, 0, &enabled); + if (ret == 0) + return enabled; + return -EINVAL; +} + +int32_t +rte_service_enable_on_lcore(struct rte_service_spec *service, uint32_t lcore) +{ + uint32_t on = 1; + return service_update(service, lcore, &on, 0); +} + +int32_t +rte_service_disable_on_lcore(struct rte_service_spec *service, uint32_t lcore) +{ + uint32_t off = 0; + return service_update(service, lcore, &off, 0); +} + +int32_t rte_service_lcore_reset_all(void) +{ + /* loop over cores, reset all to mask 0 */ + uint32_t i; + for (i = 0; i < RTE_MAX_LCORE; i++) { + lcore_states[i].service_mask = 0; + lcore_states[i].is_service_core = 0; + lcore_states[i].runstate = RUNSTATE_STOPPED; + } + for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) + rte_services[i].num_mapped_cores = 0; + + rte_smp_wmb(); + + return 0; +} + +static void +set_lcore_state(uint32_t lcore, int32_t state) +{ + /* mark core state in hugepage backed config */ + struct rte_config *cfg = rte_eal_get_configuration(); + cfg->lcore_role[lcore] = state; + + /* mark state in process local lcore_config */ + lcore_config[lcore].core_role = state; + + /* update per-lcore optimized state tracking */ + lcore_states[lcore].is_service_core = (state == ROLE_SERVICE); +} + +int32_t +rte_service_lcore_add(uint32_t lcore) +{ + if (lcore >= RTE_MAX_LCORE) + return -EINVAL; + if (lcore_states[lcore].is_service_core) + return -EALREADY; + + set_lcore_state(lcore, ROLE_SERVICE); + + /* ensure that after adding a core the mask and state are defaults */ + lcore_states[lcore].service_mask = 0; + lcore_states[lcore].runstate = RUNSTATE_STOPPED; + + rte_smp_wmb(); + return 0; +} + +int32_t +rte_service_lcore_del(uint32_t lcore) +{ + if (lcore >= RTE_MAX_LCORE) + return -EINVAL; + + struct core_state *cs = &lcore_states[lcore]; + if (!cs->is_service_core) + return -EINVAL; + + if (cs->runstate != RUNSTATE_STOPPED) + return -EBUSY; + + set_lcore_state(lcore, ROLE_RTE); + + rte_smp_wmb(); + return 0; +} + +int32_t +rte_service_lcore_start(uint32_t lcore) +{ + if (lcore >= RTE_MAX_LCORE) + return -EINVAL; + + struct core_state *cs = &lcore_states[lcore]; + if (!cs->is_service_core) + return -EINVAL; + + if (cs->runstate == RUNSTATE_RUNNING) + return -EALREADY; + + /* set core to run state first, and then launch otherwise it will + * return immediately as runstate keeps it in the service poll loop + */ + lcore_states[lcore].runstate = RUNSTATE_RUNNING; + + int ret = rte_eal_remote_launch(rte_service_runner_func, 0, lcore); + /* returns -EBUSY if the core is already launched, 0 on success */ + return ret; +} + +int32_t +rte_service_lcore_stop(uint32_t lcore) +{ + if (lcore >= RTE_MAX_LCORE) + return -EINVAL; + + if (lcore_states[lcore].runstate == RUNSTATE_STOPPED) + return -EALREADY; + + uint32_t i; + for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) { + int32_t enabled = + lcore_states[i].service_mask & (UINT64_C(1) << i); + int32_t service_running = rte_services[i].runstate != + RUNSTATE_STOPPED; + int32_t only_core = rte_services[i].num_mapped_cores == 1; + + /* if the core is mapped, and the service is running, and this + * is the only core that is mapped, the service would cease to + * run if this core stopped, so fail instead. + */ + if (enabled && service_running && only_core) + return -EBUSY; + } + + lcore_states[lcore].runstate = RUNSTATE_STOPPED; + + return 0; +} + +static void +rte_service_dump_one(FILE *f, struct rte_service_spec_impl *s, + uint64_t all_cycles, uint32_t reset) +{ + /* avoid divide by zero */ + if (all_cycles == 0) + all_cycles = 1; + + int calls = 1; + if (s->calls != 0) + calls = s->calls; + + fprintf(f, " %s: stats %d\tcalls %"PRIu64"\tcycles %" + PRIu64"\tavg: %"PRIu64"\n", + s->spec.name, service_stats_enabled(s), s->calls, + s->cycles_spent, s->cycles_spent / calls); + + if (reset) { + s->cycles_spent = 0; + s->calls = 0; + } +} + +static void +service_dump_calls_per_lcore(FILE *f, uint32_t lcore, uint32_t reset) +{ + uint32_t i; + struct core_state *cs = &lcore_states[lcore]; + + fprintf(f, "%02d\t", lcore); + for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) { + if (!service_valid(i)) + continue; + fprintf(f, "%"PRIu64"\t", cs->calls_per_service[i]); + if (reset) + cs->calls_per_service[i] = 0; + } + fprintf(f, "\n"); +} + +int32_t rte_service_dump(FILE *f, struct rte_service_spec *service) +{ + uint32_t i; + + uint64_t total_cycles = 0; + for (i = 0; i < rte_service_count; i++) { + if (!service_valid(i)) + continue; + total_cycles += rte_services[i].cycles_spent; + } + + if (service) { + struct rte_service_spec_impl *s = + (struct rte_service_spec_impl *)service; + fprintf(f, "Service %s Summary\n", s->spec.name); + uint32_t reset = 0; + rte_service_dump_one(f, s, total_cycles, reset); + return 0; + } + + fprintf(f, "Services Summary\n"); + for (i = 0; i < rte_service_count; i++) { + uint32_t reset = 1; + rte_service_dump_one(f, &rte_services[i], total_cycles, reset); + } + + fprintf(f, "Service Cores Summary\n"); + for (i = 0; i < RTE_MAX_LCORE; i++) { + if (lcore_config[i].core_role != ROLE_SERVICE) + continue; + + uint32_t reset = 0; + service_dump_calls_per_lcore(f, i, reset); + } + + return 0; +} diff --git a/lib/librte_eal/linuxapp/eal/Makefile b/lib/librte_eal/linuxapp/eal/Makefile index 640afd08..90bca4d6 100644 --- a/lib/librte_eal/linuxapp/eal/Makefile +++ b/lib/librte_eal/linuxapp/eal/Makefile @@ -37,7 +37,7 @@ ARCH_DIR ?= $(RTE_ARCH) EXPORT_MAP := rte_eal_version.map VPATH += $(RTE_SDK)/lib/librte_eal/common/arch/$(ARCH_DIR) -LIBABIVER := 4 +LIBABIVER := 5 VPATH += $(RTE_SDK)/lib/librte_eal/common @@ -50,6 +50,9 @@ LDLIBS += -ldl LDLIBS += -lpthread LDLIBS += -lgcc_s LDLIBS += -lrt +ifeq ($(CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES),y) +LDLIBS += -lnuma +endif # specific to linuxapp exec-env SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) := eal.c @@ -96,6 +99,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_malloc.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += malloc_elem.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += malloc_heap.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_keepalive.c +SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_service.c # from arch dir SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_cpuflags.c diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c index 7c78f2dc..48f12f44 100644 --- a/lib/librte_eal/linuxapp/eal/eal.c +++ b/lib/librte_eal/linuxapp/eal/eal.c @@ -46,7 +46,6 @@ #include <stddef.h> #include <errno.h> #include <limits.h> -#include <errno.h> #include <sys/mman.h> #include <sys/queue.h> #include <sys/stat.h> @@ -64,6 +63,7 @@ #include <rte_errno.h> #include <rte_per_lcore.h> #include <rte_lcore.h> +#include <rte_service_component.h> #include <rte_log.h> #include <rte_random.h> #include <rte_cycles.h> @@ -74,7 +74,6 @@ #include <rte_pci.h> #include <rte_dev.h> #include <rte_devargs.h> -#include <rte_common.h> #include <rte_version.h> #include <rte_atomic.h> #include <malloc_heap.h> @@ -890,6 +889,11 @@ rte_eal_init(int argc, char **argv) return -1; } + if (eal_option_device_parse()) { + rte_errno = ENODEV; + return -1; + } + if (rte_bus_scan()) { rte_eal_init_alert("Cannot scan the buses for devices\n"); rte_errno = ENODEV; @@ -932,6 +936,14 @@ rte_eal_init(int argc, char **argv) rte_eal_mp_remote_launch(sync_func, NULL, SKIP_MASTER); rte_eal_mp_wait_lcore(); + /* initialize services so vdevs register service during bus_probe. */ + ret = rte_service_init(); + if (ret) { + rte_eal_init_alert("rte_service_init() failed\n"); + rte_errno = ENOEXEC; + return -1; + } + /* Probe all the buses and devices/drivers on them */ if (rte_bus_probe()) { rte_eal_init_alert("Cannot probe devices\n"); @@ -939,6 +951,15 @@ rte_eal_init(int argc, char **argv) return -1; } + /* initialize default service/lcore mappings and start running. Ignore + * -ENOTSUP, as it indicates no service coremask passed to EAL. + */ + ret = rte_service_start_with_defaults(); + if (ret < 0 && ret != -ENOTSUP) { + rte_errno = ENOEXEC; + return -1; + } + rte_eal_mcfg_complete(); return fctret; diff --git a/lib/librte_eal/linuxapp/eal/eal_interrupts.c b/lib/librte_eal/linuxapp/eal/eal_interrupts.c index 2e3bd12a..3e9ac41e 100644 --- a/lib/librte_eal/linuxapp/eal/eal_interrupts.c +++ b/lib/librte_eal/linuxapp/eal/eal_interrupts.c @@ -64,6 +64,7 @@ #include <rte_malloc.h> #include <rte_errno.h> #include <rte_spinlock.h> +#include <rte_pause.h> #include "eal_private.h" #include "eal_vfio.h" diff --git a/lib/librte_eal/linuxapp/eal/eal_memory.c b/lib/librte_eal/linuxapp/eal/eal_memory.c index ebe06833..52791282 100644 --- a/lib/librte_eal/linuxapp/eal/eal_memory.c +++ b/lib/librte_eal/linuxapp/eal/eal_memory.c @@ -2,6 +2,7 @@ * BSD LICENSE * * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * Copyright(c) 2013 6WIND. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -30,36 +31,6 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -/* BSD LICENSE - * - * Copyright(c) 2013 6WIND. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ #define _FILE_OFFSET_BITS 64 #include <errno.h> @@ -70,7 +41,6 @@ #include <stdint.h> #include <inttypes.h> #include <string.h> -#include <stdarg.h> #include <sys/mman.h> #include <sys/types.h> #include <sys/stat.h> @@ -78,11 +48,14 @@ #include <sys/file.h> #include <unistd.h> #include <limits.h> -#include <errno.h> #include <sys/ioctl.h> #include <sys/time.h> #include <signal.h> #include <setjmp.h> +#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES +#include <numa.h> +#include <numaif.h> +#endif #include <rte_log.h> #include <rte_memory.h> @@ -137,6 +110,13 @@ test_phys_addrs_available(void) if (rte_xen_dom0_supported()) return; + if (!rte_eal_has_hugepages()) { + RTE_LOG(ERR, EAL, + "Started without hugepages support, physical addresses not available\n"); + phys_addrs_available = false; + return; + } + physaddr = rte_mem_virt2phy(&tmp); if (physaddr == RTE_BAD_PHYS_ADDR) { RTE_LOG(ERR, EAL, @@ -147,16 +127,6 @@ test_phys_addrs_available(void) } } -/* Lock page in physical memory and prevent from swapping. */ -int -rte_mem_lock_page(const void *virt) -{ - unsigned long virtual = (unsigned long)virt; - int page_size = getpagesize(); - unsigned long aligned = (virtual & ~ (page_size - 1)); - return mlock((void*)aligned, page_size); -} - /* * Get physical address of any mapped virtual address in the current process. */ @@ -387,6 +357,14 @@ static int huge_wrap_sigsetjmp(void) return sigsetjmp(huge_jmpenv, 1); } +#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES +/* Callback for numa library. */ +void numa_error(char *where) +{ + RTE_LOG(ERR, EAL, "%s failed: %s\n", where, strerror(errno)); +} +#endif + /* * Mmap all hugepages of hugepage table: it first open a file in * hugetlbfs, then mmap() hugepage_sz data in it. If orig is set, the @@ -395,18 +373,78 @@ static int huge_wrap_sigsetjmp(void) * map continguous physical blocks in contiguous virtual blocks. */ static unsigned -map_all_hugepages(struct hugepage_file *hugepg_tbl, - struct hugepage_info *hpi, int orig) +map_all_hugepages(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi, + uint64_t *essential_memory __rte_unused, int orig) { int fd; unsigned i; void *virtaddr; void *vma_addr = NULL; size_t vma_len = 0; +#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES + int node_id = -1; + int essential_prev = 0; + int oldpolicy; + struct bitmask *oldmask = numa_allocate_nodemask(); + bool have_numa = true; + unsigned long maxnode = 0; + + /* Check if kernel supports NUMA. */ + if (numa_available() != 0) { + RTE_LOG(DEBUG, EAL, "NUMA is not supported.\n"); + have_numa = false; + } + + if (orig && have_numa) { + RTE_LOG(DEBUG, EAL, "Trying to obtain current memory policy.\n"); + if (get_mempolicy(&oldpolicy, oldmask->maskp, + oldmask->size + 1, 0, 0) < 0) { + RTE_LOG(ERR, EAL, + "Failed to get current mempolicy: %s. " + "Assuming MPOL_DEFAULT.\n", strerror(errno)); + oldpolicy = MPOL_DEFAULT; + } + for (i = 0; i < RTE_MAX_NUMA_NODES; i++) + if (internal_config.socket_mem[i]) + maxnode = i + 1; + } +#endif for (i = 0; i < hpi->num_pages[0]; i++) { uint64_t hugepage_sz = hpi->hugepage_sz; +#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES + if (maxnode) { + unsigned int j; + + for (j = 0; j < maxnode; j++) + if (essential_memory[j]) + break; + + if (j == maxnode) { + node_id = (node_id + 1) % maxnode; + while (!internal_config.socket_mem[node_id]) { + node_id++; + node_id %= maxnode; + } + essential_prev = 0; + } else { + node_id = j; + essential_prev = essential_memory[j]; + + if (essential_memory[j] < hugepage_sz) + essential_memory[j] = 0; + else + essential_memory[j] -= hugepage_sz; + } + + RTE_LOG(DEBUG, EAL, + "Setting policy MPOL_PREFERRED for socket %d\n", + node_id); + numa_set_preferred(node_id); + } +#endif + if (orig) { hugepg_tbl[i].file_id = i; hugepg_tbl[i].size = hugepage_sz; @@ -461,7 +499,7 @@ map_all_hugepages(struct hugepage_file *hugepg_tbl, if (fd < 0) { RTE_LOG(DEBUG, EAL, "%s(): open failed: %s\n", __func__, strerror(errno)); - return i; + goto out; } /* map the segment, and populate page tables, @@ -472,7 +510,7 @@ map_all_hugepages(struct hugepage_file *hugepg_tbl, RTE_LOG(DEBUG, EAL, "%s(): mmap failed: %s\n", __func__, strerror(errno)); close(fd); - return i; + goto out; } if (orig) { @@ -497,7 +535,12 @@ map_all_hugepages(struct hugepage_file *hugepg_tbl, munmap(virtaddr, hugepage_sz); close(fd); unlink(hugepg_tbl[i].filepath); - return i; +#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES + if (maxnode) + essential_memory[node_id] = + essential_prev; +#endif + goto out; } *(int *)virtaddr = 0; } @@ -508,7 +551,7 @@ map_all_hugepages(struct hugepage_file *hugepg_tbl, RTE_LOG(DEBUG, EAL, "%s(): Locking file failed:%s \n", __func__, strerror(errno)); close(fd); - return i; + goto out; } close(fd); @@ -517,6 +560,22 @@ map_all_hugepages(struct hugepage_file *hugepg_tbl, vma_len -= hugepage_sz; } +out: +#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES + if (maxnode) { + RTE_LOG(DEBUG, EAL, + "Restoring previous memory policy: %d\n", oldpolicy); + if (oldpolicy == MPOL_DEFAULT) { + numa_set_localalloc(); + } else if (set_mempolicy(oldpolicy, oldmask->maskp, + oldmask->size + 1) < 0) { + RTE_LOG(ERR, EAL, "Failed to restore mempolicy: %s\n", + strerror(errno)); + numa_set_localalloc(); + } + } + numa_free_cpumask(oldmask); +#endif return i; } @@ -551,8 +610,8 @@ find_numasocket(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi) f = fopen("/proc/self/numa_maps", "r"); if (f == NULL) { - RTE_LOG(NOTICE, EAL, "cannot open /proc/self/numa_maps," - " consider that all memory is in socket_id 0\n"); + RTE_LOG(NOTICE, EAL, "NUMA support not available" + " consider that all memory is in socket_id 0\n"); return 0; } @@ -601,6 +660,11 @@ find_numasocket(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi) if (hugepg_tbl[i].orig_va == va) { hugepg_tbl[i].socket_id = socket_id; hp_count++; +#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES + RTE_LOG(DEBUG, EAL, + "Hugepage %s is on socket %d\n", + hugepg_tbl[i].filepath, socket_id); +#endif } } } @@ -995,7 +1059,7 @@ rte_eal_hugepage_init(void) strerror(errno)); return -1; } - mcfg->memseg[0].phys_addr = (phys_addr_t)(uintptr_t)addr; + mcfg->memseg[0].phys_addr = RTE_BAD_PHYS_ADDR; mcfg->memseg[0].addr = addr; mcfg->memseg[0].hugepage_sz = RTE_PGSIZE_4K; mcfg->memseg[0].len = internal_config.memory; @@ -1039,6 +1103,11 @@ rte_eal_hugepage_init(void) huge_register_sigbus(); + /* make a copy of socket_mem, needed for balanced allocation. */ + for (i = 0; i < RTE_MAX_NUMA_NODES; i++) + memory[i] = internal_config.socket_mem[i]; + + /* map all hugepages and sort them */ for (i = 0; i < (int)internal_config.num_hugepage_sizes; i ++){ unsigned pages_old, pages_new; @@ -1056,7 +1125,8 @@ rte_eal_hugepage_init(void) /* map all hugepages available */ pages_old = hpi->num_pages[0]; - pages_new = map_all_hugepages(&tmp_hp[hp_offset], hpi, 1); + pages_new = map_all_hugepages(&tmp_hp[hp_offset], hpi, + memory, 1); if (pages_new < pages_old) { RTE_LOG(DEBUG, EAL, "%d not %d hugepages of size %u MB allocated\n", @@ -1099,7 +1169,7 @@ rte_eal_hugepage_init(void) sizeof(struct hugepage_file), cmp_physaddr); /* remap all hugepages */ - if (map_all_hugepages(&tmp_hp[hp_offset], hpi, 0) != + if (map_all_hugepages(&tmp_hp[hp_offset], hpi, NULL, 0) != hpi->num_pages[0]) { RTE_LOG(ERR, EAL, "Failed to remap %u MB pages\n", (unsigned)(hpi->hugepage_sz / 0x100000)); diff --git a/lib/librte_eal/linuxapp/eal/eal_pci.c b/lib/librte_eal/linuxapp/eal/eal_pci.c index 595622b2..8951ce74 100644 --- a/lib/librte_eal/linuxapp/eal/eal_pci.c +++ b/lib/librte_eal/linuxapp/eal/eal_pci.c @@ -310,22 +310,20 @@ pci_scan_one(const char *dirname, const struct rte_pci_addr *addr) dev->max_vfs = (uint16_t)tmp; } - /* get numa node */ + /* get numa node, default to 0 if not present */ snprintf(filename, sizeof(filename), "%s/numa_node", dirname); - if (access(filename, R_OK) != 0) { - /* if no NUMA support, set default to 0 */ - dev->device.numa_node = 0; + + if (access(filename, F_OK) != -1) { + if (eal_parse_sysfs_value(filename, &tmp) == 0) + dev->device.numa_node = tmp; + else + dev->device.numa_node = -1; } else { - if (eal_parse_sysfs_value(filename, &tmp) < 0) { - free(dev); - return -1; - } - dev->device.numa_node = tmp; + dev->device.numa_node = 0; } - rte_pci_device_name(addr, dev->name, sizeof(dev->name)); - dev->device.name = dev->name; + pci_name_set(dev); /* parse resources */ snprintf(filename, sizeof(filename), "%s/resource", dirname); @@ -373,6 +371,7 @@ pci_scan_one(const char *dirname, const struct rte_pci_addr *addr) } else { /* already registered */ dev2->kdrv = dev->kdrv; dev2->max_vfs = dev->max_vfs; + pci_name_set(dev2); memmove(dev2->mem_resource, dev->mem_resource, sizeof(dev->mem_resource)); free(dev); @@ -430,10 +429,10 @@ parse_pci_addr_format(const char *buf, int bufsize, struct rte_pci_addr *addr) /* now convert to int values */ errno = 0; - addr->domain = (uint16_t)strtoul(splitaddr.domain, NULL, 16); - addr->bus = (uint8_t)strtoul(splitaddr.bus, NULL, 16); - addr->devid = (uint8_t)strtoul(splitaddr.devid, NULL, 16); - addr->function = (uint8_t)strtoul(splitaddr.function, NULL, 10); + addr->domain = strtoul(splitaddr.domain, NULL, 16); + addr->bus = strtoul(splitaddr.bus, NULL, 16); + addr->devid = strtoul(splitaddr.devid, NULL, 16); + addr->function = strtoul(splitaddr.function, NULL, 10); if (errno != 0) goto error; diff --git a/lib/librte_eal/linuxapp/eal/eal_pci_vfio.c b/lib/librte_eal/linuxapp/eal/eal_pci_vfio.c index 2be13195..aa9d96ed 100644 --- a/lib/librte_eal/linuxapp/eal/eal_pci_vfio.c +++ b/lib/librte_eal/linuxapp/eal/eal_pci_vfio.c @@ -214,7 +214,7 @@ pci_vfio_setup_interrupts(struct rte_pci_device *dev, int vfio_dev_fd) intr_idx = VFIO_PCI_NUM_IRQS; /* get interrupt type from internal config (MSI-X by default, can be - * overriden from the command line + * overridden from the command line */ switch (internal_config.vfio_intr_mode) { case RTE_INTR_MODE_MSIX: diff --git a/lib/librte_eal/linuxapp/eal/eal_thread.c b/lib/librte_eal/linuxapp/eal/eal_thread.c index 9f88530e..6481eeea 100644 --- a/lib/librte_eal/linuxapp/eal/eal_thread.c +++ b/lib/librte_eal/linuxapp/eal/eal_thread.c @@ -49,7 +49,6 @@ #include <rte_memzone.h> #include <rte_per_lcore.h> #include <rte_eal.h> -#include <rte_per_lcore.h> #include <rte_lcore.h> #include "eal_private.h" @@ -184,7 +183,14 @@ eal_thread_loop(__attribute__((unused)) void *arg) ret = lcore_config[lcore_id].f(fct_arg); lcore_config[lcore_id].ret = ret; rte_wmb(); - lcore_config[lcore_id].state = FINISHED; + + /* when a service core returns, it should go directly to WAIT + * state, because the application will not lcore_wait() for it. + */ + if (lcore_config[lcore_id].core_role == ROLE_SERVICE) + lcore_config[lcore_id].state = WAIT; + else + lcore_config[lcore_id].state = FINISHED; } /* never reached */ diff --git a/lib/librte_eal/linuxapp/eal/eal_xen_memory.c b/lib/librte_eal/linuxapp/eal/eal_xen_memory.c index bddbdb07..19db1cb5 100644 --- a/lib/librte_eal/linuxapp/eal/eal_xen_memory.c +++ b/lib/librte_eal/linuxapp/eal/eal_xen_memory.c @@ -38,7 +38,6 @@ #include <stdint.h> #include <inttypes.h> #include <string.h> -#include <stdarg.h> #include <sys/mman.h> #include <sys/types.h> #include <sys/stat.h> @@ -46,7 +45,6 @@ #include <sys/file.h> #include <unistd.h> #include <limits.h> -#include <errno.h> #include <sys/ioctl.h> #include <sys/time.h> diff --git a/lib/librte_eal/linuxapp/eal/rte_eal_version.map b/lib/librte_eal/linuxapp/eal/rte_eal_version.map index 670bab3a..3a8f1540 100644 --- a/lib/librte_eal/linuxapp/eal/rte_eal_version.map +++ b/lib/librte_eal/linuxapp/eal/rte_eal_version.map @@ -198,3 +198,47 @@ DPDK_17.05 { vfio_get_group_no; } DPDK_17.02; + +DPDK_17.08 { + global: + + rte_bus_find; + rte_bus_find_by_device; + rte_bus_find_by_name; + rte_log_get_level; + +} DPDK_17.05; + +EXPERIMENTAL { + global: + + rte_eal_devargs_insert; + rte_eal_devargs_parse; + rte_eal_devargs_remove; + rte_eal_hotplug_add; + rte_eal_hotplug_remove; + rte_service_disable_on_lcore; + rte_service_dump; + rte_service_enable_on_lcore; + rte_service_get_by_id; + rte_service_get_by_name; + rte_service_get_count; + rte_service_get_enabled_on_lcore; + rte_service_is_running; + rte_service_lcore_add; + rte_service_lcore_count; + rte_service_lcore_del; + rte_service_lcore_list; + rte_service_lcore_reset_all; + rte_service_lcore_start; + rte_service_lcore_stop; + rte_service_probe_capability; + rte_service_register; + rte_service_reset; + rte_service_set_stats_enable; + rte_service_start; + rte_service_start_with_defaults; + rte_service_stop; + rte_service_unregister; + +} DPDK_17.08; diff --git a/lib/librte_eal/linuxapp/igb_uio/igb_uio.c b/lib/librte_eal/linuxapp/igb_uio/igb_uio.c index b9d427c5..07a19a31 100644 --- a/lib/librte_eal/linuxapp/igb_uio/igb_uio.c +++ b/lib/librte_eal/linuxapp/igb_uio/igb_uio.c @@ -170,6 +170,37 @@ igbuio_pci_irqhandler(int irq, struct uio_info *info) return IRQ_HANDLED; } +/** + * This gets called while opening uio device file. + */ +static int +igbuio_pci_open(struct uio_info *info, struct inode *inode) +{ + struct rte_uio_pci_dev *udev = info->priv; + struct pci_dev *dev = udev->pdev; + + pci_reset_function(dev); + + /* set bus master, which was cleared by the reset function */ + pci_set_master(dev); + + return 0; +} + +static int +igbuio_pci_release(struct uio_info *info, struct inode *inode) +{ + struct rte_uio_pci_dev *udev = info->priv; + struct pci_dev *dev = udev->pdev; + + /* stop the device from further DMA */ + pci_clear_master(dev); + + pci_reset_function(dev); + + return 0; +} + #ifdef CONFIG_XEN_DOM0 static int igbuio_dom0_mmap_phys(struct uio_info *info, struct vm_area_struct *vma) @@ -372,6 +403,8 @@ igbuio_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) udev->info.version = "0.1"; udev->info.handler = igbuio_pci_irqhandler; udev->info.irqcontrol = igbuio_pci_irqcontrol; + udev->info.open = igbuio_pci_open; + udev->info.release = igbuio_pci_release; #ifdef CONFIG_XEN_DOM0 /* check if the driver run on Xen Dom0 */ if (xen_initial_domain()) diff --git a/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_82575.c b/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_82575.c index d558af20..1c30d12b 100644 --- a/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_82575.c +++ b/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_82575.c @@ -1357,7 +1357,7 @@ static s32 e1000_get_pcs_speed_and_duplex_82575(struct e1000_hw *hw, * @hw: pointer to the HW structure * * In the case of serdes shut down sfp and PCS on driver unload - * when management pass thru is not enabled. + * when management pass through is not enabled. **/ void e1000_shutdown_serdes_link_82575(struct e1000_hw *hw) { diff --git a/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_main.c b/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_main.c index 5f1f3a6b..99338c5c 100644 --- a/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_main.c +++ b/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_main.c @@ -1133,7 +1133,7 @@ static int igb_alloc_q_vector(struct igb_adapter *adapter, /* initialize pointer to rings */ ring = q_vector->ring; - /* intialize ITR */ + /* initialize ITR */ if (rxr_count) { /* rx or rx/tx vector */ if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3) diff --git a/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat.h b/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat.h index 4c52da3c..e0a03542 100644 --- a/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat.h +++ b/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat.h @@ -1165,7 +1165,7 @@ static inline u32 _kc_netif_msg_init(int debug_value, int default_msg_enable_bit #define pci_register_driver pci_module_init /* - * Most of the dma compat code is copied/modifed from the 2.4.37 + * Most of the dma compat code is copied/modified from the 2.4.37 * /include/linux/libata-compat.h header file */ /* These definitions mirror those in pci.h, so they can be used diff --git a/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_api.c b/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_api.c index f00fe796..4808d06e 100644 --- a/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_api.c +++ b/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_api.c @@ -718,7 +718,7 @@ s32 ixgbe_update_eeprom_checksum(struct ixgbe_hw *hw) * @vmdq: VMDq pool to assign * * Puts an ethernet address into a receive address register, or - * finds the rar that it is aleady in; adds to the pool list + * finds the rar that it is already in; adds to the pool list **/ s32 ixgbe_insert_mac_addr(struct ixgbe_hw *hw, u8 *addr, u32 vmdq) { diff --git a/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_common.c b/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_common.c index 88b33fa0..2c861de5 100644 --- a/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_common.c +++ b/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_common.c @@ -3007,7 +3007,7 @@ u16 ixgbe_get_pcie_msix_count_generic(struct ixgbe_hw *hw) * @vmdq: VMDq pool to assign * * Puts an ethernet address into a receive address register, or - * finds the rar that it is aleady in; adds to the pool list + * finds the rar that it is already in; adds to the pool list **/ s32 ixgbe_insert_mac_addr_generic(struct ixgbe_hw *hw, u8 *addr, u32 vmdq) { diff --git a/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/kcompat.h b/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/kcompat.h index 4c7a6408..f62a7b56 100644 --- a/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/kcompat.h +++ b/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/kcompat.h @@ -1108,7 +1108,7 @@ static inline u32 _kc_netif_msg_init(int debug_value, int default_msg_enable_bit #define pci_register_driver pci_module_init /* - * Most of the dma compat code is copied/modifed from the 2.4.37 + * Most of the dma compat code is copied/modified from the 2.4.37 * /include/linux/libata-compat.h header file */ /* These definitions mirror those in pci.h, so they can be used |