diff options
Diffstat (limited to 'lib/librte_eal/common')
25 files changed, 330 insertions, 146 deletions
diff --git a/lib/librte_eal/common/eal_common_memory.c b/lib/librte_eal/common/eal_common_memory.c index 999ba24b..e3ef3714 100644 --- a/lib/librte_eal/common/eal_common_memory.c +++ b/lib/librte_eal/common/eal_common_memory.c @@ -55,6 +55,7 @@ static uint64_t system_page_sz; static uint64_t baseaddr = 0x100000000; #endif +#define MAX_MMAP_WITH_DEFINED_ADDR_TRIES 5 void * eal_get_virtual_area(void *requested_addr, size_t *size, size_t page_sz, int flags, int mmap_flags) @@ -62,6 +63,7 @@ eal_get_virtual_area(void *requested_addr, size_t *size, bool addr_is_hint, allow_shrink, unmap, no_align; uint64_t map_sz; void *mapped_addr, *aligned_addr; + uint8_t try = 0; if (system_page_sz == 0) system_page_sz = sysconf(_SC_PAGESIZE); @@ -117,11 +119,14 @@ eal_get_virtual_area(void *requested_addr, size_t *size, if (mapped_addr != MAP_FAILED && addr_is_hint && mapped_addr != requested_addr) { - /* hint was not used. Try with another offset */ - munmap(mapped_addr, map_sz); - mapped_addr = MAP_FAILED; + try++; next_baseaddr = RTE_PTR_ADD(next_baseaddr, page_sz); - requested_addr = next_baseaddr; + if (try <= MAX_MMAP_WITH_DEFINED_ADDR_TRIES) { + /* hint was not used. Try with another offset */ + munmap(mapped_addr, map_sz); + mapped_addr = MAP_FAILED; + requested_addr = next_baseaddr; + } } } while ((allow_shrink || addr_is_hint) && mapped_addr == MAP_FAILED && *size > 0); diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c index f6dfbc73..d4ab5e23 100644 --- a/lib/librte_eal/common/eal_common_options.c +++ b/lib/librte_eal/common/eal_common_options.c @@ -216,6 +216,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg) internal_cfg->create_uio_dev = 0; internal_cfg->iova_mode = RTE_IOVA_DC; internal_cfg->user_mbuf_pool_ops_name = NULL; + CPU_ZERO(&internal_cfg->ctrl_cpuset); internal_cfg->init_complete = 0; } @@ -417,21 +418,44 @@ eal_service_cores_parsed(void) } static int -eal_parse_coremask(const char *coremask) +update_lcore_config(int *cores) { struct rte_config *cfg = rte_eal_get_configuration(); - int i, j, idx = 0; + unsigned int count = 0; + unsigned int i; + int ret = 0; + + for (i = 0; i < RTE_MAX_LCORE; i++) { + if (cores[i] != -1) { + if (!lcore_config[i].detected) { + RTE_LOG(ERR, EAL, "lcore %u unavailable\n", i); + ret = -1; + continue; + } + cfg->lcore_role[i] = ROLE_RTE; + count++; + } else { + cfg->lcore_role[i] = ROLE_OFF; + } + lcore_config[i].core_index = cores[i]; + } + if (!ret) + cfg->lcore_count = count; + return ret; +} + +static int +eal_parse_coremask(const char *coremask, int *cores) +{ unsigned count = 0; - char c; + int i, j, idx; int val; + char c; - if (eal_service_cores_parsed()) - RTE_LOG(WARNING, EAL, - "Service cores parsed before dataplane cores. " - "Please ensure -c is before -s or -S\n"); + for (idx = 0; idx < RTE_MAX_LCORE; idx++) + cores[idx] = -1; + idx = 0; - if (coremask == NULL) - return -1; /* Remove all blank characters ahead and after . * Remove 0x/0X if exists. */ @@ -456,32 +480,16 @@ eal_parse_coremask(const char *coremask) for (j = 0; j < BITS_PER_HEX && idx < RTE_MAX_LCORE; j++, idx++) { if ((1 << j) & val) { - if (!lcore_config[idx].detected) { - RTE_LOG(ERR, EAL, "lcore %u " - "unavailable\n", idx); - return -1; - } - - cfg->lcore_role[idx] = ROLE_RTE; - lcore_config[idx].core_index = count; + cores[idx] = count; count++; - } else { - cfg->lcore_role[idx] = ROLE_OFF; - lcore_config[idx].core_index = -1; } } } for (; i >= 0; i--) if (coremask[i] != '0') return -1; - for (; idx < RTE_MAX_LCORE; idx++) { - cfg->lcore_role[idx] = ROLE_OFF; - lcore_config[idx].core_index = -1; - } if (count == 0) return -1; - /* Update the count of enabled logical cores of the EAL configuration */ - cfg->lcore_count = count; return 0; } @@ -562,34 +570,19 @@ eal_parse_service_corelist(const char *corelist) } static int -eal_parse_corelist(const char *corelist) +eal_parse_corelist(const char *corelist, int *cores) { - struct rte_config *cfg = rte_eal_get_configuration(); - int i, idx = 0; unsigned count = 0; char *end = NULL; int min, max; + int idx; - if (eal_service_cores_parsed()) - RTE_LOG(WARNING, EAL, - "Service cores parsed before dataplane cores. " - "Please ensure -l is before -s or -S\n"); - - if (corelist == NULL) - return -1; + for (idx = 0; idx < RTE_MAX_LCORE; idx++) + cores[idx] = -1; - /* Remove all blank characters ahead and after */ + /* Remove all blank characters ahead */ while (isblank(*corelist)) corelist++; - i = strlen(corelist); - while ((i > 0) && isblank(corelist[i - 1])) - i--; - - /* Reset config */ - for (idx = 0; idx < RTE_MAX_LCORE; idx++) { - cfg->lcore_role[idx] = ROLE_OFF; - lcore_config[idx].core_index = -1; - } /* Get list of cores */ min = RTE_MAX_LCORE; @@ -600,10 +593,10 @@ eal_parse_corelist(const char *corelist) return -1; errno = 0; idx = strtol(corelist, &end, 10); - if (idx < 0 || idx >= (int)cfg->lcore_count) - return -1; if (errno || end == NULL) return -1; + if (idx < 0 || idx >= RTE_MAX_LCORE) + return -1; while (isblank(*end)) end++; if (*end == '-') { @@ -613,9 +606,8 @@ eal_parse_corelist(const char *corelist) if (min == RTE_MAX_LCORE) min = idx; for (idx = min; idx <= max; idx++) { - if (cfg->lcore_role[idx] != ROLE_RTE) { - cfg->lcore_role[idx] = ROLE_RTE; - lcore_config[idx].core_index = count; + if (cores[idx] == -1) { + cores[idx] = count; count++; } } @@ -627,10 +619,6 @@ eal_parse_corelist(const char *corelist) if (count == 0) return -1; - - /* Update the count of enabled logical cores of the EAL configuration */ - cfg->lcore_count = count; - return 0; } @@ -1106,13 +1094,81 @@ eal_parse_iova_mode(const char *name) return 0; } +/* caller is responsible for freeing the returned string */ +static char * +available_cores(void) +{ + char *str = NULL; + int previous; + int sequence; + char *tmp; + int idx; + + /* find the first available cpu */ + for (idx = 0; idx < RTE_MAX_LCORE; idx++) { + if (!lcore_config[idx].detected) + continue; + break; + } + if (idx >= RTE_MAX_LCORE) + return NULL; + + /* first sequence */ + if (asprintf(&str, "%d", idx) < 0) + return NULL; + previous = idx; + sequence = 0; + + for (idx++ ; idx < RTE_MAX_LCORE; idx++) { + if (!lcore_config[idx].detected) + continue; + + if (idx == previous + 1) { + previous = idx; + sequence = 1; + continue; + } + + /* finish current sequence */ + if (sequence) { + if (asprintf(&tmp, "%s-%d", str, previous) < 0) { + free(str); + return NULL; + } + free(str); + str = tmp; + } + + /* new sequence */ + if (asprintf(&tmp, "%s,%d", str, idx) < 0) { + free(str); + return NULL; + } + free(str); + str = tmp; + previous = idx; + sequence = 0; + } + + /* finish last sequence */ + if (sequence) { + if (asprintf(&tmp, "%s-%d", str, previous) < 0) { + free(str); + return NULL; + } + free(str); + str = tmp; + } + + return str; +} + int eal_parse_common_option(int opt, const char *optarg, struct internal_config *conf) { static int b_used; static int w_used; - struct rte_config *cfg = rte_eal_get_configuration(); switch (opt) { /* blacklist */ @@ -1136,9 +1192,23 @@ eal_parse_common_option(int opt, const char *optarg, w_used = 1; break; /* coremask */ - case 'c': - if (eal_parse_coremask(optarg) < 0) { - RTE_LOG(ERR, EAL, "invalid coremask\n"); + case 'c': { + int lcore_indexes[RTE_MAX_LCORE]; + + if (eal_service_cores_parsed()) + RTE_LOG(WARNING, EAL, + "Service cores parsed before dataplane cores. Please ensure -c is before -s or -S\n"); + if (eal_parse_coremask(optarg, lcore_indexes) < 0) { + RTE_LOG(ERR, EAL, "invalid coremask syntax\n"); + return -1; + } + if (update_lcore_config(lcore_indexes) < 0) { + char *available = available_cores(); + + RTE_LOG(ERR, EAL, + "invalid coremask, please check specified cores are part of %s\n", + available); + free(available); return -1; } @@ -1152,12 +1222,26 @@ eal_parse_common_option(int opt, const char *optarg, core_parsed = LCORE_OPT_MSK; break; + } /* corelist */ - case 'l': - if (eal_parse_corelist(optarg) < 0) { + case 'l': { + int lcore_indexes[RTE_MAX_LCORE]; + + if (eal_service_cores_parsed()) + RTE_LOG(WARNING, EAL, + "Service cores parsed before dataplane cores. Please ensure -l is before -s or -S\n"); + + if (eal_parse_corelist(optarg, lcore_indexes) < 0) { + RTE_LOG(ERR, EAL, "invalid core list syntax\n"); + return -1; + } + if (update_lcore_config(lcore_indexes) < 0) { + char *available = available_cores(); + RTE_LOG(ERR, EAL, - "invalid core list, please check core numbers are in [0, %u] range\n", - cfg->lcore_count-1); + "invalid core list, please check specified cores are part of %s\n", + available); + free(available); return -1; } @@ -1171,6 +1255,7 @@ eal_parse_common_option(int opt, const char *optarg, core_parsed = LCORE_OPT_LST; break; + } /* service coremask */ case 's': if (eal_parse_service_coremask(optarg) < 0) { @@ -1342,10 +1427,9 @@ eal_auto_detect_cores(struct rte_config *cfg) unsigned int lcore_id; unsigned int removed = 0; rte_cpuset_t affinity_set; - pthread_t tid = pthread_self(); - if (pthread_getaffinity_np(tid, sizeof(rte_cpuset_t), - &affinity_set) < 0) + if (pthread_getaffinity_np(pthread_self(), sizeof(rte_cpuset_t), + &affinity_set)) CPU_ZERO(&affinity_set); for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { @@ -1359,6 +1443,31 @@ eal_auto_detect_cores(struct rte_config *cfg) cfg->lcore_count -= removed; } +static void +compute_ctrl_threads_cpuset(struct internal_config *internal_cfg) +{ + rte_cpuset_t *cpuset = &internal_cfg->ctrl_cpuset; + rte_cpuset_t default_set; + unsigned int lcore_id; + + for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { + if (eal_cpu_detected(lcore_id) && + rte_lcore_has_role(lcore_id, ROLE_OFF)) { + CPU_SET(lcore_id, cpuset); + } + } + + if (pthread_getaffinity_np(pthread_self(), sizeof(rte_cpuset_t), + &default_set)) + CPU_ZERO(&default_set); + + RTE_CPU_AND(cpuset, cpuset, &default_set); + + /* if no detected CPU is off, use master core */ + if (!CPU_COUNT(cpuset)) + CPU_SET(rte_get_master_lcore(), cpuset); +} + int eal_cleanup_config(struct internal_config *internal_cfg) { @@ -1392,6 +1501,8 @@ eal_adjust_config(struct internal_config *internal_cfg) lcore_config[cfg->master_lcore].core_role = ROLE_RTE; } + compute_ctrl_threads_cpuset(internal_cfg); + /* if no memory amounts were requested, this will result in 0 and * will be overridden later, right after eal_hugepage_info_init() */ for (i = 0; i < RTE_MAX_NUMA_NODES; i++) diff --git a/lib/librte_eal/common/eal_common_proc.c b/lib/librte_eal/common/eal_common_proc.c index b46d644b..852e52e0 100644 --- a/lib/librte_eal/common/eal_common_proc.c +++ b/lib/librte_eal/common/eal_common_proc.c @@ -285,7 +285,15 @@ read_msg(struct mp_msg_internal *m, struct sockaddr_un *s) break; } } - + /* sanity-check the response */ + if (m->msg.num_fds < 0 || m->msg.num_fds > RTE_MP_MAX_FD_NUM) { + RTE_LOG(ERR, EAL, "invalid number of fd's received\n"); + return -1; + } + if (m->msg.len_param < 0 || m->msg.len_param > RTE_MP_MAX_PARAM_LEN) { + RTE_LOG(ERR, EAL, "invalid received data length\n"); + return -1; + } return 0; } @@ -678,11 +686,6 @@ send_msg(const char *dst_path, struct rte_mp_msg *msg, int type) unlink(dst_path); return 0; } - if (errno == ENOBUFS) { - RTE_LOG(ERR, EAL, "Peer cannot receive message %s\n", - dst_path); - return 0; - } RTE_LOG(ERR, EAL, "failed to send to (%s) due to %s\n", dst_path, strerror(errno)); return -1; @@ -758,6 +761,18 @@ check_input(const struct rte_mp_msg *msg) if (validate_action_name(msg->name)) return false; + if (msg->len_param < 0) { + RTE_LOG(ERR, EAL, "Message data length is negative\n"); + rte_errno = EINVAL; + return false; + } + + if (msg->num_fds < 0) { + RTE_LOG(ERR, EAL, "Number of fd's is negative\n"); + rte_errno = EINVAL; + return false; + } + if (msg->len_param > RTE_MP_MAX_PARAM_LEN) { RTE_LOG(ERR, EAL, "Message data is too long\n"); rte_errno = E2BIG; @@ -919,7 +934,7 @@ int __rte_experimental rte_mp_request_sync(struct rte_mp_msg *req, struct rte_mp_reply *reply, const struct timespec *ts) { - int dir_fd, ret = 0; + int dir_fd, ret = -1; DIR *mp_dir; struct dirent *ent; struct timeval now; @@ -927,13 +942,13 @@ rte_mp_request_sync(struct rte_mp_msg *req, struct rte_mp_reply *reply, RTE_LOG(DEBUG, EAL, "request: %s\n", req->name); - if (check_input(req) == false) - return -1; - reply->nb_sent = 0; reply->nb_received = 0; reply->msgs = NULL; + if (check_input(req) == false) + goto end; + if (internal_config.no_shconf) { RTE_LOG(DEBUG, EAL, "No shared files mode enabled, IPC is disabled\n"); return 0; @@ -942,7 +957,7 @@ rte_mp_request_sync(struct rte_mp_msg *req, struct rte_mp_reply *reply, if (gettimeofday(&now, NULL) < 0) { RTE_LOG(ERR, EAL, "Failed to get current time\n"); rte_errno = errno; - return -1; + goto end; } end.tv_nsec = (now.tv_usec * 1000 + ts->tv_nsec) % 1000000000; @@ -954,7 +969,7 @@ rte_mp_request_sync(struct rte_mp_msg *req, struct rte_mp_reply *reply, pthread_mutex_lock(&pending_requests.lock); ret = mp_request_sync(eal_mp_socket_path(), req, reply, &end); pthread_mutex_unlock(&pending_requests.lock); - return ret; + goto end; } /* for primary process, broadcast request, and collect reply 1 by 1 */ @@ -962,7 +977,7 @@ rte_mp_request_sync(struct rte_mp_msg *req, struct rte_mp_reply *reply, if (!mp_dir) { RTE_LOG(ERR, EAL, "Unable to open directory %s\n", mp_dir_path); rte_errno = errno; - return -1; + goto end; } dir_fd = dirfd(mp_dir); @@ -970,9 +985,8 @@ rte_mp_request_sync(struct rte_mp_msg *req, struct rte_mp_reply *reply, if (flock(dir_fd, LOCK_SH)) { RTE_LOG(ERR, EAL, "Unable to lock directory %s\n", mp_dir_path); - closedir(mp_dir); rte_errno = errno; - return -1; + goto close_end; } pthread_mutex_lock(&pending_requests.lock); @@ -989,14 +1003,25 @@ rte_mp_request_sync(struct rte_mp_msg *req, struct rte_mp_reply *reply, * locks on receive */ if (mp_request_sync(path, req, reply, &end)) - ret = -1; + goto unlock_end; } + ret = 0; + +unlock_end: pthread_mutex_unlock(&pending_requests.lock); /* unlock the directory */ flock(dir_fd, LOCK_UN); +close_end: /* dir_fd automatically closed on closedir */ closedir(mp_dir); + +end: + if (ret) { + free(reply->msgs); + reply->nb_received = 0; + reply->msgs = NULL; + } return ret; } diff --git a/lib/librte_eal/common/eal_common_thread.c b/lib/librte_eal/common/eal_common_thread.c index 48ef4d6d..14f206c0 100644 --- a/lib/librte_eal/common/eal_common_thread.c +++ b/lib/librte_eal/common/eal_common_thread.c @@ -16,6 +16,7 @@ #include <rte_memory.h> #include <rte_log.h> +#include "eal_internal_cfg.h" #include "eal_private.h" #include "eal_thread.h" @@ -168,10 +169,9 @@ rte_ctrl_thread_create(pthread_t *thread, const char *name, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg) { + rte_cpuset_t *cpuset = &internal_config.ctrl_cpuset; struct rte_thread_ctrl_params *params; - unsigned int lcore_id; - rte_cpuset_t cpuset; - int cpu_found, ret; + int ret; params = malloc(sizeof(*params)); if (!params) @@ -195,21 +195,8 @@ rte_ctrl_thread_create(pthread_t *thread, const char *name, "Cannot set name for ctrl thread\n"); } - cpu_found = 0; - CPU_ZERO(&cpuset); - for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { - if (eal_cpu_detected(lcore_id) && - rte_lcore_has_role(lcore_id, ROLE_OFF)) { - CPU_SET(lcore_id, &cpuset); - cpu_found = 1; - } - } - /* if no detected cpu is off, use master core */ - if (!cpu_found) - CPU_SET(rte_get_master_lcore(), &cpuset); - - ret = pthread_setaffinity_np(*thread, sizeof(cpuset), &cpuset); - if (ret < 0) + ret = pthread_setaffinity_np(*thread, sizeof(*cpuset), cpuset); + if (ret) goto fail; ret = pthread_barrier_wait(¶ms->configured); diff --git a/lib/librte_eal/common/eal_internal_cfg.h b/lib/librte_eal/common/eal_internal_cfg.h index 783ce7de..189d4f5b 100644 --- a/lib/librte_eal/common/eal_internal_cfg.h +++ b/lib/librte_eal/common/eal_internal_cfg.h @@ -13,6 +13,8 @@ #include <rte_eal.h> #include <rte_pci_dev_feature_defs.h> +#include "eal_thread.h" + #define MAX_HUGEPAGE_SIZES 3 /**< support up to 3 page sizes */ /* @@ -71,6 +73,7 @@ struct internal_config { unsigned num_hugepage_sizes; /**< how many sizes on this system */ struct hugepage_info hugepage_info[MAX_HUGEPAGE_SIZES]; enum rte_iova_mode iova_mode ; /**< Set IOVA mode on this system */ + rte_cpuset_t ctrl_cpuset; /**< cpuset for ctrl threads */ volatile unsigned int init_complete; /**< indicates whether EAL has completed initialization */ }; diff --git a/lib/librte_eal/common/eal_options.h b/lib/librte_eal/common/eal_options.h index 327c95e9..1623ae8c 100644 --- a/lib/librte_eal/common/eal_options.h +++ b/lib/librte_eal/common/eal_options.h @@ -5,6 +5,8 @@ #ifndef EAL_OPTIONS_H #define EAL_OPTIONS_H +#include "getopt.h" + enum { /* long options mapped to a short option */ #define OPT_HELP "help" diff --git a/lib/librte_eal/common/hotplug_mp.c b/lib/librte_eal/common/hotplug_mp.c index 9d610a8a..7c3f38db 100644 --- a/lib/librte_eal/common/hotplug_mp.c +++ b/lib/librte_eal/common/hotplug_mp.c @@ -361,7 +361,7 @@ int eal_dev_hotplug_request_to_primary(struct eal_dev_mp_req *req) ret = rte_mp_request_sync(&mp_req, &mp_reply, &ts); if (ret || mp_reply.nb_received != 1) { - RTE_LOG(ERR, EAL, "cannot send request to primary"); + RTE_LOG(ERR, EAL, "Cannot send request to primary\n"); if (!ret) return -1; return ret; diff --git a/lib/librte_eal/common/include/arch/ppc_64/rte_atomic.h b/lib/librte_eal/common/include/arch/ppc_64/rte_atomic.h index ce38350b..797381c0 100644 --- a/lib/librte_eal/common/include/arch/ppc_64/rte_atomic.h +++ b/lib/librte_eal/common/include/arch/ppc_64/rte_atomic.h @@ -63,11 +63,7 @@ extern "C" { * Guarantees that the STORE operations generated before the barrier * occur before the STORE operations generated after. */ -#ifdef RTE_ARCH_64 -#define rte_wmb() asm volatile("lwsync" : : : "memory") -#else #define rte_wmb() asm volatile("sync" : : : "memory") -#endif /** * Read memory barrier. @@ -75,11 +71,7 @@ extern "C" { * Guarantees that the LOAD operations generated before the barrier * occur before the LOAD operations generated after. */ -#ifdef RTE_ARCH_64 -#define rte_rmb() asm volatile("lwsync" : : : "memory") -#else #define rte_rmb() asm volatile("sync" : : : "memory") -#endif #define rte_smp_mb() rte_mb() diff --git a/lib/librte_eal/common/include/generic/rte_cycles.h b/lib/librte_eal/common/include/generic/rte_cycles.h index ac379e87..d318b91a 100644 --- a/lib/librte_eal/common/include/generic/rte_cycles.h +++ b/lib/librte_eal/common/include/generic/rte_cycles.h @@ -173,7 +173,7 @@ rte_delay_us_sleep(unsigned int us); * * @param userfunc * User function which replaces rte_delay_us. rte_delay_us_block restores - * buildin block delay function. + * builtin block delay function. */ void rte_delay_us_callback_register(void(*userfunc)(unsigned int)); diff --git a/lib/librte_eal/common/include/generic/rte_rwlock.h b/lib/librte_eal/common/include/generic/rte_rwlock.h index 5751a0e6..2c284f0b 100644 --- a/lib/librte_eal/common/include/generic/rte_rwlock.h +++ b/lib/librte_eal/common/include/generic/rte_rwlock.h @@ -64,14 +64,14 @@ rte_rwlock_read_lock(rte_rwlock_t *rwl) int success = 0; while (success == 0) { - x = rwl->cnt; + x = __atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED); /* write lock is held */ if (x < 0) { rte_pause(); continue; } - success = rte_atomic32_cmpset((volatile uint32_t *)&rwl->cnt, - (uint32_t)x, (uint32_t)(x + 1)); + success = __atomic_compare_exchange_n(&rwl->cnt, &x, x + 1, 1, + __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); } } @@ -84,7 +84,7 @@ rte_rwlock_read_lock(rte_rwlock_t *rwl) static inline void rte_rwlock_read_unlock(rte_rwlock_t *rwl) { - rte_atomic32_dec((rte_atomic32_t *)(intptr_t)&rwl->cnt); + __atomic_fetch_sub(&rwl->cnt, 1, __ATOMIC_RELEASE); } /** @@ -100,14 +100,14 @@ rte_rwlock_write_lock(rte_rwlock_t *rwl) int success = 0; while (success == 0) { - x = rwl->cnt; + x = __atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED); /* a lock is held */ if (x != 0) { rte_pause(); continue; } - success = rte_atomic32_cmpset((volatile uint32_t *)&rwl->cnt, - 0, (uint32_t)-1); + success = __atomic_compare_exchange_n(&rwl->cnt, &x, -1, 1, + __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); } } @@ -120,7 +120,7 @@ rte_rwlock_write_lock(rte_rwlock_t *rwl) static inline void rte_rwlock_write_unlock(rte_rwlock_t *rwl) { - rte_atomic32_inc((rte_atomic32_t *)(intptr_t)&rwl->cnt); + __atomic_store_n(&rwl->cnt, 0, __ATOMIC_RELEASE); } /** diff --git a/lib/librte_eal/common/include/generic/rte_spinlock.h b/lib/librte_eal/common/include/generic/rte_spinlock.h index c4c3fc31..87ae7a4f 100644 --- a/lib/librte_eal/common/include/generic/rte_spinlock.h +++ b/lib/librte_eal/common/include/generic/rte_spinlock.h @@ -61,9 +61,14 @@ rte_spinlock_lock(rte_spinlock_t *sl); static inline void rte_spinlock_lock(rte_spinlock_t *sl) { - while (__sync_lock_test_and_set(&sl->locked, 1)) - while(sl->locked) + int exp = 0; + + while (!__atomic_compare_exchange_n(&sl->locked, &exp, 1, 0, + __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) { + while (__atomic_load_n(&sl->locked, __ATOMIC_RELAXED)) rte_pause(); + exp = 0; + } } #endif @@ -80,7 +85,7 @@ rte_spinlock_unlock (rte_spinlock_t *sl); static inline void rte_spinlock_unlock (rte_spinlock_t *sl) { - __sync_lock_release(&sl->locked); + __atomic_store_n(&sl->locked, 0, __ATOMIC_RELEASE); } #endif @@ -99,7 +104,10 @@ rte_spinlock_trylock (rte_spinlock_t *sl); static inline int rte_spinlock_trylock (rte_spinlock_t *sl) { - return __sync_lock_test_and_set(&sl->locked,1) == 0; + int exp = 0; + return __atomic_compare_exchange_n(&sl->locked, &exp, 1, + 0, /* disallow spurious failure */ + __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); } #endif @@ -113,7 +121,7 @@ rte_spinlock_trylock (rte_spinlock_t *sl) */ static inline int rte_spinlock_is_locked (rte_spinlock_t *sl) { - return sl->locked; + return __atomic_load_n(&sl->locked, __ATOMIC_ACQUIRE); } /** diff --git a/lib/librte_eal/common/include/generic/rte_vect.h b/lib/librte_eal/common/include/generic/rte_vect.h index 11c6475b..3fc47979 100644 --- a/lib/librte_eal/common/include/generic/rte_vect.h +++ b/lib/librte_eal/common/include/generic/rte_vect.h @@ -55,7 +55,7 @@ typedef uint16_t rte_v128u16_t __attribute__((vector_size(16), aligned(16))); /** * 128 bits vector size to use with unsigned 32 bits elements. * - * a = (rte_v128u32_t){ a0, a1, a2, a3, a4 } + * a = (rte_v128u32_t){ a0, a1, a2, a3 } */ typedef uint32_t rte_v128u32_t __attribute__((vector_size(16), aligned(16))); diff --git a/lib/librte_eal/common/include/rte_class.h b/lib/librte_eal/common/include/rte_class.h index 276c91e9..856d09b2 100644 --- a/lib/librte_eal/common/include/rte_class.h +++ b/lib/librte_eal/common/include/rte_class.h @@ -15,7 +15,7 @@ * * A device class defines the type of function a device * will be used for e.g.: Ethernet adapter (eth), - * cryptographic coprocessor (crypto), etc. + * cryptographic co-processor (crypto), etc. */ #ifdef __cplusplus diff --git a/lib/librte_eal/common/include/rte_common.h b/lib/librte_eal/common/include/rte_common.h index 66cdf60b..48bf28ca 100644 --- a/lib/librte_eal/common/include/rte_common.h +++ b/lib/librte_eal/common/include/rte_common.h @@ -340,7 +340,7 @@ rte_is_power_of_2(uint32_t n) * Aligns input parameter to the next power of 2 * * @param x - * The integer value to algin + * The integer value to align * * @return * Input parameter aligned to the next power of 2 @@ -358,7 +358,7 @@ rte_align32pow2(uint32_t x) * Aligns input parameter to the previous power of 2 * * @param x - * The integer value to algin + * The integer value to align * * @return * Input parameter aligned to the previous power of 2 diff --git a/lib/librte_eal/common/include/rte_eal.h b/lib/librte_eal/common/include/rte_eal.h index a0cedd57..9951228e 100644 --- a/lib/librte_eal/common/include/rte_eal.h +++ b/lib/librte_eal/common/include/rte_eal.h @@ -228,6 +228,13 @@ struct rte_mp_reply { * * As we create socket channel for primary/secondary communication, use * this function typedef to register action for coming messages. + * + * @note When handling IPC request callbacks, the reply must be sent even in + * cases of error handling. Simply returning success or failure will *not* + * send a response to the requestor. + * Implementation of error signalling mechanism is up to the application. + * + * @note No memory allocations should take place inside the callback. */ typedef int (*rte_mp_t)(const struct rte_mp_msg *msg, const void *peer); @@ -237,6 +244,13 @@ typedef int (*rte_mp_t)(const struct rte_mp_msg *msg, const void *peer); * As we create socket channel for primary/secondary communication, use * this function typedef to register action for coming responses to asynchronous * requests. + * + * @note When handling IPC request callbacks, the reply must be sent even in + * cases of error handling. Simply returning success or failure will *not* + * send a response to the requestor. + * Implementation of error signalling mechanism is up to the application. + * + * @note No memory allocations should take place inside the callback. */ typedef int (*rte_mp_async_reply_t)(const struct rte_mp_msg *request, const struct rte_mp_reply *reply); @@ -287,7 +301,7 @@ rte_mp_action_unregister(const char *name); * * Send a message to the peer process. * - * This function will send a message which will be responsed by the action + * This function will send a message which will be responded by the action * identified by name in the peer process. * * @param msg @@ -311,6 +325,9 @@ rte_mp_sendmsg(struct rte_mp_msg *msg); * * @note The caller is responsible to free reply->replies. * + * @note This API must not be used inside memory-related or IPC callbacks, and + * no memory allocations should take place inside such callback. + * * @param req * The req argument contains the customized request message. * @@ -364,6 +381,11 @@ rte_mp_request_async(struct rte_mp_msg *req, const struct timespec *ts, * This function will send a reply message in response to a request message * received previously. * + * @note When handling IPC request callbacks, the reply must be sent even in + * cases of error handling. Simply returning success or failure will *not* + * send a response to the requestor. + * Implementation of error signalling mechanism is up to the application. + * * @param msg * The msg argument contains the customized message. * @@ -424,7 +446,7 @@ rte_set_application_usage_hook(rte_usage_hook_t usage_func); #define RTE_EAL_TAILQ_RWLOCK (&rte_eal_get_configuration()->mem_config->qlock) /** - * macro to get the multiple lock of mempool shared by mutiple-instance + * macro to get the multiple lock of mempool shared by multiple-instance */ #define RTE_EAL_MEMPOOL_RWLOCK (&rte_eal_get_configuration()->mem_config->mplock) diff --git a/lib/librte_eal/common/include/rte_lcore.h b/lib/librte_eal/common/include/rte_lcore.h index 6e09d918..dea17f50 100644 --- a/lib/librte_eal/common/include/rte_lcore.h +++ b/lib/librte_eal/common/include/rte_lcore.h @@ -23,10 +23,18 @@ extern "C" { #define LCORE_ID_ANY UINT32_MAX /**< Any lcore. */ #if defined(__linux__) - typedef cpu_set_t rte_cpuset_t; +typedef cpu_set_t rte_cpuset_t; +#define RTE_CPU_AND(dst, src1, src2) CPU_AND(dst, src1, src2) #elif defined(__FreeBSD__) #include <pthread_np.h> - typedef cpuset_t rte_cpuset_t; +typedef cpuset_t rte_cpuset_t; +#define RTE_CPU_AND(dst, src1, src2) do \ +{ \ + cpuset_t tmp; \ + CPU_COPY(src1, &tmp); \ + CPU_AND(&tmp, src2); \ + CPU_COPY(&tmp, dst); \ +} while (0) #endif /** @@ -280,8 +288,9 @@ int rte_thread_setname(pthread_t id, const char *name); * Create a control thread. * * Wrapper to pthread_create(), pthread_setname_np() and - * pthread_setaffinity_np(). The dataplane and service lcores are - * excluded from the affinity of the new thread. + * pthread_setaffinity_np(). The affinity of the new thread is based + * on the CPU affinity retrieved at the time rte_eal_init() was called, + * the dataplane and service lcores are then excluded. * * @param thread * Filled with the thread id of the new created thread. diff --git a/lib/librte_eal/common/include/rte_log.h b/lib/librte_eal/common/include/rte_log.h index 2f789cb9..213043c4 100644 --- a/lib/librte_eal/common/include/rte_log.h +++ b/lib/librte_eal/common/include/rte_log.h @@ -36,7 +36,7 @@ struct rte_logs { struct rte_log_dynamic_type *dynamic_types; }; -/** Global log informations */ +/** Global log information */ extern struct rte_logs rte_logs; /* SDK log type */ diff --git a/lib/librte_eal/common/include/rte_malloc.h b/lib/librte_eal/common/include/rte_malloc.h index 54a12467..e0be13ca 100644 --- a/lib/librte_eal/common/include/rte_malloc.h +++ b/lib/librte_eal/common/include/rte_malloc.h @@ -111,7 +111,7 @@ rte_calloc(const char *type, size_t num, size_t size, unsigned align); /** * Replacement function for realloc(), using huge-page memory. Reserved area * memory is resized, preserving contents. In NUMA systems, the new area - * resides on the same NUMA socket as the old area. + * may not reside on the same NUMA node as the old one. * * @param ptr * Pointer to already allocated memory diff --git a/lib/librte_eal/common/include/rte_service.h b/lib/librte_eal/common/include/rte_service.h index 34b41aff..11f67350 100644 --- a/lib/librte_eal/common/include/rte_service.h +++ b/lib/librte_eal/common/include/rte_service.h @@ -337,7 +337,7 @@ int32_t rte_service_set_stats_enable(uint32_t id, int32_t enable); int32_t rte_service_lcore_list(uint32_t array[], uint32_t n); /** - * Get the numer of services running on the supplied lcore. + * Get the number of services running on the supplied lcore. * * @param lcore Id of the service core. * @retval >=0 Number of services registered to this core. diff --git a/lib/librte_eal/common/include/rte_string_fns.h b/lib/librte_eal/common/include/rte_string_fns.h index 9a2a1ff9..35c6b003 100644 --- a/lib/librte_eal/common/include/rte_string_fns.h +++ b/lib/librte_eal/common/include/rte_string_fns.h @@ -59,10 +59,25 @@ rte_strlcpy(char *dst, const char *src, size_t size) return (size_t)snprintf(dst, size, "%s", src); } +/** + * @internal + * DPDK-specific version of strlcat for systems without + * libc or libbsd copies of the function + */ +static inline size_t +rte_strlcat(char *dst, const char *src, size_t size) +{ + size_t l = strnlen(dst, size); + if (l < size) + return l + rte_strlcpy(&dst[l], src, size - l); + return l + strlen(src); +} + /* pull in a strlcpy function */ #ifdef RTE_EXEC_ENV_BSDAPP #ifndef __BSD_VISIBLE /* non-standard functions are hidden */ #define strlcpy(dst, src, size) rte_strlcpy(dst, src, size) +#define strlcat(dst, src, size) rte_strlcat(dst, src, size) #endif #else /* non-BSD platforms */ @@ -71,6 +86,7 @@ rte_strlcpy(char *dst, const char *src, size_t size) #else /* no BSD header files, create own */ #define strlcpy(dst, src, size) rte_strlcpy(dst, src, size) +#define strlcat(dst, src, size) rte_strlcat(dst, src, size) #endif /* RTE_USE_LIBBSD */ #endif /* BSDAPP */ diff --git a/lib/librte_eal/common/include/rte_tailq.h b/lib/librte_eal/common/include/rte_tailq.h index 9b01abb2..b6fe4e5f 100644 --- a/lib/librte_eal/common/include/rte_tailq.h +++ b/lib/librte_eal/common/include/rte_tailq.h @@ -53,7 +53,7 @@ struct rte_tailq_elem { }; /** - * Return the first tailq entry casted to the right struct. + * Return the first tailq entry cast to the right struct. */ #define RTE_TAILQ_CAST(tailq_entry, struct_name) \ (struct struct_name *)&(tailq_entry)->tailq_head diff --git a/lib/librte_eal/common/include/rte_uuid.h b/lib/librte_eal/common/include/rte_uuid.h index 2c846b5f..16bbed32 100644 --- a/lib/librte_eal/common/include/rte_uuid.h +++ b/lib/librte_eal/common/include/rte_uuid.h @@ -43,7 +43,7 @@ extern "C" { #include <stdbool.h> /** - * Struct describing a Universal Unique Identifer + * Struct describing a Universal Unique Identifier */ typedef unsigned char rte_uuid_t[16]; @@ -105,7 +105,7 @@ int rte_uuid_compare(const rte_uuid_t a, const rte_uuid_t b); * @param uu * Destination UUID * @return - * Returns 0 on succes, and -1 if string is not a valid UUID. + * Returns 0 on success, and -1 if string is not a valid UUID. */ int rte_uuid_parse(const char *in, rte_uuid_t uu); diff --git a/lib/librte_eal/common/include/rte_version.h b/lib/librte_eal/common/include/rte_version.h index b4c6dd3c..7c0b13b5 100644 --- a/lib/librte_eal/common/include/rte_version.h +++ b/lib/librte_eal/common/include/rte_version.h @@ -37,7 +37,7 @@ extern "C" { /** * Patch level number i.e. the z in yy.mm.z */ -#define RTE_VER_MINOR 1 +#define RTE_VER_MINOR 2 /** * Extra string to be appended to version number diff --git a/lib/librte_eal/common/include/rte_vfio.h b/lib/librte_eal/common/include/rte_vfio.h index cae96fab..d837f1e7 100644 --- a/lib/librte_eal/common/include/rte_vfio.h +++ b/lib/librte_eal/common/include/rte_vfio.h @@ -178,7 +178,7 @@ int rte_vfio_noiommu_is_enabled(void); * an error on BSD. * * @param vfio_group_fd - * VFIO Grouup FD. + * VFIO Group FD. * * @return * 0 on success. @@ -291,6 +291,10 @@ rte_vfio_get_group_fd(int iommu_group_num); * containers by default, user needs to manage DMA mappings for * any container created by this API. * + * @note When creating containers using this API, the container will only be + * available in the process that has created it. Sharing containers and + * devices between multiple processes is not supported. + * * @return * the container fd if successful * <0 if failed diff --git a/lib/librte_eal/common/malloc_mp.c b/lib/librte_eal/common/malloc_mp.c index f3a13353..b470565e 100644 --- a/lib/librte_eal/common/malloc_mp.c +++ b/lib/librte_eal/common/malloc_mp.c @@ -501,7 +501,7 @@ handle_rollback_response(const struct rte_mp_msg *request, /* lock the request */ pthread_mutex_lock(&mp_request_list.lock); - memset(&msg, 0, sizeof(0)); + memset(&msg, 0, sizeof(msg)); entry = find_request_by_id(mpreq->id); if (entry == NULL) { |