From 8a853e3f0275efc8b05cb195085d45946942744a Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Wed, 14 Nov 2018 11:13:11 +0000 Subject: New upstream version 18.11-rc3 Change-Id: I958b9d019027ef049bd992b3968a667f3ae382ae Signed-off-by: Luca Boccassi --- lib/librte_bpf/bpf_jit_x86.c | 28 +- lib/librte_eal/common/arch/x86/rte_memcpy.c | 29 -- lib/librte_eal/common/eal_common_dev.c | 3 +- lib/librte_eal/common/eal_common_devargs.c | 36 +- lib/librte_eal/common/eal_common_memory.c | 7 +- lib/librte_eal/common/eal_common_proc.c | 31 +- lib/librte_eal/common/include/arch/x86/rte_rtm.h | 19 +- .../common/include/arch/x86/rte_spinlock.h | 21 +- lib/librte_eal/common/include/rte_common.h | 19 + lib/librte_eal/common/include/rte_devargs.h | 4 +- lib/librte_eal/common/include/rte_version.h | 2 +- lib/librte_eal/common/rte_reciprocal.c | 17 +- lib/librte_eal/linuxapp/eal/eal_alarm.c | 2 + lib/librte_eal/linuxapp/eal/eal_memory.c | 1 + lib/librte_ethdev/rte_ethdev.c | 93 +++-- lib/librte_hash/rte_cmp_x86.h | 2 + lib/librte_hash/rte_cuckoo_hash.c | 307 +++++++++++++- lib/librte_pci/rte_pci.c | 4 + lib/librte_pipeline/rte_table_action.c | 5 +- lib/librte_ring/rte_ring_c11_mem.h | 14 +- lib/librte_vhost/vhost_crypto.c | 460 ++++++++++++++++----- lib/librte_vhost/vhost_user.c | 2 +- lib/librte_vhost/virtio_net.c | 4 +- 23 files changed, 845 insertions(+), 265 deletions(-) delete mode 100644 lib/librte_eal/common/arch/x86/rte_memcpy.c (limited to 'lib') diff --git a/lib/librte_bpf/bpf_jit_x86.c b/lib/librte_bpf/bpf_jit_x86.c index 68ea389f..f70cd6be 100644 --- a/lib/librte_bpf/bpf_jit_x86.c +++ b/lib/librte_bpf/bpf_jit_x86.c @@ -208,6 +208,19 @@ emit_sib(struct bpf_jit_state *st, uint32_t scale, uint32_t idx, uint32_t base) emit_bytes(st, &v, sizeof(v)); } +/* + * emit OPCODE+REGIDX byte + */ +static void +emit_opcode(struct bpf_jit_state *st, uint8_t ops, uint32_t reg) +{ + uint8_t v; + + v = ops | (reg & 7); + emit_bytes(st, &v, sizeof(v)); +} + + /* * emit xchg %, % */ @@ -472,19 +485,18 @@ static void emit_ld_imm64(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm0, uint32_t imm1) { + uint32_t op; + const uint8_t ops = 0xB8; - if (imm1 == 0) { - emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, dreg, imm0); - return; - } + op = (imm1 == 0) ? BPF_ALU : EBPF_ALU64; - emit_rex(st, EBPF_ALU64, 0, dreg); - emit_bytes(st, &ops, sizeof(ops)); - emit_modregrm(st, MOD_DIRECT, 0, dreg); + emit_rex(st, op, 0, dreg); + emit_opcode(st, ops, dreg); emit_imm(st, imm0, sizeof(imm0)); - emit_imm(st, imm1, sizeof(imm1)); + if (imm1 != 0) + emit_imm(st, imm1, sizeof(imm1)); } /* diff --git a/lib/librte_eal/common/arch/x86/rte_memcpy.c b/lib/librte_eal/common/arch/x86/rte_memcpy.c deleted file mode 100644 index 648c8f68..00000000 --- a/lib/librte_eal/common/arch/x86/rte_memcpy.c +++ /dev/null @@ -1,29 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright(c) 2010-2017 Intel Corporation - */ - -#include -#include -#include - -void *(*rte_memcpy_ptr)(void *dst, const void *src, size_t n) = NULL; - -RTE_INIT(rte_memcpy_init) -{ -#ifdef CC_SUPPORT_AVX512F - if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F)) { - rte_memcpy_ptr = rte_memcpy_avx512f; - RTE_LOG(DEBUG, EAL, "AVX512 memcpy is using!\n"); - return; - } -#endif -#ifdef CC_SUPPORT_AVX2 - if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) { - rte_memcpy_ptr = rte_memcpy_avx2; - RTE_LOG(DEBUG, EAL, "AVX2 memcpy is using!\n"); - return; - } -#endif - rte_memcpy_ptr = rte_memcpy_sse; - RTE_LOG(DEBUG, EAL, "Default SSE/AVX memcpy is using!\n"); -} diff --git a/lib/librte_eal/common/eal_common_dev.c b/lib/librte_eal/common/eal_common_dev.c index 5759ec2d..1fdc9ab1 100644 --- a/lib/librte_eal/common/eal_common_dev.c +++ b/lib/librte_eal/common/eal_common_dev.c @@ -150,10 +150,11 @@ local_dev_probe(const char *devargs, struct rte_device **new_dev) goto err_devarg; } - ret = rte_devargs_insert(da); + ret = rte_devargs_insert(&da); if (ret) goto err_devarg; + /* the rte_devargs will be referenced in the matching rte_device */ ret = da->bus->scan(); if (ret) goto err_devarg; diff --git a/lib/librte_eal/common/eal_common_devargs.c b/lib/librte_eal/common/eal_common_devargs.c index b7b9cb69..1ccf12dc 100644 --- a/lib/librte_eal/common/eal_common_devargs.c +++ b/lib/librte_eal/common/eal_common_devargs.c @@ -263,14 +263,38 @@ rte_devargs_parsef(struct rte_devargs *da, const char *format, ...) } int __rte_experimental -rte_devargs_insert(struct rte_devargs *da) +rte_devargs_insert(struct rte_devargs **da) { - int ret; + struct rte_devargs *listed_da; + void *tmp; + + if (*da == NULL || (*da)->bus == NULL) + return -1; - ret = rte_devargs_remove(da); - if (ret < 0) - return ret; - TAILQ_INSERT_TAIL(&devargs_list, da, next); + TAILQ_FOREACH_SAFE(listed_da, &devargs_list, next, tmp) { + if (listed_da == *da) + /* devargs already in the list */ + return 0; + if (strcmp(listed_da->bus->name, (*da)->bus->name) == 0 && + strcmp(listed_da->name, (*da)->name) == 0) { + /* device already in devargs list, must be updated */ + listed_da->type = (*da)->type; + listed_da->policy = (*da)->policy; + free(listed_da->args); + listed_da->args = (*da)->args; + listed_da->bus = (*da)->bus; + listed_da->cls = (*da)->cls; + listed_da->bus_str = (*da)->bus_str; + listed_da->cls_str = (*da)->cls_str; + listed_da->data = (*da)->data; + /* replace provided devargs with found one */ + free(*da); + *da = listed_da; + return 0; + } + } + /* new device in the list */ + TAILQ_INSERT_TAIL(&devargs_list, *da, next); return 0; } diff --git a/lib/librte_eal/common/eal_common_memory.c b/lib/librte_eal/common/eal_common_memory.c index 87fd9921..d47ea493 100644 --- a/lib/librte_eal/common/eal_common_memory.c +++ b/lib/librte_eal/common/eal_common_memory.c @@ -439,11 +439,7 @@ check_iova(const struct rte_memseg_list *msl __rte_unused, return 1; } -#if defined(RTE_ARCH_64) #define MAX_DMA_MASK_BITS 63 -#else -#define MAX_DMA_MASK_BITS 31 -#endif /* check memseg iovas are within the required range based on dma mask */ static int __rte_experimental @@ -453,7 +449,8 @@ check_dma_mask(uint8_t maskbits, bool thread_unsafe) uint64_t mask; int ret; - /* sanity check */ + /* Sanity check. We only check width can be managed with 64 bits + * variables. Indeed any higher value is likely wrong. */ if (maskbits > MAX_DMA_MASK_BITS) { RTE_LOG(ERR, EAL, "wrong dma mask size %u (Max: %u)\n", maskbits, MAX_DMA_MASK_BITS); diff --git a/lib/librte_eal/common/eal_common_proc.c b/lib/librte_eal/common/eal_common_proc.c index 97663d3b..f65ef56c 100644 --- a/lib/librte_eal/common/eal_common_proc.c +++ b/lib/librte_eal/common/eal_common_proc.c @@ -800,7 +800,7 @@ mp_request_async(const char *dst, struct rte_mp_msg *req, { struct rte_mp_msg *reply_msg; struct pending_request *pending_req, *exist; - int ret; + int ret = -1; pending_req = calloc(1, sizeof(*pending_req)); reply_msg = calloc(1, sizeof(*reply_msg)); @@ -827,6 +827,28 @@ mp_request_async(const char *dst, struct rte_mp_msg *req, goto fail; } + /* + * set the alarm before sending message. there are two possible error + * scenarios to consider here: + * + * - if the alarm set fails, we free the memory right there + * - if the alarm set succeeds but sending message fails, then the alarm + * will trigger and clean up the memory + * + * Even if the alarm triggers too early (i.e. immediately), we're still + * holding the lock to pending requests queue, so the interrupt thread + * will just spin until we release the lock, and either release the + * memory, or doesn't find any pending requests in the queue because we + * never added any due to send message failure. + */ + if (rte_eal_alarm_set(ts->tv_sec * 1000000 + ts->tv_nsec / 1000, + async_reply_handle, pending_req) < 0) { + RTE_LOG(ERR, EAL, "Fail to set alarm for request %s:%s\n", + dst, req->name); + ret = -1; + goto fail; + } + ret = send_msg(dst, req, MP_REQ); if (ret < 0) { RTE_LOG(ERR, EAL, "Fail to send request %s:%s\n", @@ -841,13 +863,6 @@ mp_request_async(const char *dst, struct rte_mp_msg *req, param->user_reply.nb_sent++; - if (rte_eal_alarm_set(ts->tv_sec * 1000000 + ts->tv_nsec / 1000, - async_reply_handle, pending_req) < 0) { - RTE_LOG(ERR, EAL, "Fail to set alarm for request %s:%s\n", - dst, req->name); - rte_panic("Fix the above shit to properly free all memory\n"); - } - return 0; fail: free(pending_req); diff --git a/lib/librte_eal/common/include/arch/x86/rte_rtm.h b/lib/librte_eal/common/include/arch/x86/rte_rtm.h index ab099952..eb0f8e81 100644 --- a/lib/librte_eal/common/include/arch/x86/rte_rtm.h +++ b/lib/librte_eal/common/include/arch/x86/rte_rtm.h @@ -1,21 +1,10 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2012,2013 Intel Corporation + */ + #ifndef _RTE_RTM_H_ #define _RTE_RTM_H_ 1 -/* - * Copyright (c) 2012,2013 Intel Corporation - * Author: Andi Kleen - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that: (1) source code distributions - * retain the above copyright notice and this paragraph in its entirety, (2) - * distributions including binary code include the above copyright notice and - * this paragraph in its entirety in the documentation or other materials - * provided with the distribution - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. - */ /* Official RTM intrinsics interface matching gcc/icc, but works on older gcc compatible compilers and binutils. */ diff --git a/lib/librte_eal/common/include/arch/x86/rte_spinlock.h b/lib/librte_eal/common/include/arch/x86/rte_spinlock.h index 60321da0..e2e2b264 100644 --- a/lib/librte_eal/common/include/arch/x86/rte_spinlock.h +++ b/lib/librte_eal/common/include/arch/x86/rte_spinlock.h @@ -15,8 +15,9 @@ extern "C" { #include "rte_branch_prediction.h" #include "rte_common.h" #include "rte_pause.h" +#include "rte_cycles.h" -#define RTE_RTM_MAX_RETRIES (10) +#define RTE_RTM_MAX_RETRIES (20) #define RTE_XABORT_LOCK_BUSY (0xff) #ifndef RTE_FORCE_INTRINSICS @@ -76,7 +77,7 @@ static inline int rte_tm_supported(void) static inline int rte_try_tm(volatile int *lock) { - int retries; + int i, retries; if (!rte_rtm_supported) return 0; @@ -96,9 +97,21 @@ rte_try_tm(volatile int *lock) while (*lock) rte_pause(); - if ((status & RTE_XABORT_EXPLICIT) && - (RTE_XABORT_CODE(status) == RTE_XABORT_LOCK_BUSY)) + if ((status & RTE_XABORT_CONFLICT) || + ((status & RTE_XABORT_EXPLICIT) && + (RTE_XABORT_CODE(status) == RTE_XABORT_LOCK_BUSY))) { + /* add a small delay before retrying, basing the + * delay on the number of times we've already tried, + * to give a back-off type of behaviour. We + * randomize trycount by taking bits from the tsc count + */ + int try_count = RTE_RTM_MAX_RETRIES - retries; + int pause_count = (rte_rdtsc() & 0x7) | 1; + pause_count <<= try_count; + for (i = 0; i < pause_count; i++) + rte_pause(); continue; + } if ((status & RTE_XABORT_RETRY) == 0) /* do not retry */ break; diff --git a/lib/librte_eal/common/include/rte_common.h b/lib/librte_eal/common/include/rte_common.h index cba7bbc1..87f0f630 100644 --- a/lib/librte_eal/common/include/rte_common.h +++ b/lib/librte_eal/common/include/rte_common.h @@ -473,6 +473,25 @@ rte_log2_u32(uint32_t v) return rte_bsf32(v); } + +/** + * Return the last (most-significant) bit set. + * + * @note The last (most significant) bit is at position 32. + * @note rte_fls_u32(0) = 0, rte_fls_u32(1) = 1, rte_fls_u32(0x80000000) = 32 + * + * @param x + * The input parameter. + * @return + * The last (most-significant) bit set, or 0 if the input is 0. + */ +static inline int +rte_fls_u32(uint32_t x) +{ + return (x == 0) ? 0 : 32 - __builtin_clz(x); +} + + #ifndef offsetof /** Return the offset of a field in a structure. */ #define offsetof(TYPE, MEMBER) __builtin_offsetof (TYPE, MEMBER) diff --git a/lib/librte_eal/common/include/rte_devargs.h b/lib/librte_eal/common/include/rte_devargs.h index b1f121f8..29b3fb7c 100644 --- a/lib/librte_eal/common/include/rte_devargs.h +++ b/lib/librte_eal/common/include/rte_devargs.h @@ -146,6 +146,8 @@ __attribute__((format(printf, 2, 0))); * * @param da * The devargs structure to insert. + * If a devargs for the same device is already inserted, + * it will be updated and returned. It means *da pointer can change. * * @return * - 0 on success @@ -153,7 +155,7 @@ __attribute__((format(printf, 2, 0))); */ __rte_experimental int -rte_devargs_insert(struct rte_devargs *da); +rte_devargs_insert(struct rte_devargs **da); /** * Add a device to the user device list diff --git a/lib/librte_eal/common/include/rte_version.h b/lib/librte_eal/common/include/rte_version.h index 80c516d3..fc26e97a 100644 --- a/lib/librte_eal/common/include/rte_version.h +++ b/lib/librte_eal/common/include/rte_version.h @@ -49,7 +49,7 @@ extern "C" { * 0-15 = release candidates * 16 = release */ -#define RTE_VER_RELEASE 2 +#define RTE_VER_RELEASE 3 /** * Macro to compute a version number usable for comparisons diff --git a/lib/librte_eal/common/rte_reciprocal.c b/lib/librte_eal/common/rte_reciprocal.c index d81b11db..f017d0c2 100644 --- a/lib/librte_eal/common/rte_reciprocal.c +++ b/lib/librte_eal/common/rte_reciprocal.c @@ -41,28 +41,13 @@ #include "rte_reciprocal.h" -/* find largest set bit. - * portable and slow but does not matter for this usage. - */ -static inline int fls(uint32_t x) -{ - int b; - - for (b = 31; b >= 0; --b) { - if (x & (1u << b)) - return b + 1; - } - - return 0; -} - struct rte_reciprocal rte_reciprocal_value(uint32_t d) { struct rte_reciprocal R; uint64_t m; int l; - l = fls(d - 1); + l = rte_fls_u32(d - 1); m = ((1ULL << 32) * ((1ULL << l) - d)); m /= d; diff --git a/lib/librte_eal/linuxapp/eal/eal_alarm.c b/lib/librte_eal/linuxapp/eal/eal_alarm.c index 391d2a65..840ede78 100644 --- a/lib/librte_eal/linuxapp/eal/eal_alarm.c +++ b/lib/librte_eal/linuxapp/eal/eal_alarm.c @@ -30,7 +30,9 @@ #define NS_PER_US 1000 #define US_PER_MS 1000 #define MS_PER_S 1000 +#ifndef US_PER_S #define US_PER_S (US_PER_MS * MS_PER_S) +#endif #ifdef CLOCK_MONOTONIC_RAW /* Defined in glibc bits/time.h */ #define CLOCK_TYPE_ID CLOCK_MONOTONIC_RAW diff --git a/lib/librte_eal/linuxapp/eal/eal_memory.c b/lib/librte_eal/linuxapp/eal/eal_memory.c index c1b5e079..48b23ce1 100644 --- a/lib/librte_eal/linuxapp/eal/eal_memory.c +++ b/lib/librte_eal/linuxapp/eal/eal_memory.c @@ -1617,6 +1617,7 @@ eal_legacy_hugepage_init(void) tmp_hp = NULL; munmap(hugepage, nr_hugefiles * sizeof(struct hugepage_file)); + hugepage = NULL; /* we're not going to allocate more pages, so release VA space for * unused memseg lists diff --git a/lib/librte_ethdev/rte_ethdev.c b/lib/librte_ethdev/rte_ethdev.c index 8eaa5fcc..5f858174 100644 --- a/lib/librte_ethdev/rte_ethdev.c +++ b/lib/librte_ethdev/rte_ethdev.c @@ -1092,8 +1092,9 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, { struct rte_eth_dev *dev; struct rte_eth_dev_info dev_info; - struct rte_eth_conf local_conf = *dev_conf; + struct rte_eth_conf orig_conf; int diag; + int ret; RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL); @@ -1102,6 +1103,22 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP); RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP); + if (dev->data->dev_started) { + RTE_ETHDEV_LOG(ERR, + "Port %u must be stopped to allow configuration\n", + port_id); + return -EBUSY; + } + + /* Store original config, as rollback required on failure */ + memcpy(&orig_conf, &dev->data->dev_conf, sizeof(dev->data->dev_conf)); + + /* + * Copy the dev_conf parameter into the dev structure. + * rte_eth_dev_info_get() requires dev_conf, copy it before dev_info get + */ + memcpy(&dev->data->dev_conf, dev_conf, sizeof(dev->data->dev_conf)); + rte_eth_dev_info_get(port_id, &dev_info); /* If number of queues specified by application for both Rx and Tx is @@ -1123,26 +1140,18 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, RTE_ETHDEV_LOG(ERR, "Number of RX queues requested (%u) is greater than max supported(%d)\n", nb_rx_q, RTE_MAX_QUEUES_PER_PORT); - return -EINVAL; + ret = -EINVAL; + goto rollback; } if (nb_tx_q > RTE_MAX_QUEUES_PER_PORT) { RTE_ETHDEV_LOG(ERR, "Number of TX queues requested (%u) is greater than max supported(%d)\n", nb_tx_q, RTE_MAX_QUEUES_PER_PORT); - return -EINVAL; - } - - if (dev->data->dev_started) { - RTE_ETHDEV_LOG(ERR, - "Port %u must be stopped to allow configuration\n", - port_id); - return -EBUSY; + ret = -EINVAL; + goto rollback; } - /* Copy the dev_conf parameter into the dev structure */ - memcpy(&dev->data->dev_conf, &local_conf, sizeof(dev->data->dev_conf)); - /* * Check that the numbers of RX and TX queues are not greater * than the maximum number of RX and TX queues supported by the @@ -1151,13 +1160,15 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, if (nb_rx_q > dev_info.max_rx_queues) { RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u nb_rx_queues=%u > %u\n", port_id, nb_rx_q, dev_info.max_rx_queues); - return -EINVAL; + ret = -EINVAL; + goto rollback; } if (nb_tx_q > dev_info.max_tx_queues) { RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u nb_tx_queues=%u > %u\n", port_id, nb_tx_q, dev_info.max_tx_queues); - return -EINVAL; + ret = -EINVAL; + goto rollback; } /* Check that the device supports requested interrupts */ @@ -1165,32 +1176,36 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC))) { RTE_ETHDEV_LOG(ERR, "Driver %s does not support lsc\n", dev->device->driver->name); - return -EINVAL; + ret = -EINVAL; + goto rollback; } if ((dev_conf->intr_conf.rmv == 1) && (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_RMV))) { RTE_ETHDEV_LOG(ERR, "Driver %s does not support rmv\n", dev->device->driver->name); - return -EINVAL; + ret = -EINVAL; + goto rollback; } /* * If jumbo frames are enabled, check that the maximum RX packet * length is supported by the configured device. */ - if (local_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) { + if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) { if (dev_conf->rxmode.max_rx_pkt_len > dev_info.max_rx_pktlen) { RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u max_rx_pkt_len %u > max valid value %u\n", port_id, dev_conf->rxmode.max_rx_pkt_len, dev_info.max_rx_pktlen); - return -EINVAL; + ret = -EINVAL; + goto rollback; } else if (dev_conf->rxmode.max_rx_pkt_len < ETHER_MIN_LEN) { RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u max_rx_pkt_len %u < min valid value %u\n", port_id, dev_conf->rxmode.max_rx_pkt_len, (unsigned)ETHER_MIN_LEN); - return -EINVAL; + ret = -EINVAL; + goto rollback; } } else { if (dev_conf->rxmode.max_rx_pkt_len < ETHER_MIN_LEN || @@ -1201,25 +1216,27 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, } /* Any requested offloading must be within its device capabilities */ - if ((local_conf.rxmode.offloads & dev_info.rx_offload_capa) != - local_conf.rxmode.offloads) { + if ((dev_conf->rxmode.offloads & dev_info.rx_offload_capa) != + dev_conf->rxmode.offloads) { RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u requested Rx offloads 0x%"PRIx64" doesn't match Rx offloads " "capabilities 0x%"PRIx64" in %s()\n", - port_id, local_conf.rxmode.offloads, + port_id, dev_conf->rxmode.offloads, dev_info.rx_offload_capa, __func__); - return -EINVAL; + ret = -EINVAL; + goto rollback; } - if ((local_conf.txmode.offloads & dev_info.tx_offload_capa) != - local_conf.txmode.offloads) { + if ((dev_conf->txmode.offloads & dev_info.tx_offload_capa) != + dev_conf->txmode.offloads) { RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u requested Tx offloads 0x%"PRIx64" doesn't match Tx offloads " "capabilities 0x%"PRIx64" in %s()\n", - port_id, local_conf.txmode.offloads, + port_id, dev_conf->txmode.offloads, dev_info.tx_offload_capa, __func__); - return -EINVAL; + ret = -EINVAL; + goto rollback; } /* Check that device supports requested rss hash functions. */ @@ -1230,7 +1247,8 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, "Ethdev port_id=%u invalid rss_hf: 0x%"PRIx64", valid value: 0x%"PRIx64"\n", port_id, dev_conf->rx_adv_conf.rss_conf.rss_hf, dev_info.flow_type_rss_offloads); - return -EINVAL; + ret = -EINVAL; + goto rollback; } /* @@ -1241,7 +1259,8 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, RTE_ETHDEV_LOG(ERR, "Port%u rte_eth_dev_rx_queue_config = %d\n", port_id, diag); - return diag; + ret = diag; + goto rollback; } diag = rte_eth_dev_tx_queue_config(dev, nb_tx_q); @@ -1250,7 +1269,8 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, "Port%u rte_eth_dev_tx_queue_config = %d\n", port_id, diag); rte_eth_dev_rx_queue_config(dev, 0); - return diag; + ret = diag; + goto rollback; } diag = (*dev->dev_ops->dev_configure)(dev); @@ -1259,7 +1279,8 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, port_id, diag); rte_eth_dev_rx_queue_config(dev, 0); rte_eth_dev_tx_queue_config(dev, 0); - return eth_err(port_id, diag); + ret = eth_err(port_id, diag); + goto rollback; } /* Initialize Rx profiling if enabled at compilation time. */ @@ -1269,10 +1290,16 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, port_id, diag); rte_eth_dev_rx_queue_config(dev, 0); rte_eth_dev_tx_queue_config(dev, 0); - return eth_err(port_id, diag); + ret = eth_err(port_id, diag); + goto rollback; } return 0; + +rollback: + memcpy(&dev->data->dev_conf, &orig_conf, sizeof(dev->data->dev_conf)); + + return ret; } void diff --git a/lib/librte_hash/rte_cmp_x86.h b/lib/librte_hash/rte_cmp_x86.h index e82b4c08..13a58363 100644 --- a/lib/librte_hash/rte_cmp_x86.h +++ b/lib/librte_hash/rte_cmp_x86.h @@ -2,6 +2,8 @@ * Copyright(c) 2015 Intel Corporation */ +#include + /* Functions to compare multiple of 16 byte keys (up to 128 bytes) */ static int rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused) diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c index 5ddcccd8..c55a4f26 100644 --- a/lib/librte_hash/rte_cuckoo_hash.c +++ b/lib/librte_hash/rte_cuckoo_hash.c @@ -13,7 +13,6 @@ #include #include /* for definition of RTE_CACHE_LINE_SIZE */ #include -#include #include #include #include @@ -982,7 +981,7 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, new_k = RTE_PTR_ADD(keys, (uintptr_t)slot_id * h->key_entry_size); new_idx = (uint32_t)((uintptr_t) slot_id); /* Copy key */ - rte_memcpy(new_k->key, key, h->key_len); + memcpy(new_k->key, key, h->key_len); /* Key can be of arbitrary length, so it is not possible to store * it atomically. Hence the new key element's memory stores * (key as well as data) should be complete before it is referenced. @@ -1129,9 +1128,38 @@ rte_hash_add_key_data(const struct rte_hash *h, const void *key, void *data) return ret; } +/* Search one bucket to find the match key - uses rw lock */ +static inline int32_t +search_one_bucket_l(const struct rte_hash *h, const void *key, + uint16_t sig, void **data, + const struct rte_hash_bucket *bkt) +{ + int i; + struct rte_hash_key *k, *keys = h->key_store; + + for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) { + if (bkt->sig_current[i] == sig && + bkt->key_idx[i] != EMPTY_SLOT) { + k = (struct rte_hash_key *) ((char *)keys + + bkt->key_idx[i] * h->key_entry_size); + + if (rte_hash_cmp_eq(key, k->key, h) == 0) { + if (data != NULL) + *data = k->pdata; + /* + * Return index where key is stored, + * subtracting the first dummy index + */ + return bkt->key_idx[i] - 1; + } + } + } + return -1; +} + /* Search one bucket to find the match key */ static inline int32_t -search_one_bucket(const struct rte_hash *h, const void *key, uint16_t sig, +search_one_bucket_lf(const struct rte_hash *h, const void *key, uint16_t sig, void **data, const struct rte_hash_bucket *bkt) { int i; @@ -1163,12 +1191,11 @@ search_one_bucket(const struct rte_hash *h, const void *key, uint16_t sig, } static inline int32_t -__rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key, - hash_sig_t sig, void **data) +__rte_hash_lookup_with_hash_l(const struct rte_hash *h, const void *key, + hash_sig_t sig, void **data) { uint32_t prim_bucket_idx, sec_bucket_idx; struct rte_hash_bucket *bkt, *cur_bkt; - uint32_t cnt_b, cnt_a; int ret; uint16_t short_sig; @@ -1176,8 +1203,48 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key, prim_bucket_idx = get_prim_bucket_index(h, sig); sec_bucket_idx = get_alt_bucket_index(h, prim_bucket_idx, short_sig); + bkt = &h->buckets[prim_bucket_idx]; + __hash_rw_reader_lock(h); + /* Check if key is in primary location */ + ret = search_one_bucket_l(h, key, short_sig, data, bkt); + if (ret != -1) { + __hash_rw_reader_unlock(h); + return ret; + } + /* Calculate secondary hash */ + bkt = &h->buckets[sec_bucket_idx]; + + /* Check if key is in secondary location */ + FOR_EACH_BUCKET(cur_bkt, bkt) { + ret = search_one_bucket_l(h, key, short_sig, + data, cur_bkt); + if (ret != -1) { + __hash_rw_reader_unlock(h); + return ret; + } + } + + __hash_rw_reader_unlock(h); + + return -ENOENT; +} + +static inline int32_t +__rte_hash_lookup_with_hash_lf(const struct rte_hash *h, const void *key, + hash_sig_t sig, void **data) +{ + uint32_t prim_bucket_idx, sec_bucket_idx; + struct rte_hash_bucket *bkt, *cur_bkt; + uint32_t cnt_b, cnt_a; + int ret; + uint16_t short_sig; + + short_sig = get_short_sig(sig); + prim_bucket_idx = get_prim_bucket_index(h, sig); + sec_bucket_idx = get_alt_bucket_index(h, prim_bucket_idx, short_sig); + do { /* Load the table change counter before the lookup * starts. Acquire semantics will make sure that @@ -1188,7 +1255,7 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key, /* Check if key is in primary location */ bkt = &h->buckets[prim_bucket_idx]; - ret = search_one_bucket(h, key, short_sig, data, bkt); + ret = search_one_bucket_lf(h, key, short_sig, data, bkt); if (ret != -1) { __hash_rw_reader_unlock(h); return ret; @@ -1198,7 +1265,7 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key, /* Check if key is in secondary location */ FOR_EACH_BUCKET(cur_bkt, bkt) { - ret = search_one_bucket(h, key, short_sig, + ret = search_one_bucket_lf(h, key, short_sig, data, cur_bkt); if (ret != -1) { __hash_rw_reader_unlock(h); @@ -1222,11 +1289,19 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key, __ATOMIC_ACQUIRE); } while (cnt_b != cnt_a); - __hash_rw_reader_unlock(h); - return -ENOENT; } +static inline int32_t +__rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key, + hash_sig_t sig, void **data) +{ + if (h->readwrite_concur_lf_support) + return __rte_hash_lookup_with_hash_lf(h, key, sig, data); + else + return __rte_hash_lookup_with_hash_l(h, key, sig, data); +} + int32_t rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key, hash_sig_t sig) @@ -1528,7 +1603,197 @@ compare_signatures(uint32_t *prim_hash_matches, uint32_t *sec_hash_matches, #define PREFETCH_OFFSET 4 static inline void -__rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys, +__rte_hash_lookup_bulk_l(const struct rte_hash *h, const void **keys, + int32_t num_keys, int32_t *positions, + uint64_t *hit_mask, void *data[]) +{ + uint64_t hits = 0; + int32_t i; + int32_t ret; + uint32_t prim_hash[RTE_HASH_LOOKUP_BULK_MAX]; + uint32_t prim_index[RTE_HASH_LOOKUP_BULK_MAX]; + uint32_t sec_index[RTE_HASH_LOOKUP_BULK_MAX]; + uint16_t sig[RTE_HASH_LOOKUP_BULK_MAX]; + const struct rte_hash_bucket *primary_bkt[RTE_HASH_LOOKUP_BULK_MAX]; + const struct rte_hash_bucket *secondary_bkt[RTE_HASH_LOOKUP_BULK_MAX]; + uint32_t prim_hitmask[RTE_HASH_LOOKUP_BULK_MAX] = {0}; + uint32_t sec_hitmask[RTE_HASH_LOOKUP_BULK_MAX] = {0}; + struct rte_hash_bucket *cur_bkt, *next_bkt; + + /* Prefetch first keys */ + for (i = 0; i < PREFETCH_OFFSET && i < num_keys; i++) + rte_prefetch0(keys[i]); + + /* + * Prefetch rest of the keys, calculate primary and + * secondary bucket and prefetch them + */ + for (i = 0; i < (num_keys - PREFETCH_OFFSET); i++) { + rte_prefetch0(keys[i + PREFETCH_OFFSET]); + + prim_hash[i] = rte_hash_hash(h, keys[i]); + + sig[i] = get_short_sig(prim_hash[i]); + prim_index[i] = get_prim_bucket_index(h, prim_hash[i]); + sec_index[i] = get_alt_bucket_index(h, prim_index[i], sig[i]); + + primary_bkt[i] = &h->buckets[prim_index[i]]; + secondary_bkt[i] = &h->buckets[sec_index[i]]; + + rte_prefetch0(primary_bkt[i]); + rte_prefetch0(secondary_bkt[i]); + } + + /* Calculate and prefetch rest of the buckets */ + for (; i < num_keys; i++) { + prim_hash[i] = rte_hash_hash(h, keys[i]); + + sig[i] = get_short_sig(prim_hash[i]); + prim_index[i] = get_prim_bucket_index(h, prim_hash[i]); + sec_index[i] = get_alt_bucket_index(h, prim_index[i], sig[i]); + + primary_bkt[i] = &h->buckets[prim_index[i]]; + secondary_bkt[i] = &h->buckets[sec_index[i]]; + + rte_prefetch0(primary_bkt[i]); + rte_prefetch0(secondary_bkt[i]); + } + + __hash_rw_reader_lock(h); + + /* Compare signatures and prefetch key slot of first hit */ + for (i = 0; i < num_keys; i++) { + compare_signatures(&prim_hitmask[i], &sec_hitmask[i], + primary_bkt[i], secondary_bkt[i], + sig[i], h->sig_cmp_fn); + + if (prim_hitmask[i]) { + uint32_t first_hit = + __builtin_ctzl(prim_hitmask[i]) + >> 1; + uint32_t key_idx = + primary_bkt[i]->key_idx[first_hit]; + const struct rte_hash_key *key_slot = + (const struct rte_hash_key *)( + (const char *)h->key_store + + key_idx * h->key_entry_size); + rte_prefetch0(key_slot); + continue; + } + + if (sec_hitmask[i]) { + uint32_t first_hit = + __builtin_ctzl(sec_hitmask[i]) + >> 1; + uint32_t key_idx = + secondary_bkt[i]->key_idx[first_hit]; + const struct rte_hash_key *key_slot = + (const struct rte_hash_key *)( + (const char *)h->key_store + + key_idx * h->key_entry_size); + rte_prefetch0(key_slot); + } + } + + /* Compare keys, first hits in primary first */ + for (i = 0; i < num_keys; i++) { + positions[i] = -ENOENT; + while (prim_hitmask[i]) { + uint32_t hit_index = + __builtin_ctzl(prim_hitmask[i]) + >> 1; + uint32_t key_idx = + primary_bkt[i]->key_idx[hit_index]; + const struct rte_hash_key *key_slot = + (const struct rte_hash_key *)( + (const char *)h->key_store + + key_idx * h->key_entry_size); + + /* + * If key index is 0, do not compare key, + * as it is checking the dummy slot + */ + if (!!key_idx & + !rte_hash_cmp_eq( + key_slot->key, keys[i], h)) { + if (data != NULL) + data[i] = key_slot->pdata; + + hits |= 1ULL << i; + positions[i] = key_idx - 1; + goto next_key; + } + prim_hitmask[i] &= ~(3ULL << (hit_index << 1)); + } + + while (sec_hitmask[i]) { + uint32_t hit_index = + __builtin_ctzl(sec_hitmask[i]) + >> 1; + uint32_t key_idx = + secondary_bkt[i]->key_idx[hit_index]; + const struct rte_hash_key *key_slot = + (const struct rte_hash_key *)( + (const char *)h->key_store + + key_idx * h->key_entry_size); + + /* + * If key index is 0, do not compare key, + * as it is checking the dummy slot + */ + + if (!!key_idx & + !rte_hash_cmp_eq( + key_slot->key, keys[i], h)) { + if (data != NULL) + data[i] = key_slot->pdata; + + hits |= 1ULL << i; + positions[i] = key_idx - 1; + goto next_key; + } + sec_hitmask[i] &= ~(3ULL << (hit_index << 1)); + } +next_key: + continue; + } + + /* all found, do not need to go through ext bkt */ + if ((hits == ((1ULL << num_keys) - 1)) || !h->ext_table_support) { + if (hit_mask != NULL) + *hit_mask = hits; + __hash_rw_reader_unlock(h); + return; + } + + /* need to check ext buckets for match */ + for (i = 0; i < num_keys; i++) { + if ((hits & (1ULL << i)) != 0) + continue; + next_bkt = secondary_bkt[i]->next; + FOR_EACH_BUCKET(cur_bkt, next_bkt) { + if (data != NULL) + ret = search_one_bucket_l(h, keys[i], + sig[i], &data[i], cur_bkt); + else + ret = search_one_bucket_l(h, keys[i], + sig[i], NULL, cur_bkt); + if (ret != -1) { + positions[i] = ret; + hits |= 1ULL << i; + break; + } + } + } + + __hash_rw_reader_unlock(h); + + if (hit_mask != NULL) + *hit_mask = hits; +} + +static inline void +__rte_hash_lookup_bulk_lf(const struct rte_hash *h, const void **keys, int32_t num_keys, int32_t *positions, uint64_t *hit_mask, void *data[]) { @@ -1586,7 +1851,6 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys, rte_prefetch0(secondary_bkt[i]); } - __hash_rw_reader_lock(h); do { /* Load the table change counter before the lookup * starts. Acquire semantics will make sure that @@ -1735,10 +1999,10 @@ next_key: next_bkt = secondary_bkt[i]->next; FOR_EACH_BUCKET(cur_bkt, next_bkt) { if (data != NULL) - ret = search_one_bucket(h, keys[i], + ret = search_one_bucket_lf(h, keys[i], sig[i], &data[i], cur_bkt); else - ret = search_one_bucket(h, keys[i], + ret = search_one_bucket_lf(h, keys[i], sig[i], NULL, cur_bkt); if (ret != -1) { positions[i] = ret; @@ -1748,12 +2012,23 @@ next_key: } } - __hash_rw_reader_unlock(h); - if (hit_mask != NULL) *hit_mask = hits; } +static inline void +__rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys, + int32_t num_keys, int32_t *positions, + uint64_t *hit_mask, void *data[]) +{ + if (h->readwrite_concur_lf_support) + return __rte_hash_lookup_bulk_lf(h, keys, num_keys, + positions, hit_mask, data); + else + return __rte_hash_lookup_bulk_l(h, keys, num_keys, + positions, hit_mask, data); +} + int rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys, uint32_t num_keys, int32_t *positions) diff --git a/lib/librte_pci/rte_pci.c b/lib/librte_pci/rte_pci.c index 530738db..f400178b 100644 --- a/lib/librte_pci/rte_pci.c +++ b/lib/librte_pci/rte_pci.c @@ -30,6 +30,10 @@ get_u8_pciaddr_field(const char *in, void *_u8, char dlm) uint8_t *u8 = _u8; char *end; + /* empty string is an error though strtoul() returns 0 */ + if (*in == '\0') + return NULL; + errno = 0; val = strtoul(in, &end, 16); if (errno != 0 || end[0] != dlm || val > UINT8_MAX) { diff --git a/lib/librte_pipeline/rte_table_action.c b/lib/librte_pipeline/rte_table_action.c index 537e6593..7c7c8dd8 100644 --- a/lib/librte_pipeline/rte_table_action.c +++ b/lib/librte_pipeline/rte_table_action.c @@ -1694,10 +1694,9 @@ get_block_size(const struct rte_crypto_sym_xform *xform, uint8_t cdev_id) rte_cryptodev_info_get(cdev_id, &dev_info); - for (i = 0;; i++) { + for (i = 0; dev_info.capabilities[i].op != RTE_CRYPTO_OP_TYPE_UNDEFINED; + i++) { cap = &dev_info.capabilities[i]; - if (!cap) - break; if (cap->sym.xform_type != xform->type) continue; diff --git a/lib/librte_ring/rte_ring_c11_mem.h b/lib/librte_ring/rte_ring_c11_mem.h index 7bc74a4c..0fb73a33 100644 --- a/lib/librte_ring/rte_ring_c11_mem.h +++ b/lib/librte_ring/rte_ring_c11_mem.h @@ -61,11 +61,14 @@ __rte_ring_move_prod_head(struct rte_ring *r, unsigned int is_sp, unsigned int max = n; int success; - *old_head = __atomic_load_n(&r->prod.head, __ATOMIC_ACQUIRE); + *old_head = __atomic_load_n(&r->prod.head, __ATOMIC_RELAXED); do { /* Reset n to the initial burst count */ n = max; + /* Ensure the head is read before tail */ + __atomic_thread_fence(__ATOMIC_ACQUIRE); + /* load-acquire synchronize with store-release of ht->tail * in update_tail. */ @@ -94,7 +97,7 @@ __rte_ring_move_prod_head(struct rte_ring *r, unsigned int is_sp, /* on failure, *old_head is updated */ success = __atomic_compare_exchange_n(&r->prod.head, old_head, *new_head, - 0, __ATOMIC_ACQUIRE, + 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED); } while (unlikely(success == 0)); return n; @@ -134,11 +137,14 @@ __rte_ring_move_cons_head(struct rte_ring *r, int is_sc, int success; /* move cons.head atomically */ - *old_head = __atomic_load_n(&r->cons.head, __ATOMIC_ACQUIRE); + *old_head = __atomic_load_n(&r->cons.head, __ATOMIC_RELAXED); do { /* Restore n as it may change every loop */ n = max; + /* Ensure the head is read before tail */ + __atomic_thread_fence(__ATOMIC_ACQUIRE); + /* this load-acquire synchronize with store-release of ht->tail * in update_tail. */ @@ -166,7 +172,7 @@ __rte_ring_move_cons_head(struct rte_ring *r, int is_sc, /* on failure, *old_head will be updated */ success = __atomic_compare_exchange_n(&r->cons.head, old_head, *new_head, - 0, __ATOMIC_ACQUIRE, + 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED); } while (unlikely(success == 0)); return n; diff --git a/lib/librte_vhost/vhost_crypto.c b/lib/librte_vhost/vhost_crypto.c index 5472bead..dd01afc0 100644 --- a/lib/librte_vhost/vhost_crypto.c +++ b/lib/librte_vhost/vhost_crypto.c @@ -198,6 +198,7 @@ struct vhost_crypto { struct rte_hash *session_map; struct rte_mempool *mbuf_pool; struct rte_mempool *sess_pool; + struct rte_mempool *wb_pool; /** DPDK cryptodev ID */ uint8_t cid; @@ -215,13 +216,20 @@ struct vhost_crypto { uint8_t option; } __rte_cache_aligned; +struct vhost_crypto_writeback_data { + uint8_t *src; + uint8_t *dst; + uint64_t len; + struct vhost_crypto_writeback_data *next; +}; + struct vhost_crypto_data_req { struct vring_desc *head; struct virtio_net *dev; struct virtio_crypto_inhdr *inhdr; struct vhost_virtqueue *vq; - struct vring_desc *wb_desc; - uint16_t wb_len; + struct vhost_crypto_writeback_data *wb; + struct rte_mempool *wb_pool; uint16_t desc_idx; uint16_t len; uint16_t zero_copy; @@ -506,15 +514,29 @@ move_desc(struct vring_desc *head, struct vring_desc **cur_desc, left -= desc->len; } - if (unlikely(left > 0)) { - VC_LOG_ERR("Incorrect virtio descriptor"); + if (unlikely(left > 0)) return -1; - } *cur_desc = &head[desc->next]; return 0; } +static __rte_always_inline void * +get_data_ptr(struct vhost_crypto_data_req *vc_req, struct vring_desc *cur_desc, + uint8_t perm) +{ + void *data; + uint64_t dlen = cur_desc->len; + + data = IOVA_TO_VVA(void *, vc_req, cur_desc->addr, &dlen, perm); + if (unlikely(!data || dlen != cur_desc->len)) { + VC_LOG_ERR("Failed to map object"); + return NULL; + } + + return data; +} + static int copy_data(void *dst_data, struct vhost_crypto_data_req *vc_req, struct vring_desc **cur_desc, uint32_t size) @@ -531,10 +553,8 @@ copy_data(void *dst_data, struct vhost_crypto_data_req *vc_req, dlen = to_copy; src = IOVA_TO_VVA(uint8_t *, vc_req, desc->addr, &dlen, VHOST_ACCESS_RO); - if (unlikely(!src || !dlen)) { - VC_LOG_ERR("Failed to map descriptor"); + if (unlikely(!src || !dlen)) return -1; - } rte_memcpy((uint8_t *)data, src, dlen); data += dlen; @@ -609,73 +629,158 @@ copy_data(void *dst_data, struct vhost_crypto_data_req *vc_req, return 0; } -static __rte_always_inline void * -get_data_ptr(struct vhost_crypto_data_req *vc_req, struct vring_desc **cur_desc, - uint32_t size, uint8_t perm) +static void +write_back_data(struct vhost_crypto_data_req *vc_req) { - void *data; - uint64_t dlen = (*cur_desc)->len; - - data = IOVA_TO_VVA(void *, vc_req, (*cur_desc)->addr, &dlen, perm); - if (unlikely(!data || dlen != (*cur_desc)->len)) { - VC_LOG_ERR("Failed to map object"); - return NULL; + struct vhost_crypto_writeback_data *wb_data = vc_req->wb, *wb_last; + + while (wb_data) { + rte_prefetch0(wb_data->next); + rte_memcpy(wb_data->dst, wb_data->src, wb_data->len); + wb_last = wb_data; + wb_data = wb_data->next; + rte_mempool_put(vc_req->wb_pool, wb_last); } +} - if (unlikely(move_desc(vc_req->head, cur_desc, size) < 0)) - return NULL; +static void +free_wb_data(struct vhost_crypto_writeback_data *wb_data, + struct rte_mempool *mp) +{ + while (wb_data->next != NULL) + free_wb_data(wb_data->next, mp); - return data; + rte_mempool_put(mp, wb_data); } -static int -write_back_data(struct rte_crypto_op *op, struct vhost_crypto_data_req *vc_req) +/** + * The function will allocate a vhost_crypto_writeback_data linked list + * containing the source and destination data pointers for the write back + * operation after dequeued from Cryptodev PMD queues. + * + * @param vc_req + * The vhost crypto data request pointer + * @param cur_desc + * The pointer of the current in use descriptor pointer. The content of + * cur_desc is expected to be updated after the function execution. + * @param end_wb_data + * The last write back data element to be returned. It is used only in cipher + * and hash chain operations. + * @param src + * The source data pointer + * @param offset + * The offset to both source and destination data. For source data the offset + * is the number of bytes between src and start point of cipher operation. For + * destination data the offset is the number of bytes from *cur_desc->addr + * to the point where the src will be written to. + * @param write_back_len + * The size of the write back length. + * @return + * The pointer to the start of the write back data linked list. + */ +static struct vhost_crypto_writeback_data * +prepare_write_back_data(struct vhost_crypto_data_req *vc_req, + struct vring_desc **cur_desc, + struct vhost_crypto_writeback_data **end_wb_data, + uint8_t *src, + uint32_t offset, + uint64_t write_back_len) { - struct rte_mbuf *mbuf = op->sym->m_dst; - struct vring_desc *head = vc_req->head; - struct vring_desc *desc = vc_req->wb_desc; - int left = vc_req->wb_len; - uint32_t to_write; - uint8_t *src_data = mbuf->buf_addr, *dst; + struct vhost_crypto_writeback_data *wb_data, *head; + struct vring_desc *desc = *cur_desc; uint64_t dlen; + uint8_t *dst; + int ret; - rte_prefetch0(&head[desc->next]); - to_write = RTE_MIN(desc->len, (uint32_t)left); - dlen = desc->len; - dst = IOVA_TO_VVA(uint8_t *, vc_req, desc->addr, &dlen, - VHOST_ACCESS_RW); - if (unlikely(!dst || dlen != desc->len)) { - VC_LOG_ERR("Failed to map descriptor"); - return -1; + ret = rte_mempool_get(vc_req->wb_pool, (void **)&head); + if (unlikely(ret < 0)) { + VC_LOG_ERR("no memory"); + goto error_exit; } - rte_memcpy(dst, src_data, to_write); - left -= to_write; - src_data += to_write; + wb_data = head; - while ((desc->flags & VRING_DESC_F_NEXT) && left > 0) { - desc = &head[desc->next]; - rte_prefetch0(&head[desc->next]); - to_write = RTE_MIN(desc->len, (uint32_t)left); + if (likely(desc->len > offset)) { + wb_data->src = src + offset; dlen = desc->len; - dst = IOVA_TO_VVA(uint8_t *, vc_req, desc->addr, &dlen, - VHOST_ACCESS_RW); + dst = IOVA_TO_VVA(uint8_t *, vc_req, desc->addr, + &dlen, VHOST_ACCESS_RW) + offset; if (unlikely(!dst || dlen != desc->len)) { VC_LOG_ERR("Failed to map descriptor"); - return -1; + goto error_exit; } - rte_memcpy(dst, src_data, to_write); - left -= to_write; - src_data += to_write; - } + wb_data->dst = dst; + wb_data->len = desc->len - offset; + write_back_len -= wb_data->len; + src += offset + wb_data->len; + offset = 0; + + if (unlikely(write_back_len)) { + ret = rte_mempool_get(vc_req->wb_pool, + (void **)&(wb_data->next)); + if (unlikely(ret < 0)) { + VC_LOG_ERR("no memory"); + goto error_exit; + } - if (unlikely(left < 0)) { - VC_LOG_ERR("Incorrect virtio descriptor"); - return -1; + wb_data = wb_data->next; + } else + wb_data->next = NULL; + } else + offset -= desc->len; + + while (write_back_len) { + desc = &vc_req->head[desc->next]; + if (unlikely(!(desc->flags & VRING_DESC_F_WRITE))) { + VC_LOG_ERR("incorrect descriptor"); + goto error_exit; + } + + if (desc->len <= offset) { + offset -= desc->len; + continue; + } + + dlen = desc->len; + dst = IOVA_TO_VVA(uint8_t *, vc_req, desc->addr, &dlen, + VHOST_ACCESS_RW) + offset; + if (unlikely(dst == NULL || dlen != desc->len)) { + VC_LOG_ERR("Failed to map descriptor"); + goto error_exit; + } + + wb_data->src = src; + wb_data->dst = dst; + wb_data->len = RTE_MIN(desc->len - offset, write_back_len); + write_back_len -= wb_data->len; + src += wb_data->len; + offset = 0; + + if (write_back_len) { + ret = rte_mempool_get(vc_req->wb_pool, + (void **)&(wb_data->next)); + if (unlikely(ret < 0)) { + VC_LOG_ERR("no memory"); + goto error_exit; + } + + wb_data = wb_data->next; + } else + wb_data->next = NULL; } - return 0; + *cur_desc = &vc_req->head[desc->next]; + + *end_wb_data = wb_data; + + return head; + +error_exit: + if (head) + free_wb_data(head, vc_req->wb_pool); + + return NULL; } static uint8_t @@ -685,6 +790,7 @@ prepare_sym_cipher_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, struct vring_desc *cur_desc) { struct vring_desc *desc = cur_desc; + struct vhost_crypto_writeback_data *ewb = NULL; struct rte_mbuf *m_src = op->sym->m_src, *m_dst = op->sym->m_dst; uint8_t *iv_data = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET); uint8_t ret = 0; @@ -703,16 +809,25 @@ prepare_sym_cipher_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE: m_src->buf_iova = gpa_to_hpa(vcrypto->dev, desc->addr, cipher->para.src_data_len); - m_src->buf_addr = get_data_ptr(vc_req, &desc, - cipher->para.src_data_len, VHOST_ACCESS_RO); + m_src->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RO); if (unlikely(m_src->buf_iova == 0 || m_src->buf_addr == NULL)) { VC_LOG_ERR("zero_copy may fail due to cross page data"); ret = VIRTIO_CRYPTO_ERR; goto error_exit; } + + if (unlikely(move_desc(vc_req->head, &desc, + cipher->para.src_data_len) < 0)) { + VC_LOG_ERR("Incorrect descriptor"); + ret = VIRTIO_CRYPTO_ERR; + goto error_exit; + } + break; case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE: + vc_req->wb_pool = vcrypto->wb_pool; + if (unlikely(cipher->para.src_data_len > RTE_MBUF_DEFAULT_BUF_SIZE)) { VC_LOG_ERR("Not enough space to do data copy"); @@ -743,24 +858,31 @@ prepare_sym_cipher_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE: m_dst->buf_iova = gpa_to_hpa(vcrypto->dev, desc->addr, cipher->para.dst_data_len); - m_dst->buf_addr = get_data_ptr(vc_req, &desc, - cipher->para.dst_data_len, VHOST_ACCESS_RW); + m_dst->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RW); if (unlikely(m_dst->buf_iova == 0 || m_dst->buf_addr == NULL)) { VC_LOG_ERR("zero_copy may fail due to cross page data"); ret = VIRTIO_CRYPTO_ERR; goto error_exit; } + if (unlikely(move_desc(vc_req->head, &desc, + cipher->para.dst_data_len) < 0)) { + VC_LOG_ERR("Incorrect descriptor"); + ret = VIRTIO_CRYPTO_ERR; + goto error_exit; + } + m_dst->data_len = cipher->para.dst_data_len; break; case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE: - vc_req->wb_desc = desc; - vc_req->wb_len = cipher->para.dst_data_len; - if (unlikely(move_desc(vc_req->head, &desc, - vc_req->wb_len) < 0)) { + vc_req->wb = prepare_write_back_data(vc_req, &desc, &ewb, + rte_pktmbuf_mtod(m_src, uint8_t *), 0, + cipher->para.dst_data_len); + if (unlikely(vc_req->wb == NULL)) { ret = VIRTIO_CRYPTO_ERR; goto error_exit; } + break; default: ret = VIRTIO_CRYPTO_BADMSG; @@ -774,7 +896,7 @@ prepare_sym_cipher_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, op->sym->cipher.data.offset = 0; op->sym->cipher.data.length = cipher->para.src_data_len; - vc_req->inhdr = get_data_ptr(vc_req, &desc, INHDR_LEN, VHOST_ACCESS_WO); + vc_req->inhdr = get_data_ptr(vc_req, desc, VHOST_ACCESS_WO); if (unlikely(vc_req->inhdr == NULL)) { ret = VIRTIO_CRYPTO_BADMSG; goto error_exit; @@ -786,6 +908,9 @@ prepare_sym_cipher_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, return 0; error_exit: + if (vc_req->wb) + free_wb_data(vc_req->wb, vc_req->wb_pool); + vc_req->len = INHDR_LEN; return ret; } @@ -796,7 +921,8 @@ prepare_sym_chain_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, struct virtio_crypto_alg_chain_data_req *chain, struct vring_desc *cur_desc) { - struct vring_desc *desc = cur_desc; + struct vring_desc *desc = cur_desc, *digest_desc; + struct vhost_crypto_writeback_data *ewb = NULL, *ewb2 = NULL; struct rte_mbuf *m_src = op->sym->m_src, *m_dst = op->sym->m_dst; uint8_t *iv_data = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET); uint32_t digest_offset; @@ -812,21 +938,30 @@ prepare_sym_chain_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, } m_src->data_len = chain->para.src_data_len; - m_dst->data_len = chain->para.dst_data_len; switch (vcrypto->option) { case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE: + m_dst->data_len = chain->para.dst_data_len; + m_src->buf_iova = gpa_to_hpa(vcrypto->dev, desc->addr, chain->para.src_data_len); - m_src->buf_addr = get_data_ptr(vc_req, &desc, - chain->para.src_data_len, VHOST_ACCESS_RO); + m_src->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RO); if (unlikely(m_src->buf_iova == 0 || m_src->buf_addr == NULL)) { VC_LOG_ERR("zero_copy may fail due to cross page data"); ret = VIRTIO_CRYPTO_ERR; goto error_exit; } + + if (unlikely(move_desc(vc_req->head, &desc, + chain->para.src_data_len) < 0)) { + VC_LOG_ERR("Incorrect descriptor"); + ret = VIRTIO_CRYPTO_ERR; + goto error_exit; + } break; case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE: + vc_req->wb_pool = vcrypto->wb_pool; + if (unlikely(chain->para.src_data_len > RTE_MBUF_DEFAULT_BUF_SIZE)) { VC_LOG_ERR("Not enough space to do data copy"); @@ -838,6 +973,7 @@ prepare_sym_chain_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, ret = VIRTIO_CRYPTO_BADMSG; goto error_exit; } + break; default: ret = VIRTIO_CRYPTO_BADMSG; @@ -856,46 +992,70 @@ prepare_sym_chain_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE: m_dst->buf_iova = gpa_to_hpa(vcrypto->dev, desc->addr, chain->para.dst_data_len); - m_dst->buf_addr = get_data_ptr(vc_req, &desc, - chain->para.dst_data_len, VHOST_ACCESS_RW); + m_dst->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RW); if (unlikely(m_dst->buf_iova == 0 || m_dst->buf_addr == NULL)) { VC_LOG_ERR("zero_copy may fail due to cross page data"); ret = VIRTIO_CRYPTO_ERR; goto error_exit; } + if (unlikely(move_desc(vc_req->head, &desc, + chain->para.dst_data_len) < 0)) { + VC_LOG_ERR("Incorrect descriptor"); + ret = VIRTIO_CRYPTO_ERR; + goto error_exit; + } + op->sym->auth.digest.phys_addr = gpa_to_hpa(vcrypto->dev, desc->addr, chain->para.hash_result_len); - op->sym->auth.digest.data = get_data_ptr(vc_req, &desc, - chain->para.hash_result_len, VHOST_ACCESS_RW); + op->sym->auth.digest.data = get_data_ptr(vc_req, desc, + VHOST_ACCESS_RW); if (unlikely(op->sym->auth.digest.phys_addr == 0)) { VC_LOG_ERR("zero_copy may fail due to cross page data"); ret = VIRTIO_CRYPTO_ERR; goto error_exit; } + + if (unlikely(move_desc(vc_req->head, &desc, + chain->para.hash_result_len) < 0)) { + VC_LOG_ERR("Incorrect descriptor"); + ret = VIRTIO_CRYPTO_ERR; + goto error_exit; + } + break; case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE: - digest_offset = m_dst->data_len; - digest_addr = rte_pktmbuf_mtod_offset(m_dst, void *, - digest_offset); + vc_req->wb = prepare_write_back_data(vc_req, &desc, &ewb, + rte_pktmbuf_mtod(m_src, uint8_t *), + chain->para.cipher_start_src_offset, + chain->para.dst_data_len - + chain->para.cipher_start_src_offset); + if (unlikely(vc_req->wb == NULL)) { + ret = VIRTIO_CRYPTO_ERR; + goto error_exit; + } - vc_req->wb_desc = desc; - vc_req->wb_len = m_dst->data_len + chain->para.hash_result_len; + digest_offset = m_src->data_len; + digest_addr = rte_pktmbuf_mtod_offset(m_src, void *, + digest_offset); + digest_desc = desc; - if (unlikely(move_desc(vc_req->head, &desc, - chain->para.dst_data_len) < 0)) { - ret = VIRTIO_CRYPTO_BADMSG; + /** create a wb_data for digest */ + ewb->next = prepare_write_back_data(vc_req, &desc, &ewb2, + digest_addr, 0, chain->para.hash_result_len); + if (unlikely(ewb->next == NULL)) { + ret = VIRTIO_CRYPTO_ERR; goto error_exit; } - if (unlikely(copy_data(digest_addr, vc_req, &desc, + if (unlikely(copy_data(digest_addr, vc_req, &digest_desc, chain->para.hash_result_len)) < 0) { ret = VIRTIO_CRYPTO_BADMSG; goto error_exit; } op->sym->auth.digest.data = digest_addr; - op->sym->auth.digest.phys_addr = rte_pktmbuf_iova_offset(m_dst, + op->sym->auth.digest.phys_addr = rte_pktmbuf_iova_offset(m_src, digest_offset); break; default: @@ -904,7 +1064,7 @@ prepare_sym_chain_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, } /* record inhdr */ - vc_req->inhdr = get_data_ptr(vc_req, &desc, INHDR_LEN, VHOST_ACCESS_WO); + vc_req->inhdr = get_data_ptr(vc_req, desc, VHOST_ACCESS_WO); if (unlikely(vc_req->inhdr == NULL)) { ret = VIRTIO_CRYPTO_BADMSG; goto error_exit; @@ -927,6 +1087,8 @@ prepare_sym_chain_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, return 0; error_exit: + if (vc_req->wb) + free_wb_data(vc_req->wb, vc_req->wb_pool); vc_req->len = INHDR_LEN; return ret; } @@ -967,7 +1129,7 @@ vhost_crypto_process_one_req(struct vhost_crypto *vcrypto, vc_req->head = head; vc_req->zero_copy = vcrypto->option; - req = get_data_ptr(vc_req, &desc, sizeof(*req), VHOST_ACCESS_RO); + req = get_data_ptr(vc_req, desc, VHOST_ACCESS_RO); if (unlikely(req == NULL)) { switch (vcrypto->option) { case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE: @@ -988,6 +1150,12 @@ vhost_crypto_process_one_req(struct vhost_crypto *vcrypto, VC_LOG_ERR("Invalid option"); goto error_exit; } + } else { + if (unlikely(move_desc(vc_req->head, &desc, + sizeof(*req)) < 0)) { + VC_LOG_ERR("Incorrect descriptor"); + goto error_exit; + } } switch (req->header.opcode) { @@ -1062,7 +1230,6 @@ vhost_crypto_finalize_one_request(struct rte_crypto_op *op, struct rte_mbuf *m_dst = op->sym->m_dst; struct vhost_crypto_data_req *vc_req = rte_mbuf_to_priv(m_src); uint16_t desc_idx; - int ret = 0; if (unlikely(!vc_req)) { VC_LOG_ERR("Failed to retrieve vc_req"); @@ -1077,19 +1244,18 @@ vhost_crypto_finalize_one_request(struct rte_crypto_op *op, if (unlikely(op->status != RTE_CRYPTO_OP_STATUS_SUCCESS)) vc_req->inhdr->status = VIRTIO_CRYPTO_ERR; else { - if (vc_req->zero_copy == 0) { - ret = write_back_data(op, vc_req); - if (unlikely(ret != 0)) - vc_req->inhdr->status = VIRTIO_CRYPTO_ERR; - } + if (vc_req->zero_copy == 0) + write_back_data(vc_req); } vc_req->vq->used->ring[desc_idx].id = desc_idx; vc_req->vq->used->ring[desc_idx].len = vc_req->len; - rte_mempool_put(m_dst->pool, (void *)m_dst); rte_mempool_put(m_src->pool, (void *)m_src); + if (m_dst) + rte_mempool_put(m_dst->pool, (void *)m_dst); + return vc_req->vq; } @@ -1186,6 +1352,18 @@ rte_vhost_crypto_create(int vid, uint8_t cryptodev_id, goto error_exit; } + snprintf(name, 127, "WB_POOL_VM_%u", (uint32_t)vid); + vcrypto->wb_pool = rte_mempool_create(name, + VHOST_CRYPTO_MBUF_POOL_SIZE, + sizeof(struct vhost_crypto_writeback_data), + 128, 0, NULL, NULL, NULL, NULL, + rte_socket_id(), 0); + if (!vcrypto->wb_pool) { + VC_LOG_ERR("Failed to creath mempool"); + ret = -ENOMEM; + goto error_exit; + } + dev->extern_data = vcrypto; dev->extern_ops.pre_msg_handle = NULL; dev->extern_ops.post_msg_handle = vhost_crypto_msg_post_handler; @@ -1222,6 +1400,7 @@ rte_vhost_crypto_free(int vid) rte_hash_free(vcrypto->session_map); rte_mempool_free(vcrypto->mbuf_pool); + rte_mempool_free(vcrypto->wb_pool); rte_free(vcrypto); dev->extern_data = NULL; @@ -1257,11 +1436,30 @@ rte_vhost_crypto_set_zero_copy(int vid, enum rte_vhost_crypto_zero_copy option) if (vcrypto->option == (uint8_t)option) return 0; - if (!(rte_mempool_full(vcrypto->mbuf_pool))) { + if (!(rte_mempool_full(vcrypto->mbuf_pool)) || + !(rte_mempool_full(vcrypto->wb_pool))) { VC_LOG_ERR("Cannot update zero copy as mempool is not full"); return -EINVAL; } + if (option == RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE) { + char name[128]; + + snprintf(name, 127, "WB_POOL_VM_%u", (uint32_t)vid); + vcrypto->wb_pool = rte_mempool_create(name, + VHOST_CRYPTO_MBUF_POOL_SIZE, + sizeof(struct vhost_crypto_writeback_data), + 128, 0, NULL, NULL, NULL, NULL, + rte_socket_id(), 0); + if (!vcrypto->wb_pool) { + VC_LOG_ERR("Failed to creath mbuf pool"); + return -ENOMEM; + } + } else { + rte_mempool_free(vcrypto->wb_pool); + vcrypto->wb_pool = NULL; + } + vcrypto->option = (uint8_t)option; return 0; @@ -1277,9 +1475,8 @@ rte_vhost_crypto_fetch_requests(int vid, uint32_t qid, struct vhost_virtqueue *vq; uint16_t avail_idx; uint16_t start_idx; - uint16_t required; uint16_t count; - uint16_t i; + uint16_t i = 0; if (unlikely(dev == NULL)) { VC_LOG_ERR("Invalid vid %i", vid); @@ -1311,27 +1508,66 @@ rte_vhost_crypto_fetch_requests(int vid, uint32_t qid, /* for zero copy, we need 2 empty mbufs for src and dst, otherwise * we need only 1 mbuf as src and dst */ - required = count * 2; - if (unlikely(rte_mempool_get_bulk(vcrypto->mbuf_pool, (void **)mbufs, - required) < 0)) { - VC_LOG_ERR("Insufficient memory"); - return -ENOMEM; - } + switch (vcrypto->option) { + case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE: + if (unlikely(rte_mempool_get_bulk(vcrypto->mbuf_pool, + (void **)mbufs, count * 2) < 0)) { + VC_LOG_ERR("Insufficient memory"); + return -ENOMEM; + } - for (i = 0; i < count; i++) { - uint16_t used_idx = (start_idx + i) & (vq->size - 1); - uint16_t desc_idx = vq->avail->ring[used_idx]; - struct vring_desc *head = &vq->desc[desc_idx]; - struct rte_crypto_op *op = ops[i]; + for (i = 0; i < count; i++) { + uint16_t used_idx = (start_idx + i) & (vq->size - 1); + uint16_t desc_idx = vq->avail->ring[used_idx]; + struct vring_desc *head = &vq->desc[desc_idx]; + struct rte_crypto_op *op = ops[i]; - op->sym->m_src = mbufs[i * 2]; - op->sym->m_dst = mbufs[i * 2 + 1]; - op->sym->m_src->data_off = 0; - op->sym->m_dst->data_off = 0; + op->sym->m_src = mbufs[i * 2]; + op->sym->m_dst = mbufs[i * 2 + 1]; + op->sym->m_src->data_off = 0; + op->sym->m_dst->data_off = 0; + + if (unlikely(vhost_crypto_process_one_req(vcrypto, vq, + op, head, desc_idx)) < 0) + break; + } + + if (unlikely(i < count)) + rte_mempool_put_bulk(vcrypto->mbuf_pool, + (void **)&mbufs[i * 2], + (count - i) * 2); + + break; + + case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE: + if (unlikely(rte_mempool_get_bulk(vcrypto->mbuf_pool, + (void **)mbufs, count) < 0)) { + VC_LOG_ERR("Insufficient memory"); + return -ENOMEM; + } + + for (i = 0; i < count; i++) { + uint16_t used_idx = (start_idx + i) & (vq->size - 1); + uint16_t desc_idx = vq->avail->ring[used_idx]; + struct vring_desc *head = &vq->desc[desc_idx]; + struct rte_crypto_op *op = ops[i]; + + op->sym->m_src = mbufs[i]; + op->sym->m_dst = NULL; + op->sym->m_src->data_off = 0; + + if (unlikely(vhost_crypto_process_one_req(vcrypto, vq, + op, head, desc_idx)) < 0) + break; + } + + if (unlikely(i < count)) + rte_mempool_put_bulk(vcrypto->mbuf_pool, + (void **)&mbufs[i], + count - i); + + break; - if (unlikely(vhost_crypto_process_one_req(vcrypto, vq, op, head, - desc_idx)) < 0) - break; } vq->last_used_idx += i; diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c index cc154f31..3ea64eba 100644 --- a/lib/librte_vhost/vhost_user.c +++ b/lib/librte_vhost/vhost_user.c @@ -1732,7 +1732,7 @@ read_vhost_message(int sockfd, struct VhostUserMsg *msg) if (ret <= 0) return ret; - if (msg && msg->size) { + if (msg->size) { if (msg->size > sizeof(msg->payload)) { RTE_LOG(ERR, VHOST_CONFIG, "invalid msg size: %d\n", msg->size); diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c index 8ad30c94..5e1a1a72 100644 --- a/lib/librte_vhost/virtio_net.c +++ b/lib/librte_vhost/virtio_net.c @@ -598,7 +598,7 @@ reserve_avail_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq, avail_idx, &desc_count, buf_vec, &vec_idx, &buf_id, &len, - VHOST_ACCESS_RO) < 0)) + VHOST_ACCESS_RW) < 0)) return -1; len = RTE_MIN(len, size); @@ -1503,7 +1503,7 @@ virtio_dev_tx_packed(struct virtio_net *dev, struct vhost_virtqueue *vq, vq->last_avail_idx, &desc_count, buf_vec, &nr_vec, &buf_id, &dummy_len, - VHOST_ACCESS_RW) < 0)) + VHOST_ACCESS_RO) < 0)) break; if (likely(dev->dequeue_zero_copy == 0)) -- cgit 1.2.3-korg