diff options
Diffstat (limited to 'lib/librte_eventdev')
| -rw-r--r-- | lib/librte_eventdev/Makefile | 4 | ||||
| -rw-r--r-- | lib/librte_eventdev/meson.build | 8 | ||||
| -rw-r--r-- | lib/librte_eventdev/rte_event_eth_rx_adapter.c | 6 | ||||
| -rw-r--r-- | lib/librte_eventdev/rte_event_eth_rx_adapter.h | 4 | ||||
| -rw-r--r-- | lib/librte_eventdev/rte_event_eth_tx_adapter.c | 1138 | ||||
| -rw-r--r-- | lib/librte_eventdev/rte_event_eth_tx_adapter.h | 462 | ||||
| -rw-r--r-- | lib/librte_eventdev/rte_eventdev.c | 70 | ||||
| -rw-r--r-- | lib/librte_eventdev/rte_eventdev.h | 74 | ||||
| -rw-r--r-- | lib/librte_eventdev/rte_eventdev_pmd.h | 225 | ||||
| -rw-r--r-- | lib/librte_eventdev/rte_eventdev_version.map | 13 |
10 files changed, 1973 insertions, 31 deletions
diff --git a/lib/librte_eventdev/Makefile b/lib/librte_eventdev/Makefile index 47f599a6..94961870 100644 --- a/lib/librte_eventdev/Makefile +++ b/lib/librte_eventdev/Makefile @@ -8,7 +8,7 @@ include $(RTE_SDK)/mk/rte.vars.mk LIB = librte_eventdev.a # library version -LIBABIVER := 5 +LIBABIVER := 6 # build flags CFLAGS += -DALLOW_EXPERIMENTAL_API @@ -28,6 +28,7 @@ SRCS-y += rte_event_ring.c SRCS-y += rte_event_eth_rx_adapter.c SRCS-y += rte_event_timer_adapter.c SRCS-y += rte_event_crypto_adapter.c +SRCS-y += rte_event_eth_tx_adapter.c # export include files SYMLINK-y-include += rte_eventdev.h @@ -39,6 +40,7 @@ SYMLINK-y-include += rte_event_eth_rx_adapter.h SYMLINK-y-include += rte_event_timer_adapter.h SYMLINK-y-include += rte_event_timer_adapter_pmd.h SYMLINK-y-include += rte_event_crypto_adapter.h +SYMLINK-y-include += rte_event_eth_tx_adapter.h # versioning export map EXPORT_MAP := rte_eventdev_version.map diff --git a/lib/librte_eventdev/meson.build b/lib/librte_eventdev/meson.build index 3cbaf298..6becfe86 100644 --- a/lib/librte_eventdev/meson.build +++ b/lib/librte_eventdev/meson.build @@ -1,7 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2017 Intel Corporation -version = 5 +version = 6 allow_experimental_apis = true if host_machine.system() == 'linux' @@ -14,7 +14,8 @@ sources = files('rte_eventdev.c', 'rte_event_ring.c', 'rte_event_eth_rx_adapter.c', 'rte_event_timer_adapter.c', - 'rte_event_crypto_adapter.c') + 'rte_event_crypto_adapter.c', + 'rte_event_eth_tx_adapter.c') headers = files('rte_eventdev.h', 'rte_eventdev_pmd.h', 'rte_eventdev_pmd_pci.h', @@ -23,5 +24,6 @@ headers = files('rte_eventdev.h', 'rte_event_eth_rx_adapter.h', 'rte_event_timer_adapter.h', 'rte_event_timer_adapter_pmd.h', - 'rte_event_crypto_adapter.h') + 'rte_event_crypto_adapter.h', + 'rte_event_eth_tx_adapter.h') deps += ['ring', 'ethdev', 'hash', 'mempool', 'mbuf', 'timer', 'cryptodev'] diff --git a/lib/librte_eventdev/rte_event_eth_rx_adapter.c b/lib/librte_eventdev/rte_event_eth_rx_adapter.c index f5e5a0b5..71d008cd 100644 --- a/lib/librte_eventdev/rte_event_eth_rx_adapter.c +++ b/lib/librte_eventdev/rte_event_eth_rx_adapter.c @@ -1125,7 +1125,6 @@ rxa_poll(struct rte_event_eth_rx_adapter *rx_adapter) wrr_pos = rx_adapter->wrr_pos; max_nb_rx = rx_adapter->max_nb_rx; buf = &rx_adapter->event_enqueue_buffer; - stats = &rx_adapter->stats; /* Iterate through a WRR sequence */ for (num_queue = 0; num_queue < rx_adapter->wrr_len; num_queue++) { @@ -1998,8 +1997,7 @@ rte_event_eth_rx_adapter_create_ext(uint8_t id, uint8_t dev_id, rx_adapter->id = id; strcpy(rx_adapter->mem_name, mem_name); rx_adapter->eth_devices = rte_zmalloc_socket(rx_adapter->mem_name, - /* FIXME: incompatible with hotplug */ - rte_eth_dev_count_total() * + RTE_MAX_ETHPORTS * sizeof(struct eth_device_info), 0, socket_id); rte_convert_rss_key((const uint32_t *)default_rss_key, @@ -2012,7 +2010,7 @@ rte_event_eth_rx_adapter_create_ext(uint8_t id, uint8_t dev_id, return -ENOMEM; } rte_spinlock_init(&rx_adapter->rx_lock); - RTE_ETH_FOREACH_DEV(i) + for (i = 0; i < RTE_MAX_ETHPORTS; i++) rx_adapter->eth_devices[i].dev = &rte_eth_devices[i]; event_eth_rx_adapter[id] = rx_adapter; diff --git a/lib/librte_eventdev/rte_event_eth_rx_adapter.h b/lib/librte_eventdev/rte_event_eth_rx_adapter.h index 332ee216..863b72a1 100644 --- a/lib/librte_eventdev/rte_event_eth_rx_adapter.h +++ b/lib/librte_eventdev/rte_event_eth_rx_adapter.h @@ -76,10 +76,6 @@ * rte_event_eth_rx_adapter_cb_register() function allows the * application to register a callback that selects which packets to enqueue * to the event device. - * - * Note: - * 1) Devices created after an instance of rte_event_eth_rx_adapter_create - * should be added to a new instance of the rx adapter. */ #ifdef __cplusplus diff --git a/lib/librte_eventdev/rte_event_eth_tx_adapter.c b/lib/librte_eventdev/rte_event_eth_tx_adapter.c new file mode 100644 index 00000000..3a21defb --- /dev/null +++ b/lib/librte_eventdev/rte_event_eth_tx_adapter.c @@ -0,0 +1,1138 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation. + */ +#include <rte_spinlock.h> +#include <rte_service_component.h> +#include <rte_ethdev.h> + +#include "rte_eventdev_pmd.h" +#include "rte_event_eth_tx_adapter.h" + +#define TXA_BATCH_SIZE 32 +#define TXA_SERVICE_NAME_LEN 32 +#define TXA_MEM_NAME_LEN 32 +#define TXA_FLUSH_THRESHOLD 1024 +#define TXA_RETRY_CNT 100 +#define TXA_MAX_NB_TX 128 +#define TXA_INVALID_DEV_ID INT32_C(-1) +#define TXA_INVALID_SERVICE_ID INT64_C(-1) + +#define txa_evdev(id) (&rte_eventdevs[txa_dev_id_array[(id)]]) + +#define txa_dev_caps_get(id) txa_evdev((id))->dev_ops->eth_tx_adapter_caps_get + +#define txa_dev_adapter_create(t) txa_evdev(t)->dev_ops->eth_tx_adapter_create + +#define txa_dev_adapter_create_ext(t) \ + txa_evdev(t)->dev_ops->eth_tx_adapter_create + +#define txa_dev_adapter_free(t) txa_evdev(t)->dev_ops->eth_tx_adapter_free + +#define txa_dev_queue_add(id) txa_evdev(id)->dev_ops->eth_tx_adapter_queue_add + +#define txa_dev_queue_del(t) txa_evdev(t)->dev_ops->eth_tx_adapter_queue_del + +#define txa_dev_start(t) txa_evdev(t)->dev_ops->eth_tx_adapter_start + +#define txa_dev_stop(t) txa_evdev(t)->dev_ops->eth_tx_adapter_stop + +#define txa_dev_stats_reset(t) txa_evdev(t)->dev_ops->eth_tx_adapter_stats_reset + +#define txa_dev_stats_get(t) txa_evdev(t)->dev_ops->eth_tx_adapter_stats_get + +#define RTE_EVENT_ETH_TX_ADAPTER_ID_VALID_OR_ERR_RET(id, retval) \ +do { \ + if (!txa_valid_id(id)) { \ + RTE_EDEV_LOG_ERR("Invalid eth Rx adapter id = %d", id); \ + return retval; \ + } \ +} while (0) + +#define TXA_CHECK_OR_ERR_RET(id) \ +do {\ + int ret; \ + RTE_EVENT_ETH_TX_ADAPTER_ID_VALID_OR_ERR_RET((id), -EINVAL); \ + ret = txa_init(); \ + if (ret != 0) \ + return ret; \ + if (!txa_adapter_exist((id))) \ + return -EINVAL; \ +} while (0) + +/* Tx retry callback structure */ +struct txa_retry { + /* Ethernet port id */ + uint16_t port_id; + /* Tx queue */ + uint16_t tx_queue; + /* Adapter ID */ + uint8_t id; +}; + +/* Per queue structure */ +struct txa_service_queue_info { + /* Queue has been added */ + uint8_t added; + /* Retry callback argument */ + struct txa_retry txa_retry; + /* Tx buffer */ + struct rte_eth_dev_tx_buffer *tx_buf; +}; + +/* PMD private structure */ +struct txa_service_data { + /* Max mbufs processed in any service function invocation */ + uint32_t max_nb_tx; + /* Number of Tx queues in adapter */ + uint32_t nb_queues; + /* Synchronization with data path */ + rte_spinlock_t tx_lock; + /* Event port ID */ + uint8_t port_id; + /* Event device identifier */ + uint8_t eventdev_id; + /* Highest port id supported + 1 */ + uint16_t dev_count; + /* Loop count to flush Tx buffers */ + int loop_cnt; + /* Per ethernet device structure */ + struct txa_service_ethdev *txa_ethdev; + /* Statistics */ + struct rte_event_eth_tx_adapter_stats stats; + /* Adapter Identifier */ + uint8_t id; + /* Conf arg must be freed */ + uint8_t conf_free; + /* Configuration callback */ + rte_event_eth_tx_adapter_conf_cb conf_cb; + /* Configuration callback argument */ + void *conf_arg; + /* socket id */ + int socket_id; + /* Per adapter EAL service */ + int64_t service_id; + /* Memory allocation name */ + char mem_name[TXA_MEM_NAME_LEN]; +} __rte_cache_aligned; + +/* Per eth device structure */ +struct txa_service_ethdev { + /* Pointer to ethernet device */ + struct rte_eth_dev *dev; + /* Number of queues added */ + uint16_t nb_queues; + /* PMD specific queue data */ + void *queues; +}; + +/* Array of adapter instances, initialized with event device id + * when adapter is created + */ +static int *txa_dev_id_array; + +/* Array of pointers to service implementation data */ +static struct txa_service_data **txa_service_data_array; + +static int32_t txa_service_func(void *args); +static int txa_service_adapter_create_ext(uint8_t id, + struct rte_eventdev *dev, + rte_event_eth_tx_adapter_conf_cb conf_cb, + void *conf_arg); +static int txa_service_queue_del(uint8_t id, + const struct rte_eth_dev *dev, + int32_t tx_queue_id); + +static int +txa_adapter_exist(uint8_t id) +{ + return txa_dev_id_array[id] != TXA_INVALID_DEV_ID; +} + +static inline int +txa_valid_id(uint8_t id) +{ + return id < RTE_EVENT_ETH_TX_ADAPTER_MAX_INSTANCE; +} + +static void * +txa_memzone_array_get(const char *name, unsigned int elt_size, int nb_elems) +{ + const struct rte_memzone *mz; + unsigned int sz; + + sz = elt_size * nb_elems; + sz = RTE_ALIGN(sz, RTE_CACHE_LINE_SIZE); + + mz = rte_memzone_lookup(name); + if (mz == NULL) { + mz = rte_memzone_reserve_aligned(name, sz, rte_socket_id(), 0, + RTE_CACHE_LINE_SIZE); + if (mz == NULL) { + RTE_EDEV_LOG_ERR("failed to reserve memzone" + " name = %s err = %" + PRId32, name, rte_errno); + return NULL; + } + } + + return mz->addr; +} + +static int +txa_dev_id_array_init(void) +{ + if (txa_dev_id_array == NULL) { + int i; + + txa_dev_id_array = txa_memzone_array_get("txa_adapter_array", + sizeof(int), + RTE_EVENT_ETH_TX_ADAPTER_MAX_INSTANCE); + if (txa_dev_id_array == NULL) + return -ENOMEM; + + for (i = 0; i < RTE_EVENT_ETH_TX_ADAPTER_MAX_INSTANCE; i++) + txa_dev_id_array[i] = TXA_INVALID_DEV_ID; + } + + return 0; +} + +static int +txa_init(void) +{ + return txa_dev_id_array_init(); +} + +static int +txa_service_data_init(void) +{ + if (txa_service_data_array == NULL) { + txa_service_data_array = + txa_memzone_array_get("txa_service_data_array", + sizeof(int), + RTE_EVENT_ETH_TX_ADAPTER_MAX_INSTANCE); + if (txa_service_data_array == NULL) + return -ENOMEM; + } + + return 0; +} + +static inline struct txa_service_data * +txa_service_id_to_data(uint8_t id) +{ + return txa_service_data_array[id]; +} + +static inline struct txa_service_queue_info * +txa_service_queue(struct txa_service_data *txa, uint16_t port_id, + uint16_t tx_queue_id) +{ + struct txa_service_queue_info *tqi; + + if (unlikely(txa->txa_ethdev == NULL || txa->dev_count < port_id + 1)) + return NULL; + + tqi = txa->txa_ethdev[port_id].queues; + + return likely(tqi != NULL) ? tqi + tx_queue_id : NULL; +} + +static int +txa_service_conf_cb(uint8_t __rte_unused id, uint8_t dev_id, + struct rte_event_eth_tx_adapter_conf *conf, void *arg) +{ + int ret; + struct rte_eventdev *dev; + struct rte_event_port_conf *pc; + struct rte_event_dev_config dev_conf; + int started; + uint8_t port_id; + + pc = arg; + dev = &rte_eventdevs[dev_id]; + dev_conf = dev->data->dev_conf; + + started = dev->data->dev_started; + if (started) + rte_event_dev_stop(dev_id); + + port_id = dev_conf.nb_event_ports; + dev_conf.nb_event_ports += 1; + + ret = rte_event_dev_configure(dev_id, &dev_conf); + if (ret) { + RTE_EDEV_LOG_ERR("failed to configure event dev %u", + dev_id); + if (started) { + if (rte_event_dev_start(dev_id)) + return -EIO; + } + return ret; + } + + pc->disable_implicit_release = 0; + ret = rte_event_port_setup(dev_id, port_id, pc); + if (ret) { + RTE_EDEV_LOG_ERR("failed to setup event port %u\n", + port_id); + if (started) { + if (rte_event_dev_start(dev_id)) + return -EIO; + } + return ret; + } + + conf->event_port_id = port_id; + conf->max_nb_tx = TXA_MAX_NB_TX; + if (started) + ret = rte_event_dev_start(dev_id); + return ret; +} + +static int +txa_service_ethdev_alloc(struct txa_service_data *txa) +{ + struct txa_service_ethdev *txa_ethdev; + uint16_t i, dev_count; + + dev_count = rte_eth_dev_count_avail(); + if (txa->txa_ethdev && dev_count == txa->dev_count) + return 0; + + txa_ethdev = rte_zmalloc_socket(txa->mem_name, + dev_count * sizeof(*txa_ethdev), + 0, + txa->socket_id); + if (txa_ethdev == NULL) { + RTE_EDEV_LOG_ERR("Failed to alloc txa::txa_ethdev "); + return -ENOMEM; + } + + if (txa->dev_count) + memcpy(txa_ethdev, txa->txa_ethdev, + txa->dev_count * sizeof(*txa_ethdev)); + + RTE_ETH_FOREACH_DEV(i) { + if (i == dev_count) + break; + txa_ethdev[i].dev = &rte_eth_devices[i]; + } + + txa->txa_ethdev = txa_ethdev; + txa->dev_count = dev_count; + return 0; +} + +static int +txa_service_queue_array_alloc(struct txa_service_data *txa, + uint16_t port_id) +{ + struct txa_service_queue_info *tqi; + uint16_t nb_queue; + int ret; + + ret = txa_service_ethdev_alloc(txa); + if (ret != 0) + return ret; + + if (txa->txa_ethdev[port_id].queues) + return 0; + + nb_queue = txa->txa_ethdev[port_id].dev->data->nb_tx_queues; + tqi = rte_zmalloc_socket(txa->mem_name, + nb_queue * + sizeof(struct txa_service_queue_info), 0, + txa->socket_id); + if (tqi == NULL) + return -ENOMEM; + txa->txa_ethdev[port_id].queues = tqi; + return 0; +} + +static void +txa_service_queue_array_free(struct txa_service_data *txa, + uint16_t port_id) +{ + struct txa_service_ethdev *txa_ethdev; + struct txa_service_queue_info *tqi; + + txa_ethdev = &txa->txa_ethdev[port_id]; + if (txa->txa_ethdev == NULL || txa_ethdev->nb_queues != 0) + return; + + tqi = txa_ethdev->queues; + txa_ethdev->queues = NULL; + rte_free(tqi); + + if (txa->nb_queues == 0) { + rte_free(txa->txa_ethdev); + txa->txa_ethdev = NULL; + } +} + +static void +txa_service_unregister(struct txa_service_data *txa) +{ + if (txa->service_id != TXA_INVALID_SERVICE_ID) { + rte_service_component_runstate_set(txa->service_id, 0); + while (rte_service_may_be_active(txa->service_id)) + rte_pause(); + rte_service_component_unregister(txa->service_id); + } + txa->service_id = TXA_INVALID_SERVICE_ID; +} + +static int +txa_service_register(struct txa_service_data *txa) +{ + int ret; + struct rte_service_spec service; + struct rte_event_eth_tx_adapter_conf conf; + + if (txa->service_id != TXA_INVALID_SERVICE_ID) + return 0; + + memset(&service, 0, sizeof(service)); + snprintf(service.name, TXA_SERVICE_NAME_LEN, "txa_%d", txa->id); + service.socket_id = txa->socket_id; + service.callback = txa_service_func; + service.callback_userdata = txa; + service.capabilities = RTE_SERVICE_CAP_MT_SAFE; + ret = rte_service_component_register(&service, + (uint32_t *)&txa->service_id); + if (ret) { + RTE_EDEV_LOG_ERR("failed to register service %s err = %" + PRId32, service.name, ret); + return ret; + } + + ret = txa->conf_cb(txa->id, txa->eventdev_id, &conf, txa->conf_arg); + if (ret) { + txa_service_unregister(txa); + return ret; + } + + rte_service_component_runstate_set(txa->service_id, 1); + txa->port_id = conf.event_port_id; + txa->max_nb_tx = conf.max_nb_tx; + return 0; +} + +static struct rte_eth_dev_tx_buffer * +txa_service_tx_buf_alloc(struct txa_service_data *txa, + const struct rte_eth_dev *dev) +{ + struct rte_eth_dev_tx_buffer *tb; + uint16_t port_id; + + port_id = dev->data->port_id; + tb = rte_zmalloc_socket(txa->mem_name, + RTE_ETH_TX_BUFFER_SIZE(TXA_BATCH_SIZE), + 0, + rte_eth_dev_socket_id(port_id)); + if (tb == NULL) + RTE_EDEV_LOG_ERR("Failed to allocate memory for tx buffer"); + return tb; +} + +static int +txa_service_is_queue_added(struct txa_service_data *txa, + const struct rte_eth_dev *dev, + uint16_t tx_queue_id) +{ + struct txa_service_queue_info *tqi; + + tqi = txa_service_queue(txa, dev->data->port_id, tx_queue_id); + return tqi && tqi->added; +} + +static int +txa_service_ctrl(uint8_t id, int start) +{ + int ret; + struct txa_service_data *txa; + + txa = txa_service_id_to_data(id); + if (txa->service_id == TXA_INVALID_SERVICE_ID) + return 0; + + ret = rte_service_runstate_set(txa->service_id, start); + if (ret == 0 && !start) { + while (rte_service_may_be_active(txa->service_id)) + rte_pause(); + } + return ret; +} + +static void +txa_service_buffer_retry(struct rte_mbuf **pkts, uint16_t unsent, + void *userdata) +{ + struct txa_retry *tr; + struct txa_service_data *data; + struct rte_event_eth_tx_adapter_stats *stats; + uint16_t sent = 0; + unsigned int retry = 0; + uint16_t i, n; + + tr = (struct txa_retry *)(uintptr_t)userdata; + data = txa_service_id_to_data(tr->id); + stats = &data->stats; + + do { + n = rte_eth_tx_burst(tr->port_id, tr->tx_queue, + &pkts[sent], unsent - sent); + + sent += n; + } while (sent != unsent && retry++ < TXA_RETRY_CNT); + + for (i = sent; i < unsent; i++) + rte_pktmbuf_free(pkts[i]); + + stats->tx_retry += retry; + stats->tx_packets += sent; + stats->tx_dropped += unsent - sent; +} + +static void +txa_service_tx(struct txa_service_data *txa, struct rte_event *ev, + uint32_t n) +{ + uint32_t i; + uint16_t nb_tx; + struct rte_event_eth_tx_adapter_stats *stats; + + stats = &txa->stats; + + nb_tx = 0; + for (i = 0; i < n; i++) { + struct rte_mbuf *m; + uint16_t port; + uint16_t queue; + struct txa_service_queue_info *tqi; + + m = ev[i].mbuf; + port = m->port; + queue = rte_event_eth_tx_adapter_txq_get(m); + + tqi = txa_service_queue(txa, port, queue); + if (unlikely(tqi == NULL || !tqi->added)) { + rte_pktmbuf_free(m); + continue; + } + + nb_tx += rte_eth_tx_buffer(port, queue, tqi->tx_buf, m); + } + + stats->tx_packets += nb_tx; +} + +static int32_t +txa_service_func(void *args) +{ + struct txa_service_data *txa = args; + uint8_t dev_id; + uint8_t port; + uint16_t n; + uint32_t nb_tx, max_nb_tx; + struct rte_event ev[TXA_BATCH_SIZE]; + + dev_id = txa->eventdev_id; + max_nb_tx = txa->max_nb_tx; + port = txa->port_id; + + if (txa->nb_queues == 0) + return 0; + + if (!rte_spinlock_trylock(&txa->tx_lock)) + return 0; + + for (nb_tx = 0; nb_tx < max_nb_tx; nb_tx += n) { + + n = rte_event_dequeue_burst(dev_id, port, ev, RTE_DIM(ev), 0); + if (!n) + break; + txa_service_tx(txa, ev, n); + } + + if ((txa->loop_cnt++ & (TXA_FLUSH_THRESHOLD - 1)) == 0) { + + struct txa_service_ethdev *tdi; + struct txa_service_queue_info *tqi; + struct rte_eth_dev *dev; + uint16_t i; + + tdi = txa->txa_ethdev; + nb_tx = 0; + + RTE_ETH_FOREACH_DEV(i) { + uint16_t q; + + if (i == txa->dev_count) + break; + + dev = tdi[i].dev; + if (tdi[i].nb_queues == 0) + continue; + for (q = 0; q < dev->data->nb_tx_queues; q++) { + + tqi = txa_service_queue(txa, i, q); + if (unlikely(tqi == NULL || !tqi->added)) + continue; + + nb_tx += rte_eth_tx_buffer_flush(i, q, + tqi->tx_buf); + } + } + + txa->stats.tx_packets += nb_tx; + } + rte_spinlock_unlock(&txa->tx_lock); + return 0; +} + +static int +txa_service_adapter_create(uint8_t id, struct rte_eventdev *dev, + struct rte_event_port_conf *port_conf) +{ + struct txa_service_data *txa; + struct rte_event_port_conf *cb_conf; + int ret; + + cb_conf = rte_malloc(NULL, sizeof(*cb_conf), 0); + if (cb_conf == NULL) + return -ENOMEM; + + *cb_conf = *port_conf; + ret = txa_service_adapter_create_ext(id, dev, txa_service_conf_cb, + cb_conf); + if (ret) { + rte_free(cb_conf); + return ret; + } + + txa = txa_service_id_to_data(id); + txa->conf_free = 1; + return ret; +} + +static int +txa_service_adapter_create_ext(uint8_t id, struct rte_eventdev *dev, + rte_event_eth_tx_adapter_conf_cb conf_cb, + void *conf_arg) +{ + struct txa_service_data *txa; + int socket_id; + char mem_name[TXA_SERVICE_NAME_LEN]; + int ret; + + if (conf_cb == NULL) + return -EINVAL; + + socket_id = dev->data->socket_id; + snprintf(mem_name, TXA_MEM_NAME_LEN, + "rte_event_eth_txa_%d", + id); + + ret = txa_service_data_init(); + if (ret != 0) + return ret; + + txa = rte_zmalloc_socket(mem_name, + sizeof(*txa), + RTE_CACHE_LINE_SIZE, socket_id); + if (txa == NULL) { + RTE_EDEV_LOG_ERR("failed to get mem for tx adapter"); + return -ENOMEM; + } + + txa->id = id; + txa->eventdev_id = dev->data->dev_id; + txa->socket_id = socket_id; + strncpy(txa->mem_name, mem_name, TXA_SERVICE_NAME_LEN); + txa->conf_cb = conf_cb; + txa->conf_arg = conf_arg; + txa->service_id = TXA_INVALID_SERVICE_ID; + rte_spinlock_init(&txa->tx_lock); + txa_service_data_array[id] = txa; + + return 0; +} + +static int +txa_service_event_port_get(uint8_t id, uint8_t *port) +{ + struct txa_service_data *txa; + + txa = txa_service_id_to_data(id); + if (txa->service_id == TXA_INVALID_SERVICE_ID) + return -ENODEV; + + *port = txa->port_id; + return 0; +} + +static int +txa_service_adapter_free(uint8_t id) +{ + struct txa_service_data *txa; + + txa = txa_service_id_to_data(id); + if (txa->nb_queues) { + RTE_EDEV_LOG_ERR("%" PRIu16 " Tx queues not deleted", + txa->nb_queues); + return -EBUSY; + } + + if (txa->conf_free) + rte_free(txa->conf_arg); + rte_free(txa); + return 0; +} + +static int +txa_service_queue_add(uint8_t id, + __rte_unused struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev, + int32_t tx_queue_id) +{ + struct txa_service_data *txa; + struct txa_service_ethdev *tdi; + struct txa_service_queue_info *tqi; + struct rte_eth_dev_tx_buffer *tb; + struct txa_retry *txa_retry; + int ret; + + txa = txa_service_id_to_data(id); + + if (tx_queue_id == -1) { + int nb_queues; + uint16_t i, j; + uint16_t *qdone; + + nb_queues = eth_dev->data->nb_tx_queues; + if (txa->dev_count > eth_dev->data->port_id) { + tdi = &txa->txa_ethdev[eth_dev->data->port_id]; + nb_queues -= tdi->nb_queues; + } + + qdone = rte_zmalloc(txa->mem_name, + nb_queues * sizeof(*qdone), 0); + j = 0; + for (i = 0; i < nb_queues; i++) { + if (txa_service_is_queue_added(txa, eth_dev, i)) + continue; + ret = txa_service_queue_add(id, dev, eth_dev, i); + if (ret == 0) + qdone[j++] = i; + else + break; + } + + if (i != nb_queues) { + for (i = 0; i < j; i++) + txa_service_queue_del(id, eth_dev, qdone[i]); + } + rte_free(qdone); + return ret; + } + + ret = txa_service_register(txa); + if (ret) + return ret; + + rte_spinlock_lock(&txa->tx_lock); + + if (txa_service_is_queue_added(txa, eth_dev, tx_queue_id)) { + rte_spinlock_unlock(&txa->tx_lock); + return 0; + } + + ret = txa_service_queue_array_alloc(txa, eth_dev->data->port_id); + if (ret) + goto err_unlock; + + tb = txa_service_tx_buf_alloc(txa, eth_dev); + if (tb == NULL) + goto err_unlock; + + tdi = &txa->txa_ethdev[eth_dev->data->port_id]; + tqi = txa_service_queue(txa, eth_dev->data->port_id, tx_queue_id); + + txa_retry = &tqi->txa_retry; + txa_retry->id = txa->id; + txa_retry->port_id = eth_dev->data->port_id; + txa_retry->tx_queue = tx_queue_id; + + rte_eth_tx_buffer_init(tb, TXA_BATCH_SIZE); + rte_eth_tx_buffer_set_err_callback(tb, + txa_service_buffer_retry, txa_retry); + + tqi->tx_buf = tb; + tqi->added = 1; + tdi->nb_queues++; + txa->nb_queues++; + +err_unlock: + if (txa->nb_queues == 0) { + txa_service_queue_array_free(txa, + eth_dev->data->port_id); + txa_service_unregister(txa); + } + + rte_spinlock_unlock(&txa->tx_lock); + return 0; +} + +static int +txa_service_queue_del(uint8_t id, + const struct rte_eth_dev *dev, + int32_t tx_queue_id) +{ + struct txa_service_data *txa; + struct txa_service_queue_info *tqi; + struct rte_eth_dev_tx_buffer *tb; + uint16_t port_id; + + if (tx_queue_id == -1) { + uint16_t i; + int ret = -1; + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + ret = txa_service_queue_del(id, dev, i); + if (ret != 0) + break; + } + return ret; + } + + txa = txa_service_id_to_data(id); + port_id = dev->data->port_id; + + tqi = txa_service_queue(txa, port_id, tx_queue_id); + if (tqi == NULL || !tqi->added) + return 0; + + tb = tqi->tx_buf; + tqi->added = 0; + tqi->tx_buf = NULL; + rte_free(tb); + txa->nb_queues--; + txa->txa_ethdev[port_id].nb_queues--; + + txa_service_queue_array_free(txa, port_id); + return 0; +} + +static int +txa_service_id_get(uint8_t id, uint32_t *service_id) +{ + struct txa_service_data *txa; + + txa = txa_service_id_to_data(id); + if (txa->service_id == TXA_INVALID_SERVICE_ID) + return -ESRCH; + + if (service_id == NULL) + return -EINVAL; + + *service_id = txa->service_id; + return 0; +} + +static int +txa_service_start(uint8_t id) +{ + return txa_service_ctrl(id, 1); +} + +static int +txa_service_stats_get(uint8_t id, + struct rte_event_eth_tx_adapter_stats *stats) +{ + struct txa_service_data *txa; + + txa = txa_service_id_to_data(id); + *stats = txa->stats; + return 0; +} + +static int +txa_service_stats_reset(uint8_t id) +{ + struct txa_service_data *txa; + + txa = txa_service_id_to_data(id); + memset(&txa->stats, 0, sizeof(txa->stats)); + return 0; +} + +static int +txa_service_stop(uint8_t id) +{ + return txa_service_ctrl(id, 0); +} + + +int __rte_experimental +rte_event_eth_tx_adapter_create(uint8_t id, uint8_t dev_id, + struct rte_event_port_conf *port_conf) +{ + struct rte_eventdev *dev; + int ret; + + if (port_conf == NULL) + return -EINVAL; + + RTE_EVENT_ETH_TX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL); + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + + dev = &rte_eventdevs[dev_id]; + + ret = txa_init(); + if (ret != 0) + return ret; + + if (txa_adapter_exist(id)) + return -EEXIST; + + txa_dev_id_array[id] = dev_id; + if (txa_dev_adapter_create(id)) + ret = txa_dev_adapter_create(id)(id, dev); + + if (ret != 0) { + txa_dev_id_array[id] = TXA_INVALID_DEV_ID; + return ret; + } + + ret = txa_service_adapter_create(id, dev, port_conf); + if (ret != 0) { + if (txa_dev_adapter_free(id)) + txa_dev_adapter_free(id)(id, dev); + txa_dev_id_array[id] = TXA_INVALID_DEV_ID; + return ret; + } + + txa_dev_id_array[id] = dev_id; + return 0; +} + +int __rte_experimental +rte_event_eth_tx_adapter_create_ext(uint8_t id, uint8_t dev_id, + rte_event_eth_tx_adapter_conf_cb conf_cb, + void *conf_arg) +{ + struct rte_eventdev *dev; + int ret; + + RTE_EVENT_ETH_TX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL); + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + + ret = txa_init(); + if (ret != 0) + return ret; + + if (txa_adapter_exist(id)) + return -EINVAL; + + dev = &rte_eventdevs[dev_id]; + + txa_dev_id_array[id] = dev_id; + if (txa_dev_adapter_create_ext(id)) + ret = txa_dev_adapter_create_ext(id)(id, dev); + + if (ret != 0) { + txa_dev_id_array[id] = TXA_INVALID_DEV_ID; + return ret; + } + + ret = txa_service_adapter_create_ext(id, dev, conf_cb, conf_arg); + if (ret != 0) { + if (txa_dev_adapter_free(id)) + txa_dev_adapter_free(id)(id, dev); + txa_dev_id_array[id] = TXA_INVALID_DEV_ID; + return ret; + } + + txa_dev_id_array[id] = dev_id; + return 0; +} + + +int __rte_experimental +rte_event_eth_tx_adapter_event_port_get(uint8_t id, uint8_t *event_port_id) +{ + TXA_CHECK_OR_ERR_RET(id); + + return txa_service_event_port_get(id, event_port_id); +} + +int __rte_experimental +rte_event_eth_tx_adapter_free(uint8_t id) +{ + int ret; + + TXA_CHECK_OR_ERR_RET(id); + + ret = txa_dev_adapter_free(id) ? + txa_dev_adapter_free(id)(id, txa_evdev(id)) : + 0; + + if (ret == 0) + ret = txa_service_adapter_free(id); + txa_dev_id_array[id] = TXA_INVALID_DEV_ID; + + return ret; +} + +int __rte_experimental +rte_event_eth_tx_adapter_queue_add(uint8_t id, + uint16_t eth_dev_id, + int32_t queue) +{ + struct rte_eth_dev *eth_dev; + int ret; + uint32_t caps; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_dev_id, -EINVAL); + TXA_CHECK_OR_ERR_RET(id); + + eth_dev = &rte_eth_devices[eth_dev_id]; + if (queue != -1 && (uint16_t)queue >= eth_dev->data->nb_tx_queues) { + RTE_EDEV_LOG_ERR("Invalid tx queue_id %" PRIu16, + (uint16_t)queue); + return -EINVAL; + } + + caps = 0; + if (txa_dev_caps_get(id)) + txa_dev_caps_get(id)(txa_evdev(id), eth_dev, &caps); + + if (caps & RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT) + ret = txa_dev_queue_add(id) ? + txa_dev_queue_add(id)(id, + txa_evdev(id), + eth_dev, + queue) : 0; + else + ret = txa_service_queue_add(id, txa_evdev(id), eth_dev, queue); + + return ret; +} + +int __rte_experimental +rte_event_eth_tx_adapter_queue_del(uint8_t id, + uint16_t eth_dev_id, + int32_t queue) +{ + struct rte_eth_dev *eth_dev; + int ret; + uint32_t caps; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_dev_id, -EINVAL); + TXA_CHECK_OR_ERR_RET(id); + + eth_dev = &rte_eth_devices[eth_dev_id]; + if (queue != -1 && (uint16_t)queue >= eth_dev->data->nb_tx_queues) { + RTE_EDEV_LOG_ERR("Invalid tx queue_id %" PRIu16, + (uint16_t)queue); + return -EINVAL; + } + + caps = 0; + + if (txa_dev_caps_get(id)) + txa_dev_caps_get(id)(txa_evdev(id), eth_dev, &caps); + + if (caps & RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT) + ret = txa_dev_queue_del(id) ? + txa_dev_queue_del(id)(id, txa_evdev(id), + eth_dev, + queue) : 0; + else + ret = txa_service_queue_del(id, eth_dev, queue); + + return ret; +} + +int __rte_experimental +rte_event_eth_tx_adapter_service_id_get(uint8_t id, uint32_t *service_id) +{ + TXA_CHECK_OR_ERR_RET(id); + + return txa_service_id_get(id, service_id); +} + +int __rte_experimental +rte_event_eth_tx_adapter_start(uint8_t id) +{ + int ret; + + TXA_CHECK_OR_ERR_RET(id); + + ret = txa_dev_start(id) ? txa_dev_start(id)(id, txa_evdev(id)) : 0; + if (ret == 0) + ret = txa_service_start(id); + return ret; +} + +int __rte_experimental +rte_event_eth_tx_adapter_stats_get(uint8_t id, + struct rte_event_eth_tx_adapter_stats *stats) +{ + int ret; + + TXA_CHECK_OR_ERR_RET(id); + + if (stats == NULL) + return -EINVAL; + + *stats = (struct rte_event_eth_tx_adapter_stats){0}; + + ret = txa_dev_stats_get(id) ? + txa_dev_stats_get(id)(id, txa_evdev(id), stats) : 0; + + if (ret == 0 && txa_service_id_get(id, NULL) != ESRCH) { + if (txa_dev_stats_get(id)) { + struct rte_event_eth_tx_adapter_stats service_stats; + + ret = txa_service_stats_get(id, &service_stats); + if (ret == 0) { + stats->tx_retry += service_stats.tx_retry; + stats->tx_packets += service_stats.tx_packets; + stats->tx_dropped += service_stats.tx_dropped; + } + } else + ret = txa_service_stats_get(id, stats); + } + + return ret; +} + +int __rte_experimental +rte_event_eth_tx_adapter_stats_reset(uint8_t id) +{ + int ret; + + TXA_CHECK_OR_ERR_RET(id); + + ret = txa_dev_stats_reset(id) ? + txa_dev_stats_reset(id)(id, txa_evdev(id)) : 0; + if (ret == 0) + ret = txa_service_stats_reset(id); + return ret; +} + +int __rte_experimental +rte_event_eth_tx_adapter_stop(uint8_t id) +{ + int ret; + + TXA_CHECK_OR_ERR_RET(id); + + ret = txa_dev_stop(id) ? txa_dev_stop(id)(id, txa_evdev(id)) : 0; + if (ret == 0) + ret = txa_service_stop(id); + return ret; +} diff --git a/lib/librte_eventdev/rte_event_eth_tx_adapter.h b/lib/librte_eventdev/rte_event_eth_tx_adapter.h new file mode 100644 index 00000000..81456d4a --- /dev/null +++ b/lib/librte_eventdev/rte_event_eth_tx_adapter.h @@ -0,0 +1,462 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation. + */ + +#ifndef _RTE_EVENT_ETH_TX_ADAPTER_ +#define _RTE_EVENT_ETH_TX_ADAPTER_ + +/** + * @file + * + * RTE Event Ethernet Tx Adapter + * + * The event ethernet Tx adapter provides configuration and data path APIs + * for the ethernet transmit stage of an event driven packet processing + * application. These APIs abstract the implementation of the transmit stage + * and allow the application to use eventdev PMD support or a common + * implementation. + * + * In the common implementation, the application enqueues mbufs to the adapter + * which runs as a rte_service function. The service function dequeues events + * from its event port and transmits the mbufs referenced by these events. + * + * The ethernet Tx event adapter APIs are: + * + * - rte_event_eth_tx_adapter_create() + * - rte_event_eth_tx_adapter_create_ext() + * - rte_event_eth_tx_adapter_free() + * - rte_event_eth_tx_adapter_start() + * - rte_event_eth_tx_adapter_stop() + * - rte_event_eth_tx_adapter_queue_add() + * - rte_event_eth_tx_adapter_queue_del() + * - rte_event_eth_tx_adapter_stats_get() + * - rte_event_eth_tx_adapter_stats_reset() + * - rte_event_eth_tx_adapter_enqueue() + * - rte_event_eth_tx_adapter_event_port_get() + * - rte_event_eth_tx_adapter_service_id_get() + * + * The application creates the adapter using + * rte_event_eth_tx_adapter_create() or rte_event_eth_tx_adapter_create_ext(). + * + * The adapter will use the common implementation when the eventdev PMD + * does not have the #RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT capability. + * The common implementation uses an event port that is created using the port + * configuration parameter passed to rte_event_eth_tx_adapter_create(). The + * application can get the port identifier using + * rte_event_eth_tx_adapter_event_port_get() and must link an event queue to + * this port. + * + * If the eventdev PMD has the #RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT + * flags set, Tx adapter events should be enqueued using the + * rte_event_eth_tx_adapter_enqueue() function, else the application should + * use rte_event_enqueue_burst(). + * + * Transmit queues can be added and deleted from the adapter using + * rte_event_eth_tx_adapter_queue_add()/del() APIs respectively. + * + * The application can start and stop the adapter using the + * rte_event_eth_tx_adapter_start/stop() calls. + * + * The common adapter implementation uses an EAL service function as described + * before and its execution is controlled using the rte_service APIs. The + * rte_event_eth_tx_adapter_service_id_get() + * function can be used to retrieve the adapter's service function ID. + * + * The ethernet port and transmit queue index to transmit the mbuf on are + * specified using the mbuf port and the higher 16 bits of + * struct rte_mbuf::hash::sched:hi. The application should use the + * rte_event_eth_tx_adapter_txq_set() and rte_event_eth_tx_adapter_txq_get() + * functions to access the transmit queue index since it is expected that the + * transmit queue will be eventually defined within struct rte_mbuf and using + * these macros will help with minimizing application impact due to + * a change in how the transmit queue index is specified. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +#include <rte_mbuf.h> + +#include "rte_eventdev.h" + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Adapter configuration structure + * + * @see rte_event_eth_tx_adapter_create_ext + * @see rte_event_eth_tx_adapter_conf_cb + */ +struct rte_event_eth_tx_adapter_conf { + uint8_t event_port_id; + /**< Event port identifier, the adapter service function dequeues mbuf + * events from this port. + * @see RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT + */ + uint32_t max_nb_tx; + /**< The adapter can return early if it has processed at least + * max_nb_tx mbufs. This isn't treated as a requirement; batching may + * cause the adapter to process more than max_nb_tx mbufs. + */ +}; + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Function type used for adapter configuration callback. The callback is + * used to fill in members of the struct rte_event_eth_tx_adapter_conf, this + * callback is invoked when creating a RTE service function based + * adapter implementation. + * + * @param id + * Adapter identifier. + * @param dev_id + * Event device identifier. + * @param [out] conf + * Structure that needs to be populated by this callback. + * @param arg + * Argument to the callback. This is the same as the conf_arg passed to the + * rte_event_eth_tx_adapter_create_ext(). + * + * @return + * - 0: Success + * - <0: Error code on failure + */ +typedef int (*rte_event_eth_tx_adapter_conf_cb) (uint8_t id, uint8_t dev_id, + struct rte_event_eth_tx_adapter_conf *conf, + void *arg); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * A structure used to retrieve statistics for an ethernet Tx adapter instance. + */ +struct rte_event_eth_tx_adapter_stats { + uint64_t tx_retry; + /**< Number of transmit retries */ + uint64_t tx_packets; + /**< Number of packets transmitted */ + uint64_t tx_dropped; + /**< Number of packets dropped */ +}; + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Create a new ethernet Tx adapter with the specified identifier. + * + * @param id + * The identifier of the ethernet Tx adapter. + * @param dev_id + * The event device identifier. + * @param port_config + * Event port configuration, the adapter uses this configuration to + * create an event port if needed. + * @return + * - 0: Success + * - <0: Error code on failure + */ +int __rte_experimental +rte_event_eth_tx_adapter_create(uint8_t id, uint8_t dev_id, + struct rte_event_port_conf *port_config); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Create a new ethernet Tx adapter with the specified identifier. + * + * @param id + * The identifier of the ethernet Tx adapter. + * @param dev_id + * The event device identifier. + * @param conf_cb + * Callback function that initializes members of the + * struct rte_event_eth_tx_adapter_conf struct passed into + * it. + * @param conf_arg + * Argument that is passed to the conf_cb function. + * @return + * - 0: Success + * - <0: Error code on failure + */ +int __rte_experimental +rte_event_eth_tx_adapter_create_ext(uint8_t id, uint8_t dev_id, + rte_event_eth_tx_adapter_conf_cb conf_cb, + void *conf_arg); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Free an ethernet Tx adapter + * + * @param id + * Adapter identifier. + * @return + * - 0: Success + * - <0: Error code on failure, If the adapter still has Tx queues + * added to it, the function returns -EBUSY. + */ +int __rte_experimental +rte_event_eth_tx_adapter_free(uint8_t id); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Start ethernet Tx adapter + * + * @param id + * Adapter identifier. + * @return + * - 0: Success, Adapter started correctly. + * - <0: Error code on failure. + */ +int __rte_experimental +rte_event_eth_tx_adapter_start(uint8_t id); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Stop ethernet Tx adapter + * + * @param id + * Adapter identifier. + * @return + * - 0: Success. + * - <0: Error code on failure. + */ +int __rte_experimental +rte_event_eth_tx_adapter_stop(uint8_t id); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Add a Tx queue to the adapter. + * A queue value of -1 is used to indicate all + * queues within the device. + * + * @param id + * Adapter identifier. + * @param eth_dev_id + * Ethernet Port Identifier. + * @param queue + * Tx queue index. + * @return + * - 0: Success, Queues added successfully. + * - <0: Error code on failure. + */ +int __rte_experimental +rte_event_eth_tx_adapter_queue_add(uint8_t id, + uint16_t eth_dev_id, + int32_t queue); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Delete a Tx queue from the adapter. + * A queue value of -1 is used to indicate all + * queues within the device, that have been added to this + * adapter. + * + * @param id + * Adapter identifier. + * @param eth_dev_id + * Ethernet Port Identifier. + * @param queue + * Tx queue index. + * @return + * - 0: Success, Queues deleted successfully. + * - <0: Error code on failure. + */ +int __rte_experimental +rte_event_eth_tx_adapter_queue_del(uint8_t id, + uint16_t eth_dev_id, + int32_t queue); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Set Tx queue in the mbuf. This queue is used by the adapter + * to transmit the mbuf. + * + * @param pkt + * Pointer to the mbuf. + * @param queue + * Tx queue index. + */ +static __rte_always_inline void __rte_experimental +rte_event_eth_tx_adapter_txq_set(struct rte_mbuf *pkt, uint16_t queue) +{ + uint16_t *p = (uint16_t *)&pkt->hash.sched.hi; + p[1] = queue; +} + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Retrieve Tx queue from the mbuf. + * + * @param pkt + * Pointer to the mbuf. + * @return + * Tx queue identifier. + * + * @see rte_event_eth_tx_adapter_txq_set() + */ +static __rte_always_inline uint16_t __rte_experimental +rte_event_eth_tx_adapter_txq_get(struct rte_mbuf *pkt) +{ + uint16_t *p = (uint16_t *)&pkt->hash.sched.hi; + return p[1]; +} + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Retrieve the adapter event port. The adapter creates an event port if + * the #RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT is not set in the + * ethernet Tx capabilities of the event device. + * + * @param id + * Adapter Identifier. + * @param[out] event_port_id + * Event port pointer. + * @return + * - 0: Success. + * - <0: Error code on failure. + */ +int __rte_experimental +rte_event_eth_tx_adapter_event_port_get(uint8_t id, uint8_t *event_port_id); + +/** + * Enqueue a burst of events objects or an event object supplied in *rte_event* + * structure on an event device designated by its *dev_id* through the event + * port specified by *port_id*. This function is supported if the eventdev PMD + * has the #RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT capability flag set. + * + * The *nb_events* parameter is the number of event objects to enqueue which are + * supplied in the *ev* array of *rte_event* structure. + * + * The rte_event_eth_tx_adapter_enqueue() function returns the number of + * events objects it actually enqueued. A return value equal to *nb_events* + * means that all event objects have been enqueued. + * + * @param dev_id + * The identifier of the device. + * @param port_id + * The identifier of the event port. + * @param ev + * Points to an array of *nb_events* objects of type *rte_event* structure + * which contain the event object enqueue operations to be processed. + * @param nb_events + * The number of event objects to enqueue, typically number of + * rte_event_port_enqueue_depth() available for this port. + * + * @return + * The number of event objects actually enqueued on the event device. The + * return value can be less than the value of the *nb_events* parameter when + * the event devices queue is full or if invalid parameters are specified in a + * *rte_event*. If the return value is less than *nb_events*, the remaining + * events at the end of ev[] are not consumed and the caller has to take care + * of them, and rte_errno is set accordingly. Possible errno values include: + * - -EINVAL The port ID is invalid, device ID is invalid, an event's queue + * ID is invalid, or an event's sched type doesn't match the + * capabilities of the destination queue. + * - -ENOSPC The event port was backpressured and unable to enqueue + * one or more events. This error code is only applicable to + * closed systems. + */ +static inline uint16_t __rte_experimental +rte_event_eth_tx_adapter_enqueue(uint8_t dev_id, + uint8_t port_id, + struct rte_event ev[], + uint16_t nb_events) +{ + const struct rte_eventdev *dev = &rte_eventdevs[dev_id]; + +#ifdef RTE_LIBRTE_EVENTDEV_DEBUG + if (dev_id >= RTE_EVENT_MAX_DEVS || + !rte_eventdevs[dev_id].attached) { + rte_errno = -EINVAL; + return 0; + } + + if (port_id >= dev->data->nb_ports) { + rte_errno = -EINVAL; + return 0; + } +#endif + return dev->txa_enqueue(dev->data->ports[port_id], ev, nb_events); +} + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Retrieve statistics for an adapter + * + * @param id + * Adapter identifier. + * @param [out] stats + * A pointer to structure used to retrieve statistics for an adapter. + * @return + * - 0: Success, statistics retrieved successfully. + * - <0: Error code on failure. + */ +int __rte_experimental +rte_event_eth_tx_adapter_stats_get(uint8_t id, + struct rte_event_eth_tx_adapter_stats *stats); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Reset statistics for an adapter. + * + * @param id + * Adapter identifier. + * @return + * - 0: Success, statistics reset successfully. + * - <0: Error code on failure. + */ +int __rte_experimental +rte_event_eth_tx_adapter_stats_reset(uint8_t id); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Retrieve the service ID of an adapter. If the adapter doesn't use + * a rte_service function, this function returns -ESRCH. + * + * @param id + * Adapter identifier. + * @param [out] service_id + * A pointer to a uint32_t, to be filled in with the service id. + * @return + * - 0: Success + * - <0: Error code on failure, if the adapter doesn't use a rte_service + * function, this function returns -ESRCH. + */ +int __rte_experimental +rte_event_eth_tx_adapter_service_id_get(uint8_t id, uint32_t *service_id); + +#ifdef __cplusplus +} +#endif +#endif /* _RTE_EVENT_ETH_TX_ADAPTER_ */ diff --git a/lib/librte_eventdev/rte_eventdev.c b/lib/librte_eventdev/rte_eventdev.c index 801810ed..ebaf3087 100644 --- a/lib/librte_eventdev/rte_eventdev.c +++ b/lib/librte_eventdev/rte_eventdev.c @@ -35,22 +35,20 @@ #include "rte_eventdev.h" #include "rte_eventdev_pmd.h" -struct rte_eventdev rte_event_devices[RTE_EVENT_MAX_DEVS]; +static struct rte_eventdev rte_event_devices[RTE_EVENT_MAX_DEVS]; -struct rte_eventdev *rte_eventdevs = &rte_event_devices[0]; +struct rte_eventdev *rte_eventdevs = rte_event_devices; static struct rte_eventdev_global eventdev_globals = { .nb_devs = 0 }; -struct rte_eventdev_global *rte_eventdev_globals = &eventdev_globals; - /* Event dev north bound API implementation */ uint8_t rte_event_dev_count(void) { - return rte_eventdev_globals->nb_devs; + return eventdev_globals.nb_devs; } int @@ -62,7 +60,7 @@ rte_event_dev_get_dev_id(const char *name) if (!name) return -EINVAL; - for (i = 0; i < rte_eventdev_globals->nb_devs; i++) { + for (i = 0; i < eventdev_globals.nb_devs; i++) { cmp = (strncmp(rte_event_devices[i].data->name, name, RTE_EVENTDEV_NAME_MAX_LEN) == 0) || (rte_event_devices[i].dev ? (strncmp( @@ -109,7 +107,7 @@ rte_event_dev_info_get(uint8_t dev_id, struct rte_event_dev_info *dev_info) } int -rte_event_eth_rx_adapter_caps_get(uint8_t dev_id, uint8_t eth_port_id, +rte_event_eth_rx_adapter_caps_get(uint8_t dev_id, uint16_t eth_port_id, uint32_t *caps) { struct rte_eventdev *dev; @@ -175,6 +173,31 @@ rte_event_crypto_adapter_caps_get(uint8_t dev_id, uint8_t cdev_id, (dev, cdev, caps) : -ENOTSUP; } +int __rte_experimental +rte_event_eth_tx_adapter_caps_get(uint8_t dev_id, uint16_t eth_port_id, + uint32_t *caps) +{ + struct rte_eventdev *dev; + struct rte_eth_dev *eth_dev; + + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_port_id, -EINVAL); + + dev = &rte_eventdevs[dev_id]; + eth_dev = &rte_eth_devices[eth_port_id]; + + if (caps == NULL) + return -EINVAL; + + *caps = 0; + + return dev->dev_ops->eth_tx_adapter_caps_get ? + (*dev->dev_ops->eth_tx_adapter_caps_get)(dev, + eth_dev, + caps) + : 0; +} + static inline int rte_event_dev_queue_config(struct rte_eventdev *dev, uint8_t nb_queues) { @@ -980,6 +1003,28 @@ rte_event_port_unlink(uint8_t dev_id, uint8_t port_id, return diag; } +int __rte_experimental +rte_event_port_unlinks_in_progress(uint8_t dev_id, uint8_t port_id) +{ + struct rte_eventdev *dev; + + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + dev = &rte_eventdevs[dev_id]; + if (!is_valid_port(dev, port_id)) { + RTE_EDEV_LOG_ERR("Invalid port_id=%" PRIu8, port_id); + return -EINVAL; + } + + /* Return 0 if the PMD does not implement unlinks in progress. + * This allows PMDs which handle unlink synchronously to not implement + * this function at all. + */ + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->port_unlinks_in_progress, 0); + + return (*dev->dev_ops->port_unlinks_in_progress)(dev, + dev->data->ports[port_id]); +} + int rte_event_port_links_get(uint8_t dev_id, uint8_t port_id, uint8_t queues[], uint8_t priorities[]) @@ -1275,6 +1320,15 @@ rte_eventdev_find_free_device_index(void) return RTE_EVENT_MAX_DEVS; } +static uint16_t +rte_event_tx_adapter_enqueue(__rte_unused void *port, + __rte_unused struct rte_event ev[], + __rte_unused uint16_t nb_events) +{ + rte_errno = ENOTSUP; + return 0; +} + struct rte_eventdev * rte_event_pmd_allocate(const char *name, int socket_id) { @@ -1295,6 +1349,8 @@ rte_event_pmd_allocate(const char *name, int socket_id) eventdev = &rte_eventdevs[dev_id]; + eventdev->txa_enqueue = rte_event_tx_adapter_enqueue; + if (eventdev->data == NULL) { struct rte_eventdev_data *eventdev_data = NULL; diff --git a/lib/librte_eventdev/rte_eventdev.h b/lib/librte_eventdev/rte_eventdev.h index b6fd6ee7..d7eb69d1 100644 --- a/lib/librte_eventdev/rte_eventdev.h +++ b/lib/librte_eventdev/rte_eventdev.h @@ -1112,7 +1112,7 @@ struct rte_event { * */ int -rte_event_eth_rx_adapter_caps_get(uint8_t dev_id, uint8_t eth_port_id, +rte_event_eth_rx_adapter_caps_get(uint8_t dev_id, uint16_t eth_port_id, uint32_t *caps); #define RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT (1ULL << 0) @@ -1186,6 +1186,32 @@ int __rte_experimental rte_event_crypto_adapter_caps_get(uint8_t dev_id, uint8_t cdev_id, uint32_t *caps); +/* Ethdev Tx adapter capability bitmap flags */ +#define RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT 0x1 +/**< This flag is sent when the PMD supports a packet transmit callback + */ + +/** + * Retrieve the event device's eth Tx adapter capabilities + * + * @param dev_id + * The identifier of the device. + * + * @param eth_port_id + * The identifier of the ethernet device. + * + * @param[out] caps + * A pointer to memory filled with eth Tx adapter capabilities. + * + * @return + * - 0: Success, driver provides eth Tx adapter capabilities. + * - <0: Error code returned by the driver function. + * + */ +int __rte_experimental +rte_event_eth_tx_adapter_caps_get(uint8_t dev_id, uint16_t eth_port_id, + uint32_t *caps); + struct rte_eventdev_ops; struct rte_eventdev; @@ -1204,6 +1230,10 @@ typedef uint16_t (*event_dequeue_burst_t)(void *port, struct rte_event ev[], uint16_t nb_events, uint64_t timeout_ticks); /**< @internal Dequeue burst of events from port of a device */ +typedef uint16_t (*event_tx_adapter_enqueue)(void *port, + struct rte_event ev[], uint16_t nb_events); +/**< @internal Enqueue burst of events on port of a device */ + #define RTE_EVENTDEV_NAME_MAX_LEN (64) /**< @internal Max length of name of event PMD */ @@ -1266,7 +1296,8 @@ struct rte_eventdev { /**< Pointer to PMD dequeue function. */ event_dequeue_burst_t dequeue_burst; /**< Pointer to PMD dequeue burst function. */ - + event_tx_adapter_enqueue txa_enqueue; + /**< Pointer to PMD eth Tx adapter enqueue function. */ struct rte_eventdev_data *data; /**< Pointer to device data */ struct rte_eventdev_ops *dev_ops; @@ -1656,12 +1687,13 @@ rte_event_port_link(uint8_t dev_id, uint8_t port_id, * event port designated by its *port_id* on the event device designated * by its *dev_id*. * - * The unlink establishment shall disable the event port *port_id* from - * receiving events from the specified event queue *queue_id* - * + * The unlink call issues an async request to disable the event port *port_id* + * from receiving events from the specified event queue *queue_id*. * Event queue(s) to event port unlink establishment can be changed at runtime * without re-configuring the device. * + * @see rte_event_port_unlinks_in_progress() to poll for completed unlinks. + * * @param dev_id * The identifier of the device. * @@ -1679,22 +1711,48 @@ rte_event_port_link(uint8_t dev_id, uint8_t port_id, * NULL. * * @return - * The number of unlinks actually established. The return value can be less + * The number of unlinks successfully requested. The return value can be less * than the value of the *nb_unlinks* parameter when the implementation has the * limitation on specific queue to port unlink establishment or * if invalid parameters are specified. * If the return value is less than *nb_unlinks*, the remaining queues at the - * end of queues[] are not established, and the caller has to take care of them. + * end of queues[] are not unlinked, and the caller has to take care of them. * If return value is less than *nb_unlinks* then implementation shall update * the rte_errno accordingly, Possible rte_errno values are * (-EINVAL) Invalid parameter - * */ int rte_event_port_unlink(uint8_t dev_id, uint8_t port_id, uint8_t queues[], uint16_t nb_unlinks); /** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Returns the number of unlinks in progress. + * + * This function provides the application with a method to detect when an + * unlink has been completed by the implementation. + * + * @see rte_event_port_unlink() to issue unlink requests. + * + * @param dev_id + * The indentifier of the device. + * + * @param port_id + * Event port identifier to select port to check for unlinks in progress. + * + * @return + * The number of unlinks that are in progress. A return of zero indicates that + * there are no outstanding unlink requests. A positive return value indicates + * the number of unlinks that are in progress, but are not yet complete. + * A negative return value indicates an error, -EINVAL indicates an invalid + * parameter passed for *dev_id* or *port_id*. + */ +int __rte_experimental +rte_event_port_unlinks_in_progress(uint8_t dev_id, uint8_t port_id); + +/** * Retrieve the list of source event queues and its associated service priority * linked to the destination event port designated by its *port_id* * on the event device designated by its *dev_id*. diff --git a/lib/librte_eventdev/rte_eventdev_pmd.h b/lib/librte_eventdev/rte_eventdev_pmd.h index 3fbb4d2b..1a01326b 100644 --- a/lib/librte_eventdev/rte_eventdev_pmd.h +++ b/lib/librte_eventdev/rte_eventdev_pmd.h @@ -87,8 +87,6 @@ struct rte_eventdev_global { uint8_t nb_devs; /**< Number of devices found */ }; -extern struct rte_eventdev_global *rte_eventdev_globals; -/** Pointer to global event devices data structure. */ extern struct rte_eventdev *rte_eventdevs; /** The pool of rte_eventdev structures. */ @@ -333,6 +331,23 @@ typedef int (*eventdev_port_unlink_t)(struct rte_eventdev *dev, void *port, uint8_t queues[], uint16_t nb_unlinks); /** + * Unlinks in progress. Returns number of unlinks that the PMD is currently + * performing, but have not yet been completed. + * + * @param dev + * Event device pointer + * + * @param port + * Event port pointer + * + * @return + * Returns the number of in-progress unlinks. Zero is returned if none are + * in progress. + */ +typedef int (*eventdev_port_unlinks_in_progress_t)(struct rte_eventdev *dev, + void *port); + +/** * Converts nanoseconds to *timeout_ticks* value for rte_event_dequeue() * * @param dev @@ -450,7 +465,7 @@ typedef int (*eventdev_eth_rx_adapter_caps_get_t) const struct rte_eth_dev *eth_dev, uint32_t *caps); -struct rte_event_eth_rx_adapter_queue_conf *queue_conf; +struct rte_event_eth_rx_adapter_queue_conf; /** * Retrieve the event device's timer adapter capabilities, as well as the ops @@ -575,7 +590,7 @@ typedef int (*eventdev_eth_rx_adapter_stop_t) (const struct rte_eventdev *dev, const struct rte_eth_dev *eth_dev); -struct rte_event_eth_rx_adapter_stats *stats; +struct rte_event_eth_rx_adapter_stats; /** * Retrieve ethernet Rx adapter statistics. @@ -789,6 +804,186 @@ typedef int (*eventdev_crypto_adapter_stats_reset) (const struct rte_eventdev *dev, const struct rte_cryptodev *cdev); +/** + * Retrieve the event device's eth Tx adapter capabilities. + * + * @param dev + * Event device pointer + * + * @param eth_dev + * Ethernet device pointer + * + * @param[out] caps + * A pointer to memory filled with eth Tx adapter capabilities. + * + * @return + * - 0: Success, driver provides eth Tx adapter capabilities + * - <0: Error code returned by the driver function. + * + */ +typedef int (*eventdev_eth_tx_adapter_caps_get_t) + (const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev, + uint32_t *caps); + +/** + * Create adapter callback. + * + * @param id + * Adapter identifier + * + * @param dev + * Event device pointer + * + * @return + * - 0: Success. + * - <0: Error code on failure. + */ +typedef int (*eventdev_eth_tx_adapter_create_t)(uint8_t id, + const struct rte_eventdev *dev); + +/** + * Free adapter callback. + * + * @param id + * Adapter identifier + * + * @param dev + * Event device pointer + * + * @return + * - 0: Success. + * - <0: Error code on failure. + */ +typedef int (*eventdev_eth_tx_adapter_free_t)(uint8_t id, + const struct rte_eventdev *dev); + +/** + * Add a Tx queue to the adapter. + * A queue value of -1 is used to indicate all + * queues within the device. + * + * @param id + * Adapter identifier + * + * @param dev + * Event device pointer + * + * @param eth_dev + * Ethernet device pointer + * + * @param tx_queue_id + * Transmt queue index + * + * @return + * - 0: Success. + * - <0: Error code on failure. + */ +typedef int (*eventdev_eth_tx_adapter_queue_add_t)( + uint8_t id, + const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev, + int32_t tx_queue_id); + +/** + * Delete a Tx queue from the adapter. + * A queue value of -1 is used to indicate all + * queues within the device, that have been added to this + * adapter. + * + * @param id + * Adapter identifier + * + * @param dev + * Event device pointer + * + * @param eth_dev + * Ethernet device pointer + * + * @param tx_queue_id + * Transmit queue index + * + * @return + * - 0: Success, Queues deleted successfully. + * - <0: Error code on failure. + */ +typedef int (*eventdev_eth_tx_adapter_queue_del_t)( + uint8_t id, + const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev, + int32_t tx_queue_id); + +/** + * Start the adapter. + * + * @param id + * Adapter identifier + * + * @param dev + * Event device pointer + * + * @return + * - 0: Success, Adapter started correctly. + * - <0: Error code on failure. + */ +typedef int (*eventdev_eth_tx_adapter_start_t)(uint8_t id, + const struct rte_eventdev *dev); + +/** + * Stop the adapter. + * + * @param id + * Adapter identifier + * + * @param dev + * Event device pointer + * + * @return + * - 0: Success. + * - <0: Error code on failure. + */ +typedef int (*eventdev_eth_tx_adapter_stop_t)(uint8_t id, + const struct rte_eventdev *dev); + +struct rte_event_eth_tx_adapter_stats; + +/** + * Retrieve statistics for an adapter + * + * @param id + * Adapter identifier + * + * @param dev + * Event device pointer + * + * @param [out] stats + * A pointer to structure used to retrieve statistics for an adapter + * + * @return + * - 0: Success, statistics retrieved successfully. + * - <0: Error code on failure. + */ +typedef int (*eventdev_eth_tx_adapter_stats_get_t)( + uint8_t id, + const struct rte_eventdev *dev, + struct rte_event_eth_tx_adapter_stats *stats); + +/** + * Reset statistics for an adapter + * + * @param id + * Adapter identifier + * + * @param dev + * Event device pointer + * + * @return + * - 0: Success, statistics retrieved successfully. + * - <0: Error code on failure. + */ +typedef int (*eventdev_eth_tx_adapter_stats_reset_t)(uint8_t id, + const struct rte_eventdev *dev); + /** Event device operations function pointer table */ struct rte_eventdev_ops { eventdev_info_get_t dev_infos_get; /**< Get device info. */ @@ -815,6 +1010,8 @@ struct rte_eventdev_ops { /**< Link event queues to an event port. */ eventdev_port_unlink_t port_unlink; /**< Unlink event queues from an event port. */ + eventdev_port_unlinks_in_progress_t port_unlinks_in_progress; + /**< Unlinks in progress on an event port. */ eventdev_dequeue_timeout_ticks_t timeout_ticks; /**< Converts ns to *timeout_ticks* value for rte_event_dequeue() */ eventdev_dump_t dump; @@ -862,6 +1059,26 @@ struct rte_eventdev_ops { eventdev_crypto_adapter_stats_reset crypto_adapter_stats_reset; /**< Reset crypto stats */ + eventdev_eth_tx_adapter_caps_get_t eth_tx_adapter_caps_get; + /**< Get ethernet Tx adapter capabilities */ + + eventdev_eth_tx_adapter_create_t eth_tx_adapter_create; + /**< Create adapter callback */ + eventdev_eth_tx_adapter_free_t eth_tx_adapter_free; + /**< Free adapter callback */ + eventdev_eth_tx_adapter_queue_add_t eth_tx_adapter_queue_add; + /**< Add Tx queues to the eth Tx adapter */ + eventdev_eth_tx_adapter_queue_del_t eth_tx_adapter_queue_del; + /**< Delete Tx queues from the eth Tx adapter */ + eventdev_eth_tx_adapter_start_t eth_tx_adapter_start; + /**< Start eth Tx adapter */ + eventdev_eth_tx_adapter_stop_t eth_tx_adapter_stop; + /**< Stop eth Tx adapter */ + eventdev_eth_tx_adapter_stats_get_t eth_tx_adapter_stats_get; + /**< Get eth Tx adapter statistics */ + eventdev_eth_tx_adapter_stats_reset_t eth_tx_adapter_stats_reset; + /**< Reset eth Tx adapter statistics */ + eventdev_selftest dev_selftest; /**< Start eventdev Selftest */ diff --git a/lib/librte_eventdev/rte_eventdev_version.map b/lib/librte_eventdev/rte_eventdev_version.map index 12835e9f..d558d7d5 100644 --- a/lib/librte_eventdev/rte_eventdev_version.map +++ b/lib/librte_eventdev/rte_eventdev_version.map @@ -96,6 +96,19 @@ EXPERIMENTAL { rte_event_crypto_adapter_stats_reset; rte_event_crypto_adapter_stop; rte_event_eth_rx_adapter_cb_register; + rte_event_port_unlinks_in_progress; + rte_event_eth_tx_adapter_caps_get; + rte_event_eth_tx_adapter_create; + rte_event_eth_tx_adapter_create_ext; + rte_event_eth_tx_adapter_event_port_get; + rte_event_eth_tx_adapter_free; + rte_event_eth_tx_adapter_queue_add; + rte_event_eth_tx_adapter_queue_del; + rte_event_eth_tx_adapter_service_id_get; + rte_event_eth_tx_adapter_start; + rte_event_eth_tx_adapter_stats_get; + rte_event_eth_tx_adapter_stats_reset; + rte_event_eth_tx_adapter_stop; rte_event_timer_adapter_caps_get; rte_event_timer_adapter_create; rte_event_timer_adapter_create_ext; |