diff options
author | Damjan Marion <damarion@cisco.com> | 2023-11-13 12:18:24 +0000 |
---|---|---|
committer | Damjan Marion <damarion@cisco.com> | 2023-11-13 19:05:17 +0000 |
commit | 2d725c61286ccb8625ffad5c678cee337f88bceb (patch) | |
tree | c3b70710a4f4e5b30c68a6791af151b9de787c62 /src/plugins/dev_ena/queue.c | |
parent | e596ca1b51d25c24bd06132f9824d4bd0c6a5ecc (diff) |
ena: Amazon Elastic Network Adapter (ENA) native driver
Type: feature
Change-Id: Icd9de05f2cbac0e5a6dfb1f1414f21dc4b893104
Signed-off-by: Damjan Marion <damarion@cisco.com>
Diffstat (limited to 'src/plugins/dev_ena/queue.c')
-rw-r--r-- | src/plugins/dev_ena/queue.c | 384 |
1 files changed, 384 insertions, 0 deletions
diff --git a/src/plugins/dev_ena/queue.c b/src/plugins/dev_ena/queue.c new file mode 100644 index 00000000000..08c763c8461 --- /dev/null +++ b/src/plugins/dev_ena/queue.c @@ -0,0 +1,384 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2023 Cisco Systems, Inc. + */ + +#include <vlib/vlib.h> +#include <vnet/dev/dev.h> + +#include <dev_ena/ena.h> +#include <dev_ena/ena_inlines.h> + +VLIB_REGISTER_LOG_CLASS (ena_log, static) = { + .class_name = "ena", + .subclass_name = "queue", +}; + +void +ena_rx_queue_free (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq) +{ + ena_rxq_t *eq = vnet_dev_get_rx_queue_data (rxq); + vnet_dev_port_t *port = rxq->port; + vnet_dev_t *dev = port->dev; + + ASSERT (rxq->started == 0); + ASSERT (eq->cq_created == 0); + ASSERT (eq->sq_created == 0); + + log_debug (dev, "queue %u", rxq->queue_id); + + foreach_pointer (p, eq->buffer_indices, eq->compl_sqe_indices) + if (p) + clib_mem_free (p); + + foreach_pointer (p, eq->cqes, eq->sqes) + vnet_dev_dma_mem_free (vm, dev, p); +} + +vnet_dev_rv_t +ena_rx_queue_alloc (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq) +{ + vnet_dev_port_t *port = rxq->port; + vnet_dev_t *dev = port->dev; + ena_rxq_t *eq = vnet_dev_get_rx_queue_data (rxq); + u16 size = rxq->size; + vnet_dev_rv_t rv; + + ASSERT (eq->buffer_indices == 0); + ASSERT (eq->compl_sqe_indices == 0); + ASSERT (eq->cqes == 0); + ASSERT (eq->sqes == 0); + + log_debug (dev, "queue %u", rxq->queue_id); + + eq->buffer_indices = clib_mem_alloc_aligned ( + sizeof (eq->buffer_indices[0]) * size, CLIB_CACHE_LINE_BYTES); + + eq->compl_sqe_indices = clib_mem_alloc_aligned ( + sizeof (eq->compl_sqe_indices[0]) * size, CLIB_CACHE_LINE_BYTES); + + if ((rv = vnet_dev_dma_mem_alloc (vm, dev, sizeof (eq->cqes[0]) * size, 0, + (void **) &eq->cqes))) + goto err; + + if ((rv = vnet_dev_dma_mem_alloc (vm, dev, sizeof (eq->sqes[0]) * size, 0, + (void **) &eq->sqes))) + goto err; + + return VNET_DEV_OK; + +err: + ena_rx_queue_free (vm, rxq); + return rv; +} + +void +ena_tx_queue_free (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) +{ + ena_txq_t *eq = vnet_dev_get_tx_queue_data (txq); + vnet_dev_port_t *port = txq->port; + vnet_dev_t *dev = port->dev; + + ASSERT (txq->started == 0); + + log_debug (dev, "queue %u", txq->queue_id); + + foreach_pointer (p, eq->buffer_indices, eq->sqe_templates) + if (p) + clib_mem_free (p); + + foreach_pointer (p, eq->cqes, eq->sqes) + vnet_dev_dma_mem_free (vm, dev, p); +} + +vnet_dev_rv_t +ena_tx_queue_alloc (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) +{ + vnet_dev_port_t *port = txq->port; + vnet_dev_t *dev = port->dev; + ena_txq_t *eq = vnet_dev_get_tx_queue_data (txq); + u16 size = txq->size; + vnet_dev_rv_t rv; + + ASSERT (eq->buffer_indices == 0); + ASSERT (eq->sqe_templates == 0); + ASSERT (eq->cqes == 0); + ASSERT (eq->sqes == 0); + + log_debug (dev, "queue %u", txq->queue_id); + + eq->buffer_indices = clib_mem_alloc_aligned ( + sizeof (eq->buffer_indices[0]) * size, CLIB_CACHE_LINE_BYTES); + eq->sqe_templates = clib_mem_alloc_aligned ( + sizeof (eq->sqe_templates[0]) * size, CLIB_CACHE_LINE_BYTES); + + if ((rv = vnet_dev_dma_mem_alloc (vm, dev, sizeof (eq->cqes[0]) * size, 0, + (void **) &eq->cqes))) + goto err; + + if ((rv = vnet_dev_dma_mem_alloc (vm, dev, sizeof (eq->sqes[0]) * size, 0, + (void **) &eq->sqes))) + goto err; + + return VNET_DEV_OK; + +err: + ena_tx_queue_free (vm, txq); + return rv; +} + +vnet_dev_rv_t +ena_rx_queue_start (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq) +{ + ena_rxq_t *eq = vnet_dev_get_rx_queue_data (rxq); + vnet_dev_port_t *port = rxq->port; + vnet_dev_t *dev = port->dev; + ena_device_t *ed = vnet_dev_get_data (dev); + u16 buffer_size = vnet_dev_get_rx_queue_buffer_data_size (vm, rxq); + u16 size = rxq->size; + vnet_dev_rv_t rv; + + /* Create Completion Queue */ + ena_aq_create_cq_resp_t cqresp; + ena_aq_create_cq_cmd_t cqcmd = { + .interrupt_mode_enabled = 1, + .cq_entry_size_words = sizeof (ena_rx_cdesc_t) / 4, + .cq_depth = size, + .msix_vector = ~0, + }; + + ena_set_mem_addr (vm, dev, &cqcmd.cq_ba, eq->cqes); + if ((rv = ena_aq_create_cq (vm, dev, &cqcmd, &cqresp))) + { + log_err (dev, "queue %u cq creation failed", rxq->queue_id); + goto error; + } + + eq->cq_idx = cqresp.cq_idx; + eq->cq_created = 1; + + log_debug (dev, "queue %u cq %u created", rxq->queue_id, eq->cq_idx); + + /* Create Submission Queue */ + ena_aq_create_sq_resp_t sqresp; + ena_aq_create_sq_cmd_t sqcmd = { + .sq_direction = ENA_ADMIN_SQ_DIRECTION_RX, + .placement_policy = ENA_ADMIN_SQ_PLACEMENT_POLICY_HOST, + .completion_policy = ENA_ADMIN_SQ_COMPLETION_POLICY_DESC, + .is_physically_contiguous = 1, + .sq_depth = size, + .cq_idx = cqresp.cq_idx, + }; + + ena_set_mem_addr (vm, dev, &sqcmd.sq_ba, eq->sqes); + if ((rv = ena_aq_create_sq (vm, dev, &sqcmd, &sqresp))) + { + log_err (dev, "queue %u sq creation failed", rxq->queue_id); + goto error; + } + + eq->sq_idx = sqresp.sq_idx; + eq->sq_db = (u32 *) ((u8 *) ed->reg_bar + sqresp.sq_doorbell_offset); + eq->sq_created = 1; + + log_debug (dev, "queue %u sq %u created, sq_db %p", rxq->queue_id, + eq->sq_idx, eq->sq_db); + + for (int i = 0; i < size; i++) + { + eq->sqes[i] = (ena_rx_desc_t){ + .lo = { + .length = buffer_size, + .comp_req = 1, + .first = 1, + .last = 1, + .reserved5 = 1, /* ena_com says MBO */ + .req_id = i, + }, + }; + eq->buffer_indices[i] = VLIB_BUFFER_INVALID_INDEX; + eq->compl_sqe_indices[i] = i; + } + + eq->sq_next = 0; + eq->n_compl_sqes = size; + + return VNET_DEV_OK; + +error: + ena_rx_queue_stop (vm, rxq); + return rv; +} + +vnet_dev_rv_t +ena_tx_queue_start (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) +{ + ena_txq_t *eq = vnet_dev_get_tx_queue_data (txq); + vnet_dev_port_t *port = txq->port; + vnet_dev_t *dev = port->dev; + ena_device_t *ed = vnet_dev_get_data (dev); + u16 size = txq->size; + vnet_dev_rv_t rv; + + /* Create Completion Queue */ + ena_aq_create_cq_resp_t cqresp; + ena_aq_create_cq_cmd_t cqcmd = { + .interrupt_mode_enabled = 1, + .cq_entry_size_words = sizeof (ena_tx_cdesc_t) / 4, + .cq_depth = size, + .msix_vector = ~0, + }; + + ena_set_mem_addr (vm, dev, &cqcmd.cq_ba, eq->cqes); + if ((rv = ena_aq_create_cq (vm, dev, &cqcmd, &cqresp))) + { + log_err (dev, "queue %u cq creation failed", txq->queue_id); + goto error; + } + + eq->cq_idx = cqresp.cq_idx; + eq->cq_created = 1; + + log_debug (dev, "queue %u cq %u created", txq->queue_id, eq->cq_idx); + + /* Create Submission Queue */ + ena_aq_create_sq_resp_t sqresp; + ena_aq_create_sq_cmd_t sqcmd = { + .sq_direction = ENA_ADMIN_SQ_DIRECTION_TX, + .placement_policy = eq->llq ? ENA_ADMIN_SQ_PLACEMENT_POLICY_DEVICE : + ENA_ADMIN_SQ_PLACEMENT_POLICY_HOST, + .completion_policy = ENA_ADMIN_SQ_COMPLETION_POLICY_DESC, + .is_physically_contiguous = 1, + .sq_depth = size, + .cq_idx = cqresp.cq_idx, + }; + + if (eq->llq == 0) + ena_set_mem_addr (vm, dev, &sqcmd.sq_ba, eq->sqes); + if ((rv = ena_aq_create_sq (vm, dev, &sqcmd, &sqresp))) + { + log_err (dev, "queue %u sq creation failed", txq->queue_id); + goto error; + } + + eq->sq_idx = sqresp.sq_idx; + eq->sq_db = (u32 *) ((u8 *) ed->reg_bar + sqresp.sq_doorbell_offset); + eq->sq_created = 1; + + log_debug (dev, "queue %u sq %u created, sq_db %p", txq->queue_id, + eq->sq_idx, eq->sq_db); + + for (u32 i = 0; i < size; i++) + { + eq->sqe_templates[i] = + (ena_tx_desc_t){ .req_id_lo = i, .req_id_hi = i >> 10, .comp_req = 1 } + .as_u64x2[0]; + + eq->buffer_indices[i] = VLIB_BUFFER_INVALID_INDEX; + } + + eq->sq_head = 0; + eq->sq_tail = 0; + eq->cq_next = 0; + +#if 0 + if (txq->llq) + txq->llq_descs = + (ena_tx_llq_desc128_t *) ((u8 *) ed->mem_bar + + sqresp.llq_descriptors_offset); +#endif + + log_debug (dev, "queue %u sq %u created, sq_db %p llq_desc %p", + txq->queue_id, eq->sq_idx, eq->sq_db, + eq->llq ? eq->llq_descs : 0); + return VNET_DEV_OK; + +error: + ena_tx_queue_stop (vm, txq); + return rv; +} + +static void +ena_free_sq_buffer_indices (vlib_main_t *vm, u32 *sq_buffer_indices, + u32 n_desc) +{ + u32 *to = sq_buffer_indices; + + for (u32 *from = to; from < sq_buffer_indices + n_desc; from++) + if (from[0] != VLIB_BUFFER_INVALID_INDEX) + to++[0] = from[0]; + + if (to - sq_buffer_indices > 0) + vlib_buffer_free (vm, sq_buffer_indices, to - sq_buffer_indices); +} + +void +ena_rx_queue_stop (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq) +{ + ena_rxq_t *eq = vnet_dev_get_rx_queue_data (rxq); + vnet_dev_t *dev = rxq->port->dev; + vnet_dev_rv_t rv; + + if (eq->sq_created) + { + ena_aq_destroy_sq_cmd_t cmd = { + .sq_idx = eq->sq_idx, + .sq_direction = ENA_ADMIN_SQ_DIRECTION_TX, + }; + + if ((rv = ena_aq_destroy_sq (vm, dev, &cmd))) + log_err (dev, "queue %u failed to destroy sq %u", rxq->queue_id, + eq->sq_idx); + eq->sq_created = 0; + }; + + if (eq->cq_created) + { + ena_aq_destroy_cq_cmd_t cmd = { + .cq_idx = eq->cq_idx, + }; + + if ((rv = ena_aq_destroy_cq (vm, dev, &cmd))) + log_err (dev, "queue %u failed to destroy cq %u", rxq->queue_id, + eq->cq_idx); + eq->cq_created = 0; + }; + + if (eq->n_compl_sqes < rxq->size) + ena_free_sq_buffer_indices (vm, eq->buffer_indices, rxq->size); +} + +void +ena_tx_queue_stop (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) +{ + ena_txq_t *eq = vnet_dev_get_tx_queue_data (txq); + vnet_dev_t *dev = txq->port->dev; + vnet_dev_rv_t rv; + + if (eq->sq_created) + { + ena_aq_destroy_sq_cmd_t cmd = { + .sq_idx = eq->sq_idx, + .sq_direction = ENA_ADMIN_SQ_DIRECTION_TX, + }; + + if ((rv = ena_aq_destroy_sq (vm, dev, &cmd))) + log_err (dev, "queue %u failed to destroy sq %u", txq->queue_id, + eq->sq_idx); + eq->sq_created = 0; + }; + + if (eq->cq_created) + { + ena_aq_destroy_cq_cmd_t cmd = { + .cq_idx = eq->cq_idx, + }; + + if ((rv = ena_aq_destroy_cq (vm, dev, &cmd))) + log_err (dev, "queue %u failed to destroy cq %u", txq->queue_id, + eq->cq_idx); + eq->cq_created = 0; + }; + + if (eq->sq_head != eq->sq_tail) + ena_free_sq_buffer_indices (vm, eq->buffer_indices, txq->size); +} |