aboutsummaryrefslogtreecommitdiffstats
path: root/src/plugins/dev_ena/queue.c
diff options
context:
space:
mode:
authorDamjan Marion <damarion@cisco.com>2023-11-13 12:18:24 +0000
committerDamjan Marion <damarion@cisco.com>2023-11-13 19:05:17 +0000
commit2d725c61286ccb8625ffad5c678cee337f88bceb (patch)
treec3b70710a4f4e5b30c68a6791af151b9de787c62 /src/plugins/dev_ena/queue.c
parente596ca1b51d25c24bd06132f9824d4bd0c6a5ecc (diff)
ena: Amazon Elastic Network Adapter (ENA) native driver
Type: feature Change-Id: Icd9de05f2cbac0e5a6dfb1f1414f21dc4b893104 Signed-off-by: Damjan Marion <damarion@cisco.com>
Diffstat (limited to 'src/plugins/dev_ena/queue.c')
-rw-r--r--src/plugins/dev_ena/queue.c384
1 files changed, 384 insertions, 0 deletions
diff --git a/src/plugins/dev_ena/queue.c b/src/plugins/dev_ena/queue.c
new file mode 100644
index 00000000000..08c763c8461
--- /dev/null
+++ b/src/plugins/dev_ena/queue.c
@@ -0,0 +1,384 @@
+/* SPDX-License-Identifier: Apache-2.0
+ * Copyright(c) 2023 Cisco Systems, Inc.
+ */
+
+#include <vlib/vlib.h>
+#include <vnet/dev/dev.h>
+
+#include <dev_ena/ena.h>
+#include <dev_ena/ena_inlines.h>
+
+VLIB_REGISTER_LOG_CLASS (ena_log, static) = {
+ .class_name = "ena",
+ .subclass_name = "queue",
+};
+
+void
+ena_rx_queue_free (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq)
+{
+ ena_rxq_t *eq = vnet_dev_get_rx_queue_data (rxq);
+ vnet_dev_port_t *port = rxq->port;
+ vnet_dev_t *dev = port->dev;
+
+ ASSERT (rxq->started == 0);
+ ASSERT (eq->cq_created == 0);
+ ASSERT (eq->sq_created == 0);
+
+ log_debug (dev, "queue %u", rxq->queue_id);
+
+ foreach_pointer (p, eq->buffer_indices, eq->compl_sqe_indices)
+ if (p)
+ clib_mem_free (p);
+
+ foreach_pointer (p, eq->cqes, eq->sqes)
+ vnet_dev_dma_mem_free (vm, dev, p);
+}
+
+vnet_dev_rv_t
+ena_rx_queue_alloc (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq)
+{
+ vnet_dev_port_t *port = rxq->port;
+ vnet_dev_t *dev = port->dev;
+ ena_rxq_t *eq = vnet_dev_get_rx_queue_data (rxq);
+ u16 size = rxq->size;
+ vnet_dev_rv_t rv;
+
+ ASSERT (eq->buffer_indices == 0);
+ ASSERT (eq->compl_sqe_indices == 0);
+ ASSERT (eq->cqes == 0);
+ ASSERT (eq->sqes == 0);
+
+ log_debug (dev, "queue %u", rxq->queue_id);
+
+ eq->buffer_indices = clib_mem_alloc_aligned (
+ sizeof (eq->buffer_indices[0]) * size, CLIB_CACHE_LINE_BYTES);
+
+ eq->compl_sqe_indices = clib_mem_alloc_aligned (
+ sizeof (eq->compl_sqe_indices[0]) * size, CLIB_CACHE_LINE_BYTES);
+
+ if ((rv = vnet_dev_dma_mem_alloc (vm, dev, sizeof (eq->cqes[0]) * size, 0,
+ (void **) &eq->cqes)))
+ goto err;
+
+ if ((rv = vnet_dev_dma_mem_alloc (vm, dev, sizeof (eq->sqes[0]) * size, 0,
+ (void **) &eq->sqes)))
+ goto err;
+
+ return VNET_DEV_OK;
+
+err:
+ ena_rx_queue_free (vm, rxq);
+ return rv;
+}
+
+void
+ena_tx_queue_free (vlib_main_t *vm, vnet_dev_tx_queue_t *txq)
+{
+ ena_txq_t *eq = vnet_dev_get_tx_queue_data (txq);
+ vnet_dev_port_t *port = txq->port;
+ vnet_dev_t *dev = port->dev;
+
+ ASSERT (txq->started == 0);
+
+ log_debug (dev, "queue %u", txq->queue_id);
+
+ foreach_pointer (p, eq->buffer_indices, eq->sqe_templates)
+ if (p)
+ clib_mem_free (p);
+
+ foreach_pointer (p, eq->cqes, eq->sqes)
+ vnet_dev_dma_mem_free (vm, dev, p);
+}
+
+vnet_dev_rv_t
+ena_tx_queue_alloc (vlib_main_t *vm, vnet_dev_tx_queue_t *txq)
+{
+ vnet_dev_port_t *port = txq->port;
+ vnet_dev_t *dev = port->dev;
+ ena_txq_t *eq = vnet_dev_get_tx_queue_data (txq);
+ u16 size = txq->size;
+ vnet_dev_rv_t rv;
+
+ ASSERT (eq->buffer_indices == 0);
+ ASSERT (eq->sqe_templates == 0);
+ ASSERT (eq->cqes == 0);
+ ASSERT (eq->sqes == 0);
+
+ log_debug (dev, "queue %u", txq->queue_id);
+
+ eq->buffer_indices = clib_mem_alloc_aligned (
+ sizeof (eq->buffer_indices[0]) * size, CLIB_CACHE_LINE_BYTES);
+ eq->sqe_templates = clib_mem_alloc_aligned (
+ sizeof (eq->sqe_templates[0]) * size, CLIB_CACHE_LINE_BYTES);
+
+ if ((rv = vnet_dev_dma_mem_alloc (vm, dev, sizeof (eq->cqes[0]) * size, 0,
+ (void **) &eq->cqes)))
+ goto err;
+
+ if ((rv = vnet_dev_dma_mem_alloc (vm, dev, sizeof (eq->sqes[0]) * size, 0,
+ (void **) &eq->sqes)))
+ goto err;
+
+ return VNET_DEV_OK;
+
+err:
+ ena_tx_queue_free (vm, txq);
+ return rv;
+}
+
+vnet_dev_rv_t
+ena_rx_queue_start (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq)
+{
+ ena_rxq_t *eq = vnet_dev_get_rx_queue_data (rxq);
+ vnet_dev_port_t *port = rxq->port;
+ vnet_dev_t *dev = port->dev;
+ ena_device_t *ed = vnet_dev_get_data (dev);
+ u16 buffer_size = vnet_dev_get_rx_queue_buffer_data_size (vm, rxq);
+ u16 size = rxq->size;
+ vnet_dev_rv_t rv;
+
+ /* Create Completion Queue */
+ ena_aq_create_cq_resp_t cqresp;
+ ena_aq_create_cq_cmd_t cqcmd = {
+ .interrupt_mode_enabled = 1,
+ .cq_entry_size_words = sizeof (ena_rx_cdesc_t) / 4,
+ .cq_depth = size,
+ .msix_vector = ~0,
+ };
+
+ ena_set_mem_addr (vm, dev, &cqcmd.cq_ba, eq->cqes);
+ if ((rv = ena_aq_create_cq (vm, dev, &cqcmd, &cqresp)))
+ {
+ log_err (dev, "queue %u cq creation failed", rxq->queue_id);
+ goto error;
+ }
+
+ eq->cq_idx = cqresp.cq_idx;
+ eq->cq_created = 1;
+
+ log_debug (dev, "queue %u cq %u created", rxq->queue_id, eq->cq_idx);
+
+ /* Create Submission Queue */
+ ena_aq_create_sq_resp_t sqresp;
+ ena_aq_create_sq_cmd_t sqcmd = {
+ .sq_direction = ENA_ADMIN_SQ_DIRECTION_RX,
+ .placement_policy = ENA_ADMIN_SQ_PLACEMENT_POLICY_HOST,
+ .completion_policy = ENA_ADMIN_SQ_COMPLETION_POLICY_DESC,
+ .is_physically_contiguous = 1,
+ .sq_depth = size,
+ .cq_idx = cqresp.cq_idx,
+ };
+
+ ena_set_mem_addr (vm, dev, &sqcmd.sq_ba, eq->sqes);
+ if ((rv = ena_aq_create_sq (vm, dev, &sqcmd, &sqresp)))
+ {
+ log_err (dev, "queue %u sq creation failed", rxq->queue_id);
+ goto error;
+ }
+
+ eq->sq_idx = sqresp.sq_idx;
+ eq->sq_db = (u32 *) ((u8 *) ed->reg_bar + sqresp.sq_doorbell_offset);
+ eq->sq_created = 1;
+
+ log_debug (dev, "queue %u sq %u created, sq_db %p", rxq->queue_id,
+ eq->sq_idx, eq->sq_db);
+
+ for (int i = 0; i < size; i++)
+ {
+ eq->sqes[i] = (ena_rx_desc_t){
+ .lo = {
+ .length = buffer_size,
+ .comp_req = 1,
+ .first = 1,
+ .last = 1,
+ .reserved5 = 1, /* ena_com says MBO */
+ .req_id = i,
+ },
+ };
+ eq->buffer_indices[i] = VLIB_BUFFER_INVALID_INDEX;
+ eq->compl_sqe_indices[i] = i;
+ }
+
+ eq->sq_next = 0;
+ eq->n_compl_sqes = size;
+
+ return VNET_DEV_OK;
+
+error:
+ ena_rx_queue_stop (vm, rxq);
+ return rv;
+}
+
+vnet_dev_rv_t
+ena_tx_queue_start (vlib_main_t *vm, vnet_dev_tx_queue_t *txq)
+{
+ ena_txq_t *eq = vnet_dev_get_tx_queue_data (txq);
+ vnet_dev_port_t *port = txq->port;
+ vnet_dev_t *dev = port->dev;
+ ena_device_t *ed = vnet_dev_get_data (dev);
+ u16 size = txq->size;
+ vnet_dev_rv_t rv;
+
+ /* Create Completion Queue */
+ ena_aq_create_cq_resp_t cqresp;
+ ena_aq_create_cq_cmd_t cqcmd = {
+ .interrupt_mode_enabled = 1,
+ .cq_entry_size_words = sizeof (ena_tx_cdesc_t) / 4,
+ .cq_depth = size,
+ .msix_vector = ~0,
+ };
+
+ ena_set_mem_addr (vm, dev, &cqcmd.cq_ba, eq->cqes);
+ if ((rv = ena_aq_create_cq (vm, dev, &cqcmd, &cqresp)))
+ {
+ log_err (dev, "queue %u cq creation failed", txq->queue_id);
+ goto error;
+ }
+
+ eq->cq_idx = cqresp.cq_idx;
+ eq->cq_created = 1;
+
+ log_debug (dev, "queue %u cq %u created", txq->queue_id, eq->cq_idx);
+
+ /* Create Submission Queue */
+ ena_aq_create_sq_resp_t sqresp;
+ ena_aq_create_sq_cmd_t sqcmd = {
+ .sq_direction = ENA_ADMIN_SQ_DIRECTION_TX,
+ .placement_policy = eq->llq ? ENA_ADMIN_SQ_PLACEMENT_POLICY_DEVICE :
+ ENA_ADMIN_SQ_PLACEMENT_POLICY_HOST,
+ .completion_policy = ENA_ADMIN_SQ_COMPLETION_POLICY_DESC,
+ .is_physically_contiguous = 1,
+ .sq_depth = size,
+ .cq_idx = cqresp.cq_idx,
+ };
+
+ if (eq->llq == 0)
+ ena_set_mem_addr (vm, dev, &sqcmd.sq_ba, eq->sqes);
+ if ((rv = ena_aq_create_sq (vm, dev, &sqcmd, &sqresp)))
+ {
+ log_err (dev, "queue %u sq creation failed", txq->queue_id);
+ goto error;
+ }
+
+ eq->sq_idx = sqresp.sq_idx;
+ eq->sq_db = (u32 *) ((u8 *) ed->reg_bar + sqresp.sq_doorbell_offset);
+ eq->sq_created = 1;
+
+ log_debug (dev, "queue %u sq %u created, sq_db %p", txq->queue_id,
+ eq->sq_idx, eq->sq_db);
+
+ for (u32 i = 0; i < size; i++)
+ {
+ eq->sqe_templates[i] =
+ (ena_tx_desc_t){ .req_id_lo = i, .req_id_hi = i >> 10, .comp_req = 1 }
+ .as_u64x2[0];
+
+ eq->buffer_indices[i] = VLIB_BUFFER_INVALID_INDEX;
+ }
+
+ eq->sq_head = 0;
+ eq->sq_tail = 0;
+ eq->cq_next = 0;
+
+#if 0
+ if (txq->llq)
+ txq->llq_descs =
+ (ena_tx_llq_desc128_t *) ((u8 *) ed->mem_bar +
+ sqresp.llq_descriptors_offset);
+#endif
+
+ log_debug (dev, "queue %u sq %u created, sq_db %p llq_desc %p",
+ txq->queue_id, eq->sq_idx, eq->sq_db,
+ eq->llq ? eq->llq_descs : 0);
+ return VNET_DEV_OK;
+
+error:
+ ena_tx_queue_stop (vm, txq);
+ return rv;
+}
+
+static void
+ena_free_sq_buffer_indices (vlib_main_t *vm, u32 *sq_buffer_indices,
+ u32 n_desc)
+{
+ u32 *to = sq_buffer_indices;
+
+ for (u32 *from = to; from < sq_buffer_indices + n_desc; from++)
+ if (from[0] != VLIB_BUFFER_INVALID_INDEX)
+ to++[0] = from[0];
+
+ if (to - sq_buffer_indices > 0)
+ vlib_buffer_free (vm, sq_buffer_indices, to - sq_buffer_indices);
+}
+
+void
+ena_rx_queue_stop (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq)
+{
+ ena_rxq_t *eq = vnet_dev_get_rx_queue_data (rxq);
+ vnet_dev_t *dev = rxq->port->dev;
+ vnet_dev_rv_t rv;
+
+ if (eq->sq_created)
+ {
+ ena_aq_destroy_sq_cmd_t cmd = {
+ .sq_idx = eq->sq_idx,
+ .sq_direction = ENA_ADMIN_SQ_DIRECTION_TX,
+ };
+
+ if ((rv = ena_aq_destroy_sq (vm, dev, &cmd)))
+ log_err (dev, "queue %u failed to destroy sq %u", rxq->queue_id,
+ eq->sq_idx);
+ eq->sq_created = 0;
+ };
+
+ if (eq->cq_created)
+ {
+ ena_aq_destroy_cq_cmd_t cmd = {
+ .cq_idx = eq->cq_idx,
+ };
+
+ if ((rv = ena_aq_destroy_cq (vm, dev, &cmd)))
+ log_err (dev, "queue %u failed to destroy cq %u", rxq->queue_id,
+ eq->cq_idx);
+ eq->cq_created = 0;
+ };
+
+ if (eq->n_compl_sqes < rxq->size)
+ ena_free_sq_buffer_indices (vm, eq->buffer_indices, rxq->size);
+}
+
+void
+ena_tx_queue_stop (vlib_main_t *vm, vnet_dev_tx_queue_t *txq)
+{
+ ena_txq_t *eq = vnet_dev_get_tx_queue_data (txq);
+ vnet_dev_t *dev = txq->port->dev;
+ vnet_dev_rv_t rv;
+
+ if (eq->sq_created)
+ {
+ ena_aq_destroy_sq_cmd_t cmd = {
+ .sq_idx = eq->sq_idx,
+ .sq_direction = ENA_ADMIN_SQ_DIRECTION_TX,
+ };
+
+ if ((rv = ena_aq_destroy_sq (vm, dev, &cmd)))
+ log_err (dev, "queue %u failed to destroy sq %u", txq->queue_id,
+ eq->sq_idx);
+ eq->sq_created = 0;
+ };
+
+ if (eq->cq_created)
+ {
+ ena_aq_destroy_cq_cmd_t cmd = {
+ .cq_idx = eq->cq_idx,
+ };
+
+ if ((rv = ena_aq_destroy_cq (vm, dev, &cmd)))
+ log_err (dev, "queue %u failed to destroy cq %u", txq->queue_id,
+ eq->cq_idx);
+ eq->cq_created = 0;
+ };
+
+ if (eq->sq_head != eq->sq_tail)
+ ena_free_sq_buffer_indices (vm, eq->buffer_indices, txq->size);
+}