summaryrefslogtreecommitdiffstats
path: root/src/plugins/dev_ena/rx_node.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/plugins/dev_ena/rx_node.c')
-rw-r--r--src/plugins/dev_ena/rx_node.c457
1 files changed, 457 insertions, 0 deletions
diff --git a/src/plugins/dev_ena/rx_node.c b/src/plugins/dev_ena/rx_node.c
new file mode 100644
index 00000000000..41fc5b8c943
--- /dev/null
+++ b/src/plugins/dev_ena/rx_node.c
@@ -0,0 +1,457 @@
+/* SPDX-License-Identifier: Apache-2.0
+ * Copyright (c) 2023 Cisco Systems, Inc.
+ */
+
+#include <vnet/vnet.h>
+#include <vnet/dev/dev.h>
+#include <vnet/ethernet/ethernet.h>
+#include <vppinfra/vector/mask_compare.h>
+#include <vppinfra/vector/compress.h>
+
+#include <dev_ena/ena.h>
+#include <dev_ena/ena_inlines.h>
+
+#define ENA_RX_REFILL_BATCH 32
+
+typedef struct
+{
+ u16 phase_bit;
+ u16 size;
+ u32 mask;
+ ena_rx_cdesc_status_t st_or;
+ ena_rx_cdesc_status_t st_and;
+ u16 *comp_sqe_indices;
+ u32 *sq_buffer_indices;
+} ena_rx_ctx_t;
+
+static_always_inline void
+ena_device_input_status_to_flags (ena_rx_cdesc_status_t *statuses, u32 *flags,
+ u32 n_desc, vlib_frame_bitmap_t first_bmp,
+ int maybe_chained)
+{
+ const ena_rx_cdesc_status_t mask_first = { .first = 1 },
+ match_first1 = { .first = 1 };
+
+ const ena_rx_cdesc_status_t mask_last = { .last = 1 },
+ match_last0 = { .last = 0 };
+
+ const ena_rx_cdesc_status_t mask_l4_csum = { .ipv4_frag = 1,
+ .l4_csum_checked = 1,
+ .l4_csum_err = 1 },
+ match_l4_csum_ok = { .l4_csum_checked = 1 };
+
+ clib_memset_u32 (statuses + n_desc, 0, 8);
+#if defined(CLIB_HAVE_VEC128)
+
+#if defined(CxLIB_HAVE_VEC512)
+#define N 16
+#define u32xN u32x16
+#define u32xNu u32x16u
+#define u32xN_splat u32x16_splat
+#elif defined(CxLIB_HAVE_VEC256)
+#define N 8
+#define u32xN u32x8
+#define u32xNu u32x8u
+#define u32xN_splat u32x8_splat
+#else
+#define N 4
+#define u32xN u32x4
+#define u32xNu u32x4u
+#define u32xN_splat u32x4_splat
+#endif
+
+ const u32xN st_mask_first = u32xN_splat (mask_first.as_u32);
+ const u32xN st_match_first1 = u32xN_splat (match_first1.as_u32);
+ const u32xN st_mask_last = u32xN_splat (mask_last.as_u32);
+ const u32xN st_match_last0 = u32xN_splat (match_last0.as_u32);
+ const u32xN st_mask_l4_csum = u32xN_splat (mask_l4_csum.as_u32);
+ const u32xN st_match_l4_csum_ok = u32xN_splat (match_l4_csum_ok.as_u32);
+ const u32xN f_total_len_valid = u32xN_splat (VLIB_BUFFER_TOTAL_LENGTH_VALID);
+ const u32xN f_next_preset = u32xN_splat (VLIB_BUFFER_NEXT_PRESENT);
+ const u32xN f_l4_csum = u32xN_splat (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
+ VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
+
+ for (u32 i = 0; i < round_pow2 (n_desc, 2 * N); i += 2 * N)
+ {
+ uword msk = 0;
+ u32xN f0, f1, r0, r1;
+ u32xN s0 = ((u32xNu *) (statuses + i))[0];
+ u32xN s1 = ((u32xNu *) (statuses + i))[1];
+
+ r0 = (s0 & st_mask_first) == st_match_first1;
+ r1 = (s1 & st_mask_first) == st_match_first1;
+ f0 = r0 & f_total_len_valid;
+ f1 = r1 & f_total_len_valid;
+
+ if (maybe_chained)
+ {
+#if defined(CxLIB_HAVE_VEC512)
+ u64 msb_mask = 0x1111111111111111;
+ msk = bit_extract_u64 (u8x64_msb_mask ((u8x64) r0), msb_mask);
+ msk |= bit_extract_u64 (u8x64_msb_mask ((u8x64) r1), msb_mask) << 16;
+#elif defined(CxLIB_HAVE_VEC256)
+ msk = u8x32_msb_mask ((u8x32) r0);
+ msk |= (u64) u8x32_msb_mask ((u8x32) r1) << 32;
+ msk = bit_extract_u64 (msk, 0x1111111111111111);
+#else
+ msk = u8x16_msb_mask ((u8x16) r0);
+ msk |= (u32) u8x16_msb_mask ((u8x16) r1) << 16;
+ msk = bit_extract_u32 (msk, 0x11111111);
+#endif
+ first_bmp[i / uword_bits] |= msk << (i % uword_bits);
+ }
+
+ f0 |= ((s0 & st_mask_last) == st_match_last0) & f_next_preset;
+ f1 |= ((s1 & st_mask_last) == st_match_last0) & f_next_preset;
+
+ f0 |= ((s0 & st_mask_l4_csum) == st_match_l4_csum_ok) & f_l4_csum;
+ f1 |= ((s1 & st_mask_l4_csum) == st_match_l4_csum_ok) & f_l4_csum;
+
+ ((u32xNu *) (flags + i))[0] = f0;
+ ((u32xNu *) (flags + i))[1] = f1;
+ }
+#else
+ while (n_left)
+ {
+ u16 f = 0;
+ ena_rx_cdesc_status_t st = statuses++[0];
+
+ if ((st.as_u32 & mask_first.as_u32) == match_first1.as_u32)
+ f |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
+
+ if ((st.as_u32 & mask_last.as_u32) == match_last0.as_u32)
+ f |= VLIB_BUFFER_NEXT_PRESENT;
+
+ if ((st.as_u32 & mask_l4_csum.as_u32) == match_l4_csum_ok.as_u32)
+ f |= VNET_BUFFER_F_L4_CHECKSUM_COMPUTED |
+ VNET_BUFFER_F_L4_CHECKSUM_CORRECT;
+
+ flags++[0] = f;
+ n_left--;
+ }
+#endif
+}
+
+static_always_inline u16
+ena_device_input_cq_dequeue_no_wrap (ena_rx_ctx_t *ctx, ena_rxq_t *q,
+ ena_rx_cdesc_status_t *statuses,
+ u16 *lengths, u16 *csi)
+{
+ u32 next = q->cq_next;
+ ena_rx_cdesc_t *cqes = q->cqes;
+ u32 phase = (next & ctx->size << 1) != 0;
+ u16 index = next & ctx->mask;
+ ena_rx_cdesc_t *cd = cqes + index;
+ ena_rx_cdesc_status_t st;
+ u32 n_to_check, i = 0;
+
+ st = cd->status;
+ if (st.phase == phase)
+ return 0;
+
+ n_to_check = clib_min (VLIB_FRAME_SIZE, ctx->size - index);
+
+ ctx->st_or.as_u32 |= st.as_u32;
+ ctx->st_and.as_u32 &= st.as_u32;
+ statuses[i] = st;
+ lengths[i] = cd->length;
+ csi[i] = cd->req_id;
+ i++;
+ cd++;
+
+more:
+ for (st = cd->status; i < n_to_check && st.phase != phase;
+ i++, st = (++cd)->status)
+ {
+ ctx->st_or.as_u32 |= st.as_u32;
+ ctx->st_and.as_u32 &= st.as_u32;
+ statuses[i] = st;
+ lengths[i] = cd->length;
+ csi[i] = cd->req_id;
+ }
+
+ if (i == n_to_check)
+ {
+ n_to_check = VLIB_FRAME_SIZE - n_to_check;
+ if (n_to_check)
+ {
+ phase ^= 1;
+ cd = cqes;
+ goto more;
+ }
+ }
+
+ /* revert incomplete */
+ if (PREDICT_FALSE (statuses[i - 1].last == 0))
+ {
+ i--;
+ while (i && statuses[i - 1].last == 0)
+ i--;
+ }
+
+ return i;
+}
+
+static_always_inline void
+ena_device_input_refill (vlib_main_t *vm, ena_rx_ctx_t *ctx,
+ vnet_dev_rx_queue_t *rxq, int use_va)
+{
+ ena_rxq_t *q = vnet_dev_get_rx_queue_data (rxq);
+ const u64x2 flip_phase = (ena_rx_desc_t){ .lo.phase = 1 }.as_u64x2;
+ u32 buffer_indices[ENA_RX_REFILL_BATCH];
+ uword dma_addr[ENA_RX_REFILL_BATCH];
+ u32 n_alloc, n_compl_sqes = q->n_compl_sqes;
+ u16 *csi = ctx->comp_sqe_indices;
+ ena_rx_desc_t *sqes = q->sqes;
+
+ while (n_compl_sqes > 0)
+ {
+ n_alloc = vlib_buffer_alloc_from_pool (
+ vm, buffer_indices, clib_min (ENA_RX_REFILL_BATCH, n_compl_sqes),
+ vnet_dev_get_rx_queue_buffer_pool_index (rxq));
+
+ if (PREDICT_FALSE (n_alloc == 0))
+ break;
+
+ vlib_get_buffers_with_offset (vm, buffer_indices, (void **) dma_addr,
+ ENA_RX_REFILL_BATCH,
+ STRUCT_OFFSET_OF (vlib_buffer_t, data));
+
+ if (!use_va)
+ for (u32 i = 0; i < n_alloc; i++)
+ dma_addr[i] = vlib_physmem_get_pa (vm, (void *) dma_addr[i]);
+
+ for (u32 i = 0; i < n_alloc; i++)
+ {
+ u16 slot = csi[i];
+ u64x2 r = sqes[slot].as_u64x2 ^ flip_phase;
+ ctx->sq_buffer_indices[slot] = buffer_indices[i];
+ r[1] = dma_addr[i];
+ sqes[slot].as_u64x2 = r; /* write SQE as single 16-byte store */
+ }
+
+ csi += n_alloc;
+ n_compl_sqes -= n_alloc;
+ }
+
+ if (n_compl_sqes == q->n_compl_sqes)
+ return;
+
+ q->sq_next += q->n_compl_sqes - n_compl_sqes;
+ __atomic_store_n (q->sq_db, q->sq_next, __ATOMIC_RELEASE);
+
+ if (PREDICT_FALSE (n_compl_sqes))
+ clib_memmove (ctx->comp_sqe_indices, csi, n_compl_sqes * sizeof (csi[0]));
+
+ q->n_compl_sqes = n_compl_sqes;
+}
+
+static_always_inline uword
+ena_device_input_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
+ vnet_dev_rx_queue_t *rxq)
+{
+ ena_rxq_t *q = vnet_dev_get_rx_queue_data (rxq);
+ vnet_dev_port_t *port = rxq->port;
+ vnet_main_t *vnm = vnet_get_main ();
+ vlib_buffer_t *buffers[VLIB_FRAME_SIZE], **b;
+ ena_rx_cdesc_status_t statuses[VLIB_FRAME_SIZE + 8];
+ u16 lengths[VLIB_FRAME_SIZE + 8], *l;
+ u32 flags[VLIB_FRAME_SIZE + 8], *f;
+ u16 *csi;
+ uword n_rx_packets = 0, n_rx_bytes = 0;
+ vlib_frame_bitmap_t head_bmp = {};
+ u32 sw_if_index = port->intf.sw_if_index;
+ u32 hw_if_index = port->intf.hw_if_index;
+ u32 n_trace, n_deq, n_left;
+ u32 cq_next = q->cq_next;
+ u32 next_index = rxq->next_index;
+ vlib_frame_t *next_frame;
+ vlib_buffer_template_t bt = rxq->buffer_template;
+ u32 *bi;
+ int maybe_chained;
+
+ ASSERT (count_set_bits (rxq->size) == 1);
+ ena_rx_ctx_t ctx = {
+ .size = rxq->size,
+ .mask = rxq->size - 1,
+ .st_and.as_u32 = ~0,
+ .comp_sqe_indices = q->compl_sqe_indices,
+ .sq_buffer_indices = q->buffer_indices,
+ };
+
+ /* we may have completed SQE indices from previous run */
+ csi = ctx.comp_sqe_indices + q->n_compl_sqes;
+
+ n_deq =
+ ena_device_input_cq_dequeue_no_wrap (&ctx, q, statuses, lengths, csi);
+
+ if (n_deq == 0)
+ goto refill;
+
+ q->n_compl_sqes += n_deq;
+
+ maybe_chained = ctx.st_and.first && ctx.st_and.last ? 0 : 1;
+
+ next_frame =
+ vlib_get_next_frame_internal (vm, node, next_index, /* new frame */ 1);
+ bi = vlib_frame_vector_args (next_frame);
+
+ /* move buffer indices from the ring */
+ for (u32 i = 0; i < n_deq; i++)
+ {
+ u32 slot = csi[i];
+ bi[i] = ctx.sq_buffer_indices[slot];
+ ctx.sq_buffer_indices[slot] = VLIB_BUFFER_INVALID_INDEX;
+ }
+
+ vlib_get_buffers (vm, bi, buffers, n_deq);
+
+ if (PREDICT_FALSE (maybe_chained))
+ ena_device_input_status_to_flags (statuses, flags, n_deq, head_bmp, 1);
+ else
+ ena_device_input_status_to_flags (statuses, flags, n_deq, head_bmp, 0);
+
+ for (b = buffers, l = lengths, f = flags, n_left = n_deq; n_left >= 8;
+ b += 4, f += 4, l += 4, n_left -= 4)
+ {
+ clib_prefetch_store (b[4]);
+ clib_prefetch_store (b[5]);
+ clib_prefetch_store (b[6]);
+ clib_prefetch_store (b[7]);
+ b[0]->template = bt;
+ n_rx_bytes += b[0]->current_length = l[0];
+ b[0]->flags = f[0];
+ b[1]->template = bt;
+ n_rx_bytes += b[1]->current_length = l[1];
+ b[1]->flags = f[1];
+ b[2]->template = bt;
+ n_rx_bytes += b[2]->current_length = l[2];
+ b[2]->flags = f[2];
+ b[3]->template = bt;
+ n_rx_bytes += b[3]->current_length = l[3];
+ b[3]->flags = f[3];
+ }
+
+ for (; n_left > 0; b += 1, f += 1, l += 1, n_left -= 1)
+ {
+ b[0]->template = bt;
+ n_rx_bytes += b[0]->current_length = l[0];
+ b[0]->flags = f[0];
+ }
+
+ if (maybe_chained)
+ {
+ vlib_buffer_t *hb = 0;
+ vlib_frame_bitmap_t tail_buf_bmp = {};
+ u32 i, total_len = 0, head_flags = 0, tail_flags = 0;
+ n_rx_packets = vlib_frame_bitmap_count_set_bits (head_bmp);
+
+ vlib_frame_bitmap_init (tail_buf_bmp, n_deq);
+ vlib_frame_bitmap_xor (tail_buf_bmp, head_bmp);
+
+ foreach_vlib_frame_bitmap_set_bit_index (i, tail_buf_bmp)
+ {
+ vlib_buffer_t *pb = buffers[i - 1];
+ /* only store opertations here */
+ pb->next_buffer = bi[i];
+ if (vlib_frame_bitmap_is_bit_set (tail_buf_bmp, i - 1) == 0)
+ {
+ if (hb)
+ {
+ hb->total_length_not_including_first_buffer = total_len;
+ /* tail descriptor contains protocol info so we need to
+ * combine head and tail buffer flags */
+ hb->flags = head_flags | tail_flags;
+ }
+ head_flags = flags[i - 1];
+ total_len = 0;
+ hb = pb;
+ }
+ total_len += lengths[i];
+ tail_flags = flags[i];
+ }
+
+ hb->total_length_not_including_first_buffer = total_len;
+ hb->flags = head_flags | tail_flags;
+ }
+ else
+ n_rx_packets = n_deq;
+
+ /* packet tracing */
+ if (PREDICT_FALSE ((n_trace = vlib_get_trace_count (vm, node))))
+ {
+ u32 i;
+ if (!maybe_chained)
+ vlib_frame_bitmap_init (head_bmp, n_deq);
+ foreach_vlib_frame_bitmap_set_bit_index (i, head_bmp)
+ {
+ vlib_buffer_t *b = buffers[i];
+ if (vlib_trace_buffer (vm, node, next_index, b, 0))
+ {
+ u32 j = i;
+ ena_rx_trace_t *tr = vlib_add_trace (vm, node, b, sizeof (*tr));
+ tr->next_index = next_index;
+ tr->qid = rxq->queue_id;
+ tr->hw_if_index = hw_if_index;
+ tr->n_desc = 1;
+ tr->length = lengths[i];
+ tr->req_id = csi[i];
+ tr->status = statuses[i];
+ while (statuses[j].last == 0)
+ {
+ j++;
+ tr->n_desc++;
+ tr->length += lengths[j];
+ }
+ tr->status = statuses[j];
+
+ if (-n_trace)
+ goto trace_done;
+ }
+ }
+ trace_done:
+ vlib_set_trace_count (vm, node, n_trace);
+ }
+
+ if (PREDICT_FALSE (maybe_chained))
+ clib_compress_u32 (bi, bi, head_bmp, n_deq);
+
+ if (PREDICT_TRUE (next_index == VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT))
+ {
+ ethernet_input_frame_t *ef;
+ next_frame->flags = ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX;
+
+ ef = vlib_frame_scalar_args (next_frame);
+ ef->sw_if_index = sw_if_index;
+ ef->hw_if_index = hw_if_index;
+
+ if (ctx.st_or.l3_csum_err == 0)
+ next_frame->flags |= ETH_INPUT_FRAME_F_IP4_CKSUM_OK;
+ vlib_frame_no_append (next_frame);
+ }
+
+ vlib_put_next_frame (vm, node, next_index, VLIB_FRAME_SIZE - n_rx_packets);
+
+ vlib_increment_combined_counter (
+ vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX,
+ vm->thread_index, hw_if_index, n_rx_packets, n_rx_bytes);
+
+ q->cq_next = cq_next + n_deq;
+
+refill:
+ if (rxq->port->dev->va_dma)
+ ena_device_input_refill (vm, &ctx, rxq, 1);
+ else
+ ena_device_input_refill (vm, &ctx, rxq, 0);
+
+ return n_rx_packets;
+}
+
+VNET_DEV_NODE_FN (ena_rx_node)
+(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
+{
+ u32 n_rx = 0;
+ foreach_vnet_dev_rx_queue_runtime (rxq, node)
+ n_rx += ena_device_input_inline (vm, node, rxq);
+ return n_rx;
+}