diff options
Diffstat (limited to 'src/plugins/dev_armada/pp2/rx.c')
| -rw-r--r-- | src/plugins/dev_armada/pp2/rx.c | 269 |
1 files changed, 269 insertions, 0 deletions
diff --git a/src/plugins/dev_armada/pp2/rx.c b/src/plugins/dev_armada/pp2/rx.c new file mode 100644 index 00000000000..5b0e8d35000 --- /dev/null +++ b/src/plugins/dev_armada/pp2/rx.c @@ -0,0 +1,269 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Cisco Systems, Inc. + */ + +#include <vlib/vlib.h> +#include <vnet/dev/dev.h> +#include <vnet/ethernet/ethernet.h> +#include <vppinfra/vector/mask_compare.h> +#include <vppinfra/vector/compress.h> + +#include <dev_armada/pp2/pp2.h> + +static_always_inline vlib_buffer_t * +desc_to_vlib_buffer (vlib_main_t *vm, struct pp2_ppio_desc *d) +{ + return vlib_get_buffer (vm, pp2_ppio_inq_desc_get_cookie (d)); +} + +static_always_inline u64 +mrvl_pp2_rx_one_if (vlib_main_t *vm, vlib_node_runtime_t *node, + vnet_dev_rx_queue_t *rxq, + vnet_dev_rx_queue_if_rt_data_t *if_rt_data, + struct pp2_ppio_desc **desc_ptrs, u32 n_desc, + i32 current_data, i32 len_adj, mv_dsa_tag_t tag) +{ + vnet_main_t *vnm = vnet_get_main (); + u64 n_rx_bytes = 0; + vlib_buffer_t *b0, *b1; + u32 n_trace, n_left = n_desc; + u32 buffer_indices[VLIB_FRAME_SIZE], *bi = buffer_indices; + struct pp2_ppio_desc **dp = desc_ptrs; + u32 next_index = if_rt_data->next_index; + vlib_buffer_template_t bt = if_rt_data->buffer_template; + u32 sw_if_index = if_rt_data->sw_if_index; + + bt.current_data = current_data; + + for (; n_left >= 4; dp += 2, bi += 2, n_left -= 2) + { + clib_prefetch_store (desc_to_vlib_buffer (vm, dp[2])); + clib_prefetch_store (desc_to_vlib_buffer (vm, dp[3])); + b0 = desc_to_vlib_buffer (vm, dp[0]); + b1 = desc_to_vlib_buffer (vm, dp[1]); + bi[0] = pp2_ppio_inq_desc_get_cookie (dp[0]); + bi[1] = pp2_ppio_inq_desc_get_cookie (dp[1]); + b0->template = bt; + b1->template = bt; + + n_rx_bytes += b0->current_length = + pp2_ppio_inq_desc_get_pkt_len (dp[0]) + len_adj; + n_rx_bytes += b1->current_length = + pp2_ppio_inq_desc_get_pkt_len (dp[1]) + len_adj; + } + + for (; n_left; dp++, bi++, n_left--) + { + b0 = desc_to_vlib_buffer (vm, dp[0]); + bi[0] = pp2_ppio_inq_desc_get_cookie (dp[0]); + b0->template = bt; + + n_rx_bytes += b0->current_length = + pp2_ppio_inq_desc_get_pkt_len (dp[0]) + len_adj; + } + + /* trace */ + n_trace = vlib_get_trace_count (vm, node); + if (PREDICT_FALSE (n_trace > 0)) + { + for (u32 i = 0; i < n_desc && n_trace > 0; i++) + { + vlib_buffer_t *b = desc_to_vlib_buffer (vm, desc_ptrs[i]); + if (PREDICT_TRUE (vlib_trace_buffer (vm, node, next_index, b, + /* follow_chain */ 0))) + { + mvpp2_rx_trace_t *tr; + tr = vlib_add_trace (vm, node, b, sizeof (*tr)); + tr->desc = *desc_ptrs[i]; + tr->next_index = next_index; + tr->sw_if_index = sw_if_index; + tr->dsa_tag = tag; + n_trace--; + } + } + vlib_set_trace_count (vm, node, n_trace); + } + vlib_buffer_enqueue_to_single_next (vm, node, buffer_indices, next_index, + n_desc); + + vlib_increment_combined_counter ( + vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, + vm->thread_index, sw_if_index, n_desc, n_rx_bytes); + + return n_rx_bytes; +} + +static_always_inline uword +mrvl_pp2_rx_inline (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_frame_t *frame, vnet_dev_rx_queue_t *rxq) +{ + vnet_dev_port_t *port = rxq->port; + mvpp2_port_t *mp = vnet_dev_get_port_data (port); + mv_dsa_tag_t dsa_tags[VLIB_FRAME_SIZE]; + u16 n_desc = VLIB_FRAME_SIZE; + vlib_buffer_t *b; + u32 i; + + if (PREDICT_FALSE ( + pp2_ppio_recv (mp->ppio, 0, rxq->queue_id, mp->descs, &n_desc))) + { + vlib_error_count (vm, node->node_index, MVPP2_RX_NODE_CTR_PPIO_RECV, 1); + return 0; + } + + if (mp->is_dsa) + { + for (i = 0; i < n_desc; i++) + { + b = desc_to_vlib_buffer (vm, mp->descs + i); + u8 *start = b->data; + mv_dsa_tag_t tag = mv_dsa_tag_read (start + 14); + dsa_tags[i] = tag; + clib_memmove (start + 6, start + 2, 12); + } + + vlib_frame_bitmap_t avail_bmp = {}; + vlib_frame_bitmap_init (avail_bmp, n_desc); + u32 n_avail = n_desc; + + while (n_avail) + { + vlib_frame_bitmap_t selected_bmp = {}; + struct pp2_ppio_desc *sel_descs[VLIB_FRAME_SIZE]; + mv_dsa_tag_t tag; + u32 n_sel, index; + + tag = dsa_tags[vlib_frame_bitmap_find_first_set (avail_bmp)]; + index = tag.src_dev << 5 | tag.src_port_or_lag; + + clib_mask_compare_u32 (tag.as_u32, (u32 *) dsa_tags, selected_bmp, + n_desc); + n_sel = vlib_frame_bitmap_count_set_bits (selected_bmp); + n_avail -= n_sel; + + if (uword_bitmap_is_bit_set (mp->valid_dsa_src_bitmap, index)) + { + clib_compress_u64 ((uword *) sel_descs, (uword *) mp->desc_ptrs, + selected_bmp, n_desc); + mrvl_pp2_rx_one_if (vm, node, rxq, + vnet_dev_get_rx_queue_sec_if_rt_data ( + rxq, mp->dsa_to_sec_if[index]), + sel_descs, n_sel, 6, -4, tag); + } + else + { + u32 n_free = 0, buffer_indices[VLIB_FRAME_SIZE]; + + foreach_vlib_frame_bitmap_set_bit_index (i, selected_bmp) + buffer_indices[n_free++] = + pp2_ppio_inq_desc_get_cookie (mp->descs + i); + + u32 n_trace = vlib_get_trace_count (vm, node); + if (PREDICT_FALSE (n_trace > 0)) + { + foreach_vlib_frame_bitmap_set_bit_index (i, selected_bmp) + { + vlib_buffer_t *b = + desc_to_vlib_buffer (vm, mp->descs + i); + + if (PREDICT_TRUE (vlib_trace_buffer ( + vm, node, VNET_DEV_ETH_RX_PORT_NEXT_DROP, b, + /* follow_chain */ 0))) + { + mvpp2_rx_trace_t *tr; + tr = vlib_add_trace (vm, node, b, sizeof (*tr)); + tr->desc = mp->descs[i]; + tr->next_index = VNET_DEV_ETH_RX_PORT_NEXT_DROP; + tr->sw_if_index = CLIB_U32_MAX; + tr->dsa_tag = dsa_tags[i]; + n_trace--; + } + if (n_trace == 0) + break; + } + vlib_set_trace_count (vm, node, n_trace); + } + + vlib_buffer_free (vm, buffer_indices, n_free); + vlib_error_count (vm, node->node_index, + MVPP2_RX_NODE_CTR_UNKNOWN_DSA_SRC, 1); + } + } + } + else + { + mrvl_pp2_rx_one_if (vm, node, rxq, + vnet_dev_get_rx_queue_if_rt_data (rxq), + mp->desc_ptrs, n_desc, 2, 0, (mv_dsa_tag_t){}); + } + + return n_desc; +} + +static_always_inline void +mrvl_pp2_rx_refill (vlib_main_t *vm, vlib_node_runtime_t *node, + vnet_dev_rx_queue_t *rxq) +{ + vnet_dev_port_t *port = rxq->port; + vnet_dev_t *dev = port->dev; + mvpp2_device_t *md = vnet_dev_get_data (dev); + u32 thread_index = vm->thread_index; + struct pp2_hif *hif = md->hif[thread_index]; + struct pp2_bpool *bpool = md->thread[thread_index].bpool; + struct buff_release_entry *bre = md->thread[thread_index].bre; + u32 n_bufs, *bi; + + if (PREDICT_FALSE (pp2_bpool_get_num_buffs (bpool, &n_bufs))) + { + vlib_error_count (vm, node->node_index, + MVPP2_RX_NODE_CTR_BPOOL_GET_NUM_BUFFS, 1); + return; + } + + n_bufs = rxq->size - n_bufs; + while (n_bufs >= MRVL_PP2_BUFF_BATCH_SZ) + { + u16 n_alloc, i; + struct buff_release_entry *e = bre; + u32 buffer_indices[MRVL_PP2_BUFF_BATCH_SZ]; + + n_alloc = vlib_buffer_alloc (vm, buffer_indices, MRVL_PP2_BUFF_BATCH_SZ); + + if (PREDICT_FALSE (n_alloc == 0)) + { + vlib_error_count (vm, node->node_index, + MVPP2_RX_NODE_CTR_BUFFER_ALLOC, 1); + return; + } + + for (i = n_alloc, bi = buffer_indices; i--; e++, bi++) + { + + vlib_buffer_t *b = vlib_get_buffer (vm, bi[0]); + e->buff.addr = vlib_buffer_get_pa (vm, b) - 64; + e->buff.cookie = bi[0]; + } + + if (PREDICT_FALSE (pp2_bpool_put_buffs (hif, bre, &n_alloc))) + { + vlib_error_count (vm, node->node_index, + MVPP2_RX_NODE_CTR_BPOOL_PUT_BUFFS, 1); + vlib_buffer_free (vm, buffer_indices, n_alloc); + return; + } + + n_bufs -= n_alloc; + } +} + +VNET_DEV_NODE_FN (mvpp2_rx_node) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) +{ + u32 n_rx = 0; + foreach_vnet_dev_rx_queue_runtime (rxq, node) + { + n_rx += mrvl_pp2_rx_inline (vm, node, frame, rxq); + mrvl_pp2_rx_refill (vm, node, rxq); + } + return n_rx; +} |