From 1bd6cbb7aeded36cca4402d0c970b593316c9d70 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Thu, 15 Apr 2021 13:12:51 +0200 Subject: interface: tx queue infra Type: improvement Change-Id: I415b2f980de10ca3154d2c8677c24792453eccd0 Signed-off-by: Damjan Marion --- src/vnet/interface/runtime.c | 112 +++++++++++++++++++++---------- src/vnet/interface/rx_queue_funcs.h | 1 - src/vnet/interface/tx_queue.c | 128 ++++++++++++++++++++++++++++++++++++ src/vnet/interface/tx_queue_funcs.h | 29 ++++++++ 4 files changed, 235 insertions(+), 35 deletions(-) create mode 100644 src/vnet/interface/tx_queue.c create mode 100644 src/vnet/interface/tx_queue_funcs.h (limited to 'src/vnet/interface') diff --git a/src/vnet/interface/runtime.c b/src/vnet/interface/runtime.c index 20ac51f5b29..4fb24bfeb9e 100644 --- a/src/vnet/interface/runtime.c +++ b/src/vnet/interface/runtime.c @@ -19,6 +19,7 @@ #include #include #include +#include #include VLIB_REGISTER_LOG_CLASS (if_rxq_log, static) = { @@ -62,10 +63,12 @@ vnet_hw_if_update_runtime_data (vnet_main_t *vnm, u32 hw_if_index) u32 node_index = hi->input_node_index; vnet_hw_if_rx_queue_t *rxq; vnet_hw_if_rxq_poll_vector_t *pv, **d = 0; + vnet_hw_if_output_node_runtime_t *new_out_runtimes = 0; vlib_node_state_t *per_thread_node_state = 0; u32 n_threads = vlib_get_n_threads (); u16 *per_thread_node_adaptive = 0; - int something_changed = 0; + int something_changed_on_rx = 0; + int something_changed_on_tx = 0; clib_bitmap_t *pending_int = 0; int last_int = -1; @@ -81,13 +84,14 @@ vnet_hw_if_update_runtime_data (vnet_main_t *vnm, u32 hw_if_index) pool_foreach (rxq, im->hw_if_rx_queues) { u32 ti = rxq->thread_index; + vnet_hw_interface_t *rxq_hi; ASSERT (rxq->mode != VNET_HW_IF_RX_MODE_UNKNOWN); ASSERT (rxq->mode != VNET_HW_IF_RX_MODE_DEFAULT); - hi = vnet_get_hw_interface (vnm, rxq->hw_if_index); + rxq_hi = vnet_get_hw_interface (vnm, rxq->hw_if_index); - if (hi->input_node_index != node_index) + if (rxq_hi->input_node_index != node_index) continue; if (rxq->mode == VNET_HW_IF_RX_MODE_POLLING) @@ -111,10 +115,11 @@ vnet_hw_if_update_runtime_data (vnet_main_t *vnm, u32 hw_if_index) pool_foreach (rxq, im->hw_if_rx_queues) { u32 ti = rxq->thread_index; + vnet_hw_interface_t *rxq_hi; - hi = vnet_get_hw_interface (vnm, rxq->hw_if_index); + rxq_hi = vnet_get_hw_interface (vnm, rxq->hw_if_index); - if (hi->input_node_index != node_index) + if (rxq_hi->input_node_index != node_index) continue; if (rxq->mode == VNET_HW_IF_RX_MODE_INTERRUPT || @@ -140,7 +145,7 @@ vnet_hw_if_update_runtime_data (vnet_main_t *vnm, u32 hw_if_index) old_state = vlib_node_get_state (ovm, node_index); if (per_thread_node_state[i] != old_state) { - something_changed = 1; + something_changed_on_rx = 1; log_debug ("state changed for node %U on thread %u from %s to %s", format_vlib_node_name, vm, node_index, i, node_state_str[old_state], @@ -148,21 +153,48 @@ vnet_hw_if_update_runtime_data (vnet_main_t *vnm, u32 hw_if_index) } /* check if something changed */ - if (something_changed == 0) + if (something_changed_on_rx == 0) { vnet_hw_if_rx_node_runtime_t *rt; rt = vlib_node_get_runtime_data (ovm, node_index); if (vec_len (rt->rxq_poll_vector) != vec_len (d[i])) - something_changed = 1; + something_changed_on_rx = 1; else if (memcmp (d[i], rt->rxq_poll_vector, vec_len (d[i]) * sizeof (*d))) - something_changed = 1; + something_changed_on_rx = 1; if (clib_interrupt_get_n_int (rt->rxq_interrupts) != last_int + 1) - something_changed = 1; + something_changed_on_rx = 1; } } - if (something_changed) + new_out_runtimes = + vec_dup_aligned (hi->output_node_thread_runtimes, CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (new_out_runtimes, n_threads, CLIB_CACHE_LINE_BYTES); + + for (int i = 0; i < vec_len (hi->tx_queue_indices); i++) + { + u32 thread_index; + u32 queue_index = hi->tx_queue_indices[i]; + vnet_hw_if_tx_queue_t *txq = vnet_hw_if_get_tx_queue (vnm, queue_index); + + clib_bitmap_foreach (thread_index, txq->threads) + { + vnet_hw_if_output_node_runtime_t *rt; + rt = vec_elt_at_index (new_out_runtimes, thread_index); + if ((rt->frame.queue_id != txq->queue_id) || + (rt->frame.shared_queue != txq->shared_queue)) + { + log_debug ("tx queue data changed for interface %v, thread %u " + "(queue_id %u -> %u, shared_queue %u -> %u)", + hi->name, thread_index, rt->frame.queue_id, + txq->queue_id, rt->frame.shared_queue, + txq->shared_queue); + something_changed_on_tx = 1; + } + } + } + + if (something_changed_on_rx || something_changed_on_tx) { int with_barrier; @@ -177,35 +209,46 @@ vnet_hw_if_update_runtime_data (vnet_main_t *vnm, u32 hw_if_index) if (with_barrier) vlib_worker_thread_barrier_sync (vm); - for (int i = 0; i < n_threads; i++) + if (something_changed_on_rx) { - vlib_main_t *vm = vlib_get_main_by_index (i); - vnet_hw_if_rx_node_runtime_t *rt; - rt = vlib_node_get_runtime_data (vm, node_index); - pv = rt->rxq_poll_vector; - rt->rxq_poll_vector = d[i]; - d[i] = pv; - - if (rt->rxq_interrupts) + for (int i = 0; i < n_threads; i++) { - void *in = rt->rxq_interrupts; - int int_num = -1; - while ((int_num = clib_interrupt_get_next (in, int_num)) != -1) + vlib_main_t *vm = vlib_get_main_by_index (i); + vnet_hw_if_rx_node_runtime_t *rt; + rt = vlib_node_get_runtime_data (vm, node_index); + pv = rt->rxq_poll_vector; + rt->rxq_poll_vector = d[i]; + d[i] = pv; + + if (rt->rxq_interrupts) { - clib_interrupt_clear (in, int_num); - pending_int = clib_bitmap_set (pending_int, int_num, 1); - last_int = clib_max (last_int, int_num); + void *in = rt->rxq_interrupts; + int int_num = -1; + while ((int_num = clib_interrupt_get_next (in, int_num)) != + -1) + { + clib_interrupt_clear (in, int_num); + pending_int = clib_bitmap_set (pending_int, int_num, 1); + last_int = clib_max (last_int, int_num); + } } - } - vlib_node_set_state (vm, node_index, per_thread_node_state[i]); - vlib_node_set_flag (vm, node_index, VLIB_NODE_FLAG_ADAPTIVE_MODE, - per_thread_node_adaptive[i]); + vlib_node_set_state (vm, node_index, per_thread_node_state[i]); + vlib_node_set_flag (vm, node_index, VLIB_NODE_FLAG_ADAPTIVE_MODE, + per_thread_node_adaptive[i]); - if (last_int >= 0) - clib_interrupt_resize (&rt->rxq_interrupts, last_int + 1); - else - clib_interrupt_free (&rt->rxq_interrupts); + if (last_int >= 0) + clib_interrupt_resize (&rt->rxq_interrupts, last_int + 1); + else + clib_interrupt_free (&rt->rxq_interrupts); + } + } + if (something_changed_on_tx) + { + vnet_hw_if_output_node_runtime_t *t; + t = hi->output_node_thread_runtimes; + hi->output_node_thread_runtimes = new_out_runtimes; + new_out_runtimes = t; } if (with_barrier) @@ -231,4 +274,5 @@ vnet_hw_if_update_runtime_data (vnet_main_t *vnm, u32 hw_if_index) vec_free (d); vec_free (per_thread_node_state); vec_free (per_thread_node_adaptive); + vec_free (new_out_runtimes); } diff --git a/src/vnet/interface/rx_queue_funcs.h b/src/vnet/interface/rx_queue_funcs.h index e1e6c33988a..26dc1b8777f 100644 --- a/src/vnet/interface/rx_queue_funcs.h +++ b/src/vnet/interface/rx_queue_funcs.h @@ -33,7 +33,6 @@ vnet_hw_if_rx_mode vnet_hw_if_get_rx_queue_mode (vnet_main_t *vnm, u32 queue_index); void vnet_hw_if_set_rx_queue_thread_index (vnet_main_t *vnm, u32 queue_index, u32 thread_index); -void vnet_hw_if_update_runtime_data (vnet_main_t *vnm, u32 hw_if_index); void vnet_hw_if_generate_rxq_int_poll_vector (vlib_main_t *vm, vlib_node_runtime_t *node); diff --git a/src/vnet/interface/tx_queue.c b/src/vnet/interface/tx_queue.c new file mode 100644 index 00000000000..3041a58414d --- /dev/null +++ b/src/vnet/interface/tx_queue.c @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2021 Cisco Systems, Inc. + */ + +#include +#include +#include +#include + +VLIB_REGISTER_LOG_CLASS (if_txq_log, static) = { + .class_name = "interface", + .subclass_name = "tx-queue", +}; + +#define log_debug(fmt, ...) vlib_log_debug (if_txq_log.class, fmt, __VA_ARGS__) +#define log_err(fmt, ...) vlib_log_err (if_txq_log.class, fmt, __VA_ARGS__) + +static u64 +tx_queue_key (u32 hw_if_index, u32 queue_id) +{ + return ((u64) hw_if_index << 32) | queue_id; +} + +u32 +vnet_hw_if_get_tx_queue_index_by_id (vnet_main_t *vnm, u32 hw_if_index, + u32 queue_id) +{ + vnet_interface_main_t *im = &vnm->interface_main; + u64 key = tx_queue_key (hw_if_index, queue_id); + uword *p = hash_get_mem (im->txq_index_by_hw_if_index_and_queue_id, &key); + return p ? p[0] : ~0; +} + +u32 +vnet_hw_if_register_tx_queue (vnet_main_t *vnm, u32 hw_if_index, u32 queue_id) +{ + vnet_interface_main_t *im = &vnm->interface_main; + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index); + vnet_hw_if_tx_queue_t *txq; + u64 key = tx_queue_key (hw_if_index, queue_id); + u32 queue_index; + + if (hash_get_mem (im->txq_index_by_hw_if_index_and_queue_id, &key)) + clib_panic ("Trying to register already registered queue id (%u) in the " + "interface %v\n", + queue_id, hi->name); + + pool_get_zero (im->hw_if_tx_queues, txq); + queue_index = txq - im->hw_if_tx_queues; + vec_add1 (hi->tx_queue_indices, queue_index); + hash_set_mem_alloc (&im->txq_index_by_hw_if_index_and_queue_id, &key, + queue_index); + txq->hw_if_index = hw_if_index; + txq->queue_id = queue_id; + + log_debug ("register: interface %v queue-id %u", hi->name, queue_id); + + return queue_index; +} + +void +vnet_hw_if_unregister_tx_queue (vnet_main_t *vnm, u32 queue_index) +{ + vnet_interface_main_t *im = &vnm->interface_main; + vnet_hw_if_tx_queue_t *txq; + txq = vnet_hw_if_get_tx_queue (vnm, queue_index); + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, txq->hw_if_index); + u64 key; + + key = tx_queue_key (txq->hw_if_index, txq->queue_id); + hash_unset_mem_free (&im->txq_index_by_hw_if_index_and_queue_id, &key); + + for (int i = 0; i < vec_len (hi->tx_queue_indices); i++) + if (hi->tx_queue_indices[i] == queue_index) + { + vec_del1 (hi->tx_queue_indices, i); + break; + } + + log_debug ("unregister: interface %v queue-id %u", hi->name, txq->queue_id); + clib_bitmap_free (txq->threads); + pool_put_index (im->hw_if_tx_queues, queue_index); +} + +void +vnet_hw_if_unregister_all_tx_queues (vnet_main_t *vnm, u32 hw_if_index) +{ + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index); + vnet_interface_main_t *im = &vnm->interface_main; + vnet_hw_if_tx_queue_t *txq; + u64 key; + + log_debug ("unregister_all: interface %v", hi->name); + + for (int i = 0; i < vec_len (hi->tx_queue_indices); i++) + { + txq = vnet_hw_if_get_tx_queue (vnm, hi->tx_queue_indices[i]); + key = tx_queue_key (txq->hw_if_index, txq->queue_id); + hash_unset_mem_free (&im->txq_index_by_hw_if_index_and_queue_id, &key); + + clib_bitmap_free (txq->threads); + pool_put_index (im->hw_if_tx_queues, hi->tx_queue_indices[i]); + } + + vec_free (hi->tx_queue_indices); +} + +void +vnet_hw_if_tx_queue_assign_thread (vnet_main_t *vnm, u32 queue_index, + u32 thread_index) +{ + vnet_hw_if_tx_queue_t *txq = vnet_hw_if_get_tx_queue (vnm, queue_index); + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, txq->hw_if_index); + txq->threads = clib_bitmap_set (txq->threads, thread_index, 1); + log_debug ("assign_thread: interface %v queue-id %u thread %u", hi->name, + txq->queue_id, thread_index); +} + +void +vnet_hw_if_tx_queue_unassign_thread (vnet_main_t *vnm, u32 queue_index, + u32 thread_index) +{ + vnet_hw_if_tx_queue_t *txq = vnet_hw_if_get_tx_queue (vnm, queue_index); + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, txq->hw_if_index); + txq->threads = clib_bitmap_set (txq->threads, thread_index, 0); + log_debug ("unassign_thread: interface %v queue-id %u thread %u", hi->name, + txq->queue_id, thread_index); +} diff --git a/src/vnet/interface/tx_queue_funcs.h b/src/vnet/interface/tx_queue_funcs.h new file mode 100644 index 00000000000..22956a4eb9b --- /dev/null +++ b/src/vnet/interface/tx_queue_funcs.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2021 Cisco Systems, Inc. + */ + +#include + +/* funciton declarations */ + +u32 vnet_hw_if_get_tx_queue_index_by_id (vnet_main_t *vnm, u32 hw_if_index, + u32 queue_id); +u32 vnet_hw_if_register_tx_queue (vnet_main_t *vnm, u32 hw_if_index, + u32 queue_id); +void vnet_hw_if_unregister_tx_queue (vnet_main_t *vnm, u32 queue_index); +void vnet_hw_if_unregister_all_tx_queues (vnet_main_t *vnm, u32 hw_if_index); +void vnet_hw_if_tx_queue_assign_thread (vnet_main_t *vnm, u32 queue_index, + u32 thread_index); +void vnet_hw_if_tx_queue_unassign_thread (vnet_main_t *vnm, u32 queue_index, + u32 thread_index); + +/* inline functions */ + +static_always_inline vnet_hw_if_tx_queue_t * +vnet_hw_if_get_tx_queue (vnet_main_t *vnm, u32 queue_index) +{ + vnet_interface_main_t *im = &vnm->interface_main; + if (pool_is_free_index (im->hw_if_tx_queues, queue_index)) + return 0; + return pool_elt_at_index (im->hw_if_tx_queues, queue_index); +} -- cgit 1.2.3-korg