summaryrefslogtreecommitdiffstats
path: root/src/vnet/interface
diff options
context:
space:
mode:
authorDamjan Marion <damarion@cisco.com>2021-04-15 13:12:51 +0200
committerDamjan Marion <dmarion@me.com>2021-05-11 12:40:37 +0000
commit1bd6cbb7aeded36cca4402d0c970b593316c9d70 (patch)
treed7c9809f0a5b62b427f2f108471f64c615b32263 /src/vnet/interface
parent0d39cbac6d688fb98d05c32acec699b25a599ead (diff)
interface: tx queue infra
Type: improvement Change-Id: I415b2f980de10ca3154d2c8677c24792453eccd0 Signed-off-by: Damjan Marion <damarion@cisco.com>
Diffstat (limited to 'src/vnet/interface')
-rw-r--r--src/vnet/interface/runtime.c112
-rw-r--r--src/vnet/interface/rx_queue_funcs.h1
-rw-r--r--src/vnet/interface/tx_queue.c128
-rw-r--r--src/vnet/interface/tx_queue_funcs.h29
4 files changed, 235 insertions, 35 deletions
diff --git a/src/vnet/interface/runtime.c b/src/vnet/interface/runtime.c
index 20ac51f5b29..4fb24bfeb9e 100644
--- a/src/vnet/interface/runtime.c
+++ b/src/vnet/interface/runtime.c
@@ -19,6 +19,7 @@
#include <vnet/ip/ip.h>
#include <vnet/ethernet/ethernet.h>
#include <vnet/interface/rx_queue_funcs.h>
+#include <vnet/interface/tx_queue_funcs.h>
#include <vlib/unix/unix.h>
VLIB_REGISTER_LOG_CLASS (if_rxq_log, static) = {
@@ -62,10 +63,12 @@ vnet_hw_if_update_runtime_data (vnet_main_t *vnm, u32 hw_if_index)
u32 node_index = hi->input_node_index;
vnet_hw_if_rx_queue_t *rxq;
vnet_hw_if_rxq_poll_vector_t *pv, **d = 0;
+ vnet_hw_if_output_node_runtime_t *new_out_runtimes = 0;
vlib_node_state_t *per_thread_node_state = 0;
u32 n_threads = vlib_get_n_threads ();
u16 *per_thread_node_adaptive = 0;
- int something_changed = 0;
+ int something_changed_on_rx = 0;
+ int something_changed_on_tx = 0;
clib_bitmap_t *pending_int = 0;
int last_int = -1;
@@ -81,13 +84,14 @@ vnet_hw_if_update_runtime_data (vnet_main_t *vnm, u32 hw_if_index)
pool_foreach (rxq, im->hw_if_rx_queues)
{
u32 ti = rxq->thread_index;
+ vnet_hw_interface_t *rxq_hi;
ASSERT (rxq->mode != VNET_HW_IF_RX_MODE_UNKNOWN);
ASSERT (rxq->mode != VNET_HW_IF_RX_MODE_DEFAULT);
- hi = vnet_get_hw_interface (vnm, rxq->hw_if_index);
+ rxq_hi = vnet_get_hw_interface (vnm, rxq->hw_if_index);
- if (hi->input_node_index != node_index)
+ if (rxq_hi->input_node_index != node_index)
continue;
if (rxq->mode == VNET_HW_IF_RX_MODE_POLLING)
@@ -111,10 +115,11 @@ vnet_hw_if_update_runtime_data (vnet_main_t *vnm, u32 hw_if_index)
pool_foreach (rxq, im->hw_if_rx_queues)
{
u32 ti = rxq->thread_index;
+ vnet_hw_interface_t *rxq_hi;
- hi = vnet_get_hw_interface (vnm, rxq->hw_if_index);
+ rxq_hi = vnet_get_hw_interface (vnm, rxq->hw_if_index);
- if (hi->input_node_index != node_index)
+ if (rxq_hi->input_node_index != node_index)
continue;
if (rxq->mode == VNET_HW_IF_RX_MODE_INTERRUPT ||
@@ -140,7 +145,7 @@ vnet_hw_if_update_runtime_data (vnet_main_t *vnm, u32 hw_if_index)
old_state = vlib_node_get_state (ovm, node_index);
if (per_thread_node_state[i] != old_state)
{
- something_changed = 1;
+ something_changed_on_rx = 1;
log_debug ("state changed for node %U on thread %u from %s to %s",
format_vlib_node_name, vm, node_index, i,
node_state_str[old_state],
@@ -148,21 +153,48 @@ vnet_hw_if_update_runtime_data (vnet_main_t *vnm, u32 hw_if_index)
}
/* check if something changed */
- if (something_changed == 0)
+ if (something_changed_on_rx == 0)
{
vnet_hw_if_rx_node_runtime_t *rt;
rt = vlib_node_get_runtime_data (ovm, node_index);
if (vec_len (rt->rxq_poll_vector) != vec_len (d[i]))
- something_changed = 1;
+ something_changed_on_rx = 1;
else if (memcmp (d[i], rt->rxq_poll_vector,
vec_len (d[i]) * sizeof (*d)))
- something_changed = 1;
+ something_changed_on_rx = 1;
if (clib_interrupt_get_n_int (rt->rxq_interrupts) != last_int + 1)
- something_changed = 1;
+ something_changed_on_rx = 1;
}
}
- if (something_changed)
+ new_out_runtimes =
+ vec_dup_aligned (hi->output_node_thread_runtimes, CLIB_CACHE_LINE_BYTES);
+ vec_validate_aligned (new_out_runtimes, n_threads, CLIB_CACHE_LINE_BYTES);
+
+ for (int i = 0; i < vec_len (hi->tx_queue_indices); i++)
+ {
+ u32 thread_index;
+ u32 queue_index = hi->tx_queue_indices[i];
+ vnet_hw_if_tx_queue_t *txq = vnet_hw_if_get_tx_queue (vnm, queue_index);
+
+ clib_bitmap_foreach (thread_index, txq->threads)
+ {
+ vnet_hw_if_output_node_runtime_t *rt;
+ rt = vec_elt_at_index (new_out_runtimes, thread_index);
+ if ((rt->frame.queue_id != txq->queue_id) ||
+ (rt->frame.shared_queue != txq->shared_queue))
+ {
+ log_debug ("tx queue data changed for interface %v, thread %u "
+ "(queue_id %u -> %u, shared_queue %u -> %u)",
+ hi->name, thread_index, rt->frame.queue_id,
+ txq->queue_id, rt->frame.shared_queue,
+ txq->shared_queue);
+ something_changed_on_tx = 1;
+ }
+ }
+ }
+
+ if (something_changed_on_rx || something_changed_on_tx)
{
int with_barrier;
@@ -177,35 +209,46 @@ vnet_hw_if_update_runtime_data (vnet_main_t *vnm, u32 hw_if_index)
if (with_barrier)
vlib_worker_thread_barrier_sync (vm);
- for (int i = 0; i < n_threads; i++)
+ if (something_changed_on_rx)
{
- vlib_main_t *vm = vlib_get_main_by_index (i);
- vnet_hw_if_rx_node_runtime_t *rt;
- rt = vlib_node_get_runtime_data (vm, node_index);
- pv = rt->rxq_poll_vector;
- rt->rxq_poll_vector = d[i];
- d[i] = pv;
-
- if (rt->rxq_interrupts)
+ for (int i = 0; i < n_threads; i++)
{
- void *in = rt->rxq_interrupts;
- int int_num = -1;
- while ((int_num = clib_interrupt_get_next (in, int_num)) != -1)
+ vlib_main_t *vm = vlib_get_main_by_index (i);
+ vnet_hw_if_rx_node_runtime_t *rt;
+ rt = vlib_node_get_runtime_data (vm, node_index);
+ pv = rt->rxq_poll_vector;
+ rt->rxq_poll_vector = d[i];
+ d[i] = pv;
+
+ if (rt->rxq_interrupts)
{
- clib_interrupt_clear (in, int_num);
- pending_int = clib_bitmap_set (pending_int, int_num, 1);
- last_int = clib_max (last_int, int_num);
+ void *in = rt->rxq_interrupts;
+ int int_num = -1;
+ while ((int_num = clib_interrupt_get_next (in, int_num)) !=
+ -1)
+ {
+ clib_interrupt_clear (in, int_num);
+ pending_int = clib_bitmap_set (pending_int, int_num, 1);
+ last_int = clib_max (last_int, int_num);
+ }
}
- }
- vlib_node_set_state (vm, node_index, per_thread_node_state[i]);
- vlib_node_set_flag (vm, node_index, VLIB_NODE_FLAG_ADAPTIVE_MODE,
- per_thread_node_adaptive[i]);
+ vlib_node_set_state (vm, node_index, per_thread_node_state[i]);
+ vlib_node_set_flag (vm, node_index, VLIB_NODE_FLAG_ADAPTIVE_MODE,
+ per_thread_node_adaptive[i]);
- if (last_int >= 0)
- clib_interrupt_resize (&rt->rxq_interrupts, last_int + 1);
- else
- clib_interrupt_free (&rt->rxq_interrupts);
+ if (last_int >= 0)
+ clib_interrupt_resize (&rt->rxq_interrupts, last_int + 1);
+ else
+ clib_interrupt_free (&rt->rxq_interrupts);
+ }
+ }
+ if (something_changed_on_tx)
+ {
+ vnet_hw_if_output_node_runtime_t *t;
+ t = hi->output_node_thread_runtimes;
+ hi->output_node_thread_runtimes = new_out_runtimes;
+ new_out_runtimes = t;
}
if (with_barrier)
@@ -231,4 +274,5 @@ vnet_hw_if_update_runtime_data (vnet_main_t *vnm, u32 hw_if_index)
vec_free (d);
vec_free (per_thread_node_state);
vec_free (per_thread_node_adaptive);
+ vec_free (new_out_runtimes);
}
diff --git a/src/vnet/interface/rx_queue_funcs.h b/src/vnet/interface/rx_queue_funcs.h
index e1e6c33988a..26dc1b8777f 100644
--- a/src/vnet/interface/rx_queue_funcs.h
+++ b/src/vnet/interface/rx_queue_funcs.h
@@ -33,7 +33,6 @@ vnet_hw_if_rx_mode vnet_hw_if_get_rx_queue_mode (vnet_main_t *vnm,
u32 queue_index);
void vnet_hw_if_set_rx_queue_thread_index (vnet_main_t *vnm, u32 queue_index,
u32 thread_index);
-void vnet_hw_if_update_runtime_data (vnet_main_t *vnm, u32 hw_if_index);
void vnet_hw_if_generate_rxq_int_poll_vector (vlib_main_t *vm,
vlib_node_runtime_t *node);
diff --git a/src/vnet/interface/tx_queue.c b/src/vnet/interface/tx_queue.c
new file mode 100644
index 00000000000..3041a58414d
--- /dev/null
+++ b/src/vnet/interface/tx_queue.c
@@ -0,0 +1,128 @@
+/* SPDX-License-Identifier: Apache-2.0
+ * Copyright(c) 2021 Cisco Systems, Inc.
+ */
+
+#include <vnet/vnet.h>
+#include <vnet/devices/devices.h>
+#include <vnet/interface/tx_queue_funcs.h>
+#include <vlib/unix/unix.h>
+
+VLIB_REGISTER_LOG_CLASS (if_txq_log, static) = {
+ .class_name = "interface",
+ .subclass_name = "tx-queue",
+};
+
+#define log_debug(fmt, ...) vlib_log_debug (if_txq_log.class, fmt, __VA_ARGS__)
+#define log_err(fmt, ...) vlib_log_err (if_txq_log.class, fmt, __VA_ARGS__)
+
+static u64
+tx_queue_key (u32 hw_if_index, u32 queue_id)
+{
+ return ((u64) hw_if_index << 32) | queue_id;
+}
+
+u32
+vnet_hw_if_get_tx_queue_index_by_id (vnet_main_t *vnm, u32 hw_if_index,
+ u32 queue_id)
+{
+ vnet_interface_main_t *im = &vnm->interface_main;
+ u64 key = tx_queue_key (hw_if_index, queue_id);
+ uword *p = hash_get_mem (im->txq_index_by_hw_if_index_and_queue_id, &key);
+ return p ? p[0] : ~0;
+}
+
+u32
+vnet_hw_if_register_tx_queue (vnet_main_t *vnm, u32 hw_if_index, u32 queue_id)
+{
+ vnet_interface_main_t *im = &vnm->interface_main;
+ vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
+ vnet_hw_if_tx_queue_t *txq;
+ u64 key = tx_queue_key (hw_if_index, queue_id);
+ u32 queue_index;
+
+ if (hash_get_mem (im->txq_index_by_hw_if_index_and_queue_id, &key))
+ clib_panic ("Trying to register already registered queue id (%u) in the "
+ "interface %v\n",
+ queue_id, hi->name);
+
+ pool_get_zero (im->hw_if_tx_queues, txq);
+ queue_index = txq - im->hw_if_tx_queues;
+ vec_add1 (hi->tx_queue_indices, queue_index);
+ hash_set_mem_alloc (&im->txq_index_by_hw_if_index_and_queue_id, &key,
+ queue_index);
+ txq->hw_if_index = hw_if_index;
+ txq->queue_id = queue_id;
+
+ log_debug ("register: interface %v queue-id %u", hi->name, queue_id);
+
+ return queue_index;
+}
+
+void
+vnet_hw_if_unregister_tx_queue (vnet_main_t *vnm, u32 queue_index)
+{
+ vnet_interface_main_t *im = &vnm->interface_main;
+ vnet_hw_if_tx_queue_t *txq;
+ txq = vnet_hw_if_get_tx_queue (vnm, queue_index);
+ vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, txq->hw_if_index);
+ u64 key;
+
+ key = tx_queue_key (txq->hw_if_index, txq->queue_id);
+ hash_unset_mem_free (&im->txq_index_by_hw_if_index_and_queue_id, &key);
+
+ for (int i = 0; i < vec_len (hi->tx_queue_indices); i++)
+ if (hi->tx_queue_indices[i] == queue_index)
+ {
+ vec_del1 (hi->tx_queue_indices, i);
+ break;
+ }
+
+ log_debug ("unregister: interface %v queue-id %u", hi->name, txq->queue_id);
+ clib_bitmap_free (txq->threads);
+ pool_put_index (im->hw_if_tx_queues, queue_index);
+}
+
+void
+vnet_hw_if_unregister_all_tx_queues (vnet_main_t *vnm, u32 hw_if_index)
+{
+ vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
+ vnet_interface_main_t *im = &vnm->interface_main;
+ vnet_hw_if_tx_queue_t *txq;
+ u64 key;
+
+ log_debug ("unregister_all: interface %v", hi->name);
+
+ for (int i = 0; i < vec_len (hi->tx_queue_indices); i++)
+ {
+ txq = vnet_hw_if_get_tx_queue (vnm, hi->tx_queue_indices[i]);
+ key = tx_queue_key (txq->hw_if_index, txq->queue_id);
+ hash_unset_mem_free (&im->txq_index_by_hw_if_index_and_queue_id, &key);
+
+ clib_bitmap_free (txq->threads);
+ pool_put_index (im->hw_if_tx_queues, hi->tx_queue_indices[i]);
+ }
+
+ vec_free (hi->tx_queue_indices);
+}
+
+void
+vnet_hw_if_tx_queue_assign_thread (vnet_main_t *vnm, u32 queue_index,
+ u32 thread_index)
+{
+ vnet_hw_if_tx_queue_t *txq = vnet_hw_if_get_tx_queue (vnm, queue_index);
+ vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, txq->hw_if_index);
+ txq->threads = clib_bitmap_set (txq->threads, thread_index, 1);
+ log_debug ("assign_thread: interface %v queue-id %u thread %u", hi->name,
+ txq->queue_id, thread_index);
+}
+
+void
+vnet_hw_if_tx_queue_unassign_thread (vnet_main_t *vnm, u32 queue_index,
+ u32 thread_index)
+{
+ vnet_hw_if_tx_queue_t *txq = vnet_hw_if_get_tx_queue (vnm, queue_index);
+ vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, txq->hw_if_index);
+ txq->threads = clib_bitmap_set (txq->threads, thread_index, 0);
+ log_debug ("unassign_thread: interface %v queue-id %u thread %u", hi->name,
+ txq->queue_id, thread_index);
+}
diff --git a/src/vnet/interface/tx_queue_funcs.h b/src/vnet/interface/tx_queue_funcs.h
new file mode 100644
index 00000000000..22956a4eb9b
--- /dev/null
+++ b/src/vnet/interface/tx_queue_funcs.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: Apache-2.0
+ * Copyright(c) 2021 Cisco Systems, Inc.
+ */
+
+#include <vnet/vnet.h>
+
+/* funciton declarations */
+
+u32 vnet_hw_if_get_tx_queue_index_by_id (vnet_main_t *vnm, u32 hw_if_index,
+ u32 queue_id);
+u32 vnet_hw_if_register_tx_queue (vnet_main_t *vnm, u32 hw_if_index,
+ u32 queue_id);
+void vnet_hw_if_unregister_tx_queue (vnet_main_t *vnm, u32 queue_index);
+void vnet_hw_if_unregister_all_tx_queues (vnet_main_t *vnm, u32 hw_if_index);
+void vnet_hw_if_tx_queue_assign_thread (vnet_main_t *vnm, u32 queue_index,
+ u32 thread_index);
+void vnet_hw_if_tx_queue_unassign_thread (vnet_main_t *vnm, u32 queue_index,
+ u32 thread_index);
+
+/* inline functions */
+
+static_always_inline vnet_hw_if_tx_queue_t *
+vnet_hw_if_get_tx_queue (vnet_main_t *vnm, u32 queue_index)
+{
+ vnet_interface_main_t *im = &vnm->interface_main;
+ if (pool_is_free_index (im->hw_if_tx_queues, queue_index))
+ return 0;
+ return pool_elt_at_index (im->hw_if_tx_queues, queue_index);
+}