aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/plugins/memif/memif.c4
-rw-r--r--src/vnet/devices/tap/tap.c6
-rw-r--r--src/vnet/ethernet/interface.c1
-rw-r--r--src/vnet/interface.api77
-rw-r--r--src/vnet/interface.c5
-rw-r--r--src/vnet/interface.h12
-rw-r--r--src/vnet/interface/runtime.c85
-rw-r--r--src/vnet/interface/tx_queue_funcs.h17
-rw-r--r--src/vnet/interface_api.c169
-rw-r--r--src/vnet/interface_cli.c160
-rw-r--r--src/vnet/interface_format.c3
-rw-r--r--src/vnet/interface_funcs.h3
-rw-r--r--src/vnet/interface_output.c342
-rw-r--r--src/vnet/interface_test.c122
-rw-r--r--src/vnet/pg/stream.c1
15 files changed, 869 insertions, 138 deletions
diff --git a/src/plugins/memif/memif.c b/src/plugins/memif/memif.c
index 3b01819491b..f8c5191b971 100644
--- a/src/plugins/memif/memif.c
+++ b/src/plugins/memif/memif.c
@@ -855,10 +855,10 @@ memif_delete_if (vlib_main_t * vm, memif_if_t * mif)
}
/* *INDENT-OFF* */
-VNET_HW_INTERFACE_CLASS (memif_ip_hw_if_class, static) =
-{
+VNET_HW_INTERFACE_CLASS (memif_ip_hw_if_class, static) = {
.name = "memif-ip",
.flags = VNET_HW_INTERFACE_CLASS_FLAG_P2P,
+ .tx_hash_fn_type = VNET_HASH_FN_TYPE_IP,
};
/* *INDENT-ON* */
diff --git a/src/vnet/devices/tap/tap.c b/src/vnet/devices/tap/tap.c
index 2d075f9e0fc..4c0b4e0251c 100644
--- a/src/vnet/devices/tap/tap.c
+++ b/src/vnet/devices/tap/tap.c
@@ -58,13 +58,11 @@ tap_main_t tap_main;
goto error; \
}
- /* *INDENT-OFF* */
-VNET_HW_INTERFACE_CLASS (tun_device_hw_interface_class, static) =
-{
+VNET_HW_INTERFACE_CLASS (tun_device_hw_interface_class, static) = {
.name = "tun-device",
.flags = VNET_HW_INTERFACE_CLASS_FLAG_P2P,
+ .tx_hash_fn_type = VNET_HASH_FN_TYPE_IP,
};
- /* *INDENT-ON* */
#define TUN_MAX_PACKET_BYTES 65355
#define TUN_MIN_PACKET_BYTES 64
diff --git a/src/vnet/ethernet/interface.c b/src/vnet/ethernet/interface.c
index d287748c0e6..c941f82a50b 100644
--- a/src/vnet/ethernet/interface.c
+++ b/src/vnet/ethernet/interface.c
@@ -313,6 +313,7 @@ ethernet_mac_change (vnet_hw_interface_t * hi,
/* *INDENT-OFF* */
VNET_HW_INTERFACE_CLASS (ethernet_hw_interface_class) = {
.name = "Ethernet",
+ .tx_hash_fn_type = VNET_HASH_FN_TYPE_ETHERNET,
.format_address = format_ethernet_address,
.format_header = format_ethernet_header_with_length,
.unformat_hw_address = unformat_ethernet_address,
diff --git a/src/vnet/interface.api b/src/vnet/interface.api
index d89dea4e353..172f6afb818 100644
--- a/src/vnet/interface.api
+++ b/src/vnet/interface.api
@@ -458,6 +458,29 @@ autoreply define sw_interface_set_rx_placement
bool is_main;
};
+/** \brief Set an interface's tx-placement
+ Tx-Queue placement on specific thread is operational for only hardware
+ interface. It will not set queue - thread placement for sub-interfaces,
+ p2p and pipe interfaces.
+ @param client_index - opaque cookie to identify the sender
+ @param context - sender context, to match reply w/ request
+ @param sw_if_index - the interface whose tx-placement will be set
+ @param queue_id - the queue number whose tx-placement will be set.
+ @param array_size - the size of the thread indexes array
+ @param threads - the thread indexes of main and worker(s) threads
+ whom tx-placement will be at.
+*/
+autoendian autoreply define sw_interface_set_tx_placement
+{
+ u32 client_index;
+ u32 context;
+ vl_api_interface_index_t sw_if_index;
+ u32 queue_id;
+ u32 array_size;
+ u32 threads[array_size];
+ option vat_help = "<interface | sw_if_index <index>> queue <n> [threads <list> | mask <hex>]";
+};
+
/** \brief Set custom interface name
Set custom interface name for the interface.
@param client_index - opaque cookie to identify the sender
@@ -512,6 +535,60 @@ define sw_interface_rx_placement_details
vl_api_rx_mode_t mode;
};
+service {
+ rpc sw_interface_tx_placement_get returns sw_interface_tx_placement_get_reply
+ stream sw_interface_tx_placement_details;
+};
+
+/** \brief get the tx queue placement of interface(s)
+ @param cursor - optional, it allows client to continue a dump
+ @param sw_if_index - optional interface index for which queue placement to
+ be requested. sw_if_index = ~0 will get the placement information for all
+ interfaces. It will not get information related to sub-interfaces, p2p
+ and pipe interfaces.
+*/
+autoendian define sw_interface_tx_placement_get
+{
+ u32 client_index;
+ u32 context;
+ u32 cursor;
+ vl_api_interface_index_t sw_if_index;
+ option vat_help = "[interface | sw_if_index <index>]";
+};
+
+autoendian define sw_interface_tx_placement_get_reply
+{
+ u32 context;
+ i32 retval;
+ u32 cursor;
+};
+
+/** \brief show the interface's queue - thread placement
+ This api is used to display the interface and queue worker
+ thread placement. One message per tx-queue per interface will
+ be sent to client.
+ Each message will contain information about tx-queue id of an
+ interface, interface index, thread on which this tx-queue is
+ placed and mode of tx-queue.
+ @param client_index - opaque cookie to identify the sender
+ @param context - sender context, to match reply w/ request
+ @param sw_if_index - the interface whose tx-placement will be dumped
+ @param queue_id - the queue id
+ @param shared - the queue is shared on other threads
+ @param array_size - the size of the threads array
+ @param threads - the main and worker(s) thread index(es) whom tx-placement are at.
+*/
+autoendian define sw_interface_tx_placement_details
+{
+ u32 client_index;
+ u32 context;
+ vl_api_interface_index_t sw_if_index;
+ u32 queue_id;
+ u8 shared;
+ u32 array_size;
+ u32 threads[array_size];
+};
+
/* Gross kludge, DGMS */
autoreply define interface_name_renumber
{
diff --git a/src/vnet/interface.c b/src/vnet/interface.c
index ce024993826..982abbd0199 100644
--- a/src/vnet/interface.c
+++ b/src/vnet/interface.c
@@ -863,6 +863,10 @@ vnet_register_interface (vnet_main_t * vnm,
hw->hw_if_index = hw_index;
hw->default_rx_mode = VNET_HW_IF_RX_MODE_POLLING;
+ if (hw_class->tx_hash_fn_type == VNET_HASH_FN_TYPE_ETHERNET ||
+ hw_class->tx_hash_fn_type == VNET_HASH_FN_TYPE_IP)
+ hw->hf = vnet_hash_default_function (hw_class->tx_hash_fn_type);
+
if (dev_class->format_device_name)
hw->name = format (0, "%U", dev_class->format_device_name, dev_instance);
else if (hw_class->format_interface_name)
@@ -1020,6 +1024,7 @@ vnet_register_interface (vnet_main_t * vnm,
static char *e[] = {
"interface is down",
"interface is deleted",
+ "no tx queue available",
};
r.n_errors = ARRAY_LEN (e);
diff --git a/src/vnet/interface.h b/src/vnet/interface.h
index fe42e5d33a1..822d4c3a7ad 100644
--- a/src/vnet/interface.h
+++ b/src/vnet/interface.h
@@ -44,6 +44,7 @@
#include <vppinfra/pcap.h>
#include <vnet/l3_types.h>
#include <vppinfra/lock.h>
+#include <vnet/hash/hash.h>
struct vnet_main_t;
struct vnet_hw_interface_t;
@@ -410,6 +411,9 @@ typedef struct _vnet_hw_interface_class
/* Flags */
vnet_hw_interface_class_flags_t flags;
+ /* tx hash type for interfaces of this hw class */
+ vnet_hash_fn_type_t tx_hash_fn_type;
+
/* Function to call when hardware interface is added/deleted. */
vnet_interface_function_t *interface_add_del_function;
@@ -641,8 +645,9 @@ typedef struct
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
- vnet_hw_if_tx_frame_t frame;
- u32 n_threads;
+ vnet_hw_if_tx_frame_t *frame;
+ u32 *lookup_table;
+ u32 n_queues;
} vnet_hw_if_output_node_runtime_t;
/* Hardware-interface. This corresponds to a physical wire
@@ -696,6 +701,9 @@ typedef struct vnet_hw_interface_t
used by node function vnet_per_buffer_interface_output() */
u32 output_node_next_index;
+ /* called when hw interface is using transmit side packet steering */
+ vnet_hash_fn_t hf;
+
/* Maximum transmit rate for this interface in bits/sec. */
f64 max_rate_bits_per_sec;
diff --git a/src/vnet/interface/runtime.c b/src/vnet/interface/runtime.c
index e63f1ecc2fc..5c215e88501 100644
--- a/src/vnet/interface/runtime.c
+++ b/src/vnet/interface/runtime.c
@@ -184,39 +184,73 @@ vnet_hw_if_update_runtime_data (vnet_main_t *vnm, u32 hw_if_index)
}
}
- new_out_runtimes =
- vec_dup_aligned (hi->output_node_thread_runtimes, CLIB_CACHE_LINE_BYTES);
- vec_validate_aligned (new_out_runtimes, n_threads - 1,
- CLIB_CACHE_LINE_BYTES);
-
- if (vec_len (hi->output_node_thread_runtimes) != vec_len (new_out_runtimes))
- something_changed_on_tx = 1;
-
- for (int i = 0; i < vec_len (hi->tx_queue_indices); i++)
+ if (vec_len (hi->tx_queue_indices) > 0)
{
- u32 thread_index;
- u32 queue_index = hi->tx_queue_indices[i];
- vnet_hw_if_tx_queue_t *txq = vnet_hw_if_get_tx_queue (vnm, queue_index);
- uword n_threads = clib_bitmap_count_set_bits (txq->threads);
+ new_out_runtimes = vec_dup_aligned (hi->output_node_thread_runtimes,
+ CLIB_CACHE_LINE_BYTES);
+ vec_validate_aligned (new_out_runtimes, n_threads - 1,
+ CLIB_CACHE_LINE_BYTES);
- clib_bitmap_foreach (thread_index, txq->threads)
+ for (u32 i = 0; i < vec_len (new_out_runtimes); i++)
{
vnet_hw_if_output_node_runtime_t *rt;
- rt = vec_elt_at_index (new_out_runtimes, thread_index);
- if ((rt->frame.queue_id != txq->queue_id) ||
- (rt->n_threads != n_threads))
+ rt = vec_elt_at_index (new_out_runtimes, i);
+ u32 n_queues = 0, total_queues = vec_len (hi->tx_queue_indices);
+ rt->frame = 0;
+ rt->lookup_table = 0;
+
+ for (u32 j = 0; j < total_queues; j++)
{
+ u32 queue_index = hi->tx_queue_indices[j];
+ vnet_hw_if_tx_frame_t frame = { .shared_queue = 0,
+ .hints = 7,
+ .queue_id = ~0 };
+ vnet_hw_if_tx_queue_t *txq =
+ vnet_hw_if_get_tx_queue (vnm, queue_index);
+ if (!clib_bitmap_get (txq->threads, i))
+ continue;
+
log_debug ("tx queue data changed for interface %v, thread %u "
- "(queue_id %u -> %u, n_threads %u -> %u)",
- hi->name, thread_index, rt->frame.queue_id,
- txq->queue_id, rt->n_threads, n_threads);
+ "(queue_id %u)",
+ hi->name, i, txq->queue_id);
+ something_changed_on_tx = 1;
+
+ frame.queue_id = txq->queue_id;
+ frame.shared_queue = txq->shared_queue;
+ vec_add1 (rt->frame, frame);
+ n_queues++;
+ }
+
+ // don't initialize rt->n_queues above
+ if (rt->n_queues != n_queues)
+ {
something_changed_on_tx = 1;
- rt->frame.queue_id = txq->queue_id;
- rt->frame.shared_queue = txq->shared_queue;
- rt->n_threads = n_threads;
+ rt->n_queues = n_queues;
+ }
+ /*
+ * It is only used in case of multiple txq.
+ */
+ if (rt->n_queues > 0)
+ {
+ if (!is_pow2 (n_queues))
+ n_queues = max_pow2 (n_queues);
+
+ vec_validate_aligned (rt->lookup_table, n_queues - 1,
+ CLIB_CACHE_LINE_BYTES);
+
+ for (u32 k = 0; k < vec_len (rt->lookup_table); k++)
+ {
+ rt->lookup_table[k] = rt->frame[k % rt->n_queues].queue_id;
+ log_debug ("tx queue lookup table changed for interface %v, "
+ "(lookup table [%u]=%u)",
+ hi->name, k, rt->lookup_table[k]);
+ }
}
}
}
+ else
+ /* interface deleted */
+ something_changed_on_tx = 1;
if (something_changed_on_rx || something_changed_on_tx)
{
@@ -303,6 +337,11 @@ vnet_hw_if_update_runtime_data (vnet_main_t *vnm, u32 hw_if_index)
{
vec_free (d[i]);
vec_free (a[i]);
+ if (new_out_runtimes)
+ {
+ vec_free (new_out_runtimes[i].frame);
+ vec_free (new_out_runtimes[i].lookup_table);
+ }
}
vec_free (d);
diff --git a/src/vnet/interface/tx_queue_funcs.h b/src/vnet/interface/tx_queue_funcs.h
index 22956a4eb9b..8fcf7c336a8 100644
--- a/src/vnet/interface/tx_queue_funcs.h
+++ b/src/vnet/interface/tx_queue_funcs.h
@@ -27,3 +27,20 @@ vnet_hw_if_get_tx_queue (vnet_main_t *vnm, u32 queue_index)
return 0;
return pool_elt_at_index (im->hw_if_tx_queues, queue_index);
}
+
+static_always_inline int
+vnet_hw_if_txq_cmp_cli_api (vnet_hw_if_tx_queue_t **a,
+ vnet_hw_if_tx_queue_t **b)
+{
+ if (*a == *b)
+ return 0;
+
+ if (a[0]->hw_if_index != b[0]->hw_if_index)
+ return 2 * (a[0]->hw_if_index > b[0]->hw_if_index) - 1;
+
+ if (a[0]->queue_id != b[0]->queue_id)
+ return 2 * (a[0]->queue_id > b[0]->queue_id) - 1;
+
+ ASSERT (0);
+ return ~0;
+}
diff --git a/src/vnet/interface_api.c b/src/vnet/interface_api.c
index a765c0bbc3d..5218f74b0f4 100644
--- a/src/vnet/interface_api.c
+++ b/src/vnet/interface_api.c
@@ -22,6 +22,7 @@
#include <vnet/interface.h>
#include <vnet/interface/rx_queue_funcs.h>
+#include <vnet/interface/tx_queue_funcs.h>
#include <vnet/api_errno.h>
#include <vnet/ethernet/ethernet.h>
#include <vnet/ip/ip.h>
@@ -56,7 +57,9 @@ vpe_api_main_t vpe_api_main;
_ (SW_INTERFACE_ADD_DEL_ADDRESS, sw_interface_add_del_address) \
_ (SW_INTERFACE_SET_RX_MODE, sw_interface_set_rx_mode) \
_ (SW_INTERFACE_RX_PLACEMENT_DUMP, sw_interface_rx_placement_dump) \
+ _ (SW_INTERFACE_TX_PLACEMENT_GET, sw_interface_tx_placement_get) \
_ (SW_INTERFACE_SET_RX_PLACEMENT, sw_interface_set_rx_placement) \
+ _ (SW_INTERFACE_SET_TX_PLACEMENT, sw_interface_set_tx_placement) \
_ (SW_INTERFACE_SET_TABLE, sw_interface_set_table) \
_ (SW_INTERFACE_GET_TABLE, sw_interface_get_table) \
_ (SW_INTERFACE_SET_UNNUMBERED, sw_interface_set_unnumbered) \
@@ -1216,6 +1219,168 @@ out:
}
static void
+send_interface_tx_placement_details (vnet_hw_if_tx_queue_t **all_queues,
+ u32 index, vl_api_registration_t *rp,
+ u32 context)
+{
+ vnet_main_t *vnm = vnet_get_main ();
+ vl_api_sw_interface_tx_placement_details_t *rmp;
+ u32 n_bits = 0, v = ~0;
+ vnet_hw_if_tx_queue_t **q = vec_elt_at_index (all_queues, index);
+ uword *bitmap = q[0]->threads;
+ u32 hw_if_index = q[0]->hw_if_index;
+ vnet_hw_interface_t *hw_if = vnet_get_hw_interface (vnm, hw_if_index);
+
+ n_bits = clib_bitmap_count_set_bits (bitmap);
+ u32 n = n_bits * sizeof (u32);
+
+ /*
+ * FIXME: Use the REPLY_MACRO_DETAILS5_END once endian handler is registered
+ * and available.
+ */
+ REPLY_MACRO_DETAILS5 (
+ VL_API_SW_INTERFACE_TX_PLACEMENT_DETAILS, n, rp, context, ({
+ rmp->sw_if_index = clib_host_to_net_u32 (hw_if->sw_if_index);
+ rmp->queue_id = clib_host_to_net_u32 (q[0]->queue_id);
+ rmp->shared = q[0]->shared_queue;
+ rmp->array_size = clib_host_to_net_u32 (n_bits);
+
+ v = clib_bitmap_first_set (bitmap);
+ for (u32 i = 0; i < n_bits; i++)
+ {
+ rmp->threads[i] = clib_host_to_net_u32 (v);
+ v = clib_bitmap_next_set (bitmap, v + 1);
+ }
+ }));
+}
+
+static void
+vl_api_sw_interface_tx_placement_get_t_handler (
+ vl_api_sw_interface_tx_placement_get_t *mp)
+{
+ vnet_main_t *vnm = vnet_get_main ();
+ vl_api_sw_interface_tx_placement_get_reply_t *rmp = 0;
+ vnet_hw_if_tx_queue_t **all_queues = 0;
+ vnet_hw_if_tx_queue_t *q;
+ u32 sw_if_index = mp->sw_if_index;
+ i32 rv = 0;
+
+ if (pool_elts (vnm->interface_main.hw_if_tx_queues) == 0)
+ {
+ rv = VNET_API_ERROR_NO_SUCH_ENTRY;
+ goto err;
+ }
+
+ if (sw_if_index == ~0)
+ {
+ pool_foreach (q, vnm->interface_main.hw_if_tx_queues)
+ vec_add1 (all_queues, q);
+ vec_sort_with_function (all_queues, vnet_hw_if_txq_cmp_cli_api);
+ }
+ else
+ {
+ u32 qi = ~0;
+ vnet_sw_interface_t *si;
+
+ if (!vnet_sw_if_index_is_api_valid (sw_if_index))
+ {
+ clib_warning ("sw_if_index %u does not exist", sw_if_index);
+ rv = VNET_API_ERROR_INVALID_SW_IF_INDEX;
+ goto err;
+ }
+
+ si = vnet_get_sw_interface (vnm, sw_if_index);
+ if (si->type != VNET_SW_INTERFACE_TYPE_HARDWARE)
+ {
+ clib_warning ("interface type is not HARDWARE! P2P, PIPE and SUB"
+ " interfaces are not supported");
+ rv = VNET_API_ERROR_INVALID_INTERFACE;
+ goto err;
+ }
+
+ vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, si->hw_if_index);
+ for (qi = 0; qi < vec_len (hw->tx_queue_indices); qi++)
+ {
+ q = vnet_hw_if_get_tx_queue (vnm, hw->tx_queue_indices[qi]);
+ vec_add1 (all_queues, q);
+ }
+ }
+
+ REPLY_AND_DETAILS_VEC_MACRO_END (VL_API_SW_INTERFACE_TX_PLACEMENT_GET_REPLY,
+ all_queues, mp, rmp, rv, ({
+ send_interface_tx_placement_details (
+ all_queues, cursor, rp, mp->context);
+ }));
+
+ vec_free (all_queues);
+ return;
+
+err:
+ REPLY_MACRO_END (VL_API_SW_INTERFACE_TX_PLACEMENT_GET_REPLY);
+}
+
+static void
+vl_api_sw_interface_set_tx_placement_t_handler (
+ vl_api_sw_interface_set_tx_placement_t *mp)
+{
+ vl_api_sw_interface_set_tx_placement_reply_t *rmp;
+ vnet_main_t *vnm = vnet_get_main ();
+ u32 sw_if_index = mp->sw_if_index;
+ vnet_sw_interface_t *si;
+ uword *bitmap = 0;
+ u32 queue_id = ~0;
+ u32 size = 0;
+ clib_error_t *error = 0;
+ int rv = 0;
+
+ VALIDATE_SW_IF_INDEX_END (mp);
+
+ si = vnet_get_sw_interface (vnm, sw_if_index);
+ if (si->type != VNET_SW_INTERFACE_TYPE_HARDWARE)
+ {
+ rv = VNET_API_ERROR_INVALID_VALUE;
+ goto bad_sw_if_index;
+ }
+
+ size = mp->array_size;
+ for (u32 i = 0; i < size; i++)
+ {
+ u32 thread_index = mp->threads[i];
+ bitmap = clib_bitmap_set (bitmap, thread_index, 1);
+ }
+
+ queue_id = mp->queue_id;
+ rv = set_hw_interface_tx_queue (si->hw_if_index, queue_id, bitmap);
+
+ switch (rv)
+ {
+ case VNET_API_ERROR_INVALID_VALUE:
+ error = clib_error_return (
+ 0, "please specify valid thread(s) - last thread index %u",
+ clib_bitmap_last_set (bitmap));
+ break;
+ case VNET_API_ERROR_INVALID_QUEUE:
+ error = clib_error_return (
+ 0, "unknown queue %u on interface %s", queue_id,
+ vnet_get_hw_interface (vnet_get_main (), si->hw_if_index)->name);
+ break;
+ default:
+ break;
+ }
+
+ if (error)
+ {
+ clib_error_report (error);
+ goto out;
+ }
+
+ BAD_SW_IF_INDEX_LABEL;
+out:
+ REPLY_MACRO_END (VL_API_SW_INTERFACE_SET_TX_PLACEMENT_REPLY);
+ clib_bitmap_free (bitmap);
+}
+
+static void
vl_api_create_vlan_subif_t_handler (vl_api_create_vlan_subif_t * mp)
{
vl_api_create_vlan_subif_reply_t *rmp;
@@ -1474,6 +1639,10 @@ interface_api_hookup (vlib_main_t * vm)
/* Do not replay VL_API_SW_INTERFACE_DUMP messages */
am->api_trace_cfg[VL_API_SW_INTERFACE_DUMP].replay_enable = 0;
+ /* Mark these APIs as autoendian */
+ am->is_autoendian[VL_API_SW_INTERFACE_SET_TX_PLACEMENT] = 1;
+ am->is_autoendian[VL_API_SW_INTERFACE_TX_PLACEMENT_GET] = 1;
+
/*
* Set up the (msg_name, crc, message-id) table
*/
diff --git a/src/vnet/interface_cli.c b/src/vnet/interface_cli.c
index 4f6f2cf05a5..d2e748abec2 100644
--- a/src/vnet/interface_cli.c
+++ b/src/vnet/interface_cli.c
@@ -53,6 +53,7 @@
#include <vnet/classify/vnet_classify.h>
#include <vnet/interface/rx_queue_funcs.h>
#include <vnet/interface/tx_queue_funcs.h>
+#include <vnet/hash/hash.h>
static int
compare_interface_names (void *a1, void *a2)
{
@@ -1840,28 +1841,24 @@ VLIB_CLI_COMMAND (cmd_set_if_rx_placement,static) = {
};
/* *INDENT-ON* */
-clib_error_t *
+int
set_hw_interface_tx_queue (u32 hw_if_index, u32 queue_id, uword *bitmap)
{
vnet_main_t *vnm = vnet_get_main ();
- vnet_device_main_t *vdm = &vnet_device_main;
- vnet_hw_interface_t *hw;
+ vlib_thread_main_t *vtm = vlib_get_thread_main ();
vnet_hw_if_tx_queue_t *txq;
u32 queue_index;
u32 thread_index;
- hw = vnet_get_hw_interface (vnm, hw_if_index);
-
/* highest set bit in bitmap should not exceed last worker thread index */
thread_index = clib_bitmap_last_set (bitmap);
- if ((thread_index != ~0) && (thread_index > vdm->last_worker_thread_index))
- return clib_error_return (0, "please specify valid thread(s)");
+ if ((thread_index != ~0) && (thread_index >= vtm->n_vlib_mains))
+ return VNET_API_ERROR_INVALID_VALUE;
queue_index =
vnet_hw_if_get_tx_queue_index_by_id (vnm, hw_if_index, queue_id);
if (queue_index == ~0)
- return clib_error_return (0, "unknown queue %u on interface %s", queue_id,
- hw->name);
+ return VNET_API_ERROR_INVALID_QUEUE;
txq = vnet_hw_if_get_tx_queue (vnm, queue_index);
@@ -1889,6 +1886,7 @@ set_interface_tx_queue (vlib_main_t *vm, unformat_input_t *input,
u32 hw_if_index = (u32) ~0;
u32 queue_id = (u32) 0;
uword *bitmap = 0;
+ int rv = 0;
if (!unformat_user (input, unformat_line_input, line_input))
return 0;
@@ -1920,7 +1918,23 @@ set_interface_tx_queue (vlib_main_t *vm, unformat_input_t *input,
goto error;
}
- error = set_hw_interface_tx_queue (hw_if_index, queue_id, bitmap);
+ rv = set_hw_interface_tx_queue (hw_if_index, queue_id, bitmap);
+
+ switch (rv)
+ {
+ case VNET_API_ERROR_INVALID_VALUE:
+ error = clib_error_return (
+ 0, "please specify valid thread(s) - last thread index %u",
+ clib_bitmap_last_set (bitmap));
+ break;
+ case VNET_API_ERROR_INVALID_QUEUE:
+ error = clib_error_return (
+ 0, "unknown queue %u on interface %s", queue_id,
+ vnet_get_hw_interface (vnet_get_main (), hw_if_index)->name);
+ break;
+ default:
+ break;
+ }
error:
clib_bitmap_free (bitmap);
@@ -2467,6 +2481,132 @@ VLIB_CLI_COMMAND (cmd_set_if_name, static) = {
.function = set_interface_name,
.is_mp_safe = 1,
};
+
+static clib_error_t *
+set_interface_tx_hash_cmd (vlib_main_t *vm, unformat_input_t *input,
+ vlib_cli_command_t *cmd)
+{
+ clib_error_t *error = 0;
+ unformat_input_t _line_input, *line_input = &_line_input;
+ vnet_main_t *vnm = vnet_get_main ();
+ vnet_hw_interface_t *hi;
+ u8 *hash_name = 0;
+ u32 hw_if_index = (u32) ~0;
+ vnet_hash_fn_t hf;
+ vnet_hash_fn_type_t ftype;
+
+ if (!unformat_user (input, unformat_line_input, line_input))
+ return 0;
+
+ while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+ {
+ if (unformat (line_input, "%U", unformat_vnet_hw_interface, vnm,
+ &hw_if_index))
+ ;
+ else if (unformat (line_input, "hash-name %s", &hash_name))
+ ;
+ else
+ {
+ error = clib_error_return (0, "parse error: '%U'",
+ format_unformat_error, line_input);
+ unformat_free (line_input);
+ return error;
+ }
+ }
+
+ unformat_free (line_input);
+
+ if (hw_if_index == (u32) ~0)
+ {
+ error = clib_error_return (0, "please specify valid interface name");
+ goto error;
+ }
+
+ hi = vnet_get_hw_interface (vnm, hw_if_index);
+ ftype =
+ vnet_get_hw_interface_class (vnm, hi->hw_class_index)->tx_hash_fn_type;
+ hf = vnet_hash_function_from_name ((const char *) hash_name, ftype);
+
+ if (!hf)
+ {
+ error = clib_error_return (0, "please specify valid hash name");
+ goto error;
+ }
+
+ hi->hf = hf;
+error:
+ vec_free (hash_name);
+ return (error);
+}
+
+VLIB_CLI_COMMAND (cmd_set_if_tx_hash, static) = {
+ .path = "set interface tx-hash",
+ .short_help = "set interface tx-hash <interface> hash-name <hash-name>",
+ .function = set_interface_tx_hash_cmd,
+};
+
+static clib_error_t *
+show_tx_hash (vlib_main_t *vm, unformat_input_t *input,
+ vlib_cli_command_t *cmd)
+{
+ clib_error_t *error = 0;
+ unformat_input_t _line_input, *line_input = &_line_input;
+ vnet_main_t *vnm = vnet_get_main ();
+ vnet_hw_interface_t *hi;
+ vnet_hash_function_registration_t *hash;
+ u32 hw_if_index = (u32) ~0;
+ vnet_hash_fn_type_t ftype;
+
+ if (!unformat_user (input, unformat_line_input, line_input))
+ return 0;
+
+ while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+ {
+ if (unformat (line_input, "%U", unformat_vnet_hw_interface, vnm,
+ &hw_if_index))
+ ;
+ else
+ {
+ error = clib_error_return (0, "parse error: '%U'",
+ format_unformat_error, line_input);
+ unformat_free (line_input);
+ goto error;
+ }
+ }
+
+ unformat_free (line_input);
+
+ if (hw_if_index == (u32) ~0)
+ {
+ error = clib_error_return (0, "please specify valid interface name");
+ goto error;
+ }
+
+ hi = vnet_get_hw_interface (vnm, hw_if_index);
+ ftype =
+ vnet_get_hw_interface_class (vnm, hi->hw_class_index)->tx_hash_fn_type;
+
+ if (hi->hf)
+ {
+ hash = vnet_hash_function_from_func (hi->hf, ftype);
+ if (hash)
+ vlib_cli_output (vm, "%U", format_vnet_hash, hash);
+ else
+ vlib_cli_output (vm, "no matching hash function found");
+ }
+ else
+ vlib_cli_output (vm, "no hashing function set");
+
+error:
+ return (error);
+}
+
+VLIB_CLI_COMMAND (cmd_show_tx_hash, static) = {
+ .path = "show interface tx-hash",
+ .short_help = "show interface tx-hash [interface]",
+ .function = show_tx_hash,
+};
+
/*
* fd.io coding-style-patch-verification: ON
*
diff --git a/src/vnet/interface_format.c b/src/vnet/interface_format.c
index 4acd6ab63e6..cb428e761be 100644
--- a/src/vnet/interface_format.c
+++ b/src/vnet/interface_format.c
@@ -212,6 +212,9 @@ format_vnet_hw_interface (u8 * s, va_list * args)
if (vec_len (hi->tx_queue_indices))
{
s = format (s, "\n%UTX Queues:", format_white_space, indent + 2);
+ s = format (
+ s, "\n%UTX Hash: %U", format_white_space, indent + 4, format_vnet_hash,
+ vnet_hash_function_from_func (hi->hf, hw_class->tx_hash_fn_type));
s = format (s, "\n%U%-6s%-7s%-15s", format_white_space, indent + 4,
"queue", "shared", "thread(s)");
for (int i = 0; i < vec_len (hi->tx_queue_indices); i++)
diff --git a/src/vnet/interface_funcs.h b/src/vnet/interface_funcs.h
index f253b23f819..eef5596644c 100644
--- a/src/vnet/interface_funcs.h
+++ b/src/vnet/interface_funcs.h
@@ -427,6 +427,8 @@ clib_error_t *set_hw_interface_change_rx_mode (vnet_main_t * vnm,
/* Set rx-placement on the interface */
clib_error_t *set_hw_interface_rx_placement (u32 hw_if_index, u32 queue_id,
u32 thread_index, u8 is_main);
+/* Set tx-queue placement on the interface */
+int set_hw_interface_tx_queue (u32 hw_if_index, u32 queue_id, uword *bitmap);
/* Set the MTU on the HW interface */
void vnet_hw_interface_set_mtu (vnet_main_t * vnm, u32 hw_if_index, u32 mtu);
@@ -509,6 +511,7 @@ typedef enum
{
VNET_INTERFACE_OUTPUT_ERROR_INTERFACE_DOWN,
VNET_INTERFACE_OUTPUT_ERROR_INTERFACE_DELETED,
+ VNET_INTERFACE_OUTPUT_ERROR_NO_TX_QUEUE,
} vnet_interface_output_error_t;
/* Format for interface output traces. */
diff --git a/src/vnet/interface_output.c b/src/vnet/interface_output.c
index 72ceb95132d..659273b53cd 100644
--- a/src/vnet/interface_output.c
+++ b/src/vnet/interface_output.c
@@ -46,10 +46,12 @@
#include <vnet/udp/udp_packet.h>
#include <vnet/feature/feature.h>
#include <vnet/classify/pcap_classify.h>
+#include <vnet/hash/hash.h>
#include <vnet/interface_output.h>
#include <vppinfra/vector/mask_compare.h>
#include <vppinfra/vector/compress.h>
#include <vppinfra/vector/count_equal.h>
+#include <vppinfra/vector/array_mask.h>
typedef struct
{
@@ -176,8 +178,9 @@ vnet_interface_output_handle_offload (vlib_main_t *vm, vlib_buffer_t *b)
static_always_inline uword
vnet_interface_output_node_inline (vlib_main_t *vm, u32 sw_if_index,
vlib_combined_counter_main_t *ccm,
- vlib_buffer_t **b, u32 config_index, u8 arc,
- u32 n_left, int processing_level)
+ vlib_buffer_t **b, void **p,
+ u32 config_index, u8 arc, u32 n_left,
+ int processing_level)
{
u32 n_bytes = 0;
u32 n_bytes0, n_bytes1, n_bytes2, n_bytes3;
@@ -208,6 +211,15 @@ vnet_interface_output_node_inline (vlib_main_t *vm, u32 sw_if_index,
n_bytes += n_bytes2 = vlib_buffer_length_in_chain (vm, b[2]);
n_bytes += n_bytes3 = vlib_buffer_length_in_chain (vm, b[3]);
+ if (processing_level >= 3)
+ {
+ p[0] = vlib_buffer_get_current (b[0]);
+ p[1] = vlib_buffer_get_current (b[1]);
+ p[2] = vlib_buffer_get_current (b[2]);
+ p[3] = vlib_buffer_get_current (b[3]);
+ p += 4;
+ }
+
if (processing_level >= 2)
{
u32 tx_swif0, tx_swif1, tx_swif2, tx_swif3;
@@ -262,6 +274,12 @@ vnet_interface_output_node_inline (vlib_main_t *vm, u32 sw_if_index,
n_bytes += n_bytes0 = vlib_buffer_length_in_chain (vm, b[0]);
+ if (processing_level >= 3)
+ {
+ p[0] = vlib_buffer_get_current (b[0]);
+ p += 1;
+ }
+
if (processing_level >= 2)
{
u32 tx_swif0 = vnet_buffer (b[0])->sw_if_index[VLIB_TX];
@@ -344,39 +362,71 @@ vnet_interface_pcap_tx_trace (vlib_main_t *vm, vlib_node_runtime_t *node,
}
static_always_inline void
-store_tx_frame_scalar_data (vnet_hw_if_output_node_runtime_t *r,
- vnet_hw_if_tx_frame_t *tf)
+hash_func_with_mask (void **p, u32 *hash, u32 n_packets, u32 *lookup_table,
+ u32 mask, vnet_hash_fn_t hf)
{
- if (r)
- clib_memcpy_fast (tf, &r->frame, sizeof (vnet_hw_if_tx_frame_t));
+ u32 n_left_from = n_packets;
+
+ hf (p, hash, n_packets);
+
+ clib_array_mask_u32 (hash, mask, n_packets);
+
+ while (n_left_from >= 4)
+ {
+ hash[0] = lookup_table[hash[0]];
+ hash[1] = lookup_table[hash[1]];
+ hash[2] = lookup_table[hash[2]];
+ hash[3] = lookup_table[hash[3]];
+
+ hash += 4;
+ n_left_from -= 4;
+ }
+
+ while (n_left_from > 0)
+ {
+ hash[0] = lookup_table[hash[0]];
+
+ hash += 1;
+ n_left_from -= 1;
+ }
}
static_always_inline void
-enqueu_to_tx_node (vlib_main_t *vm, vlib_node_runtime_t *node,
- vnet_hw_interface_t *hi, u32 *from, u32 n_vectors)
+store_tx_frame_scalar_data (vnet_hw_if_tx_frame_t *copy_frame,
+ vnet_hw_if_tx_frame_t *tf)
{
- u32 next_index = VNET_INTERFACE_OUTPUT_NEXT_TX;
- vnet_hw_if_output_node_runtime_t *r = 0;
- u32 n_free, n_copy, *to;
- vnet_hw_if_tx_frame_t *tf;
- vlib_frame_t *f;
-
- ASSERT (n_vectors <= VLIB_FRAME_SIZE);
+ if (copy_frame)
+ clib_memcpy_fast (tf, copy_frame, sizeof (vnet_hw_if_tx_frame_t));
+}
- if (hi->output_node_thread_runtimes)
- r = vec_elt_at_index (hi->output_node_thread_runtimes, vm->thread_index);
+static_always_inline u32
+enqueue_one_to_tx_node (vlib_main_t *vm, vlib_node_runtime_t *node, u32 *ppqi,
+ u32 *from, vnet_hw_if_tx_frame_t *copy_frame,
+ u32 n_vectors, u32 n_left, u32 next_index)
+{
+ u32 tmp[VLIB_FRAME_SIZE];
+ u64 mask[VLIB_FRAME_SIZE / 64] = {};
+ vlib_frame_t *f;
+ vnet_hw_if_tx_frame_t *tf;
+ u32 *to;
+ u32 n_copy = 0, n_free = 0;
f = vlib_get_next_frame_internal (vm, node, next_index, 0);
tf = vlib_frame_scalar_args (f);
- if (f->n_vectors > 0 && (r == 0 || tf->queue_id == r->frame.queue_id))
+ if (f->n_vectors > 0 &&
+ (!copy_frame || (tf->queue_id == copy_frame->queue_id)))
{
/* append current next frame */
n_free = VLIB_FRAME_SIZE - f->n_vectors;
- n_copy = clib_min (n_vectors, n_free);
- n_vectors -= n_copy;
- to = vlib_frame_vector_args (f);
- to += f->n_vectors;
+ /*
+ * if frame contains enough space for worst case scenario,
+ * we can avoid use of tmp
+ */
+ if (n_free >= n_left)
+ to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors;
+ else
+ to = tmp;
}
else
{
@@ -388,25 +438,113 @@ enqueu_to_tx_node (vlib_main_t *vm, vlib_node_runtime_t *node,
}
/* empty frame - store scalar data */
- store_tx_frame_scalar_data (r, tf);
+ store_tx_frame_scalar_data (copy_frame, tf);
to = vlib_frame_vector_args (f);
n_free = VLIB_FRAME_SIZE;
- n_copy = n_vectors;
- n_vectors = 0;
}
- vlib_buffer_copy_indices (to, from, n_copy);
- vlib_put_next_frame (vm, node, next_index, n_free - n_copy);
+ /*
+ * per packet queue id array
+ * compare with given queue_id, if match, copy respective buffer index from
+ * -> to
+ */
+ if (ppqi)
+ {
+ clib_mask_compare_u32 (copy_frame->queue_id, ppqi, mask, n_vectors);
+ n_copy = clib_compress_u32 (to, from, mask, n_vectors);
- if (n_vectors == 0)
- return;
+ if (n_copy == 0)
+ return n_left;
+ }
+ else
+ {
+ /*
+ * no work required, just copy all buffer indices from -> to
+ */
+ n_copy = n_left;
+ vlib_buffer_copy_indices (to, from, n_copy);
+ }
+
+ if (to != tmp)
+ {
+ /* indices already written to frame, just close it */
+ vlib_put_next_frame (vm, node, next_index, n_free - n_copy);
+ }
+ else if (n_free >= n_copy)
+ {
+ /* enough space in the existing frame */
+ to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors;
+ vlib_buffer_copy_indices (to, tmp, n_copy);
+ vlib_put_next_frame (vm, node, next_index, n_free - n_copy);
+ }
+ else
+ {
+ /* full frame */
+ to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors;
+ vlib_buffer_copy_indices (to, tmp, n_free);
+ vlib_put_next_frame (vm, node, next_index, 0);
+
+ /* second frame */
+ u32 n_2nd_frame = n_copy - n_free;
+ f = vlib_get_next_frame_internal (vm, node, next_index, 1);
+ tf = vlib_frame_scalar_args (f);
+ /* empty frame - store scalar data */
+ store_tx_frame_scalar_data (copy_frame, tf);
+ to = vlib_frame_vector_args (f);
+ vlib_buffer_copy_indices (to, tmp + n_free, n_2nd_frame);
+ vlib_put_next_frame (vm, node, next_index,
+ VLIB_FRAME_SIZE - n_2nd_frame);
+ }
+
+ return n_left - n_copy;
+}
+
+static_always_inline void
+enqueue_to_tx_node (vlib_main_t *vm, vlib_node_runtime_t *node,
+ vnet_hw_interface_t *hi, u32 next_index,
+ vnet_hw_if_output_node_runtime_t *r, u32 *from, void **p,
+ u32 n_vectors)
+{
+ u32 n_left = n_vectors;
+
+ ASSERT (n_vectors <= VLIB_FRAME_SIZE);
+
+ /*
+ * backward compatible for drivers not integrated with new tx infra.
+ */
+ if (r == 0)
+ {
+ n_left = enqueue_one_to_tx_node (vm, node, NULL, from, NULL, n_vectors,
+ n_left, next_index);
+ }
+ /*
+ * only 1 tx queue of given interface is available on given thread
+ */
+ else if (r->n_queues == 1)
+ {
+ n_left = enqueue_one_to_tx_node (vm, node, NULL, from, r->frame,
+ n_vectors, n_left, next_index);
+ }
+ /*
+ * multi tx-queues use case
+ */
+ else if (r->n_queues > 1)
+ {
+ u32 qids[VLIB_FRAME_SIZE];
+
+ hash_func_with_mask (p, qids, n_vectors, r->lookup_table,
+ vec_len (r->lookup_table) - 1, hi->hf);
- /* we have more indices to store, take empty frame */
- from += n_copy;
- f = vlib_get_next_frame_internal (vm, node, next_index, 1);
- store_tx_frame_scalar_data (r, vlib_frame_scalar_args (f));
- vlib_buffer_copy_indices (vlib_frame_vector_args (f), from, n_vectors);
- vlib_put_next_frame (vm, node, next_index, VLIB_FRAME_SIZE - n_vectors);
+ for (u32 i = 0; i < r->n_queues; i++)
+ {
+ n_left = enqueue_one_to_tx_node (vm, node, qids, from, &r->frame[i],
+ n_vectors, n_left, next_index);
+ if (n_left == 0)
+ break;
+ }
+ }
+ else
+ ASSERT (0);
}
VLIB_NODE_FN (vnet_interface_output_node)
@@ -418,6 +556,7 @@ VLIB_NODE_FN (vnet_interface_output_node)
vnet_hw_interface_t *hi;
vnet_sw_interface_t *si;
vnet_interface_output_runtime_t *rt = (void *) node->runtime_data;
+ vnet_hw_if_output_node_runtime_t *r = 0;
vlib_buffer_t *bufs[VLIB_FRAME_SIZE];
u32 n_bytes, n_buffers = frame->n_vectors;
u32 config_index = ~0;
@@ -427,6 +566,8 @@ VLIB_NODE_FN (vnet_interface_output_node)
u8 arc = im->output_feature_arc_index;
int arc_or_subif = 0;
int do_tx_offloads = 0;
+ void *ptr[VLIB_FRAME_SIZE], **p = ptr;
+ u8 is_parr = 0;
u32 *from;
if (node->flags & VLIB_NODE_FLAG_TRACE)
@@ -462,6 +603,27 @@ VLIB_NODE_FN (vnet_interface_output_node)
node->node_index, VNET_INTERFACE_OUTPUT_ERROR_INTERFACE_DOWN);
}
+ if (hi->output_node_thread_runtimes)
+ r = vec_elt_at_index (hi->output_node_thread_runtimes, vm->thread_index);
+
+ if (r)
+ {
+ /*
+ * tx queue of given interface is not available on given thread
+ */
+ if (r->n_queues == 0)
+ return vlib_error_drop_buffers (
+ vm, node, from,
+ /* buffer stride */ 1, n_buffers, VNET_INTERFACE_OUTPUT_NEXT_DROP,
+ node->node_index, VNET_INTERFACE_OUTPUT_ERROR_NO_TX_QUEUE);
+ /*
+ * multiple tx queues available on given thread
+ */
+ else if (r->n_queues > 1)
+ /* construct array of pointer */
+ is_parr = 1;
+ }
+
/* interface-output feature arc handling */
if (PREDICT_FALSE (vnet_have_features (arc, sw_if_index)))
{
@@ -482,20 +644,28 @@ VLIB_NODE_FN (vnet_interface_output_node)
VNET_HW_INTERFACE_CAP_SUPPORTS_TX_CKSUM)
do_tx_offloads = 1;
- if (do_tx_offloads == 0 && arc_or_subif == 0)
+ // basic processing
+ if (do_tx_offloads == 0 && arc_or_subif == 0 && is_parr == 0)
n_bytes = vnet_interface_output_node_inline (
- vm, sw_if_index, ccm, bufs, config_index, arc, n_buffers, 0);
- else if (do_tx_offloads == 1 && arc_or_subif == 0)
+ vm, sw_if_index, ccm, bufs, NULL, config_index, arc, n_buffers, 0);
+ // basic processing + tx offloads
+ else if (do_tx_offloads == 1 && arc_or_subif == 0 && is_parr == 0)
n_bytes = vnet_interface_output_node_inline (
- vm, sw_if_index, ccm, bufs, config_index, arc, n_buffers, 1);
+ vm, sw_if_index, ccm, bufs, NULL, config_index, arc, n_buffers, 1);
+ // basic processing + tx offloads + vlans + arcs
+ else if (do_tx_offloads == 1 && arc_or_subif == 1 && is_parr == 0)
+ n_bytes = vnet_interface_output_node_inline (
+ vm, sw_if_index, ccm, bufs, NULL, config_index, arc, n_buffers, 2);
+ // basic processing + tx offloads + vlans + arcs + multi-txqs
else
n_bytes = vnet_interface_output_node_inline (
- vm, sw_if_index, ccm, bufs, config_index, arc, n_buffers, 2);
+ vm, sw_if_index, ccm, bufs, p, config_index, arc, n_buffers, 3);
from = vlib_frame_vector_args (frame);
if (PREDICT_TRUE (next_index == VNET_INTERFACE_OUTPUT_NEXT_TX))
{
- enqueu_to_tx_node (vm, node, hi, from, frame->n_vectors);
+ enqueue_to_tx_node (vm, node, hi, next_index, r, from, ptr,
+ frame->n_vectors);
}
else
{
@@ -1087,16 +1257,14 @@ VLIB_NODE_FN (vnet_interface_output_arc_end_node)
{
vnet_main_t *vnm = vnet_get_main ();
vnet_interface_main_t *im = &vnm->interface_main;
- vnet_hw_if_output_node_runtime_t *r = 0;
vnet_hw_interface_t *hi;
- vnet_hw_if_tx_frame_t *tf;
vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
u32 sw_if_indices[VLIB_FRAME_SIZE], *sw_if_index = sw_if_indices;
u64 used_elts[VLIB_FRAME_SIZE / 64] = {};
u64 mask[VLIB_FRAME_SIZE / 64] = {};
- u32 *tmp, *from, n_left, n_free, n_comp, *to, swif, off;
+ u32 *tmp, *from, n_left, n_comp, n_p_comp, swif, off;
u16 next_index;
- vlib_frame_t *f;
+ void *ptr[VLIB_FRAME_SIZE], **p = ptr;
from = vlib_frame_vector_args (frame);
n_left = frame->n_vectors;
@@ -1108,11 +1276,17 @@ VLIB_NODE_FN (vnet_interface_output_arc_end_node)
vlib_prefetch_buffer_header (b[5], LOAD);
vlib_prefetch_buffer_header (b[6], LOAD);
vlib_prefetch_buffer_header (b[7], LOAD);
+
+ p[0] = vlib_buffer_get_current (b[0]);
+ p[1] = vlib_buffer_get_current (b[1]);
+ p[2] = vlib_buffer_get_current (b[2]);
+ p[3] = vlib_buffer_get_current (b[3]);
sw_if_index[0] = vnet_buffer (b[0])->sw_if_index[VLIB_TX];
sw_if_index[1] = vnet_buffer (b[1])->sw_if_index[VLIB_TX];
sw_if_index[2] = vnet_buffer (b[2])->sw_if_index[VLIB_TX];
sw_if_index[3] = vnet_buffer (b[3])->sw_if_index[VLIB_TX];
+ p += 4;
b += 4;
sw_if_index += 4;
n_left -= 4;
@@ -1120,7 +1294,9 @@ VLIB_NODE_FN (vnet_interface_output_arc_end_node)
while (n_left)
{
+ p[0] = vlib_buffer_get_current (b[0]);
sw_if_index[0] = vnet_buffer (b[0])->sw_if_index[VLIB_TX];
+ p++;
b++;
sw_if_index++;
n_left--;
@@ -1137,68 +1313,40 @@ VLIB_NODE_FN (vnet_interface_output_arc_end_node)
more:
next_index = vec_elt (im->if_out_arc_end_next_index_by_sw_if_index, swif);
hi = vnet_get_sup_hw_interface (vnm, swif);
+ vnet_hw_if_output_node_runtime_t *r = 0;
+ void *ptr_tmp[VLIB_FRAME_SIZE], **p_tmp = ptr_tmp;
+
if (hi->output_node_thread_runtimes)
r = vec_elt_at_index (hi->output_node_thread_runtimes, vm->thread_index);
- f = vlib_get_next_frame_internal (vm, node, next_index, 0);
- tf = vlib_frame_scalar_args (f);
-
- if (f->n_vectors > 0 && (r == 0 || r->frame.queue_id == tf->queue_id))
- {
- /* append frame */
- n_free = VLIB_FRAME_SIZE - f->n_vectors;
- if (n_free >= f->n_vectors)
- to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors;
- else
- to = tmp;
- }
- else
- {
- if (f->n_vectors > 0)
- {
- /* current frame doesn't fit - grab empty one */
- f = vlib_get_next_frame_internal (vm, node, next_index, 1);
- tf = vlib_frame_scalar_args (f);
- }
-
- /* empty frame - store scalar data */
- store_tx_frame_scalar_data (r, tf);
- n_free = VLIB_FRAME_SIZE;
- to = vlib_frame_vector_args (f);
- }
/* compare and compress based on comparison mask */
clib_mask_compare_u32 (swif, sw_if_indices, mask, frame->n_vectors);
- n_comp = clib_compress_u32 (to, from, mask, frame->n_vectors);
+ n_comp = clib_compress_u32 (tmp, from, mask, frame->n_vectors);
- if (tmp != to)
- {
- /* indices already written to frame, just close it */
- vlib_put_next_frame (vm, node, next_index, n_free - n_comp);
- }
- else if (n_free >= n_comp)
+ /*
+ * tx queue of given interface is not available on given thread
+ */
+ if (r)
{
- /* enough space in the existing frame */
- to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors;
- vlib_buffer_copy_indices (to, tmp, n_comp);
- vlib_put_next_frame (vm, node, next_index, n_free - n_comp);
+ if (r->n_queues == 0)
+ {
+ vlib_error_drop_buffers (
+ vm, node, tmp,
+ /* buffer stride */ 1, n_comp, VNET_INTERFACE_OUTPUT_NEXT_DROP,
+ node->node_index, VNET_INTERFACE_OUTPUT_ERROR_NO_TX_QUEUE);
+ goto drop;
+ }
+ else if (r->n_queues > 1)
+ {
+ n_p_comp = clib_compress_u64 ((u64 *) p_tmp, (u64 *) ptr, mask,
+ frame->n_vectors);
+ ASSERT (n_p_comp == n_comp);
+ }
}
- else
- {
- /* full frame */
- to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors;
- vlib_buffer_copy_indices (to, tmp, n_free);
- vlib_put_next_frame (vm, node, next_index, 0);
- /* second frame */
- u32 n_frame2 = n_comp - n_free;
- f = vlib_get_next_frame_internal (vm, node, next_index, 1);
- to = vlib_frame_vector_args (f);
- vlib_buffer_copy_indices (to, tmp + n_free, n_frame2);
- tf = vlib_frame_scalar_args (f);
- store_tx_frame_scalar_data (r, tf);
- vlib_put_next_frame (vm, node, next_index, VLIB_FRAME_SIZE - n_frame2);
- }
+ enqueue_to_tx_node (vm, node, hi, next_index, r, tmp, ptr_tmp, n_comp);
+drop:
n_left -= n_comp;
if (n_left)
{
diff --git a/src/vnet/interface_test.c b/src/vnet/interface_test.c
index 4a1681f4eac..c3ddcd74cc4 100644
--- a/src/vnet/interface_test.c
+++ b/src/vnet/interface_test.c
@@ -570,6 +570,63 @@ api_sw_interface_set_rx_placement (vat_main_t *vam)
}
static int
+api_sw_interface_set_tx_placement (vat_main_t *vam)
+{
+ unformat_input_t *i = vam->input;
+ vl_api_sw_interface_set_tx_placement_t *mp;
+ u32 sw_if_index;
+ u8 sw_if_index_set = 0;
+ int ret;
+ uword *bitmap = 0;
+ u32 queue_id, n_bits = 0;
+ u32 v;
+
+ /* Parse args required to build the message */
+ while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+ {
+ if (unformat (i, "queue %d", &queue_id))
+ ;
+ else if (unformat (i, "threads %U", unformat_bitmap_list, &bitmap))
+ ;
+ else if (unformat (i, "mask %U", unformat_bitmap_mask, &bitmap))
+ ;
+ else if (unformat (i, "%U", api_unformat_sw_if_index, vam, &sw_if_index))
+ sw_if_index_set = 1;
+ else if (unformat (i, "sw_if_index %d", &sw_if_index))
+ sw_if_index_set = 1;
+ else
+ break;
+ }
+
+ if (sw_if_index_set == 0)
+ {
+ errmsg ("missing interface name or sw_if_index");
+ return -99;
+ }
+
+ n_bits = clib_bitmap_count_set_bits (bitmap);
+ /* Construct the API message */
+ M2 (SW_INTERFACE_SET_TX_PLACEMENT, mp, sizeof (u32) * n_bits);
+ mp->sw_if_index = htonl (sw_if_index);
+ mp->queue_id = htonl (queue_id);
+ mp->array_size = htonl (n_bits);
+
+ v = clib_bitmap_first_set (bitmap);
+ for (u32 j = 0; j < n_bits; j++)
+ {
+ mp->threads[j] = htonl (v);
+ v = clib_bitmap_next_set (bitmap, v + 1);
+ }
+
+ /* send it... */
+ S (mp);
+ /* Wait for a reply, return the good/bad news... */
+ W (ret);
+ clib_bitmap_free (bitmap);
+ return ret;
+}
+
+static int
api_interface_name_renumber (vat_main_t *vam)
{
unformat_input_t *line_input = vam->input;
@@ -844,6 +901,25 @@ vl_api_sw_interface_rx_placement_details_t_handler (
((mp->mode == 2) ? "interrupt" : "adaptive"));
}
+static __clib_unused void
+vl_api_sw_interface_tx_placement_details_t_handler (
+ vl_api_sw_interface_tx_placement_details_t *mp)
+{
+ vat_main_t *vam = interface_test_main.vat_main;
+ u32 size = ntohl (mp->array_size);
+ uword *bitmap = 0;
+
+ for (u32 i = 0; i < size; i++)
+ {
+ u32 thread_index = ntohl (mp->threads[i]);
+ bitmap = clib_bitmap_set (bitmap, thread_index, 1);
+ }
+
+ print (vam->ofp, "\n%-11d %-6d %-7s %U", ntohl (mp->sw_if_index),
+ ntohl (mp->queue_id), (mp->shared == 1) ? "yes" : "no",
+ format_bitmap_list, bitmap);
+}
+
static void
vl_api_create_vlan_subif_reply_t_handler (vl_api_create_vlan_subif_reply_t *mp)
{
@@ -961,6 +1037,52 @@ api_sw_interface_rx_placement_dump (vat_main_t *vam)
}
static int
+api_sw_interface_tx_placement_get (vat_main_t *vam)
+{
+ unformat_input_t *i = vam->input;
+ vl_api_sw_interface_tx_placement_get_t *mp;
+ vl_api_control_ping_t *mp_ping;
+ int ret;
+ u32 sw_if_index;
+ u8 sw_if_index_set = 0;
+
+ while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+ {
+ if (unformat (i, "%U", api_unformat_sw_if_index, vam, &sw_if_index))
+ sw_if_index_set++;
+ else if (unformat (i, "sw_if_index %d", &sw_if_index))
+ sw_if_index_set++;
+ else
+ break;
+ }
+
+ fformat (vam->ofp, "\n%-11s %-6s %-7s %-11s", "sw_if_index", "queue",
+ "shared", "threads");
+
+ /* Dump Interface tx placement */
+ M (SW_INTERFACE_TX_PLACEMENT_GET, mp);
+
+ if (sw_if_index_set)
+ mp->sw_if_index = htonl (sw_if_index);
+ else
+ mp->sw_if_index = ~0;
+
+ S (mp);
+
+ /* Use a control ping for synchronization */
+ PING (&interface_test_main, mp_ping);
+ S (mp_ping);
+
+ W (ret);
+ return ret;
+}
+
+static void
+vl_api_sw_interface_tx_placement_get_reply_t_handler ()
+{
+}
+
+static int
api_sw_interface_clear_stats (vat_main_t *vam)
{
unformat_input_t *i = vam->input;
diff --git a/src/vnet/pg/stream.c b/src/vnet/pg/stream.c
index 605567c1296..2c75c2b2bc3 100644
--- a/src/vnet/pg/stream.c
+++ b/src/vnet/pg/stream.c
@@ -245,6 +245,7 @@ VNET_HW_INTERFACE_CLASS (pg_tun_hw_interface_class) = {
.build_rewrite = NULL,
//.update_adjacency = gre_update_adj,
.flags = VNET_HW_INTERFACE_CLASS_FLAG_P2P,
+ .tx_hash_fn_type = VNET_HASH_FN_TYPE_IP,
};
u32