diff options
Diffstat (limited to 'src/vnet/devices/virtio')
| -rw-r--r-- | src/vnet/devices/virtio/node.c | 48 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/pci.c | 21 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/vhost_user.c | 51 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/vhost_user.h | 2 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/vhost_user_input.c | 76 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/virtio.c | 58 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/virtio.h | 4 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/virtio_process.c | 31 |
8 files changed, 128 insertions, 163 deletions
diff --git a/src/vnet/devices/virtio/node.c b/src/vnet/devices/virtio/node.c index 1e75ca47586..788cc617f61 100644 --- a/src/vnet/devices/virtio/node.c +++ b/src/vnet/devices/virtio/node.c @@ -26,15 +26,14 @@ #include <vlib/vlib.h> #include <vlib/unix/unix.h> #include <vnet/ethernet/ethernet.h> -#include <vnet/devices/devices.h> #include <vnet/feature/feature.h> #include <vnet/gso/gro_func.h> +#include <vnet/interface/rx_queue_funcs.h> #include <vnet/ip/ip4_packet.h> #include <vnet/ip/ip6_packet.h> #include <vnet/udp/udp_packet.h> #include <vnet/devices/virtio/virtio.h> - #define foreach_virtio_input_error \ _(BUFFER_ALLOC, "buffer alloc error") \ _(UNKNOWN, "unknown") @@ -638,30 +637,27 @@ VLIB_NODE_FN (virtio_input_node) (vlib_main_t * vm, vlib_frame_t * frame) { u32 n_rx = 0; - virtio_main_t *nm = &virtio_main; - vnet_device_input_runtime_t *rt = (void *) node->runtime_data; - vnet_device_and_queue_t *dq; - - foreach_device_and_queue (dq, rt->devices_and_queues) - { - virtio_if_t *vif; - vif = vec_elt_at_index (nm->interfaces, dq->dev_instance); - if (vif->flags & VIRTIO_IF_FLAG_ADMIN_UP) - { - if (vif->type == VIRTIO_IF_TYPE_TAP) - n_rx += virtio_device_input_inline (vm, node, frame, vif, - dq->queue_id, - VIRTIO_IF_TYPE_TAP); - else if (vif->type == VIRTIO_IF_TYPE_PCI) - n_rx += virtio_device_input_inline (vm, node, frame, vif, - dq->queue_id, - VIRTIO_IF_TYPE_PCI); - else if (vif->type == VIRTIO_IF_TYPE_TUN) - n_rx += virtio_device_input_inline (vm, node, frame, vif, - dq->queue_id, - VIRTIO_IF_TYPE_TUN); - } - } + virtio_main_t *vim = &virtio_main; + vnet_hw_if_rxq_poll_vector_t *p, + *pv = vnet_hw_if_get_rxq_poll_vector (vm, node); + + vec_foreach (p, pv) + { + virtio_if_t *vif; + vif = vec_elt_at_index (vim->interfaces, p->dev_instance); + if (vif->flags & VIRTIO_IF_FLAG_ADMIN_UP) + { + if (vif->type == VIRTIO_IF_TYPE_TAP) + n_rx += virtio_device_input_inline ( + vm, node, frame, vif, p->queue_id, VIRTIO_IF_TYPE_TAP); + else if (vif->type == VIRTIO_IF_TYPE_PCI) + n_rx += virtio_device_input_inline ( + vm, node, frame, vif, p->queue_id, VIRTIO_IF_TYPE_PCI); + else if (vif->type == VIRTIO_IF_TYPE_TUN) + n_rx += virtio_device_input_inline ( + vm, node, frame, vif, p->queue_id, VIRTIO_IF_TYPE_TUN); + } + } return n_rx; } diff --git a/src/vnet/devices/virtio/pci.c b/src/vnet/devices/virtio/pci.c index 93ea70b3b53..908aba75962 100644 --- a/src/vnet/devices/virtio/pci.c +++ b/src/vnet/devices/virtio/pci.c @@ -24,6 +24,7 @@ #include <vnet/ip/ip6_packet.h> #include <vnet/devices/virtio/virtio.h> #include <vnet/devices/virtio/pci.h> +#include <vnet/interface/rx_queue_funcs.h> #define PCI_VENDOR_ID_VIRTIO 0x1af4 #define PCI_DEVICE_ID_VIRTIO_NIC 0x1000 @@ -115,7 +116,8 @@ virtio_pci_irq_queue_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h, line--; u16 qid = line; - vnet_device_input_set_interrupt_pending (vnm, vif->hw_if_index, qid); + virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, qid); + vnet_hw_if_rx_queue_set_int_pending (vnm, vring->queue_index); } static void @@ -1519,17 +1521,8 @@ virtio_pci_create_if (vlib_main_t * vm, virtio_pci_create_if_args_t * args) } } - vnet_hw_interface_set_input_node (vnm, vif->hw_if_index, - virtio_input_node.index); - u32 i = 0; - vec_foreach_index (i, vif->rxq_vrings) - { - vnet_hw_interface_assign_rx_thread (vnm, vif->hw_if_index, i, ~0); - virtio_vring_set_numa_node (vm, vif, RX_QUEUE (i)); - /* Set default rx mode to POLLING */ - vnet_hw_interface_set_rx_mode (vnm, vif->hw_if_index, i, - VNET_HW_IF_RX_MODE_POLLING); - } + virtio_vring_set_rx_queues (vm, vif); + if (virtio_pci_is_link_up (vm, vif) & VIRTIO_NET_S_LINK_UP) { vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP; @@ -1584,10 +1577,6 @@ virtio_pci_delete_if (vlib_main_t * vm, virtio_if_t * vif) if (vif->hw_if_index) { vnet_hw_interface_set_flags (vnm, vif->hw_if_index, 0); - vec_foreach_index (i, vif->rxq_vrings) - { - vnet_hw_interface_unassign_rx_thread (vnm, vif->hw_if_index, i); - } ethernet_delete_interface (vnm, vif->hw_if_index); } diff --git a/src/vnet/devices/virtio/vhost_user.c b/src/vnet/devices/virtio/vhost_user.c index daa126064c5..b45b18b8433 100644 --- a/src/vnet/devices/virtio/vhost_user.c +++ b/src/vnet/devices/virtio/vhost_user.c @@ -36,6 +36,7 @@ #include <vnet/ethernet/ethernet.h> #include <vnet/devices/devices.h> #include <vnet/feature/feature.h> +#include <vnet/interface/rx_queue_funcs.h> #include <vnet/devices/virtio/vhost_user.h> #include <vnet/devices/virtio/vhost_user_inline.h> @@ -165,17 +166,19 @@ vhost_user_rx_thread_placement (vhost_user_intf_t * vui, u32 qid) ASSERT ((qid & 1) == 1); // should be odd // Assign new queue mappings for the interface - vnet_hw_interface_set_input_node (vnm, vui->hw_if_index, - vhost_user_input_node.index); - vnet_hw_interface_assign_rx_thread (vnm, vui->hw_if_index, q, ~0); + vnet_hw_if_set_input_node (vnm, vui->hw_if_index, + vhost_user_input_node.index); + txvq->queue_index = vnet_hw_if_register_rx_queue (vnm, vui->hw_if_index, q, + VNET_HW_IF_RXQ_THREAD_ANY); if (txvq->mode == VNET_HW_IF_RX_MODE_UNKNOWN) /* Set polling as the default */ txvq->mode = VNET_HW_IF_RX_MODE_POLLING; txvq->qid = q; - rv = vnet_hw_interface_set_rx_mode (vnm, vui->hw_if_index, q, txvq->mode); + rv = vnet_hw_if_set_rx_queue_mode (vnm, txvq->queue_index, txvq->mode); if (rv) vu_log_warn (vui, "unable to set rx mode for interface %d, " "queue %d: rc=%d", vui->hw_if_index, q, rv); + vnet_hw_if_update_runtime_data (vnm, vui->hw_if_index); } /** @brief Returns whether at least one TX and one RX vring are enabled */ @@ -213,15 +216,20 @@ vhost_user_set_interrupt_pending (vhost_user_intf_t * vui, u32 ifq) { u32 qid; vnet_main_t *vnm = vnet_get_main (); + vhost_user_vring_t *txvq; qid = ifq & 0xff; if ((qid & 1) == 0) /* Only care about the odd number, or TX, virtqueue */ return; - if (vhost_user_intf_ready (vui)) - // qid >> 1 is to convert virtqueue number to vring queue index - vnet_device_input_set_interrupt_pending (vnm, vui->hw_if_index, qid >> 1); + // qid >> 1 is to convert virtqueue number to vring queue index + qid >>= 1; + txvq = &vui->vrings[VHOST_VRING_IDX_TX (qid)]; + if (vhost_user_intf_ready (vui) && + ((txvq->mode == VNET_HW_IF_RX_MODE_ADAPTIVE) || + (txvq->mode == VNET_HW_IF_RX_MODE_INTERRUPT))) + vnet_hw_if_rx_queue_set_int_pending (vnm, txvq->queue_index); } static clib_error_t * @@ -1370,24 +1378,6 @@ vhost_user_term_if (vhost_user_intf_t * vui) for (q = 0; q < vui->num_qid; q++) { - // Remove existing queue mapping for the interface - if (q & 1) - { - int rv; - vnet_main_t *vnm = vnet_get_main (); - vhost_user_vring_t *txvq = &vui->vrings[q]; - - if (txvq->qid != -1) - { - rv = vnet_hw_interface_unassign_rx_thread (vnm, - vui->hw_if_index, - q >> 1); - if (rv) - vu_log_warn (vui, "unable to unassign interface %d, " - "queue %d: rc=%d", vui->hw_if_index, q >> 1, rv); - } - } - clib_spinlock_free (&vui->vrings[q].vring_lock); } @@ -2224,19 +2214,14 @@ show_vhost_user_command_fn (vlib_main_t * vm, { vnet_main_t *vnm = vnet_get_main (); uword thread_index; - vnet_hw_if_rx_mode mode; vhost_user_vring_t *txvq = &vui->vrings[qid]; if (txvq->qid == -1) continue; thread_index = - vnet_get_device_input_thread_index (vnm, vui->hw_if_index, - qid >> 1); - vnet_hw_interface_get_rx_mode (vnm, vui->hw_if_index, qid >> 1, - &mode); - vlib_cli_output (vm, " thread %d on vring %d, %U\n", - thread_index, qid, - format_vnet_hw_if_rx_mode, mode); + vnet_hw_if_get_rx_queue_thread_index (vnm, txvq->queue_index); + vlib_cli_output (vm, " thread %d on vring %d, %U\n", thread_index, + qid, format_vnet_hw_if_rx_mode, txvq->mode); } vlib_cli_output (vm, " tx placement: %s\n", diff --git a/src/vnet/devices/virtio/vhost_user.h b/src/vnet/devices/virtio/vhost_user.h index 604e5571141..06c78bce857 100644 --- a/src/vnet/devices/virtio/vhost_user.h +++ b/src/vnet/devices/virtio/vhost_user.h @@ -227,9 +227,9 @@ typedef struct u16 used_wrap_counter; u16 avail_wrap_counter; - u16 last_kick; u8 first_kick; + u32 queue_index; } vhost_user_vring_t; #define VHOST_USER_EVENT_START_TIMER 1 diff --git a/src/vnet/devices/virtio/vhost_user_input.c b/src/vnet/devices/virtio/vhost_user_input.c index 7ea70c629f8..62b59f69ba9 100644 --- a/src/vnet/devices/virtio/vhost_user_input.c +++ b/src/vnet/devices/virtio/vhost_user_input.c @@ -37,6 +37,7 @@ #include <vnet/devices/devices.h> #include <vnet/feature/feature.h> #include <vnet/udp/udp_packet.h> +#include <vnet/interface/rx_queue_funcs.h> #include <vnet/devices/virtio/vhost_user.h> #include <vnet/devices/virtio/vhost_user_inline.h> @@ -372,11 +373,9 @@ vhost_user_input_setup_frame (vlib_main_t * vm, vlib_node_runtime_t * node, } static_always_inline u32 -vhost_user_if_input (vlib_main_t * vm, - vhost_user_main_t * vum, - vhost_user_intf_t * vui, - u16 qid, vlib_node_runtime_t * node, - vnet_hw_if_rx_mode mode, u8 enable_csum) +vhost_user_if_input (vlib_main_t *vm, vhost_user_main_t *vum, + vhost_user_intf_t *vui, u16 qid, + vlib_node_runtime_t *node, u8 enable_csum) { vhost_user_vring_t *txvq = &vui->vrings[VHOST_VRING_IDX_TX (qid)]; vnet_feature_main_t *fm = &feature_main; @@ -411,7 +410,7 @@ vhost_user_if_input (vlib_main_t * vm, * When the traffic subsides, the scheduler switches the node back to * interrupt mode. We must tell the driver we want interrupt. */ - if (PREDICT_FALSE (mode == VNET_HW_IF_RX_MODE_ADAPTIVE)) + if (PREDICT_FALSE (txvq->mode == VNET_HW_IF_RX_MODE_ADAPTIVE)) { if ((node->flags & VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE) || @@ -1081,10 +1080,9 @@ vhost_user_assemble_packet (vring_packed_desc_t * desc_table, } static_always_inline u32 -vhost_user_if_input_packed (vlib_main_t * vm, vhost_user_main_t * vum, - vhost_user_intf_t * vui, u16 qid, - vlib_node_runtime_t * node, - vnet_hw_if_rx_mode mode, u8 enable_csum) +vhost_user_if_input_packed (vlib_main_t *vm, vhost_user_main_t *vum, + vhost_user_intf_t *vui, u16 qid, + vlib_node_runtime_t *node, u8 enable_csum) { vhost_user_vring_t *txvq = &vui->vrings[VHOST_VRING_IDX_TX (qid)]; vnet_feature_main_t *fm = &feature_main; @@ -1126,7 +1124,7 @@ vhost_user_if_input_packed (vlib_main_t * vm, vhost_user_main_t * vum, * When the traffic subsides, the scheduler switches the node back to * interrupt mode. We must tell the driver we want interrupt. */ - if (PREDICT_FALSE (mode == VNET_HW_IF_RX_MODE_ADAPTIVE)) + if (PREDICT_FALSE (txvq->mode == VNET_HW_IF_RX_MODE_ADAPTIVE)) { if ((node->flags & VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE) || @@ -1415,39 +1413,31 @@ VLIB_NODE_FN (vhost_user_input_node) (vlib_main_t * vm, vhost_user_main_t *vum = &vhost_user_main; uword n_rx_packets = 0; vhost_user_intf_t *vui; - vnet_device_input_runtime_t *rt = - (vnet_device_input_runtime_t *) node->runtime_data; - vnet_device_and_queue_t *dq; + vnet_hw_if_rxq_poll_vector_t *pv = vnet_hw_if_get_rxq_poll_vector (vm, node); + vnet_hw_if_rxq_poll_vector_t *pve; - vec_foreach (dq, rt->devices_and_queues) - { - if ((node->state == VLIB_NODE_STATE_POLLING) || - clib_atomic_swap_acq_n (&dq->interrupt_pending, 0)) - { - vui = - pool_elt_at_index (vum->vhost_user_interfaces, dq->dev_instance); - if (vhost_user_is_packed_ring_supported (vui)) - { - if (vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_CSUM)) - n_rx_packets += vhost_user_if_input_packed (vm, vum, vui, - dq->queue_id, node, - dq->mode, 1); - else - n_rx_packets += vhost_user_if_input_packed (vm, vum, vui, - dq->queue_id, node, - dq->mode, 0); - } - else - { - if (vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_CSUM)) - n_rx_packets += vhost_user_if_input (vm, vum, vui, dq->queue_id, - node, dq->mode, 1); - else - n_rx_packets += vhost_user_if_input (vm, vum, vui, dq->queue_id, - node, dq->mode, 0); - } - } - } + vec_foreach (pve, pv) + { + vui = pool_elt_at_index (vum->vhost_user_interfaces, pve->dev_instance); + if (vhost_user_is_packed_ring_supported (vui)) + { + if (vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_CSUM)) + n_rx_packets += vhost_user_if_input_packed ( + vm, vum, vui, pve->queue_id, node, 1); + else + n_rx_packets += vhost_user_if_input_packed ( + vm, vum, vui, pve->queue_id, node, 0); + } + else + { + if (vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_CSUM)) + n_rx_packets += + vhost_user_if_input (vm, vum, vui, pve->queue_id, node, 1); + else + n_rx_packets += + vhost_user_if_input (vm, vum, vui, pve->queue_id, node, 0); + } + } return n_rx_packets; } diff --git a/src/vnet/devices/virtio/virtio.c b/src/vnet/devices/virtio/virtio.c index 99f581a1f1b..e84490ba575 100644 --- a/src/vnet/devices/virtio/virtio.c +++ b/src/vnet/devices/virtio/virtio.c @@ -31,6 +31,7 @@ #include <vnet/ip/ip6_packet.h> #include <vnet/devices/virtio/virtio.h> #include <vnet/devices/virtio/pci.h> +#include <vnet/interface/rx_queue_funcs.h> virtio_main_t virtio_main; @@ -44,17 +45,11 @@ virtio_main_t virtio_main; static clib_error_t * call_read_ready (clib_file_t * uf) { - virtio_main_t *nm = &virtio_main; vnet_main_t *vnm = vnet_get_main (); - u16 qid = uf->private_data & 0xFFFF; - virtio_if_t *vif = - vec_elt_at_index (nm->interfaces, uf->private_data >> 16); u64 b; CLIB_UNUSED (ssize_t size) = read (uf->file_descriptor, &b, sizeof (b)); - if ((qid & 1) == 0) - vnet_device_input_set_interrupt_pending (vnm, vif->hw_if_index, - RX_QUEUE_ACCESS (qid)); + vnet_hw_if_rx_queue_set_int_pending (vnm, uf->private_data); return 0; } @@ -64,7 +59,6 @@ clib_error_t * virtio_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 idx, u16 sz) { virtio_vring_t *vring; - clib_file_t t = { 0 }; int i; if (!is_pow2 (sz)) @@ -123,13 +117,6 @@ virtio_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 idx, u16 sz) virtio_log_debug (vif, "vring %u size %u call_fd %d kick_fd %d", idx, vring->size, vring->call_fd, vring->kick_fd); - t.read_function = call_read_ready; - t.file_descriptor = vring->call_fd; - t.private_data = vif->dev_instance << 16 | idx; - t.description = format (0, "%U vring %u", format_virtio_device_name, - vif->dev_instance, idx); - vring->call_file_index = clib_file_add (&file_main, &t); - return 0; } @@ -233,19 +220,38 @@ virtio_set_packet_buffering (virtio_if_t * vif, u16 buffering_size) } void -virtio_vring_set_numa_node (vlib_main_t * vm, virtio_if_t * vif, u32 idx) +virtio_vring_set_rx_queues (vlib_main_t *vm, virtio_if_t *vif) { vnet_main_t *vnm = vnet_get_main (); - u32 thread_index; - virtio_vring_t *vring = - vec_elt_at_index (vif->rxq_vrings, RX_QUEUE_ACCESS (idx)); - thread_index = - vnet_get_device_input_thread_index (vnm, vif->hw_if_index, - RX_QUEUE_ACCESS (idx)); - vring->buffer_pool_index = - vlib_buffer_pool_get_default_for_numa (vm, - vlib_mains - [thread_index]->numa_node); + virtio_vring_t *vring; + + vnet_hw_if_set_input_node (vnm, vif->hw_if_index, virtio_input_node.index); + + vec_foreach (vring, vif->rxq_vrings) + { + vring->queue_index = vnet_hw_if_register_rx_queue ( + vnm, vif->hw_if_index, RX_QUEUE_ACCESS (vring->queue_id), + VNET_HW_IF_RXQ_THREAD_ANY); + vring->buffer_pool_index = vlib_buffer_pool_get_default_for_numa ( + vm, vnet_hw_if_get_rx_queue_numa_node (vnm, vring->queue_index)); + if (vif->type == VIRTIO_IF_TYPE_TAP || vif->type == VIRTIO_IF_TYPE_TUN) + { + + clib_file_t f = { + .read_function = call_read_ready, + .flags = UNIX_FILE_EVENT_EDGE_TRIGGERED, + .file_descriptor = vring->call_fd, + .private_data = vring->queue_index, + .description = format (0, "%U vring %u", format_virtio_device_name, + vif->dev_instance, vring->queue_id), + }; + + vring->call_file_index = clib_file_add (&file_main, &f); + vnet_hw_if_set_rx_queue_file_index (vnm, vring->queue_index, + vring->call_file_index); + } + } + vnet_hw_if_update_runtime_data (vnm, vif->hw_if_index); } inline void diff --git a/src/vnet/devices/virtio/virtio.h b/src/vnet/devices/virtio/virtio.h index 035dc9ca40d..c149ce482e6 100644 --- a/src/vnet/devices/virtio/virtio.h +++ b/src/vnet/devices/virtio/virtio.h @@ -82,6 +82,7 @@ typedef struct u32 *buffers; u16 size; u16 queue_id; + u32 queue_index; u16 desc_in_use; u16 desc_next; u16 last_used_idx; @@ -230,8 +231,7 @@ clib_error_t *virtio_vring_free_rx (vlib_main_t * vm, virtio_if_t * vif, u32 idx); clib_error_t *virtio_vring_free_tx (vlib_main_t * vm, virtio_if_t * vif, u32 idx); -void virtio_vring_set_numa_node (vlib_main_t * vm, virtio_if_t * vif, - u32 idx); +void virtio_vring_set_rx_queues (vlib_main_t *vm, virtio_if_t *vif); extern void virtio_free_buffers (vlib_main_t * vm, virtio_vring_t * vring); extern void virtio_set_net_hdr_size (virtio_if_t * vif); extern void virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, diff --git a/src/vnet/devices/virtio/virtio_process.c b/src/vnet/devices/virtio/virtio_process.c index 1689459f1cc..f347ef2ab57 100644 --- a/src/vnet/devices/virtio/virtio_process.c +++ b/src/vnet/devices/virtio/virtio_process.c @@ -18,6 +18,7 @@ #include <vlib/vlib.h> #include <vnet/devices/virtio/virtio.h> #include <vnet/gso/gro_func.h> +#include <vnet/interface/rx_queue_funcs.h> static uword virtio_send_interrupt_process (vlib_main_t * vm, @@ -45,22 +46,20 @@ virtio_send_interrupt_process (vlib_main_t * vm, break; case ~0: - /* *INDENT-OFF* */ - pool_foreach (vif, vim->interfaces) { - if (vif->packet_coalesce || vif->packet_buffering) - { - virtio_vring_t *vring; - vec_foreach (vring, vif->rxq_vrings) - { - if (vring->mode == VNET_HW_IF_RX_MODE_INTERRUPT || - vring->mode == VNET_HW_IF_RX_MODE_ADAPTIVE) - vnet_device_input_set_interrupt_pending ( - vnet_get_main (), vif->hw_if_index, - RX_QUEUE_ACCESS (vring->queue_id)); - } - } - } - /* *INDENT-ON* */ + pool_foreach (vif, vim->interfaces) + { + if (vif->packet_coalesce || vif->packet_buffering) + { + virtio_vring_t *vring; + vec_foreach (vring, vif->rxq_vrings) + { + if (vring->mode == VNET_HW_IF_RX_MODE_INTERRUPT || + vring->mode == VNET_HW_IF_RX_MODE_ADAPTIVE) + vnet_hw_if_rx_queue_set_int_pending ( + vnet_get_main (), vring->queue_index); + } + } + } break; default: |