From 9e2a78564f0fc07f1ea3d15a31fa7ca3a0f6424d Mon Sep 17 00:00:00 2001 From: Mohsin Kazmi Date: Thu, 13 Aug 2020 18:57:26 +0200 Subject: tap: add gro support Type: feature Change-Id: I5868dd267aa26aa97aec5fd70e70c5956ac52277 Signed-off-by: Mohsin Kazmi --- src/vnet/devices/virtio/device.c | 33 ++++++++++++++++++++++++++------- src/vnet/devices/virtio/node.c | 9 +++++++++ src/vnet/devices/virtio/pci.c | 1 + src/vnet/devices/virtio/virtio.c | 21 +++++++++++++++++++++ src/vnet/devices/virtio/virtio.h | 4 ++++ 5 files changed, 61 insertions(+), 7 deletions(-) (limited to 'src/vnet/devices/virtio') diff --git a/src/vnet/devices/virtio/device.c b/src/vnet/devices/virtio/device.c index 45c390d415d..99157cd6166 100644 --- a/src/vnet/devices/virtio/device.c +++ b/src/vnet/devices/virtio/device.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -502,7 +503,7 @@ static_always_inline uword virtio_interface_tx_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame, virtio_if_t * vif, virtio_if_type_t type, int do_gso, - int csum_offload) + int csum_offload, int do_gro) { u16 n_left = frame->n_vectors; virtio_vring_t *vring; @@ -513,6 +514,7 @@ virtio_interface_tx_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, u16 mask = sz - 1; u16 retry_count = 2; u32 *buffers = vlib_frame_vector_args (frame); + u32 to[GRO_TO_VECTOR_SIZE (n_left)]; clib_spinlock_lock_if_init (&vring->lockp); @@ -520,6 +522,12 @@ virtio_interface_tx_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, (vring->last_kick_avail_idx != vring->avail->idx)) virtio_kick (vm, vring, vif); + if (do_gro) + { + n_left = vnet_gro_inline (vm, vring->flow_table, buffers, n_left, to); + buffers = to; + } + retry: /* free consumed buffers */ virtio_free_used_device_desc (vm, vring, node->node_index); @@ -621,7 +629,8 @@ retry: if (retry_count--) goto retry; - virtio_interface_drop_inline (vm, node->node_index, buffers, n_left, + virtio_interface_drop_inline (vm, node->node_index, + buffers, n_left, VIRTIO_TX_ERROR_NO_FREE_SLOTS); } @@ -641,15 +650,18 @@ virtio_interface_tx_inline (vlib_main_t * vm, vlib_node_runtime_t * node, if (hw->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO) return virtio_interface_tx_gso_inline (vm, node, frame, vif, type, 1 /* do_gso */ , - 1 /* checksum offload */ ); + 1 /* checksum offload */ , + vif->packet_coalesce); else if (hw->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD) return virtio_interface_tx_gso_inline (vm, node, frame, vif, type, 0 /* no do_gso */ , - 1 /* checksum offload */ ); + 1 /* checksum offload */ , + 0 /* do_gro */ ); else return virtio_interface_tx_gso_inline (vm, node, frame, vif, type, 0 /* no do_gso */ , - 0 /* no checksum offload */ ); + 0 /* no checksum offload */ , + 0 /* do_gro */ ); } VNET_DEVICE_CLASS_TX_FN (virtio_device_class) (vlib_main_t * vm, @@ -717,9 +729,16 @@ virtio_interface_rx_mode_change (vnet_main_t * vnm, u32 hw_if_index, u32 qid, } if (mode == VNET_HW_INTERFACE_RX_MODE_POLLING) - vring->avail->flags |= VIRTIO_RING_FLAG_MASK_INT; + { + /* only enable packet coalesce in poll mode */ + gro_flow_table_set_is_enable (vring->flow_table, 1); + vring->avail->flags |= VIRTIO_RING_FLAG_MASK_INT; + } else - vring->avail->flags &= ~VIRTIO_RING_FLAG_MASK_INT; + { + gro_flow_table_set_is_enable (vring->flow_table, 0); + vring->avail->flags &= ~VIRTIO_RING_FLAG_MASK_INT; + } return 0; } diff --git a/src/vnet/devices/virtio/node.c b/src/vnet/devices/virtio/node.c index 63e40da53b7..a27131a6398 100644 --- a/src/vnet/devices/virtio/node.c +++ b/src/vnet/devices/virtio/node.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -267,6 +268,8 @@ virtio_device_input_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, u32 thread_index = vm->thread_index; uword n_trace = vlib_get_trace_count (vm, node); virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, qid); + u16 txq_id = thread_index % vif->num_txqs; + virtio_vring_t *txq_vring = vec_elt_at_index (vif->txq_vrings, txq_id); u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; const int hdr_sz = vif->virtio_net_hdr_sz; u32 *to_next = 0; @@ -276,6 +279,12 @@ virtio_device_input_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, u16 last = vring->last_used_idx; u16 n_left = vring->used->idx - last; + if (vif->packet_coalesce) + { + vnet_gro_flow_table_schedule_node_on_dispatcher (vm, + txq_vring->flow_table); + } + if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0 && vring->last_kick_avail_idx != vring->avail->idx) virtio_kick (vm, vring, vif); diff --git a/src/vnet/devices/virtio/pci.c b/src/vnet/devices/virtio/pci.c index cb3a3a73632..d28c7464e5a 100644 --- a/src/vnet/devices/virtio/pci.c +++ b/src/vnet/devices/virtio/pci.c @@ -824,6 +824,7 @@ virtio_pci_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 queue_num) vring->used = vr.used; vring->queue_id = queue_num; vring->avail->flags = VIRTIO_RING_FLAG_MASK_INT; + vring->flow_table = 0; ASSERT (vring->buffers == 0); vec_validate_aligned (vring->buffers, queue_size, CLIB_CACHE_LINE_BYTES); diff --git a/src/vnet/devices/virtio/virtio.c b/src/vnet/devices/virtio/virtio.c index 17fbd7920da..f8b6a39ea47 100644 --- a/src/vnet/devices/virtio/virtio.c +++ b/src/vnet/devices/virtio/virtio.c @@ -217,10 +217,25 @@ virtio_vring_free_tx (vlib_main_t * vm, virtio_if_t * vif, u32 idx) if (vring->avail) clib_mem_free (vring->avail); vec_free (vring->buffers); + gro_flow_table_free (vring->flow_table); clib_spinlock_free (&vring->lockp); return 0; } +void +virtio_set_packet_coalesce (virtio_if_t * vif) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vif->hw_if_index); + virtio_vring_t *vring; + vec_foreach (vring, vif->txq_vrings) + { + gro_flow_table_init (&vring->flow_table, + vif->type & (VIRTIO_IF_TYPE_TAP | + VIRTIO_IF_TYPE_PCI), hw->tx_node_index); + } +} + void virtio_vring_set_numa_node (vlib_main_t * vm, virtio_if_t * vif, u32 idx) { @@ -320,6 +335,7 @@ virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type) } vlib_cli_output (vm, " gso-enabled %d", vif->gso_enabled); vlib_cli_output (vm, " csum-enabled %d", vif->csum_offload_enabled); + vlib_cli_output (vm, " packet-coalesce %d", vif->packet_coalesce); if (type & (VIRTIO_IF_TYPE_TAP | VIRTIO_IF_TYPE_PCI)) vlib_cli_output (vm, " Mac Address: %U", format_ethernet_address, vif->mac_addr); @@ -412,6 +428,11 @@ virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type) vlib_cli_output (vm, " kickfd %d, callfd %d", vring->kick_fd, vring->call_fd); } + if (vring->flow_table) + { + vlib_cli_output (vm, " %U", gro_flow_table_format, + vring->flow_table); + } if (show_descr) { vlib_cli_output (vm, "\n descriptor table:\n"); diff --git a/src/vnet/devices/virtio/virtio.h b/src/vnet/devices/virtio/virtio.h index 04896ed6e2a..b57c6ebd77d 100644 --- a/src/vnet/devices/virtio/virtio.h +++ b/src/vnet/devices/virtio/virtio.h @@ -22,6 +22,7 @@ #include #include #include +#include #define foreach_virtio_net_features \ _ (VIRTIO_NET_F_CSUM, 0) /* Host handles pkts w/ partial csum */ \ @@ -123,6 +124,7 @@ typedef struct u16 last_used_idx; u16 last_kick_avail_idx; u32 call_file_index; + gro_flow_table_t *flow_table; } virtio_vring_t; typedef union @@ -165,6 +167,7 @@ typedef struct u32 sw_if_index; CLIB_CACHE_LINE_ALIGN_MARK (cacheline1); + int packet_coalesce; union { u32 id; @@ -221,6 +224,7 @@ extern void virtio_free_rx_buffers (vlib_main_t * vm, virtio_vring_t * vring); extern void virtio_set_net_hdr_size (virtio_if_t * vif); extern void virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type); +extern void virtio_set_packet_coalesce (virtio_if_t * vif); extern void virtio_pci_legacy_notify_queue (vlib_main_t * vm, virtio_if_t * vif, u16 queue_id); format_function_t format_virtio_device_name; -- cgit 1.2.3-korg