From 38b09681a0e68b3711206516bc220599f6a73281 Mon Sep 17 00:00:00 2001 From: Mohsin Kazmi Date: Wed, 3 Jun 2020 18:20:17 +0200 Subject: virtio: interface type based improvements Type: improvement Change-Id: I220ea6ab609e3c1628f5210be441d0d5e825a32c Signed-off-by: Mohsin Kazmi --- src/vnet/devices/virtio/device.c | 64 ++++++++++++++++------- src/vnet/devices/virtio/node.c | 110 +++++++++++++++++++++++++-------------- src/vnet/devices/virtio/virtio.h | 43 +++++++++------ 3 files changed, 142 insertions(+), 75 deletions(-) diff --git a/src/vnet/devices/virtio/device.c b/src/vnet/devices/virtio/device.c index 9a35bb8162f..87fd6fdcf5f 100644 --- a/src/vnet/devices/virtio/device.c +++ b/src/vnet/devices/virtio/device.c @@ -255,7 +255,8 @@ set_gso_offsets (vlib_buffer_t * b, struct virtio_net_hdr_v1 *hdr, int is_l2) static_always_inline u16 add_buffer_to_slot (vlib_main_t * vm, virtio_if_t * vif, - virtio_vring_t * vring, u32 bi, u16 free_desc_count, + virtio_if_type_t type, virtio_vring_t * vring, + u32 bi, u16 free_desc_count, u16 avail, u16 next, u16 mask, int do_gso, int csum_offload, uword node_index) { @@ -265,7 +266,7 @@ add_buffer_to_slot (vlib_main_t * vm, virtio_if_t * vif, d = &vring->desc[next]; vlib_buffer_t *b = vlib_get_buffer (vm, bi); struct virtio_net_hdr_v1 *hdr = vlib_buffer_get_current (b) - hdr_sz; - int is_l2 = (vif->type & (VIRTIO_IF_TYPE_TAP | VIRTIO_IF_TYPE_PCI)); + int is_l2 = (type & (VIRTIO_IF_TYPE_TAP | VIRTIO_IF_TYPE_PCI)); clib_memset (hdr, 0, hdr_sz); @@ -296,8 +297,8 @@ add_buffer_to_slot (vlib_main_t * vm, virtio_if_t * vif, if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0)) { d->addr = - ((vif->type == VIRTIO_IF_TYPE_PCI) ? vlib_buffer_get_current_pa (vm, - b) : + ((type == VIRTIO_IF_TYPE_PCI) ? vlib_buffer_get_current_pa (vm, + b) : pointer_to_uword (vlib_buffer_get_current (b))) - hdr_sz; d->len = b->current_length + hdr_sz; d->flags = 0; @@ -329,7 +330,7 @@ add_buffer_to_slot (vlib_main_t * vm, virtio_if_t * vif, struct vring_desc *id = (struct vring_desc *) vlib_buffer_get_current (indirect_desc); u32 count = 1; - if (vif->type == VIRTIO_IF_TYPE_PCI) + if (type == VIRTIO_IF_TYPE_PCI) { d->addr = vlib_physmem_get_pa (vm, id); id->addr = vlib_buffer_get_current_pa (vm, b) - hdr_sz; @@ -386,7 +387,7 @@ add_buffer_to_slot (vlib_main_t * vm, virtio_if_t * vif, d->len = count * sizeof (struct vring_desc); d->flags = VRING_DESC_F_INDIRECT; } - else if (vif->type == VIRTIO_IF_TYPE_PCI) + else if (type == VIRTIO_IF_TYPE_PCI) { u16 count = next; vlib_buffer_t *b_temp = b; @@ -478,9 +479,10 @@ virtio_find_free_desc (virtio_vring_t * vring, u16 size, u16 mask, } static_always_inline uword -virtio_interface_tx_inline (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_frame_t * frame, virtio_if_t * vif, - int do_gso, int csum_offload) +virtio_interface_tx_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * frame, virtio_if_t * vif, + virtio_if_type_t type, int do_gso, + int csum_offload) { u16 n_left = frame->n_vectors; virtio_vring_t *vring; @@ -525,7 +527,7 @@ retry: { u16 n_added = 0; n_added = - add_buffer_to_slot (vm, vif, vring, buffers[0], free_desc_count, + add_buffer_to_slot (vm, vif, type, vring, buffers[0], free_desc_count, avail, next, mask, do_gso, csum_offload, node->node_index); @@ -570,25 +572,49 @@ retry: return frame->n_vectors - n_left; } +static_always_inline uword +virtio_interface_tx_inline (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * frame, virtio_if_t * vif, + virtio_if_type_t type) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vif->hw_if_index); + + if (hw->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO) + return virtio_interface_tx_gso_inline (vm, node, frame, vif, type, + 1 /* do_gso */ , + 1 /* checksum offload */ ); + else if (hw->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD) + return virtio_interface_tx_gso_inline (vm, node, frame, vif, type, + 0 /* no do_gso */ , + 1 /* checksum offload */ ); + else + return virtio_interface_tx_gso_inline (vm, node, frame, vif, type, + 0 /* no do_gso */ , + 0 /* no checksum offload */ ); +} + VNET_DEVICE_CLASS_TX_FN (virtio_device_class) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) { - vnet_main_t *vnm = vnet_get_main (); virtio_main_t *nm = &virtio_main; vnet_interface_output_runtime_t *rund = (void *) node->runtime_data; virtio_if_t *vif = pool_elt_at_index (nm->interfaces, rund->dev_instance); - vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vif->hw_if_index); - if (hw->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO) - return virtio_interface_tx_inline (vm, node, frame, vif, 1 /* do_gso */ , - 1); - else if (hw->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD) + if (vif->type == VIRTIO_IF_TYPE_TAP) return virtio_interface_tx_inline (vm, node, frame, vif, - 0 /* no do_gso */ , 1); - else + VIRTIO_IF_TYPE_TAP); + else if (vif->type == VIRTIO_IF_TYPE_PCI) + return virtio_interface_tx_inline (vm, node, frame, vif, + VIRTIO_IF_TYPE_PCI); + else if (vif->type == VIRTIO_IF_TYPE_TUN) return virtio_interface_tx_inline (vm, node, frame, vif, - 0 /* no do_gso */ , 0); + VIRTIO_IF_TYPE_TUN); + else + ASSERT (0); + + return 0; } static void diff --git a/src/vnet/devices/virtio/node.c b/src/vnet/devices/virtio/node.c index 8d7c9360b00..06641852316 100644 --- a/src/vnet/devices/virtio/node.c +++ b/src/vnet/devices/virtio/node.c @@ -80,7 +80,8 @@ format_virtio_input_trace (u8 * s, va_list * args) static_always_inline void virtio_refill_vring (vlib_main_t * vm, virtio_if_t * vif, - virtio_vring_t * vring, const int hdr_sz) + virtio_if_type_t type, virtio_vring_t * vring, + const int hdr_sz) { u16 used, next, avail, n_slots; u16 sz = vring->size; @@ -114,12 +115,11 @@ more: * previous offset. Here we want to make sure, it should be 0 * initialized. */ - b->current_data = 0; - b->current_data -= hdr_sz; + b->current_data = -hdr_sz; memset (vlib_buffer_get_current (b), 0, hdr_sz); d->addr = - ((vif->type == VIRTIO_IF_TYPE_PCI) ? vlib_buffer_get_current_pa (vm, - b) : + ((type == VIRTIO_IF_TYPE_PCI) ? vlib_buffer_get_current_pa (vm, + b) : pointer_to_uword (vlib_buffer_get_current (b))); d->len = vlib_buffer_get_default_data_size (vm) + hdr_sz; d->flags = VRING_DESC_F_WRITE; @@ -146,10 +146,22 @@ virtio_needs_csum (vlib_buffer_t * b0, struct virtio_net_hdr_v1 *hdr, u8 * l4_proto, u8 * l4_hdr_sz, virtio_if_type_t type) { if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) - { u16 ethertype = 0, l2hdr_sz = 0; - if (type != VIRTIO_IF_TYPE_TUN) + + if (type == VIRTIO_IF_TYPE_TUN) + { + switch (b0->data[0] & 0xf0) + { + case 0x40: + ethertype = ETHERNET_TYPE_IP4; + break; + case 0x60: + ethertype = ETHERNET_TYPE_IP6; + break; + } + } + else { ethernet_header_t *eh = (ethernet_header_t *) vlib_buffer_get_current (b0); @@ -171,21 +183,10 @@ virtio_needs_csum (vlib_buffer_t * b0, struct virtio_net_hdr_v1 *hdr, } } } - else - { - switch (b0->data[0] & 0xf0) - { - case 0x40: - ethertype = ETHERNET_TYPE_IP4; - break; - case 0x60: - ethertype = ETHERNET_TYPE_IP6; - break; - } - } vnet_buffer (b0)->l2_hdr_offset = 0; vnet_buffer (b0)->l3_hdr_offset = l2hdr_sz; + if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP4)) { ip4_header_t *ip4 = @@ -252,9 +253,10 @@ fill_gso_buffer_flags (vlib_buffer_t * b0, struct virtio_net_hdr_v1 *hdr, } static_always_inline uword -virtio_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_frame_t * frame, virtio_if_t * vif, u16 qid, - int gso_enabled, int checksum_offload_enabled) +virtio_device_input_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * frame, virtio_if_t * vif, + u16 qid, virtio_if_type_t type, + int gso_enabled, int checksum_offload_enabled) { vnet_main_t *vnm = vnet_get_main (); u32 thread_index = vm->thread_index; @@ -276,10 +278,14 @@ virtio_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, if (n_left == 0) goto refill; + if (type == VIRTIO_IF_TYPE_TUN) + next_index = VNET_DEVICE_INPUT_NEXT_IP4_INPUT; + while (n_left) { u32 n_left_to_next; u32 next0 = next_index; + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); while (n_left && n_left_to_next) @@ -296,13 +302,11 @@ virtio_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, if (hdr_sz == sizeof (struct virtio_net_hdr_v1)) num_buffers = hdr->num_buffers; - b0->current_data += hdr_sz; + b0->current_data = 0; b0->current_length = len; - b0->total_length_not_including_first_buffer = 0; - b0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID; if (checksum_offload_enabled) - virtio_needs_csum (b0, hdr, &l4_proto, &l4_hdr_sz, vif->type); + virtio_needs_csum (b0, hdr, &l4_proto, &l4_hdr_sz, type); if (gso_enabled) fill_gso_buffer_flags (b0, hdr, l4_proto, l4_hdr_sz); @@ -315,6 +319,8 @@ virtio_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, { vlib_buffer_t *pb, *cb; pb = b0; + b0->total_length_not_including_first_buffer = 0; + b0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID; while (num_buffers > 1) { last++; @@ -337,8 +343,10 @@ virtio_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, num_buffers--; n_left--; } + len += b0->total_length_not_including_first_buffer; } - if (PREDICT_FALSE (vif->type == VIRTIO_IF_TYPE_TUN)) + + if (type == VIRTIO_IF_TYPE_TUN) { switch (b0->data[0] & 0xf0) { @@ -354,12 +362,15 @@ virtio_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, } } - if (PREDICT_FALSE (vif->per_interface_next_index != ~0)) next0 = vif->per_interface_next_index; - /* redirect if feature path enabled */ - vnet_feature_start_device_input_x1 (vif->sw_if_index, &next0, b0); + if (type != VIRTIO_IF_TYPE_TUN) + { + /* only for l2, redirect if feature path enabled */ + vnet_feature_start_device_input_x1 (vif->sw_if_index, &next0, + b0); + } /* trace */ VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0); @@ -373,7 +384,7 @@ virtio_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); tr->next_index = next0; tr->hw_if_index = vif->hw_if_index; - tr->len = len + b0->total_length_not_including_first_buffer; + tr->len = len; clib_memcpy_fast (&tr->hdr, hdr, hdr_sz); } @@ -391,7 +402,7 @@ virtio_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, /* next packet */ n_rx_packets++; - n_rx_bytes += (len + b0->total_length_not_including_first_buffer); + n_rx_bytes += len; } vlib_put_next_frame (vm, node, next_index, n_left_to_next); } @@ -403,11 +414,29 @@ virtio_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, n_rx_bytes); refill: - virtio_refill_vring (vm, vif, vring, hdr_sz); + virtio_refill_vring (vm, vif, type, vring, hdr_sz); return n_rx_packets; } +static_always_inline uword +virtio_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * frame, virtio_if_t * vif, u16 qid, + virtio_if_type_t type) +{ + + if (vif->gso_enabled) + return virtio_device_input_gso_inline (vm, node, frame, vif, + qid, type, 1, 1); + else if (vif->csum_offload_enabled) + return virtio_device_input_gso_inline (vm, node, frame, vif, + qid, type, 0, 1); + else + return virtio_device_input_gso_inline (vm, node, frame, vif, + qid, type, 0, 0); + return 0; +} + VLIB_NODE_FN (virtio_input_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) @@ -423,15 +452,18 @@ VLIB_NODE_FN (virtio_input_node) (vlib_main_t * vm, vif = vec_elt_at_index (nm->interfaces, dq->dev_instance); if (vif->flags & VIRTIO_IF_FLAG_ADMIN_UP) { - if (vif->gso_enabled) + if (vif->type == VIRTIO_IF_TYPE_TAP) n_rx += virtio_device_input_inline (vm, node, frame, vif, - dq->queue_id, 1, 1); - else if (vif->csum_offload_enabled) + dq->queue_id, + VIRTIO_IF_TYPE_TAP); + else if (vif->type == VIRTIO_IF_TYPE_PCI) n_rx += virtio_device_input_inline (vm, node, frame, vif, - dq->queue_id, 0, 1); - else + dq->queue_id, + VIRTIO_IF_TYPE_PCI); + else if (vif->type == VIRTIO_IF_TYPE_TUN) n_rx += virtio_device_input_inline (vm, node, frame, vif, - dq->queue_id, 0, 0); + dq->queue_id, + VIRTIO_IF_TYPE_TUN); } } diff --git a/src/vnet/devices/virtio/virtio.h b/src/vnet/devices/virtio/virtio.h index f5dc1c86930..68462d8075d 100644 --- a/src/vnet/devices/virtio/virtio.h +++ b/src/vnet/devices/virtio/virtio.h @@ -106,10 +106,10 @@ typedef struct typedef struct { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + clib_spinlock_t lockp; struct vring_desc *desc; struct vring_used *used; struct vring_avail *avail; - clib_spinlock_t lockp; u16 desc_in_use; u16 desc_next; int kick_fd; @@ -119,10 +119,10 @@ typedef struct u16 queue_id; #define VRING_TX_OUT_OF_ORDER 1 u16 flags; - u32 call_file_index; u32 *buffers; u16 last_used_idx; u16 last_kick_avail_idx; + u32 call_file_index; } virtio_vring_t; typedef union @@ -140,34 +140,45 @@ typedef union typedef struct { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + u64 features; u32 flags; + u32 per_interface_next_index; + u16 num_rxqs; + u16 num_txqs; + virtio_vring_t *rxq_vrings; + virtio_vring_t *txq_vrings; + int gso_enabled; + int csum_offload_enabled; + union + { + int *tap_fds; + struct + { + u32 pci_dev_handle; + u32 msix_enabled; + }; + }; + u16 virtio_net_hdr_sz; + virtio_if_type_t type; - u32 dev_instance; u32 hw_if_index; u32 sw_if_index; - u32 numa_node; - u16 virtio_net_hdr_sz; - virtio_if_type_t type; + + CLIB_CACHE_LINE_ALIGN_MARK (cacheline1); union { u32 id; pci_addr_t pci_addr; }; - u32 per_interface_next_index; int *vhost_fds; - int *tap_fds; - u32 msix_enabled; - u32 pci_dev_handle; - virtio_vring_t *rxq_vrings; - virtio_vring_t *txq_vrings; - u64 features, remote_features; + u32 dev_instance; + u32 numa_node; + u64 remote_features; /* error */ clib_error_t *error; u8 support_int_mode; /* support interrupt mode */ u16 max_queue_pairs; - u16 num_rxqs; - u16 num_txqs; u8 status; u8 mac_addr[6]; u8 *host_if_name; @@ -179,8 +190,6 @@ typedef struct ip6_address_t host_ip6_addr; u8 host_ip6_prefix_len; u32 host_mtu_size; - int gso_enabled; - int csum_offload_enabled; int ifindex; virtio_vring_t *cxq_vring; } virtio_if_t; -- cgit 1.2.3-korg