diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/vnet/devices/virtio/FEATURE.yaml | 1 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/cli.c | 6 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/device.c | 343 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/node.c | 216 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/pci.c | 354 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/virtio.c | 184 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/virtio.h | 43 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/virtio_pci_modern.c | 25 |
8 files changed, 1037 insertions, 135 deletions
diff --git a/src/vnet/devices/virtio/FEATURE.yaml b/src/vnet/devices/virtio/FEATURE.yaml index ac7133bae7f..7b2fb59e1ad 100644 --- a/src/vnet/devices/virtio/FEATURE.yaml +++ b/src/vnet/devices/virtio/FEATURE.yaml @@ -7,6 +7,7 @@ features: - Device mode to emulate vhost-user interface presented to VPP from the guest VM. - Support virtio 1.0 in virtio + - Support virtio 1.1 packed ring in virtio [experimental] - Support multi-queue, GSO, checksum offload, indirect descriptor, jumbo frame, and packed ring. - Support virtio 1.1 packed ring in vhost diff --git a/src/vnet/devices/virtio/cli.c b/src/vnet/devices/virtio/cli.c index 8a9c97be8cb..05f9eab18cf 100644 --- a/src/vnet/devices/virtio/cli.c +++ b/src/vnet/devices/virtio/cli.c @@ -49,10 +49,12 @@ virtio_pci_create_command_fn (vlib_main_t * vm, unformat_input_t * input, args.checksum_offload_enabled = 1; else if (unformat (line_input, "buffering")) { - args.virtio_flags = VIRTIO_FLAG_BUFFERING; + args.virtio_flags |= VIRTIO_FLAG_BUFFERING; if (unformat (line_input, "size %u", &buffering_size)) args.buffering_size = buffering_size; } + else if (unformat (line_input, "packed")) + args.virtio_flags |= VIRTIO_FLAG_PACKED; else return clib_error_return (0, "unknown input `%U'", format_unformat_error, input); @@ -69,7 +71,7 @@ VLIB_CLI_COMMAND (virtio_pci_create_command, static) = { .path = "create interface virtio", .short_help = "create interface virtio <pci-address> " "[feature-mask <hex-mask>] [gso-enabled] [csum-enabled] " - "[buffering [size <buffering-szie>]]", + "[buffering [size <buffering-szie>]] [packed]", .function = virtio_pci_create_command_fn, }; /* *INDENT-ON* */ diff --git a/src/vnet/devices/virtio/device.c b/src/vnet/devices/virtio/device.c index add82b77634..7e57952905b 100644 --- a/src/vnet/devices/virtio/device.c +++ b/src/vnet/devices/virtio/device.c @@ -161,8 +161,8 @@ virtio_memset_ring_u32 (u32 * ring, u32 start, u32 ring_size, u32 n_buffers) } static_always_inline void -virtio_free_used_device_desc (vlib_main_t * vm, virtio_vring_t * vring, - uword node_index) +virtio_free_used_device_desc_split (vlib_main_t * vm, virtio_vring_t * vring, + uword node_index) { u16 used = vring->desc_in_use; u16 sz = vring->size; @@ -227,7 +227,60 @@ virtio_free_used_device_desc (vlib_main_t * vm, virtio_vring_t * vring, } static_always_inline void -set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr, int is_l2) +virtio_free_used_device_desc_packed (vlib_main_t * vm, virtio_vring_t * vring, + uword node_index) +{ + vring_packed_desc_t *d; + u16 sz = vring->size; + u16 last = vring->last_used_idx; + u16 n_buffers = 0, start; + u16 flags; + + if (vring->desc_in_use == 0) + return; + + d = &vring->packed_desc[last]; + flags = d->flags; + start = d->id; + + while ((flags & VRING_DESC_F_AVAIL) == (vring->used_wrap_counter << 7) && + (flags & VRING_DESC_F_USED) == (vring->used_wrap_counter << 15)) + { + last++; + n_buffers++; + + if (last >= sz) + { + last = 0; + vring->used_wrap_counter ^= 1; + } + d = &vring->packed_desc[last]; + flags = d->flags; + } + + if (n_buffers) + { + vlib_buffer_free_from_ring (vm, vring->buffers, start, sz, n_buffers); + virtio_memset_ring_u32 (vring->buffers, start, sz, n_buffers); + vring->desc_in_use -= n_buffers; + vring->last_used_idx = last; + } +} + +static_always_inline void +virtio_free_used_device_desc (vlib_main_t * vm, virtio_vring_t * vring, + uword node_index, int packed) +{ + if (packed) + virtio_free_used_device_desc_packed (vm, vring, node_index); + else + virtio_free_used_device_desc_split (vm, vring, node_index); + +} + +static_always_inline void +set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr, + const int is_l2) { if (b->flags & VNET_BUFFER_F_IS_IP4) { @@ -274,7 +327,8 @@ set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr, int is_l2) } static_always_inline void -set_gso_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr, int is_l2) +set_gso_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr, + const int is_l2) { if (b->flags & VNET_BUFFER_F_IS_IP4) { @@ -504,6 +558,238 @@ done: return n_added; } +static_always_inline u16 +add_buffer_to_slot_packed (vlib_main_t * vm, vlib_node_runtime_t * node, + virtio_vring_t * vring, u32 bi, u16 next, + int hdr_sz, int do_gso, int csum_offload, + int is_pci, int is_tun, int is_indirect, + int is_any_layout) +{ + u16 n_added = 0, flags = 0; + int is_l2 = !is_tun; + vring_packed_desc_t *d = &vring->packed_desc[next]; + vlib_buffer_t *b = vlib_get_buffer (vm, bi); + virtio_net_hdr_v1_t *hdr = vlib_buffer_get_current (b) - hdr_sz; + u32 drop_inline = ~0; + + clib_memset (hdr, 0, hdr_sz); + + if (b->flags & VNET_BUFFER_F_GSO) + { + if (do_gso) + set_gso_offsets (b, hdr, is_l2); + else + { + drop_inline = VIRTIO_TX_ERROR_GSO_PACKET_DROP; + goto done; + } + } + else if (b->flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | + VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)) + { + if (csum_offload) + set_checksum_offsets (b, hdr, is_l2); + else + { + drop_inline = VIRTIO_TX_ERROR_CSUM_OFFLOAD_PACKET_DROP; + goto done; + } + } + if (PREDICT_FALSE (b->flags & VLIB_BUFFER_IS_TRACED)) + { + virtio_tx_trace (vm, node, b, bi, is_tun); + } + + if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0)) + { + d->addr = + ((is_pci) ? vlib_buffer_get_current_pa (vm, + b) : + pointer_to_uword (vlib_buffer_get_current (b))) - hdr_sz; + d->len = b->current_length + hdr_sz; + } + else if (is_indirect) + { + /* + * We are using single vlib_buffer_t for indirect descriptor(s) + * chain. Single descriptor is 16 bytes and vlib_buffer_t + * has 2048 bytes space. So maximum long chain can have 128 + * (=2048/16) indirect descriptors. + * It can easily support 65535 bytes of Jumbo frames with + * each data buffer size of 512 bytes minimum. + */ + u32 indirect_buffer = 0; + if (PREDICT_FALSE (vlib_buffer_alloc (vm, &indirect_buffer, 1) == 0)) + { + drop_inline = VIRTIO_TX_ERROR_INDIRECT_DESC_ALLOC_FAILED; + goto done; + } + + vlib_buffer_t *indirect_desc = vlib_get_buffer (vm, indirect_buffer); + indirect_desc->current_data = 0; + indirect_desc->flags |= VLIB_BUFFER_NEXT_PRESENT; + indirect_desc->next_buffer = bi; + bi = indirect_buffer; + + vring_packed_desc_t *id = + (vring_packed_desc_t *) vlib_buffer_get_current (indirect_desc); + u32 count = 1; + if (is_pci) + { + d->addr = vlib_physmem_get_pa (vm, id); + id->addr = vlib_buffer_get_current_pa (vm, b) - hdr_sz; + + /* + * If VIRTIO_F_ANY_LAYOUT is not negotiated, then virtio_net_hdr + * should be presented in separate descriptor and data will start + * from next descriptor. + */ + if (is_any_layout) + id->len = b->current_length + hdr_sz; + else + { + id->len = hdr_sz; + id->flags = 0; + id->id = 0; + count++; + id++; + id->addr = vlib_buffer_get_current_pa (vm, b); + id->len = b->current_length; + } + while (b->flags & VLIB_BUFFER_NEXT_PRESENT) + { + id->flags = 0; + id->id = 0; + count++; + id++; + b = vlib_get_buffer (vm, b->next_buffer); + id->addr = vlib_buffer_get_current_pa (vm, b); + id->len = b->current_length; + } + } + id->flags = 0; + id->id = 0; + d->len = count * sizeof (vring_packed_desc_t); + flags = VRING_DESC_F_INDIRECT; + } + else + { + ASSERT (0); + } + if (vring->avail_wrap_counter) + { + flags |= VRING_DESC_F_AVAIL; + flags &= ~VRING_DESC_F_USED; + } + else + { + flags &= ~VRING_DESC_F_AVAIL; + flags |= VRING_DESC_F_USED; + } + + d->id = next; + d->flags = flags; + vring->buffers[next] = bi; + n_added++; + +done: + if (drop_inline != ~0) + virtio_interface_drop_inline (vm, node->node_index, &bi, 1, drop_inline); + + return n_added; +} + +static_always_inline uword +virtio_interface_tx_packed_gso_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, + virtio_if_t * vif, + virtio_if_type_t type, + virtio_vring_t * vring, + u32 * buffers, u16 n_left, + const int do_gso, + const int csum_offload) +{ + int is_pci = (type == VIRTIO_IF_TYPE_PCI); + int is_tun = (type == VIRTIO_IF_TYPE_TUN); + int is_indirect = + ((vif->features & VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC)) != 0); + int is_any_layout = + ((vif->features & VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT)) != 0); + const int hdr_sz = vif->virtio_net_hdr_sz; + u16 sz = vring->size; + u16 used, next, n_buffers = 0, n_buffers_left = 0; + u16 n_vectors = n_left; + + + used = vring->desc_in_use; + next = vring->desc_next; + + if (vif->packet_buffering) + { + n_buffers = n_buffers_left = virtio_vring_n_buffers (vring->buffering); + + while (n_buffers_left && used < sz) + { + u16 n_added = 0; + + u32 bi = virtio_vring_buffering_read_from_front (vring->buffering); + if (bi == ~0) + break; + n_added = add_buffer_to_slot_packed (vm, node, + vring, bi, next, + hdr_sz, do_gso, csum_offload, + is_pci, is_tun, is_indirect, + is_any_layout); + n_buffers_left--; + if (PREDICT_FALSE (n_added == 0)) + continue; + + used++; + next++; + if (next >= sz) + { + next = 0; + vring->avail_wrap_counter ^= 1; + } + } + } + + while (n_left && used < sz) + { + u16 n_added = 0; + + n_added = add_buffer_to_slot_packed (vm, node, + vring, buffers[0], next, + hdr_sz, do_gso, csum_offload, + is_pci, is_tun, is_indirect, + is_any_layout); + buffers++; + n_left--; + if (PREDICT_FALSE (n_added == 0)) + continue; + + used++; + next++; + if (next >= sz) + { + next = 0; + vring->avail_wrap_counter ^= 1; + } + } + + if (n_left != n_vectors || n_buffers != n_buffers_left) + { + CLIB_MEMORY_STORE_BARRIER (); + vring->desc_next = next; + vring->desc_in_use = used; + CLIB_MEMORY_BARRIER (); + if (vring->device_event->flags != VRING_EVENT_F_DISABLE) + virtio_kick (vm, vring, vif); + } + + return n_left; +} + static_always_inline void virtio_find_free_desc (virtio_vring_t * vring, u16 size, u16 mask, u16 req, u16 next, u32 * first_free_desc_index, @@ -541,11 +827,13 @@ virtio_find_free_desc (virtio_vring_t * vring, u16 size, u16 mask, } static_always_inline u16 -virtio_interface_tx_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, - virtio_if_t * vif, - virtio_if_type_t type, virtio_vring_t * vring, - u32 * buffers, u16 n_left, int do_gso, - int csum_offload) +virtio_interface_tx_split_gso_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, + virtio_if_t * vif, + virtio_if_type_t type, + virtio_vring_t * vring, u32 * buffers, + u16 n_left, int do_gso, + int csum_offload) { u16 used, next, avail, n_buffers = 0, n_buffers_left = 0; int is_pci = (type == VIRTIO_IF_TYPE_PCI); @@ -650,26 +938,46 @@ virtio_interface_tx_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, } static_always_inline u16 +virtio_interface_tx_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, + virtio_if_t * vif, + virtio_if_type_t type, virtio_vring_t * vring, + u32 * buffers, u16 n_left, int packed, + int do_gso, int csum_offload) +{ + if (packed) + return virtio_interface_tx_packed_gso_inline (vm, node, vif, type, vring, + buffers, n_left, + 1 /* do_gso */ , + 1 /* checksum offload */ ); + else + return virtio_interface_tx_split_gso_inline (vm, node, vif, type, vring, + buffers, n_left, + 1 /* do_gso */ , + 1 /* checksum offload */ ); +} + +static_always_inline u16 virtio_interface_tx_inline (vlib_main_t * vm, vlib_node_runtime_t * node, virtio_if_t * vif, virtio_vring_t * vring, virtio_if_type_t type, - u32 * buffers, u16 n_left) + u32 * buffers, u16 n_left, int packed) { vnet_main_t *vnm = vnet_get_main (); vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vif->hw_if_index); if (hw->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO) return virtio_interface_tx_gso_inline (vm, node, vif, type, vring, - buffers, n_left, 1 /* do_gso */ , + buffers, n_left, packed, + 1 /* do_gso */ , 1 /* checksum offload */ ); else if (hw->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD) return virtio_interface_tx_gso_inline (vm, node, vif, type, vring, - buffers, n_left, + buffers, n_left, packed, 0 /* no do_gso */ , 1 /* checksum offload */ ); else return virtio_interface_tx_gso_inline (vm, node, vif, type, vring, - buffers, n_left, + buffers, n_left, packed, 0 /* no do_gso */ , 0 /* no checksum offload */ ); } @@ -686,6 +994,7 @@ VNET_DEVICE_CLASS_TX_FN (virtio_device_class) (vlib_main_t * vm, u16 n_left = frame->n_vectors; u32 *buffers = vlib_frame_vector_args (frame); u32 to[GRO_TO_VECTOR_SIZE (n_left)]; + int packed = vif->is_packed; clib_spinlock_lock_if_init (&vring->lockp); @@ -703,23 +1012,23 @@ VNET_DEVICE_CLASS_TX_FN (virtio_device_class) (vlib_main_t * vm, retry: /* free consumed buffers */ - virtio_free_used_device_desc (vm, vring, node->node_index); + virtio_free_used_device_desc (vm, vring, node->node_index, packed); if (vif->type == VIRTIO_IF_TYPE_TAP) n_left = virtio_interface_tx_inline (vm, node, vif, vring, VIRTIO_IF_TYPE_TAP, &buffers[frame->n_vectors - n_left], - n_left); + n_left, packed); else if (vif->type == VIRTIO_IF_TYPE_PCI) n_left = virtio_interface_tx_inline (vm, node, vif, vring, VIRTIO_IF_TYPE_PCI, &buffers[frame->n_vectors - n_left], - n_left); + n_left, packed); else if (vif->type == VIRTIO_IF_TYPE_TUN) n_left = virtio_interface_tx_inline (vm, node, vif, vring, VIRTIO_IF_TYPE_TUN, &buffers[frame->n_vectors - n_left], - n_left); + n_left, packed); else ASSERT (0); diff --git a/src/vnet/devices/virtio/node.c b/src/vnet/devices/virtio/node.c index 91e788e4d28..a45b52c2eab 100644 --- a/src/vnet/devices/virtio/node.c +++ b/src/vnet/devices/virtio/node.c @@ -81,9 +81,9 @@ format_virtio_input_trace (u8 * s, va_list * args) } static_always_inline void -virtio_refill_vring (vlib_main_t * vm, virtio_if_t * vif, - virtio_if_type_t type, virtio_vring_t * vring, - const int hdr_sz, u32 node_index) +virtio_refill_vring_split (vlib_main_t * vm, virtio_if_t * vif, + virtio_if_type_t type, virtio_vring_t * vring, + const int hdr_sz, u32 node_index) { u16 used, next, avail, n_slots, n_refill; u16 sz = vring->size; @@ -149,6 +149,88 @@ more: } static_always_inline void +virtio_refill_vring_packed (vlib_main_t * vm, virtio_if_t * vif, + virtio_if_type_t type, virtio_vring_t * vring, + const int hdr_sz, u32 node_index) +{ + u16 used, next, n_slots, n_refill, flags = 0, first_desc_flags; + u16 sz = vring->size; + +more: + used = vring->desc_in_use; + + if (sz == used) + return; + + /* deliver free buffers in chunks of 64 */ + n_refill = clib_min (sz - used, 64); + + next = vring->desc_next; + first_desc_flags = vring->packed_desc[next].flags; + n_slots = + vlib_buffer_alloc_to_ring_from_pool (vm, vring->buffers, next, + sz, n_refill, + vring->buffer_pool_index); + + if (PREDICT_FALSE (n_slots != n_refill)) + { + vlib_error_count (vm, node_index, + VIRTIO_INPUT_ERROR_BUFFER_ALLOC, n_refill - n_slots); + if (n_slots == 0) + return; + } + + while (n_slots) + { + vring_packed_desc_t *d = &vring->packed_desc[next]; + vlib_buffer_t *b = vlib_get_buffer (vm, vring->buffers[next]); + /* + * current_data may not be initialized with 0 and may contain + * previous offset. Here we want to make sure, it should be 0 + * initialized. + */ + b->current_data = -hdr_sz; + memset (vlib_buffer_get_current (b), 0, hdr_sz); + d->addr = + ((type == VIRTIO_IF_TYPE_PCI) ? vlib_buffer_get_current_pa (vm, + b) : + pointer_to_uword (vlib_buffer_get_current (b))); + d->len = vlib_buffer_get_default_data_size (vm) + hdr_sz; + + if (vring->avail_wrap_counter) + flags = (VRING_DESC_F_AVAIL | VRING_DESC_F_WRITE); + else + flags = (VRING_DESC_F_USED | VRING_DESC_F_WRITE); + + d->id = next; + if (vring->desc_next == next) + first_desc_flags = flags; + else + d->flags = flags; + + next++; + if (next >= sz) + { + next = 0; + vring->avail_wrap_counter ^= 1; + } + n_slots--; + used++; + } + CLIB_MEMORY_STORE_BARRIER (); + vring->packed_desc[vring->desc_next].flags = first_desc_flags; + vring->desc_next = next; + vring->desc_in_use = used; + CLIB_MEMORY_BARRIER (); + if (vring->device_event->flags != VRING_EVENT_F_DISABLE) + { + virtio_kick (vm, vring, vif); + } + + goto more; +} + +static_always_inline void virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr, u8 * l4_proto, u8 * l4_hdr_sz, virtio_if_type_t type) { @@ -258,11 +340,51 @@ fill_gso_buffer_flags (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr, } } +static_always_inline u16 +virtio_n_left_to_process (virtio_vring_t * vring, const int packed) +{ + if (packed) + return vring->desc_in_use; + else + return vring->used->idx - vring->last_used_idx; +} + +static_always_inline u16 +virtio_get_slot_id (virtio_vring_t * vring, const int packed, u16 last, + u16 mask) +{ + if (packed) + return vring->packed_desc[last].id; + else + return vring->used->ring[last & mask].id; +} + +static_always_inline u16 +virtio_get_len (virtio_vring_t * vring, const int packed, const int hdr_sz, + u16 last, u16 mask) +{ + if (packed) + return vring->packed_desc[last].len - hdr_sz; + else + return vring->used->ring[last & mask].len - hdr_sz; +} + +#define increment_last(last, packed, vring) \ + do { \ + last++; \ + if (packed && last >= vring->size) \ + { \ + last = 0; \ + vring->used_wrap_counter ^= 1; \ + } \ + } while (0) + static_always_inline uword virtio_device_input_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame, virtio_if_t * vif, virtio_vring_t * vring, virtio_if_type_t type, - int gso_enabled, int checksum_offload_enabled) + int gso_enabled, int checksum_offload_enabled, + int packed) { vnet_main_t *vnm = vnet_get_main (); u32 thread_index = vm->thread_index; @@ -274,7 +396,7 @@ virtio_device_input_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, u32 n_rx_bytes = 0; u16 mask = vring->size - 1; u16 last = vring->last_used_idx; - u16 n_left = vring->used->idx - last; + u16 n_left = virtio_n_left_to_process (vring, packed); vlib_buffer_t bt; if (n_left == 0) @@ -303,12 +425,24 @@ virtio_device_input_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, while (n_left && n_left_to_next) { + if (packed) + { + vring_packed_desc_t *d = &vring->packed_desc[last]; + u16 flags = d->flags; + if ((flags & VRING_DESC_F_AVAIL) != + (vring->used_wrap_counter << 7) + || (flags & VRING_DESC_F_USED) != + (vring->used_wrap_counter << 15)) + { + n_left = 0; + break; + } + } u8 l4_proto = 0, l4_hdr_sz = 0; u16 num_buffers = 1; - vring_used_elem_t *e = &vring->used->ring[last & mask]; virtio_net_hdr_v1_t *hdr; - u16 slot = e->id; - u16 len = e->len - hdr_sz; + u16 slot = virtio_get_slot_id (vring, packed, last, mask); + u16 len = virtio_get_len (vring, packed, hdr_sz, last, mask); u32 bi0 = vring->buffers[slot]; vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0); hdr = vlib_buffer_get_current (b0); @@ -336,20 +470,22 @@ virtio_device_input_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, b0->total_length_not_including_first_buffer = 0; while (num_buffers > 1) { - last++; - e = &vring->used->ring[last & mask]; - u32 cbi = vring->buffers[e->id]; + increment_last (last, packed, vring); + u16 cslot = virtio_get_slot_id (vring, packed, last, mask); + u16 clen = + virtio_get_len (vring, packed, hdr_sz, last, mask); + u32 cbi = vring->buffers[cslot]; cb = vlib_get_buffer (vm, cbi); /* current buffer */ - cb->current_length = e->len; + cb->current_length = clen; /* previous buffer */ pb->next_buffer = cbi; pb->flags |= VLIB_BUFFER_NEXT_PRESENT; /* first buffer */ - b0->total_length_not_including_first_buffer += e->len; + b0->total_length_not_including_first_buffer += clen; pb = cb; vring->desc_in_use--; @@ -406,7 +542,7 @@ virtio_device_input_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, to_next += 1; n_left_to_next--; n_left--; - last++; + increment_last (last, packed, vring); /* only tun interfaces may have different next index */ if (type == VIRTIO_IF_TYPE_TUN) @@ -451,21 +587,49 @@ virtio_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, clib_spinlock_unlock_if_init (&txq_vring->lockp); } - if ((vring->used->flags & VRING_USED_F_NO_NOTIFY) == 0 && - vring->last_kick_avail_idx != vring->avail->idx) - virtio_kick (vm, vring, vif); + if (vif->is_packed) + { + if (vring->device_event->flags != VRING_EVENT_F_DISABLE) + virtio_kick (vm, vring, vif); + + if (vif->gso_enabled) + rv = + virtio_device_input_gso_inline (vm, node, frame, vif, vring, type, + 1, 1, 1); + else if (vif->csum_offload_enabled) + rv = + virtio_device_input_gso_inline (vm, node, frame, vif, vring, type, + 0, 1, 1); + else + rv = + virtio_device_input_gso_inline (vm, node, frame, vif, vring, type, + 0, 0, 1); - if (vif->gso_enabled) - rv = virtio_device_input_gso_inline (vm, node, frame, vif, vring, type, - 1, 1); - else if (vif->csum_offload_enabled) - rv = virtio_device_input_gso_inline (vm, node, frame, vif, vring, type, - 0, 1); + virtio_refill_vring_packed (vm, vif, type, vring, hdr_sz, + node->node_index); + } else - rv = virtio_device_input_gso_inline (vm, node, frame, vif, vring, type, - 0, 0); + { + if ((vring->used->flags & VRING_USED_F_NO_NOTIFY) == 0 && + vring->last_kick_avail_idx != vring->avail->idx) + virtio_kick (vm, vring, vif); + + if (vif->gso_enabled) + rv = + virtio_device_input_gso_inline (vm, node, frame, vif, vring, type, + 1, 1, 0); + else if (vif->csum_offload_enabled) + rv = + virtio_device_input_gso_inline (vm, node, frame, vif, vring, type, + 0, 1, 0); + else + rv = + virtio_device_input_gso_inline (vm, node, frame, vif, vring, type, + 0, 0, 0); - virtio_refill_vring (vm, vif, type, vring, hdr_sz, node->node_index); + virtio_refill_vring_split (vm, vif, type, vring, hdr_sz, + node->node_index); + } return rv; } diff --git a/src/vnet/devices/virtio/pci.c b/src/vnet/devices/virtio/pci.c index 177f764c08a..785ca4839c5 100644 --- a/src/vnet/devices/virtio/pci.c +++ b/src/vnet/devices/virtio/pci.c @@ -195,8 +195,148 @@ device_status (vlib_main_t * vm, virtio_if_t * vif) } static int -virtio_pci_send_ctrl_msg (vlib_main_t * vm, virtio_if_t * vif, - virtio_ctrl_msg_t * data, u32 len) +virtio_pci_send_ctrl_msg_packed (vlib_main_t * vm, virtio_if_t * vif, + virtio_ctrl_msg_t * data, u32 len) +{ + virtio_vring_t *vring = vif->cxq_vring; + virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR; + virtio_ctrl_msg_t result; + u32 buffer_index; + vlib_buffer_t *b; + u16 used, next; + u16 sz = vring->size; + u16 flags = 0, first_desc_flags = 0; + + used = vring->desc_in_use; + next = vring->desc_next; + vring_packed_desc_t *d = &vring->packed_desc[next]; + + if (vlib_buffer_alloc (vm, &buffer_index, 1)) + b = vlib_get_buffer (vm, buffer_index); + else + return VIRTIO_NET_ERR; + /* + * current_data may not be initialized with 0 and may contain + * previous offset. + */ + b->current_data = 0; + clib_memcpy (vlib_buffer_get_current (b), data, sizeof (virtio_ctrl_msg_t)); + + first_desc_flags = VRING_DESC_F_NEXT; + if (vring->avail_wrap_counter) + { + first_desc_flags |= VRING_DESC_F_AVAIL; + first_desc_flags &= ~VRING_DESC_F_USED; + } + else + { + first_desc_flags &= ~VRING_DESC_F_AVAIL; + first_desc_flags |= VRING_DESC_F_USED; + } + d->addr = vlib_buffer_get_current_pa (vm, b); + d->len = sizeof (virtio_net_ctrl_hdr_t); + d->id = next; + + next++; + if (next >= sz) + { + next = 0; + vring->avail_wrap_counter ^= 1; + } + used++; + + d = &vring->packed_desc[next]; + flags = VRING_DESC_F_NEXT; + if (vring->avail_wrap_counter) + { + flags |= VRING_DESC_F_AVAIL; + flags &= ~VRING_DESC_F_USED; + } + else + { + flags &= ~VRING_DESC_F_AVAIL; + flags |= VRING_DESC_F_USED; + } + d->addr = vlib_buffer_get_current_pa (vm, b) + + STRUCT_OFFSET_OF (virtio_ctrl_msg_t, data); + d->len = len; + d->id = next; + d->flags = flags; + + next++; + if (next >= sz) + { + next = 0; + vring->avail_wrap_counter ^= 1; + } + used++; + + d = &vring->packed_desc[next]; + flags = VRING_DESC_F_WRITE; + if (vring->avail_wrap_counter) + { + flags |= VRING_DESC_F_AVAIL; + flags &= ~VRING_DESC_F_USED; + } + else + { + flags &= ~VRING_DESC_F_AVAIL; + flags |= VRING_DESC_F_USED; + } + d->addr = vlib_buffer_get_current_pa (vm, b) + + STRUCT_OFFSET_OF (virtio_ctrl_msg_t, status); + d->len = sizeof (data->status); + d->id = next; + d->flags = flags; + + next++; + if (next >= sz) + { + next = 0; + vring->avail_wrap_counter ^= 1; + } + used++; + + CLIB_MEMORY_STORE_BARRIER (); + vring->packed_desc[vring->desc_next].flags = first_desc_flags; + vring->desc_next = next; + vring->desc_in_use = used; + CLIB_MEMORY_BARRIER (); + if (vring->device_event->flags != VRING_EVENT_F_DISABLE) + { + virtio_kick (vm, vring, vif); + } + + u16 last = vring->last_used_idx; + d = &vring->packed_desc[last]; + do + { + flags = d->flags; + } + while ((flags & VRING_DESC_F_AVAIL) != (vring->used_wrap_counter << 7) + || (flags & VRING_DESC_F_USED) != (vring->used_wrap_counter << 15)); + + last += 3; + if (last >= vring->size) + { + last = last - vring->size; + vring->used_wrap_counter ^= 1; + } + vring->desc_in_use -= 3; + vring->last_used_idx = last; + + CLIB_MEMORY_BARRIER (); + clib_memcpy (&result, vlib_buffer_get_current (b), + sizeof (virtio_ctrl_msg_t)); + virtio_log_debug (vif, "ctrl-queue: status %u", result.status); + status = result.status; + vlib_buffer_free (vm, &buffer_index, 1); + return status; +} + +static int +virtio_pci_send_ctrl_msg_split (vlib_main_t * vm, virtio_if_t * vif, + virtio_ctrl_msg_t * data, u32 len) { virtio_vring_t *vring = vif->cxq_vring; virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR; @@ -290,6 +430,16 @@ virtio_pci_send_ctrl_msg (vlib_main_t * vm, virtio_if_t * vif, } static int +virtio_pci_send_ctrl_msg (vlib_main_t * vm, virtio_if_t * vif, + virtio_ctrl_msg_t * data, u32 len) +{ + if (vif->is_packed) + return virtio_pci_send_ctrl_msg_packed (vm, vif, data, len); + else + return virtio_pci_send_ctrl_msg_split (vm, vif, data, len); +} + +static int virtio_pci_disable_offload (vlib_main_t * vm, virtio_if_t * vif) { virtio_ctrl_msg_t offload_hdr; @@ -440,8 +590,73 @@ virtio_pci_queue_size_valid (u16 qsz) } clib_error_t * -virtio_pci_control_vring_init (vlib_main_t * vm, virtio_if_t * vif, - u16 queue_num) +virtio_pci_control_vring_packed_init (vlib_main_t * vm, virtio_if_t * vif, + u16 queue_num) +{ + clib_error_t *error = 0; + u16 queue_size = 0; + virtio_vring_t *vring; + u32 i = 0; + void *ptr = NULL; + + queue_size = vif->virtio_pci_func->get_queue_size (vm, vif, queue_num); + + if (queue_size > 32768) + return clib_error_return (0, "ring size must be 32768 or lower"); + + if (queue_size == 0) + queue_size = 256; + + vec_validate_aligned (vif->cxq_vring, 0, CLIB_CACHE_LINE_BYTES); + vring = vec_elt_at_index (vif->cxq_vring, 0); + + i = + (((queue_size * sizeof (vring_packed_desc_t)) + + sizeof (vring_desc_event_t) + VIRTIO_PCI_VRING_ALIGN - + 1) & ~(VIRTIO_PCI_VRING_ALIGN - 1)) + sizeof (vring_desc_event_t); + + ptr = + vlib_physmem_alloc_aligned_on_numa (vm, i, VIRTIO_PCI_VRING_ALIGN, + vif->numa_node); + if (!ptr) + return vlib_physmem_last_error (vm); + clib_memset (ptr, 0, i); + + vring->packed_desc = ptr; + + vring->driver_event = ptr + (queue_size * sizeof (vring_packed_desc_t)); + vring->driver_event->off_wrap = 0; + vring->driver_event->flags = VRING_EVENT_F_DISABLE; + + vring->device_event = + ptr + + (((queue_size * sizeof (vring_packed_desc_t)) + + sizeof (vring_desc_event_t) + VIRTIO_PCI_VRING_ALIGN - + 1) & ~(VIRTIO_PCI_VRING_ALIGN - 1)); + vring->device_event->off_wrap = 0; + vring->device_event->flags = 0; + + vring->queue_id = queue_num; + vring->size = queue_size; + vring->avail_wrap_counter = 1; + vring->used_wrap_counter = 1; + + ASSERT (vring->buffers == 0); + + virtio_log_debug (vif, "control-queue: number %u, size %u", queue_num, + queue_size); + vif->virtio_pci_func->setup_queue (vm, vif, queue_num, (void *) vring); + vring->queue_notify_offset = + vif->notify_off_multiplier * + vif->virtio_pci_func->get_queue_notify_off (vm, vif, queue_num); + virtio_log_debug (vif, "queue-notify-offset: number %u, offset %u", + queue_num, vring->queue_notify_offset); + return error; +} + +clib_error_t * +virtio_pci_control_vring_split_init (vlib_main_t * vm, virtio_if_t * vif, + u16 queue_num) { clib_error_t *error = 0; u16 queue_size = 0; @@ -496,7 +711,18 @@ virtio_pci_control_vring_init (vlib_main_t * vm, virtio_if_t * vif, } clib_error_t * -virtio_pci_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 queue_num) +virtio_pci_control_vring_init (vlib_main_t * vm, virtio_if_t * vif, + u16 queue_num) +{ + if (vif->is_packed) + return virtio_pci_control_vring_packed_init (vm, vif, queue_num); + else + return virtio_pci_control_vring_split_init (vm, vif, queue_num); +} + +clib_error_t * +virtio_pci_vring_split_init (vlib_main_t * vm, virtio_if_t * vif, + u16 queue_num) { vlib_thread_main_t *vtm = vlib_get_thread_main (); clib_error_t *error = 0; @@ -574,6 +800,106 @@ virtio_pci_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 queue_num) return error; } +clib_error_t * +virtio_pci_vring_packed_init (vlib_main_t * vm, virtio_if_t * vif, + u16 queue_num) +{ + vlib_thread_main_t *vtm = vlib_get_thread_main (); + clib_error_t *error = 0; + u16 queue_size = 0; + virtio_vring_t *vring; + u32 i = 0; + void *ptr = NULL; + + queue_size = vif->virtio_pci_func->get_queue_size (vm, vif, queue_num); + + if (queue_size > 32768) + return clib_error_return (0, "ring size must be 32768 or lower"); + + if (queue_size == 0) + queue_size = 256; + + if (queue_num % 2) + { + vec_validate_aligned (vif->txq_vrings, TX_QUEUE_ACCESS (queue_num), + CLIB_CACHE_LINE_BYTES); + vring = vec_elt_at_index (vif->txq_vrings, TX_QUEUE_ACCESS (queue_num)); + if (vif->max_queue_pairs < vtm->n_vlib_mains) + clib_spinlock_init (&vring->lockp); + } + else + { + vec_validate_aligned (vif->rxq_vrings, RX_QUEUE_ACCESS (queue_num), + CLIB_CACHE_LINE_BYTES); + vring = vec_elt_at_index (vif->rxq_vrings, RX_QUEUE_ACCESS (queue_num)); + } + + i = + (((queue_size * sizeof (vring_packed_desc_t)) + + sizeof (vring_desc_event_t) + VIRTIO_PCI_VRING_ALIGN - + 1) & ~(VIRTIO_PCI_VRING_ALIGN - 1)) + sizeof (vring_desc_event_t); + + ptr = + vlib_physmem_alloc_aligned_on_numa (vm, i, VIRTIO_PCI_VRING_ALIGN, + vif->numa_node); + if (!ptr) + return vlib_physmem_last_error (vm); + + clib_memset (ptr, 0, i); + vring->packed_desc = ptr; + + vring->driver_event = ptr + (queue_size * sizeof (vring_packed_desc_t)); + vring->driver_event->off_wrap = 0; + vring->driver_event->flags = VRING_EVENT_F_DISABLE; + + vring->device_event = + ptr + + (((queue_size * sizeof (vring_packed_desc_t)) + + sizeof (vring_desc_event_t) + VIRTIO_PCI_VRING_ALIGN - + 1) & ~(VIRTIO_PCI_VRING_ALIGN - 1)); + vring->device_event->off_wrap = 0; + vring->device_event->flags = 0; + + vring->queue_id = queue_num; + + vring->avail_wrap_counter = 1; + vring->used_wrap_counter = 1; + + ASSERT (vring->buffers == 0); + vec_validate_aligned (vring->buffers, queue_size, CLIB_CACHE_LINE_BYTES); + if (queue_num % 2) + { + virtio_log_debug (vif, "tx-queue: number %u, size %u", queue_num, + queue_size); + clib_memset_u32 (vring->buffers, ~0, queue_size); + } + else + { + virtio_log_debug (vif, "rx-queue: number %u, size %u", queue_num, + queue_size); + } + vring->size = queue_size; + if (vif->virtio_pci_func->setup_queue (vm, vif, queue_num, (void *) vring)) + return clib_error_return (0, "error in queue address setup"); + + vring->queue_notify_offset = + vif->notify_off_multiplier * + vif->virtio_pci_func->get_queue_notify_off (vm, vif, queue_num); + virtio_log_debug (vif, "queue-notify-offset: number %u, offset %u", + queue_num, vring->queue_notify_offset); + + return error; +} + +clib_error_t * +virtio_pci_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 queue_num) +{ + if (vif->is_packed) + return virtio_pci_vring_packed_init (vm, vif, queue_num); + else + return virtio_pci_vring_split_init (vm, vif, queue_num); +} + static void virtio_negotiate_features (vlib_main_t * vm, virtio_if_t * vif, u64 req_features) @@ -605,6 +931,13 @@ virtio_negotiate_features (vlib_main_t * vm, virtio_if_t * vif, if (vif->is_modern) supported_features |= VIRTIO_FEATURE (VIRTIO_F_VERSION_1); + if (vif->is_packed) + { + supported_features |= + (VIRTIO_FEATURE (VIRTIO_F_RING_PACKED) | + VIRTIO_FEATURE (VIRTIO_F_IN_ORDER)); + } + if (req_features == 0) { req_features = supported_features; @@ -621,6 +954,9 @@ virtio_negotiate_features (vlib_main_t * vm, virtio_if_t * vif, vif->features &= ~VIRTIO_FEATURE (VIRTIO_NET_F_MTU); } + if ((vif->features & (VIRTIO_FEATURE (VIRTIO_F_RING_PACKED))) == 0) + vif->is_packed = 0; + vif->virtio_pci_func->set_driver_features (vm, vif, vif->features); vif->features = vif->virtio_pci_func->get_driver_features (vm, vif); } @@ -1034,6 +1370,8 @@ virtio_pci_create_if (vlib_main_t * vm, virtio_pci_create_if_args_t * args) vif->dev_instance = vif - vim->interfaces; vif->per_interface_next_index = ~0; vif->pci_addr.as_u32 = args->addr; + if (args->virtio_flags & VIRTIO_FLAG_PACKED) + vif->is_packed = 1; if ((error = vlib_pci_device_open (vm, (vlib_pci_addr_t *) & vif->pci_addr, @@ -1260,7 +1598,7 @@ virtio_pci_delete_if (vlib_main_t * vm, virtio_if_t * vif) virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, i); if (vring->used) { - virtio_free_rx_buffers (vm, vring); + virtio_free_buffers (vm, vring); } vec_free (vring->buffers); vlib_physmem_free (vm, vring->desc); @@ -1271,9 +1609,11 @@ virtio_pci_delete_if (vlib_main_t * vm, virtio_if_t * vif) virtio_vring_t *vring = vec_elt_at_index (vif->txq_vrings, i); if (vring->used) { - virtio_free_used_desc (vm, vring); + virtio_free_buffers (vm, vring); } vec_free (vring->buffers); + gro_flow_table_free (vring->flow_table); + virtio_vring_buffering_free (vm, vring->buffering); clib_spinlock_free (&vring->lockp); vlib_physmem_free (vm, vring->desc); } diff --git a/src/vnet/devices/virtio/virtio.c b/src/vnet/devices/virtio/virtio.c index 925ad092b91..99f581a1f1b 100644 --- a/src/vnet/devices/virtio/virtio.c +++ b/src/vnet/devices/virtio/virtio.c @@ -134,7 +134,7 @@ virtio_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 idx, u16 sz) } inline void -virtio_free_rx_buffers (vlib_main_t * vm, virtio_vring_t * vring) +virtio_free_buffers (vlib_main_t * vm, virtio_vring_t * vring) { u16 used = vring->desc_in_use; u16 last = vring->last_used_idx; @@ -159,7 +159,7 @@ virtio_vring_free_rx (vlib_main_t * vm, virtio_if_t * vif, u32 idx) close (vring->call_fd); if (vring->used) { - virtio_free_rx_buffers (vm, vring); + virtio_free_buffers (vm, vring); clib_mem_free (vring->used); } if (vring->desc) @@ -170,32 +170,6 @@ virtio_vring_free_rx (vlib_main_t * vm, virtio_if_t * vif, u32 idx) return 0; } -inline void -virtio_free_used_desc (vlib_main_t * vm, virtio_vring_t * vring) -{ - u16 used = vring->desc_in_use; - u16 sz = vring->size; - u16 mask = sz - 1; - u16 last = vring->last_used_idx; - u16 n_left = vring->used->idx - last; - - if (n_left == 0) - return; - - while (n_left) - { - vring_used_elem_t *e = &vring->used->ring[last & mask]; - u16 slot = e->id; - - vlib_buffer_free (vm, &vring->buffers[slot], 1); - used--; - last++; - n_left--; - } - vring->desc_in_use = used; - vring->last_used_idx = last; -} - clib_error_t * virtio_vring_free_tx (vlib_main_t * vm, virtio_if_t * vif, u32 idx) { @@ -207,7 +181,7 @@ virtio_vring_free_tx (vlib_main_t * vm, virtio_if_t * vif, u32 idx) close (vring->call_fd); if (vring->used) { - virtio_free_used_desc (vm, vring); + virtio_free_buffers (vm, vring); clib_mem_free (vring->used); } if (vring->desc) @@ -408,10 +382,24 @@ virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type) " qsz %d, last_used_idx %d, desc_next %d, desc_in_use %d", vring->size, vring->last_used_idx, vring->desc_next, vring->desc_in_use); - vlib_cli_output (vm, - " avail.flags 0x%x avail.idx %d used.flags 0x%x used.idx %d", - vring->avail->flags, vring->avail->idx, - vring->used->flags, vring->used->idx); + if (vif->is_packed) + { + vlib_cli_output (vm, + " driver_event.flags 0x%x driver_event.off_wrap %d device_event.flags 0x%x device_event.off_wrap %d", + vring->driver_event->flags, + vring->driver_event->off_wrap, + vring->device_event->flags, + vring->device_event->off_wrap); + vlib_cli_output (vm, + " avail wrap counter %d, used wrap counter %d", + vring->avail_wrap_counter, + vring->used_wrap_counter); + } + else + vlib_cli_output (vm, + " avail.flags 0x%x avail.idx %d used.flags 0x%x used.idx %d", + vring->avail->flags, vring->avail->idx, + vring->used->flags, vring->used->idx); if (type & (VIRTIO_IF_TYPE_TAP | VIRTIO_IF_TYPE_TUN)) { vlib_cli_output (vm, " kickfd %d, callfd %d", vring->kick_fd, @@ -421,17 +409,29 @@ virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type) { vlib_cli_output (vm, "\n descriptor table:\n"); vlib_cli_output (vm, - " id addr len flags next user_addr\n"); + " id addr len flags next/id user_addr\n"); vlib_cli_output (vm, - " ===== ================== ===== ====== ===== ==================\n"); + " ===== ================== ===== ====== ======= ==================\n"); for (j = 0; j < vring->size; j++) { - vring_desc_t *desc = &vring->desc[j]; - vlib_cli_output (vm, - " %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n", - j, desc->addr, - desc->len, - desc->flags, desc->next, desc->addr); + if (vif->is_packed) + { + vring_packed_desc_t *desc = &vring->packed_desc[j]; + vlib_cli_output (vm, + " %-5d 0x%016lx %-5d 0x%04x %-8d 0x%016lx\n", + j, desc->addr, + desc->len, + desc->flags, desc->id, desc->addr); + } + else + { + vring_desc_t *desc = &vring->desc[j]; + vlib_cli_output (vm, + " %-5d 0x%016lx %-5d 0x%04x %-8d 0x%016lx\n", + j, desc->addr, + desc->len, + desc->flags, desc->next, desc->addr); + } } } } @@ -443,10 +443,24 @@ virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type) " qsz %d, last_used_idx %d, desc_next %d, desc_in_use %d", vring->size, vring->last_used_idx, vring->desc_next, vring->desc_in_use); - vlib_cli_output (vm, - " avail.flags 0x%x avail.idx %d used.flags 0x%x used.idx %d", - vring->avail->flags, vring->avail->idx, - vring->used->flags, vring->used->idx); + if (vif->is_packed) + { + vlib_cli_output (vm, + " driver_event.flags 0x%x driver_event.off_wrap %d device_event.flags 0x%x device_event.off_wrap %d", + vring->driver_event->flags, + vring->driver_event->off_wrap, + vring->device_event->flags, + vring->device_event->off_wrap); + vlib_cli_output (vm, + " avail wrap counter %d, used wrap counter %d", + vring->avail_wrap_counter, + vring->used_wrap_counter); + } + else + vlib_cli_output (vm, + " avail.flags 0x%x avail.idx %d used.flags 0x%x used.idx %d", + vring->avail->flags, vring->avail->idx, + vring->used->flags, vring->used->idx); if (type & (VIRTIO_IF_TYPE_TAP | VIRTIO_IF_TYPE_TUN)) { vlib_cli_output (vm, " kickfd %d, callfd %d", vring->kick_fd, @@ -466,17 +480,29 @@ virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type) { vlib_cli_output (vm, "\n descriptor table:\n"); vlib_cli_output (vm, - " id addr len flags next user_addr\n"); + " id addr len flags next/id user_addr\n"); vlib_cli_output (vm, - " ===== ================== ===== ====== ===== ==================\n"); + " ===== ================== ===== ====== ======== ==================\n"); for (j = 0; j < vring->size; j++) { - vring_desc_t *desc = &vring->desc[j]; - vlib_cli_output (vm, - " %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n", - j, desc->addr, - desc->len, - desc->flags, desc->next, desc->addr); + if (vif->is_packed) + { + vring_packed_desc_t *desc = &vring->packed_desc[j]; + vlib_cli_output (vm, + " %-5d 0x%016lx %-5d 0x%04x %-8d 0x%016lx\n", + j, desc->addr, + desc->len, + desc->flags, desc->id, desc->addr); + } + else + { + vring_desc_t *desc = &vring->desc[j]; + vlib_cli_output (vm, + " %-5d 0x%016lx %-5d 0x%04x %-8d 0x%016lx\n", + j, desc->addr, + desc->len, + desc->flags, desc->next, desc->addr); + } } } } @@ -489,10 +515,26 @@ virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type) " qsz %d, last_used_idx %d, desc_next %d, desc_in_use %d", vring->size, vring->last_used_idx, vring->desc_next, vring->desc_in_use); - vlib_cli_output (vm, - " avail.flags 0x%x avail.idx %d used.flags 0x%x used.idx %d", - vring->avail->flags, vring->avail->idx, - vring->used->flags, vring->used->idx); + if (vif->is_packed) + { + vlib_cli_output (vm, + " driver_event.flags 0x%x driver_event.off_wrap %d device_event.flags 0x%x device_event.off_wrap %d", + vring->driver_event->flags, + vring->driver_event->off_wrap, + vring->device_event->flags, + vring->device_event->off_wrap); + vlib_cli_output (vm, + " avail wrap counter %d, used wrap counter %d", + vring->avail_wrap_counter, + vring->used_wrap_counter); + } + else + { + vlib_cli_output (vm, + " avail.flags 0x%x avail.idx %d used.flags 0x%x used.idx %d", + vring->avail->flags, vring->avail->idx, + vring->used->flags, vring->used->idx); + } if (type & (VIRTIO_IF_TYPE_TAP | VIRTIO_IF_TYPE_TUN)) { vlib_cli_output (vm, " kickfd %d, callfd %d", vring->kick_fd, @@ -502,17 +544,29 @@ virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type) { vlib_cli_output (vm, "\n descriptor table:\n"); vlib_cli_output (vm, - " id addr len flags next user_addr\n"); + " id addr len flags next/id user_addr\n"); vlib_cli_output (vm, - " ===== ================== ===== ====== ===== ==================\n"); + " ===== ================== ===== ====== ======== ==================\n"); for (j = 0; j < vring->size; j++) { - vring_desc_t *desc = &vring->desc[j]; - vlib_cli_output (vm, - " %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n", - j, desc->addr, - desc->len, - desc->flags, desc->next, desc->addr); + if (vif->is_packed) + { + vring_packed_desc_t *desc = &vring->packed_desc[j]; + vlib_cli_output (vm, + " %-5d 0x%016lx %-5d 0x%04x %-8d 0x%016lx\n", + j, desc->addr, + desc->len, + desc->flags, desc->id, desc->addr); + } + else + { + vring_desc_t *desc = &vring->desc[j]; + vlib_cli_output (vm, + " %-5d 0x%016lx %-5d 0x%04x %-8d 0x%016lx\n", + j, desc->addr, + desc->len, + desc->flags, desc->next, desc->addr); + } } } } diff --git a/src/vnet/devices/virtio/virtio.h b/src/vnet/devices/virtio/virtio.h index fda72365db9..035dc9ca40d 100644 --- a/src/vnet/devices/virtio/virtio.h +++ b/src/vnet/devices/virtio/virtio.h @@ -64,29 +64,46 @@ typedef struct { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); clib_spinlock_t lockp; - vring_desc_t *desc; - vring_used_t *used; - vring_avail_t *avail; + union + { + struct + { + vring_desc_t *desc; + vring_used_t *used; + vring_avail_t *avail; + }; + struct + { + vring_packed_desc_t *packed_desc; + vring_desc_event_t *driver_event; + vring_desc_event_t *device_event; + }; + }; + u32 *buffers; + u16 size; + u16 queue_id; u16 desc_in_use; u16 desc_next; + u16 last_used_idx; + u16 last_kick_avail_idx; union { struct { int kick_fd; int call_fd; + u32 call_file_index; + }; + struct + { + u16 avail_wrap_counter; + u16 used_wrap_counter; + u16 queue_notify_offset; }; - u16 queue_notify_offset; }; - u8 buffer_pool_index; - u16 size; - u16 queue_id; #define VRING_TX_OUT_OF_ORDER 1 u16 flags; - u32 *buffers; - u16 last_used_idx; - u16 last_kick_avail_idx; - u32 call_file_index; + u8 buffer_pool_index; vnet_hw_if_rx_mode mode; virtio_vring_buffering_t *buffering; gro_flow_table_t *flow_table; @@ -190,6 +207,7 @@ typedef struct }; }; const virtio_pci_func_t *virtio_pci_func; + int is_packed; } virtio_if_t; typedef struct @@ -214,8 +232,7 @@ clib_error_t *virtio_vring_free_tx (vlib_main_t * vm, virtio_if_t * vif, u32 idx); void virtio_vring_set_numa_node (vlib_main_t * vm, virtio_if_t * vif, u32 idx); -extern void virtio_free_used_desc (vlib_main_t * vm, virtio_vring_t * vring); -extern void virtio_free_rx_buffers (vlib_main_t * vm, virtio_vring_t * vring); +extern void virtio_free_buffers (vlib_main_t * vm, virtio_vring_t * vring); extern void virtio_set_net_hdr_size (virtio_if_t * vif); extern void virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type); diff --git a/src/vnet/devices/virtio/virtio_pci_modern.c b/src/vnet/devices/virtio/virtio_pci_modern.c index 1934f98003d..8e090ffed3a 100644 --- a/src/vnet/devices/virtio/virtio_pci_modern.c +++ b/src/vnet/devices/virtio/virtio_pci_modern.c @@ -268,24 +268,39 @@ static u8 virtio_pci_modern_setup_queue (vlib_main_t * vm, virtio_if_t * vif, u16 queue_id, void *p) { - vring_t vr; + u64 desc, avail, used; u16 queue_size = 0; virtio_pci_modern_set_queue_select (vif, queue_id); queue_size = virtio_pci_modern_get_queue_size (vm, vif, queue_id); - vring_init (&vr, queue_size, p, VIRTIO_PCI_VRING_ALIGN); - u64 desc = vlib_physmem_get_pa (vm, vr.desc); + if (vif->is_packed) + { + virtio_vring_t *vring = (virtio_vring_t *) p; + + desc = vlib_physmem_get_pa (vm, vring->packed_desc); + avail = vlib_physmem_get_pa (vm, vring->driver_event); + used = vlib_physmem_get_pa (vm, vring->device_event); + } + else + { + vring_t vr; + + vring_init (&vr, queue_size, p, VIRTIO_PCI_VRING_ALIGN); + + desc = vlib_physmem_get_pa (vm, vr.desc); + avail = vlib_physmem_get_pa (vm, vr.avail); + used = vlib_physmem_get_pa (vm, vr.used); + } + virtio_pci_modern_set_queue_desc (vif, desc); if (desc != virtio_pci_modern_get_queue_desc (vif)) return 1; - u64 avail = vlib_physmem_get_pa (vm, vr.avail); virtio_pci_modern_set_queue_driver (vif, avail); if (avail != virtio_pci_modern_get_queue_driver (vif)) return 1; - u64 used = vlib_physmem_get_pa (vm, vr.used); virtio_pci_modern_set_queue_device (vif, used); if (used != virtio_pci_modern_get_queue_device (vif)) return 1; |