diff options
Diffstat (limited to 'src/vnet/devices/virtio')
| -rw-r--r-- | src/vnet/devices/virtio/device.c | 29 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/node.c | 18 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/pci.c | 50 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/pci.h | 99 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/vhost_std.h | 87 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/vhost_user.c | 38 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/vhost_user.h | 145 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/vhost_user_api.c | 8 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/vhost_user_inline.h | 12 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/vhost_user_input.c | 39 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/vhost_user_output.c | 49 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/virtio.c | 22 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/virtio.h | 49 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/virtio_pci_modern.c | 2 | ||||
| -rw-r--r-- | src/vnet/devices/virtio/virtio_std.h | 208 |
15 files changed, 472 insertions, 383 deletions
diff --git a/src/vnet/devices/virtio/device.c b/src/vnet/devices/virtio/device.c index 99157cd6166..56c0a98491e 100644 --- a/src/vnet/devices/virtio/device.c +++ b/src/vnet/devices/virtio/device.c @@ -138,7 +138,7 @@ virtio_free_used_device_desc (vlib_main_t * vm, virtio_vring_t * vring, while (n_left) { - struct vring_used_elem *e = &vring->used->ring[last & mask]; + vring_used_elem_t *e = &vring->used->ring[last & mask]; u16 slot, n_buffers; slot = n_buffers = e->id; @@ -147,7 +147,7 @@ virtio_free_used_device_desc (vlib_main_t * vm, virtio_vring_t * vring, n_left--; last++; n_buffers++; - struct vring_desc *d = &vring->desc[e->id]; + vring_desc_t *d = &vring->desc[e->id]; u16 next; while (d->flags & VRING_DESC_F_NEXT) { @@ -189,8 +189,7 @@ virtio_free_used_device_desc (vlib_main_t * vm, virtio_vring_t * vring, } static_always_inline void -set_checksum_offsets (vlib_buffer_t * b, struct virtio_net_hdr_v1 *hdr, - int is_l2) +set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr, int is_l2) { if (b->flags & VNET_BUFFER_F_IS_IP4) { @@ -237,7 +236,7 @@ set_checksum_offsets (vlib_buffer_t * b, struct virtio_net_hdr_v1 *hdr, } static_always_inline void -set_gso_offsets (vlib_buffer_t * b, struct virtio_net_hdr_v1 *hdr, int is_l2) +set_gso_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr, int is_l2) { if (b->flags & VNET_BUFFER_F_IS_IP4) { @@ -283,10 +282,10 @@ add_buffer_to_slot (vlib_main_t * vm, virtio_if_t * vif, { u16 n_added = 0; int hdr_sz = vif->virtio_net_hdr_sz; - struct vring_desc *d; + vring_desc_t *d; d = &vring->desc[next]; vlib_buffer_t *b = vlib_get_buffer (vm, bi); - struct virtio_net_hdr_v1 *hdr = vlib_buffer_get_current (b) - hdr_sz; + virtio_net_hdr_v1_t *hdr = vlib_buffer_get_current (b) - hdr_sz; int is_l2 = (type & (VIRTIO_IF_TYPE_TAP | VIRTIO_IF_TYPE_PCI)); clib_memset (hdr, 0, hdr_sz); @@ -348,8 +347,8 @@ add_buffer_to_slot (vlib_main_t * vm, virtio_if_t * vif, indirect_desc->next_buffer = bi; bi = indirect_buffer; - struct vring_desc *id = - (struct vring_desc *) vlib_buffer_get_current (indirect_desc); + vring_desc_t *id = + (vring_desc_t *) vlib_buffer_get_current (indirect_desc); u32 count = 1; if (type == VIRTIO_IF_TYPE_PCI) { @@ -405,7 +404,7 @@ add_buffer_to_slot (vlib_main_t * vm, virtio_if_t * vif, } id->flags = 0; id->next = 0; - d->len = count * sizeof (struct vring_desc); + d->len = count * sizeof (vring_desc_t); d->flags = VRING_DESC_F_INDIRECT; } else if (type == VIRTIO_IF_TYPE_PCI) @@ -518,7 +517,7 @@ virtio_interface_tx_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, clib_spinlock_lock_if_init (&vring->lockp); - if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0 && + if ((vring->used->flags & VRING_USED_F_NO_NOTIFY) == 0 && (vring->last_kick_avail_idx != vring->avail->idx)) virtio_kick (vm, vring, vif); @@ -620,7 +619,7 @@ retry: vring->avail->idx = avail; vring->desc_next = next; vring->desc_in_use = used; - if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0) + if ((vring->used->flags & VRING_USED_F_NO_NOTIFY) == 0) virtio_kick (vm, vring, vif); } @@ -724,7 +723,7 @@ virtio_interface_rx_mode_change (vnet_main_t * vnm, u32 hw_if_index, u32 qid, if (vif->type == VIRTIO_IF_TYPE_PCI && !(vif->support_int_mode)) { - vring->avail->flags |= VIRTIO_RING_FLAG_MASK_INT; + vring->avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; return clib_error_return (0, "interrupt mode is not supported"); } @@ -732,12 +731,12 @@ virtio_interface_rx_mode_change (vnet_main_t * vnm, u32 hw_if_index, u32 qid, { /* only enable packet coalesce in poll mode */ gro_flow_table_set_is_enable (vring->flow_table, 1); - vring->avail->flags |= VIRTIO_RING_FLAG_MASK_INT; + vring->avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; } else { gro_flow_table_set_is_enable (vring->flow_table, 0); - vring->avail->flags &= ~VIRTIO_RING_FLAG_MASK_INT; + vring->avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; } return 0; diff --git a/src/vnet/devices/virtio/node.c b/src/vnet/devices/virtio/node.c index a27131a6398..7fabe36401c 100644 --- a/src/vnet/devices/virtio/node.c +++ b/src/vnet/devices/virtio/node.c @@ -59,7 +59,7 @@ typedef struct u32 hw_if_index; u16 ring; u16 len; - struct virtio_net_hdr_v1 hdr; + virtio_net_hdr_v1_t hdr; } virtio_input_trace_t; static u8 * @@ -115,7 +115,7 @@ more: while (n_slots) { - struct vring_desc *d = &vring->desc[next];; + vring_desc_t *d = &vring->desc[next];; vlib_buffer_t *b = vlib_get_buffer (vm, vring->buffers[next]); /* * current_data may not be initialized with 0 and may contain @@ -141,7 +141,7 @@ more: vring->desc_next = next; vring->desc_in_use = used; - if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0) + if ((vring->used->flags & VRING_USED_F_NO_NOTIFY) == 0) { virtio_kick (vm, vring, vif); } @@ -149,7 +149,7 @@ more: } static_always_inline void -virtio_needs_csum (vlib_buffer_t * b0, struct virtio_net_hdr_v1 *hdr, +virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr, u8 * l4_proto, u8 * l4_hdr_sz, virtio_if_type_t type) { if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) @@ -239,7 +239,7 @@ virtio_needs_csum (vlib_buffer_t * b0, struct virtio_net_hdr_v1 *hdr, } static_always_inline void -fill_gso_buffer_flags (vlib_buffer_t * b0, struct virtio_net_hdr_v1 *hdr, +fill_gso_buffer_flags (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr, u8 l4_proto, u8 l4_hdr_sz) { if (hdr->gso_type == VIRTIO_NET_HDR_GSO_TCPV4) @@ -285,7 +285,7 @@ virtio_device_input_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, txq_vring->flow_table); } - if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0 && + if ((vring->used->flags & VRING_USED_F_NO_NOTIFY) == 0 && vring->last_kick_avail_idx != vring->avail->idx) virtio_kick (vm, vring, vif); @@ -306,14 +306,14 @@ virtio_device_input_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, { u8 l4_proto = 0, l4_hdr_sz = 0; u16 num_buffers = 1; - struct vring_used_elem *e = &vring->used->ring[last & mask]; - struct virtio_net_hdr_v1 *hdr; + vring_used_elem_t *e = &vring->used->ring[last & mask]; + virtio_net_hdr_v1_t *hdr; u16 slot = e->id; u16 len = e->len - hdr_sz; u32 bi0 = vring->buffers[slot]; vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0); hdr = vlib_buffer_get_current (b0); - if (hdr_sz == sizeof (struct virtio_net_hdr_v1)) + if (hdr_sz == sizeof (virtio_net_hdr_v1_t)) num_buffers = hdr->num_buffers; b0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID; diff --git a/src/vnet/devices/virtio/pci.c b/src/vnet/devices/virtio/pci.c index 6c6edbc3836..5ba9f36de81 100644 --- a/src/vnet/devices/virtio/pci.c +++ b/src/vnet/devices/virtio/pci.c @@ -194,20 +194,13 @@ device_status (vlib_main_t * vm, virtio_if_t * vif) } } -struct virtio_ctrl_msg -{ - struct virtio_net_ctrl_hdr ctrl; - virtio_net_ctrl_ack status; - u8 data[1024]; -}; - static int virtio_pci_send_ctrl_msg (vlib_main_t * vm, virtio_if_t * vif, - struct virtio_ctrl_msg *data, u32 len) + virtio_ctrl_msg_t * data, u32 len) { virtio_vring_t *vring = vif->cxq_vring; - virtio_net_ctrl_ack status = VIRTIO_NET_ERR; - struct virtio_ctrl_msg result; + virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR; + virtio_ctrl_msg_t result; u32 buffer_index; vlib_buffer_t *b; u16 used, next, avail; @@ -217,7 +210,7 @@ virtio_pci_send_ctrl_msg (vlib_main_t * vm, virtio_if_t * vif, used = vring->desc_in_use; next = vring->desc_next; avail = vring->avail->idx; - struct vring_desc *d = &vring->desc[next]; + vring_desc_t *d = &vring->desc[next]; if (vlib_buffer_alloc (vm, &buffer_index, 1)) b = vlib_get_buffer (vm, buffer_index); @@ -228,11 +221,10 @@ virtio_pci_send_ctrl_msg (vlib_main_t * vm, virtio_if_t * vif, * previous offset. */ b->current_data = 0; - clib_memcpy (vlib_buffer_get_current (b), data, - sizeof (struct virtio_ctrl_msg)); + clib_memcpy (vlib_buffer_get_current (b), data, sizeof (virtio_ctrl_msg_t)); d->flags = VRING_DESC_F_NEXT; d->addr = vlib_buffer_get_current_pa (vm, b); - d->len = sizeof (struct virtio_net_ctrl_hdr); + d->len = sizeof (virtio_net_ctrl_hdr_t); vring->avail->ring[avail & mask] = next; avail++; next = (next + 1) & mask; @@ -242,7 +234,7 @@ virtio_pci_send_ctrl_msg (vlib_main_t * vm, virtio_if_t * vif, d = &vring->desc[next]; d->flags = VRING_DESC_F_NEXT; d->addr = vlib_buffer_get_current_pa (vm, b) + - STRUCT_OFFSET_OF (struct virtio_ctrl_msg, data); + STRUCT_OFFSET_OF (virtio_ctrl_msg_t, data); d->len = len; next = (next + 1) & mask; d->next = next; @@ -251,7 +243,7 @@ virtio_pci_send_ctrl_msg (vlib_main_t * vm, virtio_if_t * vif, d = &vring->desc[next]; d->flags = VRING_DESC_F_WRITE; d->addr = vlib_buffer_get_current_pa (vm, b) + - STRUCT_OFFSET_OF (struct virtio_ctrl_msg, status); + STRUCT_OFFSET_OF (virtio_ctrl_msg_t, status); d->len = sizeof (data->status); next = (next + 1) & mask; used++; @@ -271,7 +263,7 @@ virtio_pci_send_ctrl_msg (vlib_main_t * vm, virtio_if_t * vif, while (n_left) { - struct vring_used_elem *e = &vring->used->ring[last & mask]; + vring_used_elem_t *e = &vring->used->ring[last & mask]; u16 slot = e->id; d = &vring->desc[slot]; @@ -290,7 +282,7 @@ virtio_pci_send_ctrl_msg (vlib_main_t * vm, virtio_if_t * vif, CLIB_MEMORY_BARRIER (); clib_memcpy (&result, vlib_buffer_get_current (b), - sizeof (struct virtio_ctrl_msg)); + sizeof (virtio_ctrl_msg_t)); virtio_log_debug (vif, "ctrl-queue: status %u", result.status); status = result.status; vlib_buffer_free (vm, &buffer_index, 1); @@ -300,8 +292,8 @@ virtio_pci_send_ctrl_msg (vlib_main_t * vm, virtio_if_t * vif, static int virtio_pci_disable_offload (vlib_main_t * vm, virtio_if_t * vif) { - struct virtio_ctrl_msg offload_hdr; - virtio_net_ctrl_ack status = VIRTIO_NET_ERR; + virtio_ctrl_msg_t offload_hdr; + virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR; offload_hdr.ctrl.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS; offload_hdr.ctrl.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET; @@ -320,8 +312,8 @@ virtio_pci_disable_offload (vlib_main_t * vm, virtio_if_t * vif) static int virtio_pci_enable_checksum_offload (vlib_main_t * vm, virtio_if_t * vif) { - struct virtio_ctrl_msg csum_offload_hdr; - virtio_net_ctrl_ack status = VIRTIO_NET_ERR; + virtio_ctrl_msg_t csum_offload_hdr; + virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR; csum_offload_hdr.ctrl.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS; csum_offload_hdr.ctrl.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET; @@ -341,8 +333,8 @@ virtio_pci_enable_checksum_offload (vlib_main_t * vm, virtio_if_t * vif) static int virtio_pci_enable_gso (vlib_main_t * vm, virtio_if_t * vif) { - struct virtio_ctrl_msg gso_hdr; - virtio_net_ctrl_ack status = VIRTIO_NET_ERR; + virtio_ctrl_msg_t gso_hdr; + virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR; gso_hdr.ctrl.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS; gso_hdr.ctrl.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET; @@ -424,8 +416,8 @@ static int virtio_pci_enable_multiqueue (vlib_main_t * vm, virtio_if_t * vif, u16 num_queues) { - struct virtio_ctrl_msg mq_hdr; - virtio_net_ctrl_ack status = VIRTIO_NET_ERR; + virtio_ctrl_msg_t mq_hdr; + virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR; mq_hdr.ctrl.class = VIRTIO_NET_CTRL_MQ; mq_hdr.ctrl.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET; @@ -454,7 +446,7 @@ virtio_pci_control_vring_init (vlib_main_t * vm, virtio_if_t * vif, clib_error_t *error = 0; u16 queue_size = 0; virtio_vring_t *vring; - struct vring vr; + vring_t vr; u32 i = 0; void *ptr = NULL; @@ -505,7 +497,7 @@ virtio_pci_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 queue_num) clib_error_t *error = 0; u16 queue_size = 0; virtio_vring_t *vring; - struct vring vr; + vring_t vr; u32 i = 0; void *ptr = NULL; @@ -666,7 +658,7 @@ clib_error_t * virtio_pci_read_caps (vlib_main_t * vm, virtio_if_t * vif, void **bar) { clib_error_t *error = 0; - struct virtio_pci_cap cap; + virtio_pci_cap_t cap; u8 pos, common_cfg = 0, notify = 0, dev_cfg = 0, isr = 0, pci_cfg = 0; vlib_pci_dev_handle_t h = vif->pci_dev_handle; diff --git a/src/vnet/devices/virtio/pci.h b/src/vnet/devices/virtio/pci.h index 47c74ab2ec6..ab5c6f15ec2 100644 --- a/src/vnet/devices/virtio/pci.h +++ b/src/vnet/devices/virtio/pci.h @@ -19,20 +19,20 @@ /* VirtIO ABI version, this must match exactly. */ #define VIRTIO_PCI_ABI_VERSION 0 +/* VirtIO device IDs. */ +#define VIRTIO_ID_NETWORK 0x01 + /* * Vector value used to disable MSI for queue. * define in include/linux/virtio_pci.h - * #define VIRTIO_MSI_NO_VECTOR 0xFFFF */ +#define VIRTIO_MSI_NO_VECTOR 0xFFFF /* The bit of the ISR which indicates a device has an interrupt. */ #define VIRTIO_PCI_ISR_INTR 0x1 /* The bit of the ISR which indicates a device configuration change. */ #define VIRTIO_PCI_ISR_CONFIG 0x2 -/* VirtIO device IDs. */ -#define VIRTIO_ID_NETWORK 0x01 - /* Status byte for guest to report progress. */ #define foreach_virtio_config_status_flags \ _ (VIRTIO_CONFIG_STATUS_RESET, 0x00) \ @@ -50,46 +50,19 @@ typedef enum #undef _ } virtio_config_status_flags_t; -#define foreach_virtio_net_feature_flags \ - _ (VIRTIO_NET_F_CSUM, 0) /* Host handles pkts w/ partial csum */ \ - _ (VIRTIO_NET_F_GUEST_CSUM, 1) /* Guest handles pkts w/ partial csum */ \ - _ (VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, 2) /* Dynamic offload configuration. */ \ - _ (VIRTIO_NET_F_MTU, 3) /* Initial MTU advice. */ \ - _ (VIRTIO_NET_F_MAC, 5) /* Host has given MAC address. */ \ - _ (VIRTIO_NET_F_GSO, 6) /* Host handles pkts w/ any GSO. */ \ - _ (VIRTIO_NET_F_GUEST_TSO4, 7) /* Guest can handle TSOv4 in. */ \ - _ (VIRTIO_NET_F_GUEST_TSO6, 8) /* Guest can handle TSOv6 in. */ \ - _ (VIRTIO_NET_F_GUEST_ECN, 9) /* Guest can handle TSO[6] w/ ECN in. */ \ - _ (VIRTIO_NET_F_GUEST_UFO, 10) /* Guest can handle UFO in. */ \ - _ (VIRTIO_NET_F_HOST_TSO4, 11) /* Host can handle TSOv4 in. */ \ - _ (VIRTIO_NET_F_HOST_TSO6, 12) /* Host can handle TSOv6 in. */ \ - _ (VIRTIO_NET_F_HOST_ECN, 13) /* Host can handle TSO[6] w/ ECN in. */ \ - _ (VIRTIO_NET_F_HOST_UFO, 14) /* Host can handle UFO in. */ \ - _ (VIRTIO_NET_F_MRG_RXBUF, 15) /* Host can merge receive buffers. */ \ - _ (VIRTIO_NET_F_STATUS, 16) /* virtio_net_config.status available */ \ - _ (VIRTIO_NET_F_CTRL_VQ, 17) /* Control channel available */ \ - _ (VIRTIO_NET_F_CTRL_RX, 18) /* Control channel RX mode support */ \ - _ (VIRTIO_NET_F_CTRL_VLAN, 19) /* Control channel VLAN filtering */ \ - _ (VIRTIO_NET_F_CTRL_RX_EXTRA, 20) /* Extra RX mode control support */ \ - _ (VIRTIO_NET_F_GUEST_ANNOUNCE, 21) /* Guest can announce device on the network */ \ - _ (VIRTIO_NET_F_MQ, 22) /* Device supports Receive Flow Steering */ \ - _ (VIRTIO_NET_F_CTRL_MAC_ADDR, 23) /* Set MAC address */ \ - _ (VIRTIO_F_NOTIFY_ON_EMPTY, 24) \ - _ (VHOST_F_LOG_ALL, 26) /* Log all write descriptors */ \ - _ (VIRTIO_F_ANY_LAYOUT, 27) /* Can the device handle any descripor layout */ \ - _ (VIRTIO_RING_F_INDIRECT_DESC, 28) /* Support indirect buffer descriptors */ \ - _ (VIRTIO_RING_F_EVENT_IDX, 29) /* The Guest publishes the used index for which it expects an interrupt \ - * at the end of the avail ring. Host should ignore the avail->flags field. */ \ -/* The Host publishes the avail index for which it expects a kick \ - * at the end of the used ring. Guest should ignore the used->flags field. */ \ - _ (VHOST_USER_F_PROTOCOL_FEATURES, 30) \ - _ (VIRTIO_F_VERSION_1, 32) \ - -#define VIRTIO_NET_F_CTRL_GUEST_OFFLOADS 2 -#define VIRTIO_NET_F_MTU 3 + #define VIRTIO_NET_S_LINK_UP 1 /* Link is up */ #define VIRTIO_NET_S_ANNOUNCE 2 /* Announcement is needed */ +#define VIRTIO_NET_OK 0 +#define VIRTIO_NET_ERR 1 + +/* If multiqueue is provided by host, then we support it. */ +#define VIRTIO_NET_CTRL_MQ 4 +#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0 +#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1 +#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000 + /* * Control network offloads * Reconfigures the network offloads that Guest can handle. @@ -138,7 +111,7 @@ typedef struct typedef struct { - struct virtio_pci_cap cap; + virtio_pci_cap_t cap; u32 notify_off_multiplier; /* Multiplier for queue_notify_off. */ } virtio_pci_notify_cap_t; @@ -174,35 +147,29 @@ typedef struct u16 mtu; } virtio_net_config_t; -typedef struct -{ - u64 addr; - u32 len; - u16 flags; - u16 next; -} vring_desc_t; - -typedef struct +/* + * Control virtqueue data structures + * + * The control virtqueue expects a header in the first sg entry + * and an ack/status response in the last entry. Data for the + * command goes in between. + */ +/* *INDENT-OFF* */ +typedef CLIB_PACKED (struct { - u16 flags; - u16 idx; - u16 ring[0]; - /* u16 used_event; */ -} vring_avail_t; + u8 class; + u8 cmd; +}) virtio_net_ctrl_hdr_t; +/* *INDENT-ON* */ -typedef struct -{ - u32 id; - u32 len; -} vring_used_elem_t; +typedef u8 virtio_net_ctrl_ack_t; typedef struct { - u16 flags; - u16 idx; - vring_used_elem_t ring[0]; - /* u16 avail_event; */ -} vring_used_t; + virtio_net_ctrl_hdr_t ctrl; + virtio_net_ctrl_ack_t status; + u8 data[1024]; +} virtio_ctrl_msg_t; typedef struct _virtio_pci_func { diff --git a/src/vnet/devices/virtio/vhost_std.h b/src/vnet/devices/virtio/vhost_std.h new file mode 100644 index 00000000000..a24071faeb5 --- /dev/null +++ b/src/vnet/devices/virtio/vhost_std.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __VIRTIO_VHOST_STD_H__ +#define __VIRTIO_VHOST_STD_H__ + +typedef struct +{ + u64 guest_phys_addr; + u64 memory_size; + u64 userspace_addr; + u64 mmap_offset; +} vhost_memory_region_t; + +typedef struct +{ + u32 nregions; + u32 padding; + vhost_memory_region_t regions[0]; +} vhost_memory_t; + +typedef struct +{ + u32 index; + u32 num; +} vhost_vring_state_t; + +typedef struct +{ + u32 index; + int fd; +} vhost_vring_file_t; + +typedef struct +{ + u32 index; + u32 flags; + u64 desc_user_addr; + u64 used_user_addr; + u64 avail_user_addr; + u64 log_guest_addr; +} vhost_vring_addr_t; + +typedef struct +{ + u64 size; + u64 offset; +} vhost_user_log_t; + +/* vhost kernel ioctls */ +#define VHOST_VIRTIO 0xAF +#define VHOST_GET_FEATURES _IOR(VHOST_VIRTIO, 0x00, u64) +#define VHOST_SET_FEATURES _IOW(VHOST_VIRTIO, 0x00, u64) +#define VHOST_SET_OWNER _IO(VHOST_VIRTIO, 0x01) +#define VHOST_RESET_OWNER _IO(VHOST_VIRTIO, 0x02) +#define VHOST_SET_MEM_TABLE _IOW(VHOST_VIRTIO, 0x03, vhost_memory_t) +#define VHOST_SET_LOG_BASE _IOW(VHOST_VIRTIO, 0x04, u64) +#define VHOST_SET_LOG_FD _IOW(VHOST_VIRTIO, 0x07, int) +#define VHOST_SET_VRING_NUM _IOW(VHOST_VIRTIO, 0x10, vhost_vring_state_t) +#define VHOST_SET_VRING_ADDR _IOW(VHOST_VIRTIO, 0x11, vhost_vring_addr_t) +#define VHOST_SET_VRING_BASE _IOW(VHOST_VIRTIO, 0x12, vhost_vring_state_t) +#define VHOST_GET_VRING_BASE _IOWR(VHOST_VIRTIO, 0x12, vhost_vring_state_t) +#define VHOST_SET_VRING_KICK _IOW(VHOST_VIRTIO, 0x20, vhost_vring_file_t) +#define VHOST_SET_VRING_CALL _IOW(VHOST_VIRTIO, 0x21, vhost_vring_file_t) +#define VHOST_SET_VRING_ERR _IOW(VHOST_VIRTIO, 0x22, vhost_vring_file_t) +#define VHOST_NET_SET_BACKEND _IOW(VHOST_VIRTIO, 0x30, vhost_vring_file_t) + +#endif + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vnet/devices/virtio/vhost_user.c b/src/vnet/devices/virtio/vhost_user.c index 1883f862742..92aee00c25e 100644 --- a/src/vnet/devices/virtio/vhost_user.c +++ b/src/vnet/devices/virtio/vhost_user.c @@ -453,21 +453,21 @@ vhost_user_socket_read (clib_file_t * uf) { case VHOST_USER_GET_FEATURES: msg.flags |= 4; - msg.u64 = (1ULL << FEAT_VIRTIO_NET_F_MRG_RXBUF) | - (1ULL << FEAT_VIRTIO_NET_F_CTRL_VQ) | - (1ULL << FEAT_VIRTIO_F_ANY_LAYOUT) | - (1ULL << FEAT_VIRTIO_F_INDIRECT_DESC) | - (1ULL << FEAT_VHOST_F_LOG_ALL) | - (1ULL << FEAT_VIRTIO_NET_F_GUEST_ANNOUNCE) | - (1ULL << FEAT_VIRTIO_NET_F_MQ) | - (1ULL << FEAT_VHOST_USER_F_PROTOCOL_FEATURES) | - (1ULL << FEAT_VIRTIO_F_VERSION_1); + msg.u64 = VIRTIO_FEATURE (VIRTIO_NET_F_MRG_RXBUF) | + VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ) | + VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT) | + VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC) | + VIRTIO_FEATURE (VHOST_F_LOG_ALL) | + VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_ANNOUNCE) | + VIRTIO_FEATURE (VIRTIO_NET_F_MQ) | + VIRTIO_FEATURE (VHOST_USER_F_PROTOCOL_FEATURES) | + VIRTIO_FEATURE (VIRTIO_F_VERSION_1); msg.u64 &= vui->feature_mask; if (vui->enable_gso) msg.u64 |= FEATURE_VIRTIO_NET_F_HOST_GUEST_TSO_FEATURE_BITS; if (vui->enable_packed) - msg.u64 |= (1ULL << FEAT_VIRTIO_F_RING_PACKED); + msg.u64 |= VIRTIO_FEATURE (VIRTIO_F_RING_PACKED); msg.size = sizeof (msg.u64); vu_log_debug (vui, "if %d msg VHOST_USER_GET_FEATURES - reply " @@ -488,14 +488,14 @@ vhost_user_socket_read (clib_file_t * uf) vui->features = msg.u64; if (vui->features & - ((1 << FEAT_VIRTIO_NET_F_MRG_RXBUF) | - (1ULL << FEAT_VIRTIO_F_VERSION_1))) + (VIRTIO_FEATURE (VIRTIO_NET_F_MRG_RXBUF) | + VIRTIO_FEATURE (VIRTIO_F_VERSION_1))) vui->virtio_net_hdr_sz = 12; else vui->virtio_net_hdr_sz = 10; vui->is_any_layout = - (vui->features & (1 << FEAT_VIRTIO_F_ANY_LAYOUT)) ? 1 : 0; + (vui->features & VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT)) ? 1 : 0; ASSERT (vui->virtio_net_hdr_sz < VLIB_BUFFER_PRE_DATA_SIZE); vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vui->hw_if_index); @@ -649,7 +649,7 @@ vhost_user_socket_read (clib_file_t * uf) /* Spec says: If VHOST_USER_F_PROTOCOL_FEATURES has not been negotiated, the ring is initialized in an enabled state. */ - if (!(vui->features & (1 << FEAT_VHOST_USER_F_PROTOCOL_FEATURES))) + if (!(vui->features & VIRTIO_FEATURE (VHOST_USER_F_PROTOCOL_FEATURES))) vui->vrings[msg.state.index].enabled = 1; vui->vrings[msg.state.index].last_used_idx = @@ -807,7 +807,7 @@ vhost_user_socket_read (clib_file_t * uf) if (vui->vrings[msg.state.index].avail_wrap_counter == 1) vui->vrings[msg.state.index].avail_wrap_counter = - VIRTQ_DESC_F_AVAIL; + VRING_DESC_F_AVAIL; } vlib_worker_thread_barrier_release (vm); break; @@ -1710,7 +1710,7 @@ vhost_user_connect_command_fn (vlib_main_t * vm, /* GSO feature is disable by default */ feature_mask &= ~FEATURE_VIRTIO_NET_F_HOST_GUEST_TSO_FEATURE_BITS; /* packed-ring feature is disable by default */ - feature_mask &= ~(1ULL << FEAT_VIRTIO_F_RING_PACKED); + feature_mask &= ~VIRTIO_FEATURE (VIRTIO_F_RING_PACKED); while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) { if (unformat (line_input, "socket %s", &sock_filename)) @@ -1924,7 +1924,7 @@ vhost_user_show_desc (vlib_main_t * vm, vhost_user_intf_t * vui, int q, vlib_cli_output (vm, "%U", format_vhost_user_desc, " %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n", vui, desc_table, j, &mem_hint); - if (show_verbose && (desc_table[j].flags & VIRTQ_DESC_F_INDIRECT)) + if (show_verbose && (desc_table[j].flags & VRING_DESC_F_INDIRECT)) { n_entries = desc_table[j].len / sizeof (vring_desc_t); desc_table = map_guest_mem (vui, desc_table[j].addr, &mem_hint); @@ -2012,7 +2012,7 @@ vhost_user_show_desc_packed (vlib_main_t * vm, vhost_user_intf_t * vui, int q, vlib_cli_output (vm, "%U", format_vhost_user_packed_desc, " %-5u 0x%016lx %-5u 0x%04x %-5u 0x%016lx\n", vui, desc_table, j, &mem_hint); - if (show_verbose && (desc_table[j].flags & VIRTQ_DESC_F_INDIRECT)) + if (show_verbose && (desc_table[j].flags & VRING_DESC_F_INDIRECT)) { n_entries = desc_table[j].len >> 4; desc_table = map_guest_mem (vui, desc_table[j].addr, &mem_hint); @@ -2059,7 +2059,7 @@ show_vhost_user_command_fn (vlib_main_t * vm, static struct feat_struct feat_array[] = { #define _(s,b) { .str = #s, .bit = b, }, - foreach_virtio_net_feature + foreach_virtio_net_features #undef _ {.str = NULL} }; diff --git a/src/vnet/devices/virtio/vhost_user.h b/src/vnet/devices/virtio/vhost_user.h index b86f42e70e8..868f34434ce 100644 --- a/src/vnet/devices/virtio/vhost_user.h +++ b/src/vnet/devices/virtio/vhost_user.h @@ -14,44 +14,27 @@ */ #ifndef __VIRTIO_VHOST_USER_H__ #define __VIRTIO_VHOST_USER_H__ + +#include <vnet/devices/virtio/virtio_std.h> +#include <vnet/devices/virtio/vhost_std.h> + /* vhost-user data structures */ #define VHOST_MEMORY_MAX_NREGIONS 8 #define VHOST_USER_MSG_HDR_SZ 12 -#define VHOST_VRING_MAX_SIZE 32768 #define VHOST_VRING_MAX_N 16 //8TX + 8RX #define VHOST_VRING_IDX_RX(qid) (2*qid) #define VHOST_VRING_IDX_TX(qid) (2*qid + 1) #define VHOST_USER_VRING_NOFD_MASK 0x100 -#define VIRTQ_DESC_F_NEXT 1 -#define VIRTQ_DESC_F_WRITE 2 -#define VIRTQ_DESC_F_INDIRECT 4 - -#define VIRTQ_DESC_F_AVAIL (1 << 7) -#define VIRTQ_DESC_F_USED (1 << 15) - -#define VRING_EVENT_F_ENABLE 0x0 -#define VRING_EVENT_F_DISABLE 0x1 -#define VRING_EVENT_F_DESC 0x2 #define VHOST_USER_PROTOCOL_F_MQ 0 #define VHOST_USER_PROTOCOL_F_LOG_SHMFD 1 #define VHOST_VRING_F_LOG 0 -#define VHOST_USER_F_PROTOCOL_FEATURES 30 #define VHOST_USER_PROTOCOL_FEATURES ((1ULL << VHOST_USER_PROTOCOL_F_MQ) | \ (1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD)) -/* If multiqueue is provided by host, then we support it. */ -#define VIRTIO_NET_CTRL_MQ 4 -#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0 -#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1 -#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000 - -#define VRING_USED_F_NO_NOTIFY 1 -#define VRING_AVAIL_F_NO_INTERRUPT 1 - #define vu_log_debug(dev, f, ...) \ { \ vlib_log(VLIB_LOG_LEVEL_DEBUG, vhost_user_main.log_default, "%U: " f, \ @@ -91,46 +74,17 @@ typedef enum #undef _ } virtio_trace_flag_t; -#define foreach_virtio_net_feature \ - _ (VIRTIO_NET_F_CSUM, 0) \ - _ (VIRTIO_NET_F_GUEST_CSUM, 1) \ - _ (VIRTIO_NET_F_GUEST_TSO4, 7) \ - _ (VIRTIO_NET_F_GUEST_TSO6, 8) \ - _ (VIRTIO_NET_F_GUEST_UFO, 10) \ - _ (VIRTIO_NET_F_HOST_TSO4, 11) \ - _ (VIRTIO_NET_F_HOST_TSO6, 12) \ - _ (VIRTIO_NET_F_HOST_UFO, 14) \ - _ (VIRTIO_NET_F_MRG_RXBUF, 15) \ - _ (VIRTIO_NET_F_CTRL_VQ, 17) \ - _ (VIRTIO_NET_F_GUEST_ANNOUNCE, 21) \ - _ (VIRTIO_NET_F_MQ, 22) \ - _ (VHOST_F_LOG_ALL, 26) \ - _ (VIRTIO_F_ANY_LAYOUT, 27) \ - _ (VIRTIO_F_INDIRECT_DESC, 28) \ - _ (VIRTIO_F_EVENT_IDX, 29) \ - _ (VHOST_USER_F_PROTOCOL_FEATURES, 30) \ - _ (VIRTIO_F_VERSION_1, 32) \ - _ (VIRTIO_F_RING_PACKED, 34) \ - _ (VIRTIO_F_IN_ORDER, 35) - -typedef enum -{ -#define _(f,n) FEAT_##f = (n), - foreach_virtio_net_feature -#undef _ -} virtio_net_feature_t; - #define FEATURE_VIRTIO_NET_F_HOST_TSO_FEATURE_BITS \ - ((1ULL << FEAT_VIRTIO_NET_F_CSUM) | \ - (1ULL << FEAT_VIRTIO_NET_F_HOST_UFO) | \ - (1ULL << FEAT_VIRTIO_NET_F_HOST_TSO4) | \ - (1ULL << FEAT_VIRTIO_NET_F_HOST_TSO6)) + (VIRTIO_FEATURE (VIRTIO_NET_F_CSUM) | \ + VIRTIO_FEATURE (VIRTIO_NET_F_HOST_UFO) | \ + VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO4) | \ + VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO6)) #define FEATURE_VIRTIO_NET_F_GUEST_TSO_FEATURE_BITS \ - ((1ULL << FEAT_VIRTIO_NET_F_GUEST_CSUM) | \ - (1ULL << FEAT_VIRTIO_NET_F_GUEST_UFO) | \ - (1ULL << FEAT_VIRTIO_NET_F_GUEST_TSO4) | \ - (1ULL << FEAT_VIRTIO_NET_F_GUEST_TSO6)) + (VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM) | \ + VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_UFO) | \ + VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4) | \ + VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO6)) #define FEATURE_VIRTIO_NET_F_HOST_GUEST_TSO_FEATURE_BITS \ (FEATURE_VIRTIO_NET_F_HOST_TSO_FEATURE_BITS | \ @@ -165,23 +119,6 @@ typedef struct vhost_user_memory vhost_user_memory_region_t regions[VHOST_MEMORY_MAX_NREGIONS]; } __attribute ((packed)) vhost_user_memory_t; -typedef struct -{ - u32 index, num; -} __attribute ((packed)) vhost_vring_state_t; - -typedef struct -{ - u32 index, flags; - u64 desc_user_addr, used_user_addr, avail_user_addr, log_guest_addr; -} __attribute ((packed)) vhost_vring_addr_t; - -typedef struct vhost_user_log -{ - u64 size; - u64 offset; -} __attribute ((packed)) vhost_user_log_t; - typedef enum vhost_user_req { VHOST_USER_NONE = 0, @@ -206,64 +143,6 @@ typedef enum vhost_user_req VHOST_USER_MAX } vhost_user_req_t; -// vring_desc I/O buffer descriptor -typedef struct -{ - uint64_t addr; // packet data buffer address - uint32_t len; // packet data buffer size - uint16_t flags; // (see below) - uint16_t next; // optional index next descriptor in chain -} __attribute ((packed)) vring_desc_t; - -typedef struct -{ - uint16_t flags; - volatile uint16_t idx; - uint16_t ring[VHOST_VRING_MAX_SIZE]; -} __attribute ((packed)) vring_avail_t; - -typedef struct -{ - uint16_t flags; - uint16_t idx; - struct /* vring_used_elem */ - { - uint32_t id; - uint32_t len; - } ring[VHOST_VRING_MAX_SIZE]; -} __attribute ((packed)) vring_used_t; - -typedef CLIB_PACKED (struct -{ - u64 addr; // packet data buffer address - u32 len; // packet data buffer size - u16 id; // buffer id - u16 flags; // flags -}) vring_packed_desc_t; - -STATIC_ASSERT_SIZEOF (vring_packed_desc_t, 16); - -typedef CLIB_PACKED (struct -{ - u16 off_wrap; - u16 flags; -}) vring_desc_event_t; - -typedef struct -{ - u8 flags; - u8 gso_type; - u16 hdr_len; - u16 gso_size; - u16 csum_start; - u16 csum_offset; -} __attribute ((packed)) virtio_net_hdr_t; - -typedef struct { - virtio_net_hdr_t hdr; - u16 num_buffers; -} __attribute ((packed)) virtio_net_hdr_mrg_rxbuf_t; - typedef struct vhost_user_msg { vhost_user_req_t request; u32 flags; diff --git a/src/vnet/devices/virtio/vhost_user_api.c b/src/vnet/devices/virtio/vhost_user_api.c index 80fe5548efd..ec335c529f2 100644 --- a/src/vnet/devices/virtio/vhost_user_api.c +++ b/src/vnet/devices/virtio/vhost_user_api.c @@ -65,10 +65,10 @@ vl_api_create_vhost_user_if_t_handler (vl_api_create_vhost_user_if_t * mp) u8 *mac_p = NULL; if (mp->disable_mrg_rxbuf) - disabled_features = (1ULL << FEAT_VIRTIO_NET_F_MRG_RXBUF); + disabled_features = VIRTIO_FEATURE (VIRTIO_NET_F_MRG_RXBUF); if (mp->disable_indirect_desc) - disabled_features |= (1ULL << FEAT_VIRTIO_F_INDIRECT_DESC); + disabled_features |= VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC); /* * GSO and PACKED are not supported by feature mask via binary API. We @@ -76,7 +76,7 @@ vl_api_create_vhost_user_if_t_handler (vl_api_create_vhost_user_if_t * mp) * explicitly via enable_gso and enable_packed argument */ disabled_features |= FEATURE_VIRTIO_NET_F_HOST_GUEST_TSO_FEATURE_BITS | - (1ULL << FEAT_VIRTIO_F_RING_PACKED); + VIRTIO_FEATURE (VIRTIO_F_RING_PACKED); features &= ~disabled_features; if (mp->use_custom_mac) @@ -129,7 +129,7 @@ vl_api_modify_vhost_user_if_t_handler (vl_api_modify_vhost_user_if_t * mp) * explicitly via enable_gso and enable_packed argument */ disabled_features |= FEATURE_VIRTIO_NET_F_HOST_GUEST_TSO_FEATURE_BITS | - (1ULL << FEAT_VIRTIO_F_RING_PACKED); + VIRTIO_FEATURE (VIRTIO_F_RING_PACKED); features &= ~disabled_features; rv = vhost_user_modify_if (vnm, vm, (char *) mp->sock_filename, diff --git a/src/vnet/devices/virtio/vhost_user_inline.h b/src/vnet/devices/virtio/vhost_user_inline.h index ceaf78cf799..17b6a90618f 100644 --- a/src/vnet/devices/virtio/vhost_user_inline.h +++ b/src/vnet/devices/virtio/vhost_user_inline.h @@ -176,7 +176,7 @@ vhost_user_log_dirty_pages_2 (vhost_user_intf_t * vui, u64 addr, u64 len, u8 is_host_address) { if (PREDICT_TRUE (vui->log_base_addr == 0 - || !(vui->features & (1 << FEAT_VHOST_F_LOG_ALL)))) + || !(vui->features & VIRTIO_FEATURE (VHOST_F_LOG_ALL)))) { return; } @@ -296,7 +296,7 @@ vhost_user_update_gso_interface_count (vhost_user_intf_t * vui, u8 add) static_always_inline u8 vhost_user_packed_desc_available (vhost_user_vring_t * vring, u16 idx) { - return (((vring->packed_desc[idx].flags & VIRTQ_DESC_F_AVAIL) == + return (((vring->packed_desc[idx].flags & VRING_DESC_F_AVAIL) == vring->avail_wrap_counter)); } @@ -305,7 +305,7 @@ vhost_user_advance_last_avail_idx (vhost_user_vring_t * vring) { vring->last_avail_idx++; if (PREDICT_FALSE ((vring->last_avail_idx & vring->qsz_mask) == 0)) - vring->avail_wrap_counter ^= VIRTQ_DESC_F_AVAIL; + vring->avail_wrap_counter ^= VRING_DESC_F_AVAIL; } static_always_inline void @@ -319,7 +319,7 @@ vhost_user_advance_last_avail_table_idx (vhost_user_intf_t * vui, /* pick up the slot of the next avail idx */ while (desc_table[vring->last_avail_idx & vring->qsz_mask].flags & - VIRTQ_DESC_F_NEXT) + VRING_DESC_F_NEXT) vhost_user_advance_last_avail_idx (vring); } @@ -330,7 +330,7 @@ static_always_inline void vhost_user_undo_advanced_last_avail_idx (vhost_user_vring_t * vring) { if (PREDICT_FALSE ((vring->last_avail_idx & vring->qsz_mask) == 0)) - vring->avail_wrap_counter ^= VIRTQ_DESC_F_AVAIL; + vring->avail_wrap_counter ^= VRING_DESC_F_AVAIL; vring->last_avail_idx--; } @@ -368,7 +368,7 @@ vhost_user_advance_last_used_idx (vhost_user_vring_t * vring) static_always_inline u64 vhost_user_is_packed_ring_supported (vhost_user_intf_t * vui) { - return (vui->features & (1ULL << FEAT_VIRTIO_F_RING_PACKED)); + return (vui->features & VIRTIO_FEATURE (VIRTIO_F_RING_PACKED)); } #endif diff --git a/src/vnet/devices/virtio/vhost_user_input.c b/src/vnet/devices/virtio/vhost_user_input.c index dd899094225..ea8e7d6f777 100644 --- a/src/vnet/devices/virtio/vhost_user_input.c +++ b/src/vnet/devices/virtio/vhost_user_input.c @@ -39,7 +39,6 @@ #include <vnet/devices/devices.h> #include <vnet/feature/feature.h> -#include <vnet/devices/virtio/virtio.h> #include <vnet/devices/virtio/vhost_user.h> #include <vnet/devices/virtio/vhost_user_inline.h> @@ -108,18 +107,18 @@ vhost_user_rx_trace (vhost_trace_t * t, t->qid = qid; hdr_desc = &txvq->desc[desc_current]; - if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT) + if (txvq->desc[desc_current].flags & VRING_DESC_F_INDIRECT) { t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_INDIRECT; /* Header is the first here */ hdr_desc = map_guest_mem (vui, txvq->desc[desc_current].addr, &hint); } - if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT) + if (txvq->desc[desc_current].flags & VRING_DESC_F_NEXT) { t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SIMPLE_CHAINED; } - if (!(txvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT) && - !(txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)) + if (!(txvq->desc[desc_current].flags & VRING_DESC_F_NEXT) && + !(txvq->desc[desc_current].flags & VRING_DESC_F_INDIRECT)) { t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SINGLE_DESC; } @@ -560,7 +559,7 @@ vhost_user_if_input (vlib_main_t * vm, /* This depends on the setup but is very consistent * So I think the CPU branch predictor will make a pretty good job * at optimizing the decision. */ - if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT) + if (txvq->desc[desc_current].flags & VRING_DESC_F_INDIRECT) { desc_table = map_guest_mem (vui, txvq->desc[desc_current].addr, &map_hint); @@ -591,7 +590,7 @@ vhost_user_if_input (vlib_main_t * vm, if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { if ((desc_data_offset == desc_table[desc_current].len) && - (desc_table[desc_current].flags & VIRTQ_DESC_F_NEXT)) + (desc_table[desc_current].flags & VRING_DESC_F_NEXT)) { current = desc_table[desc_current].next; b_data = map_guest_mem (vui, desc_table[current].addr, @@ -617,7 +616,7 @@ vhost_user_if_input (vlib_main_t * vm, if (desc_data_offset == desc_table[desc_current].len) { if (PREDICT_FALSE (desc_table[desc_current].flags & - VIRTQ_DESC_F_NEXT)) + VRING_DESC_F_NEXT)) { desc_current = desc_table[desc_current].next; desc_data_offset = 0; @@ -776,10 +775,10 @@ vhost_user_mark_desc_consumed (vhost_user_intf_t * vui, { if (txvq->used_wrap_counter) desc_table[(desc_head + desc_idx) & mask].flags |= - (VIRTQ_DESC_F_AVAIL | VIRTQ_DESC_F_USED); + (VRING_DESC_F_AVAIL | VRING_DESC_F_USED); else desc_table[(desc_head + desc_idx) & mask].flags &= - ~(VIRTQ_DESC_F_AVAIL | VIRTQ_DESC_F_USED); + ~(VRING_DESC_F_AVAIL | VRING_DESC_F_USED); vhost_user_advance_last_used_idx (txvq); } } @@ -799,18 +798,18 @@ vhost_user_rx_trace_packed (vhost_trace_t * t, vhost_user_intf_t * vui, t->qid = qid; hdr_desc = &txvq->packed_desc[desc_current]; - if (txvq->packed_desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT) + if (txvq->packed_desc[desc_current].flags & VRING_DESC_F_INDIRECT) { t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_INDIRECT; /* Header is the first here */ hdr_desc = map_guest_mem (vui, txvq->packed_desc[desc_current].addr, &hint); } - if (txvq->packed_desc[desc_current].flags & VIRTQ_DESC_F_NEXT) + if (txvq->packed_desc[desc_current].flags & VRING_DESC_F_NEXT) t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SIMPLE_CHAINED; - if (!(txvq->packed_desc[desc_current].flags & VIRTQ_DESC_F_NEXT) && - !(txvq->packed_desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)) + if (!(txvq->packed_desc[desc_current].flags & VRING_DESC_F_NEXT) && + !(txvq->packed_desc[desc_current].flags & VRING_DESC_F_INDIRECT)) t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SINGLE_DESC; t->first_desc_len = hdr_desc ? hdr_desc->len : 0; @@ -1018,7 +1017,7 @@ vhost_user_compute_chained_desc_len (vhost_user_intf_t * vui, u32 desc_len = 0; u16 mask = txvq->qsz_mask; - while (desc_table[*current].flags & VIRTQ_DESC_F_NEXT) + while (desc_table[*current].flags & VRING_DESC_F_NEXT) { desc_len += desc_table[*current].len; (*n_left)++; @@ -1177,7 +1176,7 @@ vhost_user_if_input_packed (vlib_main_t * vm, vhost_user_main_t * vum, while (vhost_user_packed_desc_available (txvq, current) && (n_left < VLIB_FRAME_SIZE)) { - if (desc_table[current].flags & VIRTQ_DESC_F_INDIRECT) + if (desc_table[current].flags & VRING_DESC_F_INDIRECT) { buffers_required += vhost_user_compute_indirect_desc_len (vui, txvq, buffer_data_size, @@ -1244,7 +1243,7 @@ vhost_user_if_input_packed (vlib_main_t * vm, vhost_user_main_t * vum, desc_data_offset = vui->virtio_net_hdr_sz; n_descs_to_process = 1; - if (desc_table[desc_idx].flags & VIRTQ_DESC_F_INDIRECT) + if (desc_table[desc_idx].flags & VRING_DESC_F_INDIRECT) { n_descs = desc_table[desc_idx].len >> 4; desc_table = map_guest_mem (vui, desc_table[desc_idx].addr, @@ -1298,7 +1297,7 @@ vhost_user_if_input_packed (vlib_main_t * vm, vhost_user_main_t * vum, * loop. So count how many descriptors in the chain. */ n_descs_to_process = 1; - while (desc_table[desc_idx].flags & VIRTQ_DESC_F_NEXT) + while (desc_table[desc_idx].flags & VRING_DESC_F_NEXT) { vhost_user_assemble_packet (desc_table, &desc_idx, b_head, &b_current, &next, &b, &bi_current, @@ -1431,7 +1430,7 @@ VLIB_NODE_FN (vhost_user_input_node) (vlib_main_t * vm, pool_elt_at_index (vum->vhost_user_interfaces, dq->dev_instance); if (vhost_user_is_packed_ring_supported (vui)) { - if (vui->features & (1ULL << FEAT_VIRTIO_NET_F_CSUM)) + if (vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_CSUM)) n_rx_packets += vhost_user_if_input_packed (vm, vum, vui, dq->queue_id, node, dq->mode, 1); @@ -1442,7 +1441,7 @@ VLIB_NODE_FN (vhost_user_input_node) (vlib_main_t * vm, } else { - if (vui->features & (1ULL << FEAT_VIRTIO_NET_F_CSUM)) + if (vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_CSUM)) n_rx_packets += vhost_user_if_input (vm, vum, vui, dq->queue_id, node, dq->mode, 1); else diff --git a/src/vnet/devices/virtio/vhost_user_output.c b/src/vnet/devices/virtio/vhost_user_output.c index d48e43738a9..2d17ddfda04 100644 --- a/src/vnet/devices/virtio/vhost_user_output.c +++ b/src/vnet/devices/virtio/vhost_user_output.c @@ -40,7 +40,6 @@ #include <vnet/devices/devices.h> #include <vnet/feature/feature.h> -#include <vnet/devices/virtio/virtio.h> #include <vnet/devices/virtio/vhost_user.h> #include <vnet/devices/virtio/vhost_user_inline.h> @@ -166,18 +165,18 @@ vhost_user_tx_trace (vhost_trace_t * t, t->qid = qid; hdr_desc = &rxvq->desc[desc_current]; - if (rxvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT) + if (rxvq->desc[desc_current].flags & VRING_DESC_F_INDIRECT) { t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_INDIRECT; /* Header is the first here */ hdr_desc = map_guest_mem (vui, rxvq->desc[desc_current].addr, &hint); } - if (rxvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT) + if (rxvq->desc[desc_current].flags & VRING_DESC_F_NEXT) { t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SIMPLE_CHAINED; } - if (!(rxvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT) && - !(rxvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)) + if (!(rxvq->desc[desc_current].flags & VRING_DESC_F_NEXT) && + !(rxvq->desc[desc_current].flags & VRING_DESC_F_INDIRECT)) { t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SINGLE_DESC; } @@ -277,19 +276,19 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b, if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM) { if (is_ip4 && - (vui->features & (1ULL << FEAT_VIRTIO_NET_F_GUEST_TSO4))) + (vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4))) { hdr->gso_size = vnet_buffer2 (b)->gso_size; hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4; } else if (is_ip6 && - (vui->features & (1ULL << FEAT_VIRTIO_NET_F_GUEST_TSO6))) + (vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO6))) { hdr->gso_size = vnet_buffer2 (b)->gso_size; hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6; } } - else if ((vui->features & (1ULL << FEAT_VIRTIO_NET_F_GUEST_UFO)) && + else if ((vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_UFO)) && (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)) { hdr->gso_size = vnet_buffer2 (b)->gso_size; @@ -312,10 +311,10 @@ vhost_user_mark_desc_available (vlib_main_t * vm, vhost_user_vring_t * rxvq, if (rxvq->used_wrap_counter) flags = desc_table[last_used_idx & rxvq->qsz_mask].flags | - (VIRTQ_DESC_F_AVAIL | VIRTQ_DESC_F_USED); + (VRING_DESC_F_AVAIL | VRING_DESC_F_USED); else flags = desc_table[last_used_idx & rxvq->qsz_mask].flags & - ~(VIRTQ_DESC_F_AVAIL | VIRTQ_DESC_F_USED); + ~(VRING_DESC_F_AVAIL | VRING_DESC_F_USED); vhost_user_advance_last_used_idx (rxvq); @@ -323,10 +322,10 @@ vhost_user_mark_desc_available (vlib_main_t * vm, vhost_user_vring_t * rxvq, { if (rxvq->used_wrap_counter) desc_table[rxvq->last_used_idx & rxvq->qsz_mask].flags |= - (VIRTQ_DESC_F_AVAIL | VIRTQ_DESC_F_USED); + (VRING_DESC_F_AVAIL | VRING_DESC_F_USED); else desc_table[rxvq->last_used_idx & rxvq->qsz_mask].flags &= - ~(VIRTQ_DESC_F_AVAIL | VIRTQ_DESC_F_USED); + ~(VRING_DESC_F_AVAIL | VRING_DESC_F_USED); vhost_user_advance_last_used_idx (rxvq); } @@ -339,7 +338,7 @@ vhost_user_mark_desc_available (vlib_main_t * vm, vhost_user_vring_t * rxvq, vring_packed_desc_t *desc_table = rxvq->packed_desc; while (desc_table[rxvq->last_used_idx & rxvq->qsz_mask].flags & - VIRTQ_DESC_F_NEXT) + VRING_DESC_F_NEXT) vhost_user_advance_last_used_idx (rxvq); /* Advance past the current chained table entries */ @@ -374,19 +373,19 @@ vhost_user_tx_trace_packed (vhost_trace_t * t, vhost_user_intf_t * vui, t->qid = qid; hdr_desc = &rxvq->packed_desc[desc_current]; - if (rxvq->packed_desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT) + if (rxvq->packed_desc[desc_current].flags & VRING_DESC_F_INDIRECT) { t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_INDIRECT; /* Header is the first here */ hdr_desc = map_guest_mem (vui, rxvq->packed_desc[desc_current].addr, &hint); } - if (rxvq->packed_desc[desc_current].flags & VIRTQ_DESC_F_NEXT) + if (rxvq->packed_desc[desc_current].flags & VRING_DESC_F_NEXT) { t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SIMPLE_CHAINED; } - if (!(rxvq->packed_desc[desc_current].flags & VIRTQ_DESC_F_NEXT) && - !(rxvq->packed_desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)) + if (!(rxvq->packed_desc[desc_current].flags & VRING_DESC_F_NEXT) && + !(rxvq->packed_desc[desc_current].flags & VRING_DESC_F_INDIRECT)) { t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SINGLE_DESC; } @@ -463,7 +462,7 @@ retry: * Go deeper in case of indirect descriptor. * To test it, turn off mrg_rxbuf. */ - if (desc_table[desc_head].flags & VIRTQ_DESC_F_INDIRECT) + if (desc_table[desc_head].flags & VRING_DESC_F_INDIRECT) { indirect = 1; if (PREDICT_FALSE (desc_table[desc_head].len < @@ -482,7 +481,7 @@ retry: } desc_index = 0; } - else if (rxvq->packed_desc[desc_head].flags & VIRTQ_DESC_F_NEXT) + else if (rxvq->packed_desc[desc_head].flags & VRING_DESC_F_NEXT) chained = 1; desc_len = vui->virtio_net_hdr_sz; @@ -502,7 +501,7 @@ retry: /* Guest supports csum offload and buffer requires checksum offload? */ if (or_flags && - (vui->features & (1ULL << FEAT_VIRTIO_NET_F_GUEST_CSUM))) + (vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM))) vhost_user_handle_tx_offload (vui, b0, &hdr->hdr); /* Prepare a copy order executed later for the header */ @@ -529,7 +528,7 @@ retry: * Test it with both indirect and mrg_rxbuf off */ if (PREDICT_FALSE (!(desc_table[desc_index].flags & - VIRTQ_DESC_F_NEXT))) + VRING_DESC_F_NEXT))) { /* * Last descriptor in chain. @@ -801,7 +800,7 @@ retry: /* Go deeper in case of indirect descriptor * I don't know of any driver providing indirect for RX. */ - if (PREDICT_FALSE (rxvq->desc[desc_head].flags & VIRTQ_DESC_F_INDIRECT)) + if (PREDICT_FALSE (rxvq->desc[desc_head].flags & VRING_DESC_F_INDIRECT)) { if (PREDICT_FALSE (rxvq->desc[desc_head].len < sizeof (vring_desc_t))) @@ -838,7 +837,7 @@ retry: /* Guest supports csum offload and buffer requires checksum offload? */ if (or_flags - && (vui->features & (1ULL << FEAT_VIRTIO_NET_F_GUEST_CSUM))) + && (vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM))) vhost_user_handle_tx_offload (vui, b0, &hdr->hdr); // Prepare a copy order executed later for the header @@ -858,7 +857,7 @@ retry: { if (buffer_len == 0) { //Get new output - if (desc_table[desc_index].flags & VIRTQ_DESC_F_NEXT) + if (desc_table[desc_index].flags & VRING_DESC_F_NEXT) { //Next one is chained desc_index = desc_table[desc_index].next; @@ -898,7 +897,7 @@ retry: desc_head = desc_index = rxvq->avail->ring[rxvq->last_avail_idx & rxvq->qsz_mask]; if (PREDICT_FALSE - (rxvq->desc[desc_head].flags & VIRTQ_DESC_F_INDIRECT)) + (rxvq->desc[desc_head].flags & VRING_DESC_F_INDIRECT)) { //It is seriously unlikely that a driver will put indirect descriptor //after non-indirect descriptor. diff --git a/src/vnet/devices/virtio/virtio.c b/src/vnet/devices/virtio/virtio.c index f8b6a39ea47..ec22a0d45f4 100644 --- a/src/vnet/devices/virtio/virtio.c +++ b/src/vnet/devices/virtio/virtio.c @@ -21,8 +21,6 @@ #include <net/if.h> #include <linux/if_tun.h> #include <sys/ioctl.h> -#include <linux/virtio_net.h> -#include <linux/vhost.h> #include <sys/eventfd.h> #include <vlib/vlib.h> @@ -93,19 +91,19 @@ virtio_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 idx, u16 sz) CLIB_CACHE_LINE_BYTES); vring = vec_elt_at_index (vif->rxq_vrings, RX_QUEUE_ACCESS (idx)); } - i = sizeof (struct vring_desc) * sz; + i = sizeof (vring_desc_t) * sz; i = round_pow2 (i, CLIB_CACHE_LINE_BYTES); vring->desc = clib_mem_alloc_aligned (i, CLIB_CACHE_LINE_BYTES); clib_memset (vring->desc, 0, i); - i = sizeof (struct vring_avail) + sz * sizeof (vring->avail->ring[0]); + i = sizeof (vring_avail_t) + sz * sizeof (vring->avail->ring[0]); i = round_pow2 (i, CLIB_CACHE_LINE_BYTES); vring->avail = clib_mem_alloc_aligned (i, CLIB_CACHE_LINE_BYTES); clib_memset (vring->avail, 0, i); // tell kernel that we don't need interrupt - vring->avail->flags = VIRTIO_RING_FLAG_MASK_INT; + vring->avail->flags = VRING_AVAIL_F_NO_INTERRUPT; - i = sizeof (struct vring_used) + sz * sizeof (struct vring_used_elem); + i = sizeof (vring_used_t) + sz * sizeof (vring_used_elem_t); i = round_pow2 (i, CLIB_CACHE_LINE_BYTES); vring->used = clib_mem_alloc_aligned (i, CLIB_CACHE_LINE_BYTES); clib_memset (vring->used, 0, i); @@ -186,7 +184,7 @@ virtio_free_used_desc (vlib_main_t * vm, virtio_vring_t * vring) while (n_left) { - struct vring_used_elem *e = &vring->used->ring[last & mask]; + vring_used_elem_t *e = &vring->used->ring[last & mask]; u16 slot = e->id; vlib_buffer_free (vm, &vring->buffers[slot], 1); @@ -257,9 +255,9 @@ virtio_set_net_hdr_size (virtio_if_t * vif) { if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MRG_RXBUF) || vif->features & VIRTIO_FEATURE (VIRTIO_F_VERSION_1)) - vif->virtio_net_hdr_sz = sizeof (struct virtio_net_hdr_v1); + vif->virtio_net_hdr_sz = sizeof (virtio_net_hdr_v1_t); else - vif->virtio_net_hdr_sz = sizeof (struct virtio_net_hdr); + vif->virtio_net_hdr_sz = sizeof (virtio_net_hdr_t); } inline void @@ -402,7 +400,7 @@ virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type) " ===== ================== ===== ====== ===== ==================\n"); for (j = 0; j < vring->size; j++) { - struct vring_desc *desc = &vring->desc[j]; + vring_desc_t *desc = &vring->desc[j]; vlib_cli_output (vm, " %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n", j, desc->addr, @@ -442,7 +440,7 @@ virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type) " ===== ================== ===== ====== ===== ==================\n"); for (j = 0; j < vring->size; j++) { - struct vring_desc *desc = &vring->desc[j]; + vring_desc_t *desc = &vring->desc[j]; vlib_cli_output (vm, " %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n", j, desc->addr, @@ -478,7 +476,7 @@ virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type) " ===== ================== ===== ====== ===== ==================\n"); for (j = 0; j < vring->size; j++) { - struct vring_desc *desc = &vring->desc[j]; + vring_desc_t *desc = &vring->desc[j]; vlib_cli_output (vm, " %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n", j, desc->addr, diff --git a/src/vnet/devices/virtio/virtio.h b/src/vnet/devices/virtio/virtio.h index 7a5dcd83fe3..f1eaa07fe5d 100644 --- a/src/vnet/devices/virtio/virtio.h +++ b/src/vnet/devices/virtio/virtio.h @@ -18,47 +18,10 @@ #ifndef _VNET_DEVICES_VIRTIO_VIRTIO_H_ #define _VNET_DEVICES_VIRTIO_VIRTIO_H_ -#include <linux/virtio_config.h> -#include <linux/virtio_net.h> -#include <linux/virtio_pci.h> -#include <linux/virtio_ring.h> +#include <vnet/devices/virtio/virtio_std.h> +#include <vnet/devices/virtio/vhost_std.h> #include <vnet/gso/gro.h> -#define foreach_virtio_net_features \ - _ (VIRTIO_NET_F_CSUM, 0) /* Host handles pkts w/ partial csum */ \ - _ (VIRTIO_NET_F_GUEST_CSUM, 1) /* Guest handles pkts w/ partial csum */ \ - _ (VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, 2) /* Dynamic offload configuration. */ \ - _ (VIRTIO_NET_F_MTU, 3) /* Initial MTU advice. */ \ - _ (VIRTIO_NET_F_MAC, 5) /* Host has given MAC address. */ \ - _ (VIRTIO_NET_F_GSO, 6) /* Host handles pkts w/ any GSO. */ \ - _ (VIRTIO_NET_F_GUEST_TSO4, 7) /* Guest can handle TSOv4 in. */ \ - _ (VIRTIO_NET_F_GUEST_TSO6, 8) /* Guest can handle TSOv6 in. */ \ - _ (VIRTIO_NET_F_GUEST_ECN, 9) /* Guest can handle TSO[6] w/ ECN in. */ \ - _ (VIRTIO_NET_F_GUEST_UFO, 10) /* Guest can handle UFO in. */ \ - _ (VIRTIO_NET_F_HOST_TSO4, 11) /* Host can handle TSOv4 in. */ \ - _ (VIRTIO_NET_F_HOST_TSO6, 12) /* Host can handle TSOv6 in. */ \ - _ (VIRTIO_NET_F_HOST_ECN, 13) /* Host can handle TSO[6] w/ ECN in. */ \ - _ (VIRTIO_NET_F_HOST_UFO, 14) /* Host can handle UFO in. */ \ - _ (VIRTIO_NET_F_MRG_RXBUF, 15) /* Host can merge receive buffers. */ \ - _ (VIRTIO_NET_F_STATUS, 16) /* virtio_net_config.status available */ \ - _ (VIRTIO_NET_F_CTRL_VQ, 17) /* Control channel available */ \ - _ (VIRTIO_NET_F_CTRL_RX, 18) /* Control channel RX mode support */ \ - _ (VIRTIO_NET_F_CTRL_VLAN, 19) /* Control channel VLAN filtering */ \ - _ (VIRTIO_NET_F_CTRL_RX_EXTRA, 20) /* Extra RX mode control support */ \ - _ (VIRTIO_NET_F_GUEST_ANNOUNCE, 21) /* Guest can announce device on the network */ \ - _ (VIRTIO_NET_F_MQ, 22) /* Device supports Receive Flow Steering */ \ - _ (VIRTIO_NET_F_CTRL_MAC_ADDR, 23) /* Set MAC address */ \ - _ (VIRTIO_F_NOTIFY_ON_EMPTY, 24) \ - _ (VHOST_F_LOG_ALL, 26) /* Log all write descriptors */ \ - _ (VIRTIO_F_ANY_LAYOUT, 27) /* Can the device handle any descriptor layout */ \ - _ (VIRTIO_RING_F_INDIRECT_DESC, 28) /* Support indirect buffer descriptors */ \ - _ (VIRTIO_RING_F_EVENT_IDX, 29) /* The Guest publishes the used index for which it expects an interrupt \ - * at the end of the avail ring. Host should ignore the avail->flags field. */ \ -/* The Host publishes the avail index for which it expects a kick \ - * at the end of the used ring. Guest should ignore the used->flags field. */ \ - _ (VHOST_USER_F_PROTOCOL_FEATURES, 30) \ - _ (VIRTIO_F_VERSION_1, 32) /* v1.0 compliant. */ \ - #define foreach_virtio_if_flag \ _(0, ADMIN_UP, "admin-up") \ _(1, DELETING, "deleting") @@ -70,8 +33,6 @@ typedef enum #undef _ } virtio_if_flag_t; -#define VIRTIO_FEATURE(X) (1ULL << X) - #define TX_QUEUE(X) ((X*2) + 1) #define RX_QUEUE(X) (X*2) #define TX_QUEUE_ACCESS(X) (X/2) @@ -99,9 +60,9 @@ typedef struct { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); clib_spinlock_t lockp; - struct vring_desc *desc; - struct vring_used *used; - struct vring_avail *avail; + vring_desc_t *desc; + vring_used_t *used; + vring_avail_t *avail; u16 desc_in_use; u16 desc_next; int kick_fd; diff --git a/src/vnet/devices/virtio/virtio_pci_modern.c b/src/vnet/devices/virtio/virtio_pci_modern.c index 24e8a4fe404..0d86ff2acda 100644 --- a/src/vnet/devices/virtio/virtio_pci_modern.c +++ b/src/vnet/devices/virtio/virtio_pci_modern.c @@ -267,7 +267,7 @@ static u8 virtio_pci_modern_setup_queue (vlib_main_t * vm, virtio_if_t * vif, u16 queue_id, void *p) { - struct vring vr; + vring_t vr; u16 queue_size = 0; virtio_pci_modern_set_queue_select (vif, queue_id); diff --git a/src/vnet/devices/virtio/virtio_std.h b/src/vnet/devices/virtio/virtio_std.h new file mode 100644 index 00000000000..98befb5c820 --- /dev/null +++ b/src/vnet/devices/virtio/virtio_std.h @@ -0,0 +1,208 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __VIRTIO_STD_H__ +#define __VIRTIO_STD_H__ + +#define foreach_virtio_net_features \ + _ (VIRTIO_NET_F_CSUM, 0) /* Host handles pkts w/ partial csum */ \ + _ (VIRTIO_NET_F_GUEST_CSUM, 1) /* Guest handles pkts w/ partial csum */ \ + _ (VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, 2) /* Dynamic offload configuration. */ \ + _ (VIRTIO_NET_F_MTU, 3) /* Initial MTU advice. */ \ + _ (VIRTIO_NET_F_MAC, 5) /* Host has given MAC address. */ \ + _ (VIRTIO_NET_F_GSO, 6) /* Host handles pkts w/ any GSO. */ \ + _ (VIRTIO_NET_F_GUEST_TSO4, 7) /* Guest can handle TSOv4 in. */ \ + _ (VIRTIO_NET_F_GUEST_TSO6, 8) /* Guest can handle TSOv6 in. */ \ + _ (VIRTIO_NET_F_GUEST_ECN, 9) /* Guest can handle TSO[6] w/ ECN in. */ \ + _ (VIRTIO_NET_F_GUEST_UFO, 10) /* Guest can handle UFO in. */ \ + _ (VIRTIO_NET_F_HOST_TSO4, 11) /* Host can handle TSOv4 in. */ \ + _ (VIRTIO_NET_F_HOST_TSO6, 12) /* Host can handle TSOv6 in. */ \ + _ (VIRTIO_NET_F_HOST_ECN, 13) /* Host can handle TSO[6] w/ ECN in. */ \ + _ (VIRTIO_NET_F_HOST_UFO, 14) /* Host can handle UFO in. */ \ + _ (VIRTIO_NET_F_MRG_RXBUF, 15) /* Host can merge receive buffers. */ \ + _ (VIRTIO_NET_F_STATUS, 16) /* virtio_net_config.status available */ \ + _ (VIRTIO_NET_F_CTRL_VQ, 17) /* Control channel available */ \ + _ (VIRTIO_NET_F_CTRL_RX, 18) /* Control channel RX mode support */ \ + _ (VIRTIO_NET_F_CTRL_VLAN, 19) /* Control channel VLAN filtering */ \ + _ (VIRTIO_NET_F_CTRL_RX_EXTRA, 20) /* Extra RX mode control support */ \ + _ (VIRTIO_NET_F_GUEST_ANNOUNCE, 21) /* Guest can announce device on the network */ \ + _ (VIRTIO_NET_F_MQ, 22) /* Device supports Receive Flow Steering */ \ + _ (VIRTIO_NET_F_CTRL_MAC_ADDR, 23) /* Set MAC address */ \ + _ (VIRTIO_F_NOTIFY_ON_EMPTY, 24) \ + _ (VHOST_F_LOG_ALL, 26) /* Log all write descriptors */ \ + _ (VIRTIO_F_ANY_LAYOUT, 27) /* Can the device handle any descriptor layout */ \ + _ (VIRTIO_RING_F_INDIRECT_DESC, 28) /* Support indirect buffer descriptors */ \ + _ (VIRTIO_RING_F_EVENT_IDX, 29) /* The Guest publishes the used index for which it expects an interrupt \ + * at the end of the avail ring. Host should ignore the avail->flags field. */ \ +/* The Host publishes the avail index for which it expects a kick \ + * at the end of the used ring. Guest should ignore the used->flags field. */ \ + _ (VHOST_USER_F_PROTOCOL_FEATURES, 30) \ + _ (VIRTIO_F_VERSION_1, 32) /* v1.0 compliant. */ \ + _ (VIRTIO_F_IOMMU_PLATFORM, 33) \ + _ (VIRTIO_F_RING_PACKED, 34) \ + _ (VIRTIO_F_IN_ORDER, 35) /* all buffers are used by the device in the */ \ + /* same order in which they have been made available */ \ + _ (VIRTIO_F_ORDER_PLATFORM, 36) /* memory accesses by the driver and the */ \ + /* device are ordered in a way described by the platfor */ \ + _ (VIRTIO_F_NOTIFICATION_DATA, 38) /* the driver passes extra data (besides */ \ + /* identifying the virtqueue) in its device notifications. */ \ + _ (VIRTIO_NET_F_SPEED_DUPLEX, 63) /* Device set linkspeed and duplex */ + +typedef enum +{ +#define _(f,n) f = n, + foreach_virtio_net_features +#undef _ +} virtio_net_feature_t; + +#define VIRTIO_FEATURE(X) (1ULL << X) + +#define VRING_MAX_SIZE 32768 + +#define VRING_DESC_F_NEXT 1 +#define VRING_DESC_F_WRITE 2 +#define VRING_DESC_F_INDIRECT 4 + +#define VRING_DESC_F_AVAIL (1 << 7) +#define VRING_DESC_F_USED (1 << 15) + +#define VRING_EVENT_F_ENABLE 0x0 +#define VRING_EVENT_F_DISABLE 0x1 +#define VRING_EVENT_F_DESC 0x2 + +#define VRING_USED_F_NO_NOTIFY 1 +#define VRING_AVAIL_F_NO_INTERRUPT 1 + +typedef struct +{ + u64 addr; + u32 len; + u16 flags; + u16 next; +} vring_desc_t; + +typedef struct +{ + u16 flags; + u16 idx; + u16 ring[0]; + /* u16 used_event; */ +} vring_avail_t; + +typedef struct +{ + u32 id; + u32 len; +} vring_used_elem_t; + +typedef struct +{ + u16 flags; + u16 idx; + vring_used_elem_t ring[0]; + /* u16 avail_event; */ +} vring_used_t; + +/* *INDENT-OFF* */ +typedef CLIB_PACKED (struct +{ + u64 addr; // packet data buffer address + u32 len; // packet data buffer size + u16 id; // buffer id + u16 flags; // flags +}) vring_packed_desc_t; + +STATIC_ASSERT_SIZEOF (vring_packed_desc_t, 16); + +typedef CLIB_PACKED (struct +{ + u16 off_wrap; + u16 flags; +}) vring_desc_event_t; + +#define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* Use csum_start, csum_offset */ +#define VIRTIO_NET_HDR_F_DATA_VALID 2 /* Csum is valid */ + +#define VIRTIO_NET_HDR_GSO_NONE 0 /* Not a GSO frame */ +#define VIRTIO_NET_HDR_GSO_TCPV4 1 /* GSO frame, IPv4 TCP (TSO) */ +#define VIRTIO_NET_HDR_GSO_UDP 3 /* GSO frame, IPv4 UDP (UFO) */ +#define VIRTIO_NET_HDR_GSO_TCPV6 4 /* GSO frame, IPv6 TCP */ +#define VIRTIO_NET_HDR_GSO_ECN 0x80 /* TCP has ECN set */ + +typedef CLIB_PACKED (struct +{ + u8 flags; + u8 gso_type; + u16 hdr_len; /* Ethernet + IP + tcp/udp hdrs */ + u16 gso_size; /* Bytes to append to hdr_len per frame */ + u16 csum_start; /* Position to start checksumming from */ + u16 csum_offset; /* Offset after that to place checksum */ + u16 num_buffers; /* Number of merged rx buffers */ +}) virtio_net_hdr_v1_t; + +typedef CLIB_PACKED (struct +{ + u8 flags; + u8 gso_type; + u16 hdr_len; + u16 gso_size; + u16 csum_start; + u16 csum_offset; +}) virtio_net_hdr_t; + +typedef CLIB_PACKED (struct +{ + virtio_net_hdr_t hdr; + u16 num_buffers; +}) virtio_net_hdr_mrg_rxbuf_t; + +/* *INDENT-ON* */ + +typedef struct +{ + u16 num; + vring_desc_t *desc; + vring_avail_t *avail; + vring_used_t *used; +} vring_t; + +static_always_inline void +vring_init (vring_t * vr, u32 num, void *p, u32 align) +{ + vr->num = num; + vr->desc = p; + vr->avail = (vring_avail_t *) ((char *) p + num * sizeof (vring_desc_t)); + vr->used = + (vring_used_t *) ((char *) p + + ((sizeof (vring_desc_t) * num + + sizeof (u16) * (3 + num) + align - 1) & ~(align - + 1))); +} + +static_always_inline u16 +vring_size (u32 num, u32 align) +{ + return ((sizeof (vring_desc_t) * num + sizeof (u16) * (3 + num) + + align - 1) & ~(align - 1)) + + sizeof (u16) * 3 + sizeof (vring_used_elem_t) * num; +} +#endif + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ |