summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/vnet/devices/virtio/device.c47
-rw-r--r--src/vnet/devices/virtio/pci.c18
-rw-r--r--src/vnet/devices/virtio/virtio.c43
-rw-r--r--src/vnet/devices/virtio/virtio.h1
4 files changed, 61 insertions, 48 deletions
diff --git a/src/vnet/devices/virtio/device.c b/src/vnet/devices/virtio/device.c
index 04cf9ec49c8..e560c29e5e7 100644
--- a/src/vnet/devices/virtio/device.c
+++ b/src/vnet/devices/virtio/device.c
@@ -90,9 +90,8 @@ format_virtio_tx_trace (u8 * s, va_list * args)
return s;
}
-#ifndef CLIB_MARCH_VARIANT
-inline void
-virtio_free_used_desc (vlib_main_t * vm, virtio_vring_t * vring)
+static_always_inline void
+virtio_free_used_device_desc (vlib_main_t * vm, virtio_vring_t * vring)
{
u16 used = vring->desc_in_use;
u16 sz = vring->size;
@@ -106,17 +105,35 @@ virtio_free_used_desc (vlib_main_t * vm, virtio_vring_t * vring)
while (n_left)
{
struct vring_used_elem *e = &vring->used->ring[last & mask];
- u16 slot = e->id;
+ u16 slot, n_buffers;
+ slot = n_buffers = e->id;
- vlib_buffer_free (vm, &vring->buffers[slot], 1);
- used--;
- last++;
- n_left--;
+ while (e->id == n_buffers)
+ {
+ n_left--;
+ last++;
+ n_buffers++;
+ if (n_left == 0)
+ break;
+ e = &vring->used->ring[last & mask];
+ }
+ vlib_buffer_free_from_ring (vm, vring->buffers, slot,
+ sz, (n_buffers - slot));
+ used -= (n_buffers - slot);
+
+ if (n_left > 0)
+ {
+ slot = e->id;
+
+ vlib_buffer_free (vm, &vring->buffers[slot], 1);
+ used--;
+ last++;
+ n_left--;
+ }
}
vring->desc_in_use = used;
vring->last_used_idx = last;
}
-#endif /* CLIB_MARCH_VARIANT */
static_always_inline u16
add_buffer_to_slot (vlib_main_t * vm, virtio_if_t * vif,
@@ -170,9 +187,15 @@ add_buffer_to_slot (vlib_main_t * vm, virtio_if_t * vif,
* It can easily support 65535 bytes of Jumbo frames with
* each data buffer size of 512 bytes minimum.
*/
- vlib_buffer_t *indirect_desc =
- vlib_get_buffer (vm, vring->indirect_buffers[next]);
+ u32 indirect_buffer = 0;
+ if (PREDICT_FALSE (vlib_buffer_alloc (vm, &indirect_buffer, 1) == 0))
+ return n_added;
+
+ vlib_buffer_t *indirect_desc = vlib_get_buffer (vm, indirect_buffer);
indirect_desc->current_data = 0;
+ indirect_desc->flags |= VLIB_BUFFER_NEXT_PRESENT;
+ indirect_desc->next_buffer = bi;
+ bi = indirect_buffer;
struct vring_desc *id =
(struct vring_desc *) vlib_buffer_get_current (indirect_desc);
@@ -261,7 +284,7 @@ virtio_interface_tx_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
virtio_kick (vm, vring, vif);
/* free consumed buffers */
- virtio_free_used_desc (vm, vring);
+ virtio_free_used_device_desc (vm, vring);
used = vring->desc_in_use;
next = vring->desc_next;
diff --git a/src/vnet/devices/virtio/pci.c b/src/vnet/devices/virtio/pci.c
index 31027f7f687..5076d9f4dbd 100644
--- a/src/vnet/devices/virtio/pci.c
+++ b/src/vnet/devices/virtio/pci.c
@@ -679,20 +679,8 @@ virtio_pci_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 queue_num)
ASSERT (vring->buffers == 0);
vec_validate_aligned (vring->buffers, queue_size, CLIB_CACHE_LINE_BYTES);
- ASSERT (vring->indirect_buffers == 0);
- vec_validate_aligned (vring->indirect_buffers, queue_size,
- CLIB_CACHE_LINE_BYTES);
if (queue_num % 2)
{
- u32 n_alloc = 0;
- do
- {
- if (n_alloc < queue_size)
- n_alloc =
- vlib_buffer_alloc (vm, vring->indirect_buffers + n_alloc,
- queue_size - n_alloc);
- }
- while (n_alloc != queue_size);
vif->num_txqs++;
virtio_log_debug (vim, vif, "tx-queue: number %u, size %u", queue_num,
queue_size);
@@ -1239,7 +1227,6 @@ virtio_pci_delete_if (vlib_main_t * vm, virtio_if_t * vif)
virtio_free_rx_buffers (vm, vring);
}
vec_free (vring->buffers);
- vec_free (vring->indirect_buffers);
vlib_physmem_free (vm, vring->desc);
}
@@ -1252,12 +1239,7 @@ virtio_pci_delete_if (vlib_main_t * vm, virtio_if_t * vif)
{
virtio_free_used_desc (vm, vring);
}
- if (vring->queue_id % 2)
- {
- vlib_buffer_free_no_next (vm, vring->indirect_buffers, vring->size);
- }
vec_free (vring->buffers);
- vec_free (vring->indirect_buffers);
vlib_physmem_free (vm, vring->desc);
}
diff --git a/src/vnet/devices/virtio/virtio.c b/src/vnet/devices/virtio/virtio.c
index 9fbbc1bf508..90acb75fa6d 100644
--- a/src/vnet/devices/virtio/virtio.c
+++ b/src/vnet/devices/virtio/virtio.c
@@ -113,20 +113,6 @@ virtio_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 idx, u16 sz)
vring->queue_id = idx;
ASSERT (vring->buffers == 0);
vec_validate_aligned (vring->buffers, sz, CLIB_CACHE_LINE_BYTES);
- ASSERT (vring->indirect_buffers == 0);
- vec_validate_aligned (vring->indirect_buffers, sz, CLIB_CACHE_LINE_BYTES);
- if (idx % 2)
- {
- u32 n_alloc = 0;
- do
- {
- if (n_alloc < sz)
- n_alloc +=
- vlib_buffer_alloc (vm, vring->indirect_buffers + n_alloc,
- sz - n_alloc);
- }
- while (n_alloc != sz);
- }
vring->size = sz;
vring->call_fd = eventfd (0, EFD_NONBLOCK | EFD_CLOEXEC);
@@ -196,10 +182,35 @@ virtio_vring_free_rx (vlib_main_t * vm, virtio_if_t * vif, u32 idx)
if (vring->avail)
clib_mem_free (vring->avail);
vec_free (vring->buffers);
- vec_free (vring->indirect_buffers);
return 0;
}
+inline void
+virtio_free_used_desc (vlib_main_t * vm, virtio_vring_t * vring)
+{
+ u16 used = vring->desc_in_use;
+ u16 sz = vring->size;
+ u16 mask = sz - 1;
+ u16 last = vring->last_used_idx;
+ u16 n_left = vring->used->idx - last;
+
+ if (n_left == 0)
+ return;
+
+ while (n_left)
+ {
+ struct vring_used_elem *e = &vring->used->ring[last & mask];
+ u16 slot = e->id;
+
+ vlib_buffer_free (vm, &vring->buffers[slot], 1);
+ used--;
+ last++;
+ n_left--;
+ }
+ vring->desc_in_use = used;
+ vring->last_used_idx = last;
+}
+
clib_error_t *
virtio_vring_free_tx (vlib_main_t * vm, virtio_if_t * vif, u32 idx)
{
@@ -218,9 +229,7 @@ virtio_vring_free_tx (vlib_main_t * vm, virtio_if_t * vif, u32 idx)
clib_mem_free (vring->desc);
if (vring->avail)
clib_mem_free (vring->avail);
- vlib_buffer_free_no_next (vm, vring->indirect_buffers, vring->size);
vec_free (vring->buffers);
- vec_free (vring->indirect_buffers);
return 0;
}
diff --git a/src/vnet/devices/virtio/virtio.h b/src/vnet/devices/virtio/virtio.h
index 1de704386a8..a02d64db061 100644
--- a/src/vnet/devices/virtio/virtio.h
+++ b/src/vnet/devices/virtio/virtio.h
@@ -115,7 +115,6 @@ typedef struct
u16 flags;
u32 call_file_index;
u32 *buffers;
- u32 *indirect_buffers;
u16 last_used_idx;
u16 last_kick_avail_idx;
} virtio_vring_t;