4 files changed, 61 insertions, 48 deletions

diff --git a/src/vnet/devices/virtio/device.c b/src/vnet/devices/virtio/device.c
index 04cf9ec49c8..e560c29e5e7 100644
--- a/src/vnet/devices/virtio/device.c
+++ b/src/vnet/devices/virtio/device.c
@@ -90,9 +90,8 @@ format_virtio_tx_trace (u8 * s, va_list * args)
   return s;
 }
 
-#ifndef CLIB_MARCH_VARIANT
-inline void
-virtio_free_used_desc (vlib_main_t * vm, virtio_vring_t * vring)
+static_always_inline void
+virtio_free_used_device_desc (vlib_main_t * vm, virtio_vring_t * vring)
 {
   u16 used = vring->desc_in_use;
   u16 sz = vring->size;
@@ -106,17 +105,35 @@ virtio_free_used_desc (vlib_main_t * vm, virtio_vring_t * vring)
   while (n_left)
     {
       struct vring_used_elem *e = &vring->used->ring[last & mask];
-      u16 slot = e->id;
+      u16 slot, n_buffers;
+      slot = n_buffers = e->id;
 
-      vlib_buffer_free (vm, &vring->buffers[slot], 1);
-      used--;
-      last++;
-      n_left--;
+      while (e->id == n_buffers)
+	{
+	  n_left--;
+	  last++;
+	  n_buffers++;
+	  if (n_left == 0)
+	    break;
+	  e = &vring->used->ring[last & mask];
+	}
+      vlib_buffer_free_from_ring (vm, vring->buffers, slot,
+				  sz, (n_buffers - slot));
+      used -= (n_buffers - slot);
+
+      if (n_left > 0)
+	{
+	  slot = e->id;
+
+	  vlib_buffer_free (vm, &vring->buffers[slot], 1);
+	  used--;
+	  last++;
+	  n_left--;
+	}
     }
   vring->desc_in_use = used;
   vring->last_used_idx = last;
 }
-#endif /* CLIB_MARCH_VARIANT */
 
 static_always_inline u16
 add_buffer_to_slot (vlib_main_t * vm, virtio_if_t * vif,
@@ -170,9 +187,15 @@ add_buffer_to_slot (vlib_main_t * vm, virtio_if_t * vif,
        * It can easily support 65535 bytes of Jumbo frames with
        * each data buffer size of 512 bytes minimum.
        */
-      vlib_buffer_t *indirect_desc =
-	vlib_get_buffer (vm, vring->indirect_buffers[next]);
+      u32 indirect_buffer = 0;
+      if (PREDICT_FALSE (vlib_buffer_alloc (vm, &indirect_buffer, 1) == 0))
+	return n_added;
+
+      vlib_buffer_t *indirect_desc = vlib_get_buffer (vm, indirect_buffer);
       indirect_desc->current_data = 0;
+      indirect_desc->flags |= VLIB_BUFFER_NEXT_PRESENT;
+      indirect_desc->next_buffer = bi;
+      bi = indirect_buffer;
 
       struct vring_desc *id =
 	(struct vring_desc *) vlib_buffer_get_current (indirect_desc);
@@ -261,7 +284,7 @@ virtio_interface_tx_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
     virtio_kick (vm, vring, vif);
 
   /* free consumed buffers */
-  virtio_free_used_desc (vm, vring);
+  virtio_free_used_device_desc (vm, vring);
 
   used = vring->desc_in_use;
   next = vring->desc_next;
diff --git a/src/vnet/devices/virtio/pci.c b/src/vnet/devices/virtio/pci.c
index 31027f7f687..5076d9f4dbd 100644
--- a/src/vnet/devices/virtio/pci.c
+++ b/src/vnet/devices/virtio/pci.c
@@ -679,20 +679,8 @@ virtio_pci_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 queue_num)
 
   ASSERT (vring->buffers == 0);
   vec_validate_aligned (vring->buffers, queue_size, CLIB_CACHE_LINE_BYTES);
-  ASSERT (vring->indirect_buffers == 0);
-  vec_validate_aligned (vring->indirect_buffers, queue_size,
-			CLIB_CACHE_LINE_BYTES);
   if (queue_num % 2)
     {
-      u32 n_alloc = 0;
-      do
-	{
-	  if (n_alloc < queue_size)
-	    n_alloc =
-	      vlib_buffer_alloc (vm, vring->indirect_buffers + n_alloc,
-				 queue_size - n_alloc);
-	}
-      while (n_alloc != queue_size);
       vif->num_txqs++;
       virtio_log_debug (vim, vif, "tx-queue: number %u, size %u", queue_num,
 			queue_size);
@@ -1239,7 +1227,6 @@ virtio_pci_delete_if (vlib_main_t * vm, virtio_if_t * vif)
 	virtio_free_rx_buffers (vm, vring);
       }
     vec_free (vring->buffers);
-    vec_free (vring->indirect_buffers);
     vlib_physmem_free (vm, vring->desc);
   }
 
@@ -1252,12 +1239,7 @@ virtio_pci_delete_if (vlib_main_t * vm, virtio_if_t * vif)
       {
 	virtio_free_used_desc (vm, vring);
       }
-    if (vring->queue_id % 2)
-      {
-	vlib_buffer_free_no_next (vm, vring->indirect_buffers, vring->size);
-      }
     vec_free (vring->buffers);
-    vec_free (vring->indirect_buffers);
     vlib_physmem_free (vm, vring->desc);
   }
 
diff --git a/src/vnet/devices/virtio/virtio.c b/src/vnet/devices/virtio/virtio.c
index 9fbbc1bf508..90acb75fa6d 100644
--- a/src/vnet/devices/virtio/virtio.c
+++ b/src/vnet/devices/virtio/virtio.c
@@ -113,20 +113,6 @@ virtio_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 idx, u16 sz)
   vring->queue_id = idx;
   ASSERT (vring->buffers == 0);
   vec_validate_aligned (vring->buffers, sz, CLIB_CACHE_LINE_BYTES);
-  ASSERT (vring->indirect_buffers == 0);
-  vec_validate_aligned (vring->indirect_buffers, sz, CLIB_CACHE_LINE_BYTES);
-  if (idx % 2)
-    {
-      u32 n_alloc = 0;
-      do
-	{
-	  if (n_alloc < sz)
-	    n_alloc +=
-	      vlib_buffer_alloc (vm, vring->indirect_buffers + n_alloc,
-				 sz - n_alloc);
-	}
-      while (n_alloc != sz);
-    }
 
   vring->size = sz;
   vring->call_fd = eventfd (0, EFD_NONBLOCK | EFD_CLOEXEC);
@@ -196,10 +182,35 @@ virtio_vring_free_rx (vlib_main_t * vm, virtio_if_t * vif, u32 idx)
   if (vring->avail)
     clib_mem_free (vring->avail);
   vec_free (vring->buffers);
-  vec_free (vring->indirect_buffers);
   return 0;
 }
 
+inline void
+virtio_free_used_desc (vlib_main_t * vm, virtio_vring_t * vring)
+{
+  u16 used = vring->desc_in_use;
+  u16 sz = vring->size;
+  u16 mask = sz - 1;
+  u16 last = vring->last_used_idx;
+  u16 n_left = vring->used->idx - last;
+
+  if (n_left == 0)
+    return;
+
+  while (n_left)
+    {
+      struct vring_used_elem *e = &vring->used->ring[last & mask];
+      u16 slot = e->id;
+
+      vlib_buffer_free (vm, &vring->buffers[slot], 1);
+      used--;
+      last++;
+      n_left--;
+    }
+  vring->desc_in_use = used;
+  vring->last_used_idx = last;
+}
+
 clib_error_t *
 virtio_vring_free_tx (vlib_main_t * vm, virtio_if_t * vif, u32 idx)
 {
@@ -218,9 +229,7 @@ virtio_vring_free_tx (vlib_main_t * vm, virtio_if_t * vif, u32 idx)
     clib_mem_free (vring->desc);
   if (vring->avail)
     clib_mem_free (vring->avail);
-  vlib_buffer_free_no_next (vm, vring->indirect_buffers, vring->size);
   vec_free (vring->buffers);
-  vec_free (vring->indirect_buffers);
   return 0;
 }
 
diff --git a/src/vnet/devices/virtio/virtio.h b/src/vnet/devices/virtio/virtio.h
index 1de704386a8..a02d64db061 100644
--- a/src/vnet/devices/virtio/virtio.h
+++ b/src/vnet/devices/virtio/virtio.h
@@ -115,7 +115,6 @@ typedef struct
   u16 flags;
   u32 call_file_index;
   u32 *buffers;
-  u32 *indirect_buffers;
   u16 last_used_idx;
   u16 last_kick_avail_idx;
 } virtio_vring_t;