tapv2: deleting tap interface may leak buffers (VPP-1124)

Buffers may be allocated for indirect descriptors by tx thread and
they are freed when tx thread is invoked in the next invocation.
This is to allow the recipient (kernel) to have a chance to process
them. But if the tap interface is deleted, the tx thread may not yet
be called to clean up the indirect descriptors' buffers. In that case,
we need to remove them without waiting for the tx thread to be called.
Failure to do so may cause buffers leak when the tap interface is deleted.

For the RX ring, leakage also exists for vring->buffers when the interface
is removed.

Change-Id: I3df313a0e60334776b19daf51a9f5bf20dfdc489
Signed-off-by: Steven <sluong@cisco.com>

4 files changed, 34 insertions, 11 deletions

diff --git a/src/vnet/devices/tap/tap.c b/src/vnet/devices/tap/tap.c
index ff7bd91e8db..a7d10fe5473 100644
--- a/src/vnet/devices/tap/tap.c
+++ b/src/vnet/devices/tap/tap.c
@@ -390,7 +390,7 @@ error:
     close (vif->tap_fd);
   if (vif->fd != -1)
     close (vif->fd);
-  vec_foreach_index (i, vif->vrings) virtio_vring_free (vif, i);
+  vec_foreach_index (i, vif->vrings) virtio_vring_free (vm, vif, i);
   memset (vif, 0, sizeof (virtio_if_t));
   pool_put (vim->interfaces, vif);
 
@@ -429,7 +429,7 @@ tap_delete_if (vlib_main_t * vm, u32 sw_if_index)
   if (vif->fd != -1)
     close (vif->fd);
 
-  vec_foreach_index (i, vif->vrings) virtio_vring_free (vif, i);
+  vec_foreach_index (i, vif->vrings) virtio_vring_free (vm, vif, i);
   vec_free (vif->vrings);
 
   hash_unset (tm->dev_instance_by_interface_id, vif->id);
diff --git a/src/vnet/devices/virtio/device.c b/src/vnet/devices/virtio/device.c
index df5fcf509b8..5528ddb0429 100644
--- a/src/vnet/devices/virtio/device.c
+++ b/src/vnet/devices/virtio/device.c
@@ -88,7 +88,7 @@ format_virtio_tx_trace (u8 * s, va_list * args)
   return s;
 }
 
-static_always_inline void
+inline void
 virtio_free_used_desc (vlib_main_t * vm, virtio_vring_t * vring)
 {
   u16 used = vring->desc_in_use;
diff --git a/src/vnet/devices/virtio/virtio.c b/src/vnet/devices/virtio/virtio.c
index 7829da6b1a1..3867cf03a46 100644
--- a/src/vnet/devices/virtio/virtio.c
+++ b/src/vnet/devices/virtio/virtio.c
@@ -134,20 +134,41 @@ error:
   return err;
 }
 
+static_always_inline void
+virtio_free_rx_buffers (vlib_main_t * vm, virtio_vring_t * vring)
+{
+  u16 used = vring->desc_in_use;
+  u16 next = vring->desc_next;
+  u16 mask = vring->size - 1;
+
+  while (used)
+    {
+      vlib_buffer_free (vm, &vring->buffers[next], 1);
+      next = (next + 1) & mask;
+      used--;
+    }
+}
+
 clib_error_t *
-virtio_vring_free (virtio_if_t * vif, u32 idx)
+virtio_vring_free (vlib_main_t * vm, virtio_if_t * vif, u32 idx)
 {
-  //TODO free buffers and indirect descriptor allocs
   virtio_vring_t *vring = vec_elt_at_index (vif->vrings, idx);
+
+  clib_file_del_by_index (&file_main, vring->call_file_index);
+  close (vring->kick_fd);
+  close (vring->call_fd);
+  if (vring->used)
+    {
+      if ((idx & 1) == 1)
+	virtio_free_used_desc (vm, vring);
+      else
+	virtio_free_rx_buffers (vm, vring);
+      clib_mem_free (vring->used);
+    }
   if (vring->desc)
     clib_mem_free (vring->desc);
   if (vring->avail)
     clib_mem_free (vring->avail);
-  if (vring->used)
-    clib_mem_free (vring->used);
-  clib_file_del_by_index (&file_main, vring->call_file_index);
-  close (vring->kick_fd);
-  close (vring->call_fd);
   vec_free (vring->buffers);
   return 0;
 }
diff --git a/src/vnet/devices/virtio/virtio.h b/src/vnet/devices/virtio/virtio.h
index 079223d233e..90eeb536ab1 100644
--- a/src/vnet/devices/virtio/virtio.h
+++ b/src/vnet/devices/virtio/virtio.h
@@ -128,7 +128,9 @@ extern vlib_node_registration_t virtio_input_node;
 
 clib_error_t *virtio_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 idx,
 				 u16 sz);
-clib_error_t *virtio_vring_free (virtio_if_t * vif, u32 idx);
+clib_error_t *virtio_vring_free (vlib_main_t * vm, virtio_if_t * vif,
+				 u32 idx);
+extern void virtio_free_used_desc (vlib_main_t * vm, virtio_vring_t * vring);
 
 #endif /* _VNET_DEVICES_VIRTIO_VIRTIO_H_ */