Collect per Q stats for vhost-user interface

Change-Id: I394960c300ff7a81c4c8e05afd5a4175e66666eb
Signed-off-by: Shesha Sreenivasamurthy <shesha@cisco.com>

5 files changed, 87 insertions, 9 deletions

diff --git a/vnet/vnet/devices/dpdk/device.c b/vnet/vnet/devices/dpdk/device.c
index a93e9f4cf82..02703cc59b2 100644
--- a/vnet/vnet/devices/dpdk/device.c
+++ b/vnet/vnet/devices/dpdk/device.c
@@ -308,14 +308,24 @@ u32 tx_burst_vector_internal (vlib_main_t * vm,
 #endif
           if (PREDICT_TRUE(tx_head > tx_tail)) 
             {
+              int i; u32 bytes = 0;
+              struct rte_mbuf **pkts = &tx_vector[tx_tail];
+              for (i = 0; i < (tx_head - tx_tail); i++) {
+                  struct rte_mbuf *buff = pkts[i];
+                  bytes += rte_pktmbuf_data_len(buff);
+              } 
+                
               /* no wrap, transmit in one burst */
               rv = rte_vhost_enqueue_burst(&xd->vu_vhost_dev, offset + VIRTIO_RXQ,
                                            &tx_vector[tx_tail],
                                            (uint16_t) (tx_head-tx_tail));
               if (PREDICT_TRUE(rv > 0))
                 {
+                  dpdk_vu_vring *vring = &(xd->vu_intf->vrings[offset + VIRTIO_RXQ]);
+                  vring->packets += rv;
+                  vring->bytes += bytes;
+
                   if (dpdk_vhost_user_want_interrupt(xd, offset + VIRTIO_RXQ)) {
-                    dpdk_vu_vring *vring = &(xd->vu_intf->vrings[offset + VIRTIO_RXQ]);
                     vring->n_since_last_int += rv;
 
                     f64 now = vlib_time_now (vm);
@@ -336,14 +346,23 @@ u32 tx_burst_vector_internal (vlib_main_t * vm,
                * so we can try to transmit the rest. If we didn't transmit
                * everything, stop now.
                */
+              int i; u32 bytes = 0;
+              struct rte_mbuf **pkts = &tx_vector[tx_tail];
+              for (i = 0; i < (DPDK_TX_RING_SIZE - tx_tail); i++) {
+                  struct rte_mbuf *buff = pkts[i];
+                  bytes += rte_pktmbuf_data_len(buff);
+              }
               rv = rte_vhost_enqueue_burst(&xd->vu_vhost_dev, offset + VIRTIO_RXQ,
                                            &tx_vector[tx_tail], 
                                            (uint16_t) (DPDK_TX_RING_SIZE - tx_tail));
 
               if (PREDICT_TRUE(rv > 0))
                 {
+                  dpdk_vu_vring *vring = &(xd->vu_intf->vrings[offset + VIRTIO_RXQ]);
+                  vring->packets += rv;
+                  vring->bytes += bytes;
+
                   if (dpdk_vhost_user_want_interrupt(xd, offset + VIRTIO_RXQ)) {
-                    dpdk_vu_vring *vring = &(xd->vu_intf->vrings[offset + VIRTIO_RXQ]);
                     vring->n_since_last_int += rv;
 
                     f64 now = vlib_time_now (vm);
@@ -825,6 +844,14 @@ static void dpdk_clear_hw_interface_counters (u32 instance)
       memset (&xd->last_stats, 0, sizeof (xd->last_stats));
     }
   rte_eth_xstats_reset(xd->device_index);
+
+  if (PREDICT_FALSE(xd->dev_type == VNET_DPDK_DEV_VHOST_USER)) {
+    int i;
+    for (i = 0; i < xd->rx_q_used * VIRTIO_QNUM; i++) {
+        xd->vu_intf->vrings[i].packets = 0;
+        xd->vu_intf->vrings[i].bytes = 0;
+    }
+  }
 }
 
 #ifdef RTE_LIBRTE_KNI
diff --git a/vnet/vnet/devices/dpdk/dpdk.h b/vnet/vnet/devices/dpdk/dpdk.h
index e90f4030ce9..e0ab0c581c2 100644
--- a/vnet/vnet/devices/dpdk/dpdk.h
+++ b/vnet/vnet/devices/dpdk/dpdk.h
@@ -148,6 +148,8 @@ typedef struct {
   u32 callfd_idx;
   u32 n_since_last_int;
   f64 int_deadline;
+  u64 packets;
+  u64 bytes;
 } dpdk_vu_vring;
 
 typedef struct {
diff --git a/vnet/vnet/devices/dpdk/dpdk_priv.h b/vnet/vnet/devices/dpdk/dpdk_priv.h
index 49e69f44cc7..2e4909f99b8 100644
--- a/vnet/vnet/devices/dpdk/dpdk_priv.h
+++ b/vnet/vnet/devices/dpdk/dpdk_priv.h
@@ -96,11 +96,21 @@ dpdk_rx_burst ( dpdk_main_t * dm, dpdk_device_t * xd, u16 queue_id)
                                           bm->pktmbuf_pools[socket_id],
                                           xd->rx_vectors[queue_id], VLIB_FRAME_SIZE);
 
+      int i; u32 bytes = 0;
+      struct rte_mbuf **pkts = xd->rx_vectors[queue_id];
+      for (i = 0; i < n_buffers; i++) {
+          struct rte_mbuf *buff = pkts[i];
+          bytes += rte_pktmbuf_data_len(buff);
+      } 
+
       f64 now = vlib_time_now (vm);
 
+      dpdk_vu_vring *vring = &(xd->vu_intf->vrings[offset + VIRTIO_TXQ]);
+      vring->packets += n_buffers;
+      vring->bytes += bytes;
+
       /* send pending interrupts if needed */
       if (dpdk_vhost_user_want_interrupt(xd, offset + VIRTIO_TXQ)) {
-          dpdk_vu_vring *vring = &(xd->vu_intf->vrings[offset + VIRTIO_TXQ]);
           vring->n_since_last_int += n_buffers;
 
           if ((vring->n_since_last_int && (vring->int_deadline < now))
@@ -109,7 +119,7 @@ dpdk_rx_burst ( dpdk_main_t * dm, dpdk_device_t * xd, u16 queue_id)
       }
 
       if (dpdk_vhost_user_want_interrupt(xd, offset + VIRTIO_RXQ)) {
-          dpdk_vu_vring *vring = &(xd->vu_intf->vrings[offset + VIRTIO_RXQ]);
+          vring = &(xd->vu_intf->vrings[offset + VIRTIO_RXQ]);
           if (vring->n_since_last_int && (vring->int_deadline < now))
             dpdk_vhost_user_send_interrupt(vm, xd, offset + VIRTIO_RXQ);
       }
diff --git a/vnet/vnet/devices/dpdk/format.c b/vnet/vnet/devices/dpdk/format.c
index daf4386dbd6..447dfffb1b6 100644
--- a/vnet/vnet/devices/dpdk/format.c
+++ b/vnet/vnet/devices/dpdk/format.c
@@ -513,6 +513,13 @@ u8 * format_dpdk_device (u8 * s, va_list * args)
                  format_dpdk_rss_hf_name, di.flow_type_rss_offloads);
     }
 
+    if (verbose && xd->dev_type == VNET_DPDK_DEV_VHOST_USER) {
+        s = format(s, "%Uqueue size (max):  rx %d (%d) tx %d (%d)\n",
+                 format_white_space, indent + 2,
+                 xd->rx_q_used, xd->rx_q_used,
+                 xd->tx_q_used, xd->tx_q_used);
+    }
+
   if (xd->cpu_socket > -1)
     s = format (s, "%Ucpu socket %d",
                 format_white_space, indent + 2,
@@ -536,7 +543,7 @@ u8 * format_dpdk_device (u8 * s, va_list * args)
 
   vec_foreach(xstat, xd->xstats)
     {
-      if (xstat->value)
+      if (verbose == 2 || (verbose && xstat->value))
         {
           /* format_c_identifier don't like c strings inside vector */
           u8 * name = format(0,"%s", xstat->name);
@@ -547,6 +554,34 @@ u8 * format_dpdk_device (u8 * s, va_list * args)
         }
     }
 
+    if (verbose && xd->dev_type == VNET_DPDK_DEV_VHOST_USER) {
+        int i;
+        for (i = 0; i < xd->rx_q_used * VIRTIO_QNUM; i++) {
+            u8 * name;
+            if (verbose == 2 || xd->vu_intf->vrings[i].packets) {
+                if (i & 1) {
+                    name = format(NULL, "tx q%d packets", i >> 1);
+                } else {
+                    name = format(NULL, "rx q%d packets", i >> 1);
+                }
+                xs = format(xs, "\n%U%-38U%16Ld",
+                    format_white_space, indent + 4,
+                    format_c_identifier, name, xd->vu_intf->vrings[i].packets);
+                vec_free(name);
+
+                if (i & 1) {
+                    name = format(NULL, "tx q%d bytes", i >> 1);
+                } else {
+                    name = format(NULL, "rx q%d bytes", i >> 1);
+                }
+                xs = format(xs, "\n%U%-38U%16Ld",
+                    format_white_space, indent + 4,
+                    format_c_identifier, name, xd->vu_intf->vrings[i].bytes);
+                vec_free(name);
+            }
+        }
+    }
+
   if (xs)
     {
       s = format(s, "\n%Uextended stats:%v",
diff --git a/vnet/vnet/devices/dpdk/vhost_user.c b/vnet/vnet/devices/dpdk/vhost_user.c
index 735c17514a3..9db4a0ac163 100644
--- a/vnet/vnet/devices/dpdk/vhost_user.c
+++ b/vnet/vnet/devices/dpdk/vhost_user.c
@@ -201,6 +201,7 @@ dpdk_create_vhost_user_if_internal (u32 * hw_if_index, u32 if_id, u8 *hwaddr)
   clib_error_t * error;
   dpdk_device_and_queue_t * dq;
   int num_qpairs = 1;
+  dpdk_vu_intf_t *vui = NULL;
 
 #if RTE_VERSION >= RTE_VERSION_NUM(2, 2, 0, 0)
   num_qpairs = dm->use_rss < 1 ? 1 : tm->n_vlib_mains;
@@ -238,11 +239,14 @@ dpdk_create_vhost_user_if_internal (u32 * hw_if_index, u32 if_id, u8 *hwaddr)
           xd->vu_if_id = if_id;
 
       // reset virtqueues
+      vui = xd->vu_intf;
       for (j = 0; j < num_qpairs * VIRTIO_QNUM; j++) {
           memset(xd->vu_vhost_dev.virtqueue[j], 0, sizeof(struct vhost_virtqueue));
           xd->vu_vhost_dev.virtqueue[j]->kickfd = -1; 
           xd->vu_vhost_dev.virtqueue[j]->callfd = -1; 
           xd->vu_vhost_dev.virtqueue[j]->backend = -1; 
+          vui->vrings[j].packets = 0;
+          vui->vrings[j].bytes = 0;
        }
 
       // reset lockp
@@ -283,7 +287,7 @@ dpdk_create_vhost_user_if_internal (u32 * hw_if_index, u32 if_id, u8 *hwaddr)
 
       xd->device_index = xd - dm->devices;
       xd->per_interface_next_index = ~0;
-      xd->vu_intf = NULL;
+      xd->vu_intf = clib_mem_alloc (sizeof(*(xd->vu_intf)));
 
       xd->vu_vhost_dev.mem = clib_mem_alloc (sizeof(struct virtio_memory) +
                                              VHOST_MEMORY_MAX_NREGIONS *
@@ -296,12 +300,15 @@ dpdk_create_vhost_user_if_internal (u32 * hw_if_index, u32 if_id, u8 *hwaddr)
        * New virtqueue structure is an array of VHOST_MAX_QUEUE_PAIRS * 2
        * We need to allocate numq pairs.
        */
+      vui = xd->vu_intf;
       for (j = 0; j < num_qpairs * VIRTIO_QNUM; j++) {
           xd->vu_vhost_dev.virtqueue[j] = clib_mem_alloc (sizeof(struct vhost_virtqueue));
           memset(xd->vu_vhost_dev.virtqueue[j], 0, sizeof(struct vhost_virtqueue));
           xd->vu_vhost_dev.virtqueue[j]->kickfd = -1; 
           xd->vu_vhost_dev.virtqueue[j]->callfd = -1; 
           xd->vu_vhost_dev.virtqueue[j]->backend = -1; 
+          vui->vrings[j].packets = 0;
+          vui->vrings[j].bytes = 0;
       }
 
       dpdk_device_lock_init(xd);
@@ -361,9 +368,6 @@ dpdk_create_vhost_user_if_internal (u32 * hw_if_index, u32 if_id, u8 *hwaddr)
   sw = vnet_get_hw_sw_interface (dm->vnet_main, xd->vlib_hw_if_index);
   xd->vlib_sw_if_index = sw->sw_if_index;
 
-  if (!xd->vu_intf)
-      xd->vu_intf = clib_mem_alloc (sizeof(*(xd->vu_intf)));
-
   *hw_if_index = xd->vlib_hw_if_index;
 
   DBG_SOCK("xd->device_index: %d, dm->input_cpu_count: %d, "