Add histogram to frame queue tracing

Display it via "show frame histogram" CLI.

Change-Id: I436a2125f391af85d1743cf8765e5f27fa0ca809
Signed-off-by: Todd Foggoa (tfoggoa) <tfoggoa@cisco.com>

3 files changed, 132 insertions, 32 deletions

diff --git a/vnet/vnet/devices/dpdk/cli.c b/vnet/vnet/devices/dpdk/cli.c
index 6fa3b231..9ec19867 100644
--- a/vnet/vnet/devices/dpdk/cli.c
+++ b/vnet/vnet/devices/dpdk/cli.c
@@ -25,8 +25,6 @@
 
 #include "dpdk_priv.h"
 
-frame_queue_trace_t *frame_queue_traces;
-
 static clib_error_t *
 pcap_trace_command_fn (vlib_main_t * vm,
      unformat_input_t * input,
@@ -293,6 +291,7 @@ trace_frame_queue (vlib_main_t *vm, unformat_input_t *input,
 {
   clib_error_t * error = NULL;
   frame_queue_trace_t *fqt;
+  frame_queue_nelt_counter_t *fqh;
   u32 num_fq;
   u32 fqix;
   u32 enable = 0;
@@ -313,13 +312,16 @@ trace_frame_queue (vlib_main_t *vm, unformat_input_t *input,
     }
 
   // Allocate storage for trace if necessary
-  vec_validate_aligned(frame_queue_traces, num_fq-1, CLIB_CACHE_LINE_BYTES);
+  vec_validate_aligned(dpdk_main.frame_queue_traces, num_fq-1, CLIB_CACHE_LINE_BYTES);
+  vec_validate_aligned(dpdk_main.frame_queue_histogram, num_fq-1, CLIB_CACHE_LINE_BYTES);
 
   for (fqix=0; fqix<num_fq; fqix++) {
-    fqt = &frame_queue_traces[fqix];
+    fqt = &dpdk_main.frame_queue_traces[fqix];
+    fqh = &dpdk_main.frame_queue_histogram[fqix];
 
     memset(fqt->n_vectors, 0xff, sizeof(fqt->n_vectors));
     fqt->written = 0;
+    memset(fqh, 0, sizeof(*fqh));
     vlib_frame_queues[fqix]->trace = enable;
   }
   return error;
@@ -334,26 +336,50 @@ VLIB_CLI_COMMAND (cmd_trace_frame_queue,static) = {
 
 
 /*
+ * Adding two counters and compute percent of total
+ * Round up, e.g. 0.000001 => 1%
+ */
+static u32
+compute_percent (u64 *two_counters, u64 total)
+{
+    if (total == 0)
+      {
+        return 0;
+      }
+    else
+      {
+        return (((two_counters[0] + two_counters[1]) * 100) + (total-1)) / total;
+      }
+}
+
+/*
  * Display frame queue trace data gathered by threads.
  */
 static clib_error_t *
-show_frame_queue (vlib_main_t *vm, unformat_input_t *input,
-                  vlib_cli_command_t *cmd)
+show_frame_queue_internal (vlib_main_t *vm,
+                           u32          histogram)
 {
   clib_error_t * error = NULL;
   frame_queue_trace_t *fqt;
+  frame_queue_nelt_counter_t *fqh;
   u32 num_fq;
   u32 fqix;
 
-  num_fq = vec_len(frame_queue_traces);
+  num_fq = vec_len(dpdk_main.frame_queue_traces);
   if (num_fq == 0)
     {
       vlib_cli_output(vm, "No trace data for frame queues\n");
       return error;
     }
 
+  if (histogram)
+    {
+      vlib_cli_output(vm, "0-1   2-3   4-5   6-7   8-9   10-11 12-13 14-15 "
+                          "16-17 18-19 20-21 22-23 24-25 26-27 28-29 30-31\n");
+    }
+
   for (fqix=0; fqix<num_fq; fqix++) {
-    fqt = &frame_queue_traces[fqix];
+    fqt = &(dpdk_main.frame_queue_traces[fqix]);
 
     vlib_cli_output(vm, "Thread %d %v\n", fqix, vlib_worker_threads[fqix].name);
 
@@ -363,32 +389,93 @@ show_frame_queue (vlib_main_t *vm, unformat_input_t *input,
         continue;
       }
 
-    vlib_cli_output(vm, "  vector-threshold %d  ring size %d  in use %d\n",
-                    fqt->threshold, fqt->nelts, fqt->n_in_use);
-    vlib_cli_output(vm, "  head %12d  head_hint %12d  tail %12d\n",
-                    fqt->head, fqt->head_hint, fqt->tail);
-    vlib_cli_output(vm, "  %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d\n",
-                    fqt->n_vectors[0], fqt->n_vectors[1], fqt->n_vectors[2], fqt->n_vectors[3],
-                    fqt->n_vectors[4], fqt->n_vectors[5], fqt->n_vectors[6], fqt->n_vectors[7],
-                    fqt->n_vectors[8], fqt->n_vectors[9], fqt->n_vectors[10], fqt->n_vectors[11],
-                    fqt->n_vectors[12], fqt->n_vectors[13], fqt->n_vectors[14], fqt->n_vectors[15]);
-
-    if (fqt->nelts > 16)
+    if (histogram)
       {
+        fqh = &(dpdk_main.frame_queue_histogram[fqix]);
+        u32 nelt;
+        u64 total = 0;
+
+        for (nelt=0; nelt<MAX_NELTS; nelt++) {
+            total += fqh->count[nelt];
+        }
+
+        /*
+         * Print in pairs to condense the output.
+         * Allow entries with 0 counts to be clearly identified, by rounding up.
+         * Any non-zero value will be displayed as at least one percent. This
+         * also means the sum of percentages can be > 100, but that is fine. The
+         * histogram is counted from the last time "trace frame on" was issued.
+         */
+        vlib_cli_output(vm,
+                        "%3d%%  %3d%%  %3d%%  %3d%%  %3d%%  %3d%%  %3d%%  %3d%%  "
+                        "%3d%%  %3d%%  %3d%%  %3d%%  %3d%%  %3d%%  %3d%%  %3d%%\n",
+                        compute_percent(&fqh->count[ 0], total),
+                        compute_percent(&fqh->count[ 2], total),
+                        compute_percent(&fqh->count[ 4], total),
+                        compute_percent(&fqh->count[ 6], total),
+                        compute_percent(&fqh->count[ 8], total),
+                        compute_percent(&fqh->count[10], total),
+                        compute_percent(&fqh->count[12], total),
+                        compute_percent(&fqh->count[14], total),
+                        compute_percent(&fqh->count[16], total),
+                        compute_percent(&fqh->count[18], total),
+                        compute_percent(&fqh->count[20], total),
+                        compute_percent(&fqh->count[22], total),
+                        compute_percent(&fqh->count[24], total),
+                        compute_percent(&fqh->count[26], total),
+                        compute_percent(&fqh->count[28], total),
+                        compute_percent(&fqh->count[30], total));
+      }
+    else
+      {
+        vlib_cli_output(vm, "  vector-threshold %d  ring size %d  in use %d\n",
+                        fqt->threshold, fqt->nelts, fqt->n_in_use);
+        vlib_cli_output(vm, "  head %12d  head_hint %12d  tail %12d\n",
+                        fqt->head, fqt->head_hint, fqt->tail);
         vlib_cli_output(vm, "  %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d\n",
-                        fqt->n_vectors[16], fqt->n_vectors[17], fqt->n_vectors[18], fqt->n_vectors[19],
-                        fqt->n_vectors[20], fqt->n_vectors[21], fqt->n_vectors[22], fqt->n_vectors[23],
-                        fqt->n_vectors[24], fqt->n_vectors[25], fqt->n_vectors[26], fqt->n_vectors[27],
-                        fqt->n_vectors[28], fqt->n_vectors[29], fqt->n_vectors[30], fqt->n_vectors[31]);
+                        fqt->n_vectors[0], fqt->n_vectors[1], fqt->n_vectors[2], fqt->n_vectors[3],
+                        fqt->n_vectors[4], fqt->n_vectors[5], fqt->n_vectors[6], fqt->n_vectors[7],
+                        fqt->n_vectors[8], fqt->n_vectors[9], fqt->n_vectors[10], fqt->n_vectors[11],
+                        fqt->n_vectors[12], fqt->n_vectors[13], fqt->n_vectors[14], fqt->n_vectors[15]);
+
+        if (fqt->nelts > 16)
+          {
+            vlib_cli_output(vm, "  %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d\n",
+                            fqt->n_vectors[16], fqt->n_vectors[17], fqt->n_vectors[18], fqt->n_vectors[19],
+                            fqt->n_vectors[20], fqt->n_vectors[21], fqt->n_vectors[22], fqt->n_vectors[23],
+                            fqt->n_vectors[24], fqt->n_vectors[25], fqt->n_vectors[26], fqt->n_vectors[27],
+                            fqt->n_vectors[28], fqt->n_vectors[29], fqt->n_vectors[30], fqt->n_vectors[31]);
+          }
       }
-  }
+
+   }
   return error;
 }
 
-VLIB_CLI_COMMAND (cmd_show_frame_queue,static) = {
+static clib_error_t *
+show_frame_queue_trace (vlib_main_t *vm, unformat_input_t *input,
+                        vlib_cli_command_t *cmd)
+{
+  return show_frame_queue_internal (vm, 0);
+}
+
+static clib_error_t *
+show_frame_queue_histogram (vlib_main_t *vm, unformat_input_t *input,
+                            vlib_cli_command_t *cmd)
+{
+  return show_frame_queue_internal (vm, 1);
+}
+
+VLIB_CLI_COMMAND (cmd_show_frame_queue_trace,static) = {
     .path = "show frame-queue",
     .short_help = "show frame-queue trace",
-    .function = show_frame_queue,
+    .function = show_frame_queue_trace,
+};
+
+VLIB_CLI_COMMAND (cmd_show_frame_queue_histogram,static) = {
+    .path = "show frame-queue histogram",
+    .short_help = "show frame-queue histogram",
+    .function = show_frame_queue_histogram,
 };
 
 
diff --git a/vnet/vnet/devices/dpdk/dpdk.h b/vnet/vnet/devices/dpdk/dpdk.h
index 3761a7f7..c6b0711d 100644
--- a/vnet/vnet/devices/dpdk/dpdk.h
+++ b/vnet/vnet/devices/dpdk/dpdk.h
@@ -267,6 +267,10 @@ typedef struct {
   i32 n_vectors[MAX_NELTS];
 } frame_queue_trace_t;
 
+typedef struct {
+  u64 count[MAX_NELTS];
+} frame_queue_nelt_counter_t;
+
 #define DPDK_TX_RING_SIZE (4 * 1024)
 
 #define DPDK_STATS_POLL_INTERVAL      (10.0)
@@ -396,6 +400,10 @@ typedef struct {
   f64 link_state_poll_interval;
   f64 stat_poll_interval;
 
+  /* for frame queue tracing */
+  frame_queue_trace_t        *frame_queue_traces;
+  frame_queue_nelt_counter_t *frame_queue_histogram;
+
   /* convenience */
   vlib_main_t * vlib_main;
   vnet_main_t * vnet_main;
diff --git a/vnet/vnet/devices/dpdk/threads.c b/vnet/vnet/devices/dpdk/threads.c
index eeb440e2..d8fb698e 100644
--- a/vnet/vnet/devices/dpdk/threads.c
+++ b/vnet/vnet/devices/dpdk/threads.c
@@ -35,9 +35,6 @@
 #include <vlibmemory/vl_memory_api_h.h> 
 #undef vl_printfun
 
-vlib_thread_main_t vlib_thread_main;
-
-frame_queue_trace_t *frame_queue_traces;
 
 /*
  * Check the frame queue to see if any frames are available.
@@ -65,19 +62,27 @@ static inline int vlib_frame_queue_dequeue_internal (vlib_main_t *vm)
   if (PREDICT_FALSE(fq->trace))
     {
       frame_queue_trace_t *fqt;
+      frame_queue_nelt_counter_t *fqh;
       u32 elix;
    
-      fqt = &frame_queue_traces[thread_id];
+      fqt = &dpdk_main.frame_queue_traces[thread_id];
+
       fqt->nelts = fq->nelts;
       fqt->head = fq->head;
       fqt->head_hint = fq->head_hint;
       fqt->tail = fq->tail;
       fqt->threshold = fq->vector_threshold;
       fqt->n_in_use = fqt->tail - fqt->head;
-      if (fqt->n_in_use > fqt->nelts){
-        fqt->n_in_use = 0;
+      if (fqt->n_in_use >= fqt->nelts){
+        // if beyond max then use max
+        fqt->n_in_use = fqt->nelts-1;
       }
 
+      /* Record the number of elements in use in the histogram */
+      fqh = &dpdk_main.frame_queue_histogram[thread_id];
+      fqh->count[ fqt->n_in_use ]++;
+
+      /* Record a snapshot of the elements in use */
       for (elix=0; elix<fqt->nelts; elix++) {
         elt = fq->elts + ((fq->head+1 + elix) & (fq->nelts-1));
         if (1 || elt->valid)