vlib: convert foreach_vlib_main macro to be more gdb and clang-format friendly

Type: improvement
Change-Id: I1152e58d7bfcb3c4347147f87a834d45ad51cdfe
Signed-off-by: Damjan Marion <damarion@cisco.com>

13 files changed, 315 insertions, 350 deletions

diff --git a/.clang-format b/.clang-format
index 8b5c955cede..bce671f766c 100644
--- a/.clang-format
+++ b/.clang-format
@@ -18,4 +18,5 @@ ForEachMacros:
   - 'vec_foreach_index'
   - 'vec_foreach_index_backwards'
   - 'vlib_foreach_rx_tx'
+  - 'foreach_vlib_main'
 
diff --git a/src/plugins/dispatch-trace/main.c b/src/plugins/dispatch-trace/main.c
index 0650997889d..ce662dc0eea 100644
--- a/src/plugins/dispatch-trace/main.c
+++ b/src/plugins/dispatch-trace/main.c
@@ -257,23 +257,24 @@ vlib_pcap_dispatch_trace_configure (vlib_pcap_dispatch_trace_args_t *a)
   /* Independent of enable/disable, to allow buffer trace multi nodes */
   if (a->buffer_trace_node_index != ~0)
     {
-      foreach_vlib_main (({
-	tm = &this_vlib_main->trace_main;
-	tm->verbose = 0; /* not sure this ever did anything... */
-	vec_validate (tm->nodes, a->buffer_trace_node_index);
-	tn = tm->nodes + a->buffer_trace_node_index;
-	tn->limit += a->buffer_traces_to_capture;
-	if (a->post_mortem)
-	  {
-	    tm->filter_flag = FILTER_FLAG_POST_MORTEM;
-	    tm->filter_count = ~0;
-	  }
-	tm->trace_enable = 1;
-	if (vlib_node_set_dispatch_wrapper (this_vlib_main,
-					    dispatch_pcap_trace))
-	  clib_warning (0, "Dispatch wrapper already in use on thread %u",
-			this_vlib_main->thread_index);
-      }));
+      foreach_vlib_main ()
+	{
+	  tm = &this_vlib_main->trace_main;
+	  tm->verbose = 0; /* not sure this ever did anything... */
+	  vec_validate (tm->nodes, a->buffer_trace_node_index);
+	  tn = tm->nodes + a->buffer_trace_node_index;
+	  tn->limit += a->buffer_traces_to_capture;
+	  if (a->post_mortem)
+	    {
+	      tm->filter_flag = FILTER_FLAG_POST_MORTEM;
+	      tm->filter_count = ~0;
+	    }
+	  tm->trace_enable = 1;
+	  if (vlib_node_set_dispatch_wrapper (this_vlib_main,
+					      dispatch_pcap_trace))
+	    clib_warning (0, "Dispatch wrapper already in use on thread %u",
+			  this_vlib_main->thread_index);
+	}
       vec_add1 (dtm->dispatch_buffer_trace_nodes, a->buffer_trace_node_index);
     }
 
@@ -301,12 +302,13 @@ vlib_pcap_dispatch_trace_configure (vlib_pcap_dispatch_trace_args_t *a)
   else
     {
       dtm->enable = 0;
-      foreach_vlib_main (({
-	tm = &this_vlib_main->trace_main;
-	tm->filter_flag = 0;
-	tm->filter_count = 0;
-	vlib_node_set_dispatch_wrapper (this_vlib_main, 0);
-      }));
+      foreach_vlib_main ()
+	{
+	  tm = &this_vlib_main->trace_main;
+	  tm->filter_flag = 0;
+	  tm->filter_count = 0;
+	  vlib_node_set_dispatch_wrapper (this_vlib_main, 0);
+	}
       vec_reset_length (dtm->dispatch_buffer_trace_nodes);
       if (pm->n_packets_captured)
 	{
diff --git a/src/plugins/quic/quic.c b/src/plugins/quic/quic.c
index 03e6f64b95a..cf4d1470ffd 100644
--- a/src/plugins/quic/quic.c
+++ b/src/plugins/quic/quic.c
@@ -2649,18 +2649,17 @@ quic_get_counter_value (u32 event_code)
   em = &vm->error_main;
   n = vlib_get_node (vm, quic_input_node.index);
   code = event_code;
-  /* *INDENT-OFF* */
-  foreach_vlib_main(({
-    em = &this_vlib_main->error_main;
-    i = n->error_heap_index + code;
-    c = em->counters[i];
-
-    if (i < vec_len (em->counters_last_clear))
-       c -= em->counters_last_clear[i];
-    sum += c;
-    index++;
-  }));
-  /* *INDENT-ON* */
+  foreach_vlib_main ()
+    {
+      em = &this_vlib_main->error_main;
+      i = n->error_heap_index + code;
+      c = em->counters[i];
+
+      if (i < vec_len (em->counters_last_clear))
+	c -= em->counters_last_clear[i];
+      sum += c;
+      index++;
+    }
   return sum;
 }
 
diff --git a/src/plugins/tlsopenssl/tls_async.c b/src/plugins/tlsopenssl/tls_async.c
index 2988816a255..49987baef16 100644
--- a/src/plugins/tlsopenssl/tls_async.c
+++ b/src/plugins/tlsopenssl/tls_async.c
@@ -415,15 +415,14 @@ openssl_async_node_enable_disable (u8 is_en)
   vlib_thread_main_t *vtm = vlib_get_thread_main ();
   u8 have_workers = vtm->n_threads != 0;
 
-  /* *INDENT-OFF* */
-  foreach_vlib_main (({
-    if (have_workers && ii != 0)
-      {
-        vlib_node_set_state (this_vlib_main, tls_async_process_node.index,
-                         state);
-      }
-  }));
-  /* *INDENT-ON* */
+  foreach_vlib_main ()
+    {
+      if (have_workers && ii != 0)
+	{
+	  vlib_node_set_state (this_vlib_main, tls_async_process_node.index,
+			       state);
+	}
+    }
 }
 
 int
diff --git a/src/plugins/tracedump/tracedump.c b/src/plugins/tracedump/tracedump.c
index 7c83ba25dc9..f1073fe247d 100644
--- a/src/plugins/tracedump/tracedump.c
+++ b/src/plugins/tracedump/tracedump.c
@@ -230,26 +230,24 @@ vl_api_trace_dump_t_handler (vl_api_trace_dump_t * mp)
       vec_validate (client_trace_cache, vlib_get_n_threads () - 1);
       i = 0;
 
-      /* *INDENT-OFF* */
-      foreach_vlib_main (
-      ({
-        vlib_trace_main_t *tm = &this_vlib_main->trace_main;
-
-        /* Filter as directed */
-        trace_apply_filter(this_vlib_main);
-
-        pool_foreach (th, tm->trace_buffer_pool)
-         {
-          vec_add1 (client_trace_cache[i], th[0]);
-        }
-
-        /* Sort them by increasing time. */
-        if (vec_len (client_trace_cache[i]))
-          vec_sort_with_function (client_trace_cache[i], trace_cmp);
-
-        i++;
-      }));
-      /* *INDENT-ON* */
+      foreach_vlib_main ()
+	{
+	  vlib_trace_main_t *tm = &this_vlib_main->trace_main;
+
+	  /* Filter as directed */
+	  trace_apply_filter (this_vlib_main);
+
+	  pool_foreach (th, tm->trace_buffer_pool)
+	    {
+	      vec_add1 (client_trace_cache[i], th[0]);
+	    }
+
+	  /* Sort them by increasing time. */
+	  if (vec_len (client_trace_cache[i]))
+	    vec_sort_with_function (client_trace_cache[i], trace_cmp);
+
+	  i++;
+	}
       vlib_worker_thread_barrier_release (vlib_get_first_main ());
     }
 
diff --git a/src/vlib/cli.c b/src/vlib/cli.c
index 5cad95ee004..cad105f17c6 100644
--- a/src/vlib/cli.c
+++ b/src/vlib/cli.c
@@ -849,17 +849,14 @@ show_memory_usage (vlib_main_t * vm,
 	 */
 	was_enabled = clib_mem_trace_enable_disable (0);
 
-        /* *INDENT-OFF* */
-        foreach_vlib_main (
-        ({
-          vlib_cli_output (vm, "%sThread %d %s\n", index ? "\n":"", index,
-                           vlib_worker_threads[index].name);
-          vlib_cli_output (vm, "  %U\n", format_clib_mem_heap,
-                           mm->per_cpu_mheaps[index],
-                           verbose);
-          index++;
-        }));
-        /* *INDENT-ON* */
+	foreach_vlib_main ()
+	  {
+	    vlib_cli_output (vm, "%sThread %d %s\n", index ? "\n" : "", index,
+			     vlib_worker_threads[index].name);
+	    vlib_cli_output (vm, "  %U\n", format_clib_mem_heap,
+			     mm->per_cpu_mheaps[index], verbose);
+	    index++;
+	  }
 
 	/* Restore the trace flag */
 	clib_mem_trace_enable_disable (was_enabled);
diff --git a/src/vlib/error.c b/src/vlib/error.c
index 7d502d777a3..99664c618a9 100644
--- a/src/vlib/error.c
+++ b/src/vlib/error.c
@@ -252,42 +252,40 @@ show_errors (vlib_main_t * vm,
     vlib_cli_output (vm, "%=10s%=35s%=35s%=10s", "Count", "Node", "Reason",
 		     "Severity");
 
+  foreach_vlib_main ()
+    {
+      em = &this_vlib_main->error_main;
+
+      if (verbose)
+	vlib_cli_output (vm, "Thread %u (%v):", index,
+			 vlib_worker_threads[index].name);
 
-  /* *INDENT-OFF* */
-  foreach_vlib_main(({
-    em = &this_vlib_main->error_main;
+      for (ni = 0; ni < vec_len (this_vlib_main->node_main.nodes); ni++)
+	{
+	  n = vlib_get_node (this_vlib_main, ni);
+	  for (code = 0; code < n->n_errors; code++)
+	    {
+	      i = n->error_heap_index + code;
+	      c = em->counters[i];
+	      if (i < vec_len (em->counters_last_clear))
+		c -= em->counters_last_clear[i];
+	      sums[i] += c;
 
-    if (verbose)
-      vlib_cli_output(vm, "Thread %u (%v):", index,
-                      vlib_worker_threads[index].name);
+	      if (c == 0 && verbose < 2)
+		continue;
 
-    for (ni = 0; ni < vec_len (this_vlib_main->node_main.nodes); ni++)
-      {
-	n = vlib_get_node (this_vlib_main, ni);
-	for (code = 0; code < n->n_errors; code++)
-	  {
-	    i = n->error_heap_index + code;
-	    c = em->counters[i];
-	    if (i < vec_len (em->counters_last_clear))
-	      c -= em->counters_last_clear[i];
-	    sums[i] += c;
-
-	    if (c == 0 && verbose < 2)
-	      continue;
-
-            if (verbose)
-              vlib_cli_output (vm, "%10lu%=35v%=35s%=10s%=6d", c, n->name,
-                               em->counters_heap[i].name,
-                               sev2str(em->counters_heap[i].severity), i);
-            else
-              vlib_cli_output (vm, "%10lu%=35v%=35s%=10s", c, n->name,
-                               em->counters_heap[i].name,
-                               sev2str(em->counters_heap[i].severity));
-	  }
-      }
-    index++;
-  }));
-  /* *INDENT-ON* */
+	      if (verbose)
+		vlib_cli_output (vm, "%10lu%=35v%=35s%=10s%=6d", c, n->name,
+				 em->counters_heap[i].name,
+				 sev2str (em->counters_heap[i].severity), i);
+	      else
+		vlib_cli_output (vm, "%10lu%=35v%=35s%=10s", c, n->name,
+				 em->counters_heap[i].name,
+				 sev2str (em->counters_heap[i].severity));
+	    }
+	}
+      index++;
+    }
 
   if (verbose)
     vlib_cli_output (vm, "Total:");
@@ -335,14 +333,13 @@ clear_error_counters (vlib_main_t * vm,
   vlib_error_main_t *em;
   u32 i;
 
-  /* *INDENT-OFF* */
-  foreach_vlib_main(({
-    em = &this_vlib_main->error_main;
-    vec_validate (em->counters_last_clear, vec_len (em->counters) - 1);
-    for (i = 0; i < vec_len (em->counters); i++)
-      em->counters_last_clear[i] = em->counters[i];
-  }));
-  /* *INDENT-ON* */
+  foreach_vlib_main ()
+    {
+      em = &this_vlib_main->error_main;
+      vec_validate (em->counters_last_clear, vec_len (em->counters) - 1);
+      for (i = 0; i < vec_len (em->counters); i++)
+	em->counters_last_clear[i] = em->counters[i];
+    }
   return 0;
 }
 
diff --git a/src/vlib/threads.h b/src/vlib/threads.h
index d715ebfff58..28a81f78712 100644
--- a/src/vlib/threads.h
+++ b/src/vlib/threads.h
@@ -237,29 +237,16 @@ typedef enum
 
 void vlib_worker_thread_fork_fixup (vlib_fork_fixup_t which);
 
-#define foreach_vlib_main(body)                                               \
-  do                                                                          \
-    {                                                                         \
-      vlib_main_t **__vlib_mains = 0, *this_vlib_main;                        \
-      int ii;                                                                 \
-                                                                              \
-      for (ii = 0; ii < vec_len (vlib_global_main.vlib_mains); ii++)          \
-	{                                                                     \
-	  this_vlib_main = vlib_global_main.vlib_mains[ii];                   \
-	  ASSERT (ii == 0 || this_vlib_main->parked_at_barrier == 1);         \
-	  if (this_vlib_main)                                                 \
-	    vec_add1 (__vlib_mains, this_vlib_main);                          \
-	}                                                                     \
-                                                                              \
-      for (ii = 0; ii < vec_len (__vlib_mains); ii++)                         \
-	{                                                                     \
-	  this_vlib_main = __vlib_mains[ii];                                  \
-	  /* body uses this_vlib_main... */                                   \
-	  (body);                                                             \
-	}                                                                     \
-      vec_free (__vlib_mains);                                                \
-    }                                                                         \
-  while (0);
+#define foreach_vlib_main()                                                   \
+  for (vlib_main_t *ii = 0, *this_vlib_main = vlib_global_main.vlib_mains[0]; \
+       (ii - (vlib_main_t *) 0) < vec_len (vlib_global_main.vlib_mains);      \
+       ii++, this_vlib_main =                                                 \
+	       vlib_global_main.vlib_mains[ii - (vlib_main_t *) 0])           \
+    if (CLIB_ASSERT_ENABLE &&                                                 \
+	!(ii == 0 ||                                                          \
+	  (this_vlib_main && this_vlib_main->parked_at_barrier == 1)))        \
+      ASSERT (0);                                                             \
+    else if (this_vlib_main)
 
 #define foreach_sched_policy \
   _(SCHED_OTHER, OTHER, "other") \
diff --git a/src/vlib/trace.c b/src/vlib/trace.c
index 836e8b473a6..4bbd9505b71 100644
--- a/src/vlib/trace.c
+++ b/src/vlib/trace.c
@@ -117,25 +117,23 @@ clear_trace_buffer (void)
   int i;
   vlib_trace_main_t *tm;
 
-  /* *INDENT-OFF* */
-  foreach_vlib_main (
-  ({
-    tm = &this_vlib_main->trace_main;
-
-    tm->trace_enable = 0;
-    vec_free (tm->nodes);
-  }));
-
-  foreach_vlib_main (
-  ({
-    tm = &this_vlib_main->trace_main;
-
-    for (i = 0; i < vec_len (tm->trace_buffer_pool); i++)
-      if (! pool_is_free_index (tm->trace_buffer_pool, i))
-        vec_free (tm->trace_buffer_pool[i]);
-    pool_free (tm->trace_buffer_pool);
-  }));
-  /* *INDENT-ON* */
+  foreach_vlib_main ()
+    {
+      tm = &this_vlib_main->trace_main;
+
+      tm->trace_enable = 0;
+      vec_free (tm->nodes);
+    }
+
+  foreach_vlib_main ()
+    {
+      tm = &this_vlib_main->trace_main;
+
+      for (i = 0; i < vec_len (tm->trace_buffer_pool); i++)
+	if (!pool_is_free_index (tm->trace_buffer_pool, i))
+	  vec_free (tm->trace_buffer_pool[i]);
+      pool_free (tm->trace_buffer_pool);
+    }
 }
 
 u8 *
@@ -309,52 +307,50 @@ cli_show_trace_buffer (vlib_main_t * vm,
 
   /* Get active traces from pool. */
 
-  /* *INDENT-OFF* */
-  foreach_vlib_main (
-  ({
-    fmt = "------------------- Start of thread %d %s -------------------\n";
-    s = format (s, fmt, index, vlib_worker_threads[index].name);
+  foreach_vlib_main ()
+    {
+      fmt = "------------------- Start of thread %d %s -------------------\n";
+      s = format (s, fmt, index, vlib_worker_threads[index].name);
 
-    tm = &this_vlib_main->trace_main;
+      tm = &this_vlib_main->trace_main;
 
-    trace_apply_filter(this_vlib_main);
+      trace_apply_filter (this_vlib_main);
 
-    traces = 0;
-    pool_foreach (h, tm->trace_buffer_pool)
-     {
-      vec_add1 (traces, h[0]);
-    }
+      traces = 0;
+      pool_foreach (h, tm->trace_buffer_pool)
+	{
+	  vec_add1 (traces, h[0]);
+	}
 
-    if (vec_len (traces) == 0)
-      {
-        s = format (s, "No packets in trace buffer\n");
-        goto done;
-      }
-
-    /* Sort them by increasing time. */
-    vec_sort_with_function (traces, trace_time_cmp);
-
-    for (i = 0; i < vec_len (traces); i++)
-      {
-        if (i == max)
-          {
-            char *warn = "Limiting display to %d packets."
-                               " To display more specify max.";
-            vlib_cli_output (vm, warn, max);
-            s = format (s, warn, max);
-            goto done;
-          }
-
-        s = format (s, "Packet %d\n%U\n\n", i + 1,
-                         format_vlib_trace, vm, traces[i]);
-      }
-
-  done:
-    vec_free (traces);
-
-    index++;
-  }));
-  /* *INDENT-ON* */
+      if (vec_len (traces) == 0)
+	{
+	  s = format (s, "No packets in trace buffer\n");
+	  goto done;
+	}
+
+      /* Sort them by increasing time. */
+      vec_sort_with_function (traces, trace_time_cmp);
+
+      for (i = 0; i < vec_len (traces); i++)
+	{
+	  if (i == max)
+	    {
+	      char *warn = "Limiting display to %d packets."
+			   " To display more specify max.";
+	      vlib_cli_output (vm, warn, max);
+	      s = format (s, warn, max);
+	      goto done;
+	    }
+
+	  s = format (s, "Packet %d\n%U\n\n", i + 1, format_vlib_trace, vm,
+		      traces[i]);
+	}
+
+    done:
+      vec_free (traces);
+
+      index++;
+    }
 
   vlib_cli_output (vm, "%v", s);
   vec_free (s);
@@ -394,8 +390,7 @@ trace_update_capture_options (u32 add, u32 node_index, u32 filter, u8 verbose)
   if (add == ~0)
     add = 50;
 
-  /* *INDENT-OFF* */
-  foreach_vlib_main ((
+  foreach_vlib_main ()
     {
       tm = &this_vlib_main->trace_main;
       tm->verbose = verbose;
@@ -411,14 +406,13 @@ trace_update_capture_options (u32 add, u32 node_index, u32 filter, u8 verbose)
 	  tn->limit = tn->count = 0;
       else
 	  tn->limit += add;
-    }));
+    }
 
-  foreach_vlib_main ((
+  foreach_vlib_main ()
     {
       tm = &this_vlib_main->trace_main;
       tm->trace_enable = 1;
-    }));
-  /* *INDENT-ON* */
+    }
 
   vlib_enable_disable_pkt_trace_filter (! !filter);
 }
@@ -533,24 +527,22 @@ VLIB_CLI_COMMAND (add_trace_cli,static) = {
 void
 trace_filter_set (u32 node_index, u32 flag, u32 count)
 {
-  /* *INDENT-OFF* */
-  foreach_vlib_main (
-  ({
-    vlib_trace_main_t *tm;
-
-    tm = &this_vlib_main->trace_main;
-    tm->filter_node_index = node_index;
-    tm->filter_flag = flag;
-    tm->filter_count = count;
-
-    /*
-     * Clear the trace limits to stop any in-progress tracing
-     * Prevents runaway trace allocations when the filter changes
-     * (or is removed)
-     */
-    vec_free (tm->nodes);
-  }));
-  /* *INDENT-ON* */
+  foreach_vlib_main ()
+    {
+      vlib_trace_main_t *tm;
+
+      tm = &this_vlib_main->trace_main;
+      tm->filter_node_index = node_index;
+      tm->filter_flag = flag;
+      tm->filter_count = count;
+
+      /*
+       * Clear the trace limits to stop any in-progress tracing
+       * Prevents runaway trace allocations when the filter changes
+       * (or is removed)
+       */
+      vec_free (tm->nodes);
+    }
 }
 
 
diff --git a/src/vnet/interface.c b/src/vnet/interface.c
index 13deaccea2d..f648440fbf8 100644
--- a/src/vnet/interface.c
+++ b/src/vnet/interface.c
@@ -875,23 +875,23 @@ vnet_register_interface (vnet_main_t * vnm,
       vlib_node_rename (vm, hw->tx_node_index, "%v", tx_node_name);
       vlib_node_rename (vm, hw->output_node_index, "%v", output_node_name);
 
-      /* *INDENT-OFF* */
-      foreach_vlib_main ({
-        vnet_interface_output_runtime_t *rt;
-
-	rt = vlib_node_get_runtime_data (this_vlib_main, hw->output_node_index);
-	ASSERT (rt->is_deleted == 1);
-	rt->is_deleted = 0;
-	rt->hw_if_index = hw_index;
-	rt->sw_if_index = hw->sw_if_index;
-	rt->dev_instance = hw->dev_instance;
-
-	rt = vlib_node_get_runtime_data (this_vlib_main, hw->tx_node_index);
-	rt->hw_if_index = hw_index;
-	rt->sw_if_index = hw->sw_if_index;
-	rt->dev_instance = hw->dev_instance;
-      });
-      /* *INDENT-ON* */
+      foreach_vlib_main ()
+	{
+	  vnet_interface_output_runtime_t *rt;
+
+	  rt =
+	    vlib_node_get_runtime_data (this_vlib_main, hw->output_node_index);
+	  ASSERT (rt->is_deleted == 1);
+	  rt->is_deleted = 0;
+	  rt->hw_if_index = hw_index;
+	  rt->sw_if_index = hw->sw_if_index;
+	  rt->dev_instance = hw->dev_instance;
+
+	  rt = vlib_node_get_runtime_data (this_vlib_main, hw->tx_node_index);
+	  rt->hw_if_index = hw_index;
+	  rt->sw_if_index = hw->sw_if_index;
+	  rt->dev_instance = hw->dev_instance;
+	}
 
       /* The new class may differ from the old one.
        * Functions have to be updated. */
@@ -900,14 +900,13 @@ vnet_register_interface (vnet_main_t * vnm,
       node->node_fn_registrations = if_out_node->node_fn_registrations;
       node->function = if_out_node->function;
 
-      /* *INDENT-OFF* */
-      foreach_vlib_main ({
-        nrt = vlib_node_get_runtime (this_vlib_main, hw->output_node_index);
-        nrt->function = node->function;
-	vlib_node_runtime_perf_counter (this_vlib_main, nrt, 0, 0, 0,
-					VLIB_NODE_RUNTIME_PERF_RESET);
-      });
-      /* *INDENT-ON* */
+      foreach_vlib_main ()
+	{
+	  nrt = vlib_node_get_runtime (this_vlib_main, hw->output_node_index);
+	  nrt->function = node->function;
+	  vlib_node_runtime_perf_counter (this_vlib_main, nrt, 0, 0, 0,
+					  VLIB_NODE_RUNTIME_PERF_RESET);
+	}
 
       node = vlib_get_node (vm, hw->tx_node_index);
       if (dev_class->tx_fn_registrations)
@@ -919,14 +918,14 @@ vnet_register_interface (vnet_main_t * vnm,
       else
 	node->function = dev_class->tx_function;
       node->format_trace = dev_class->format_tx_trace;
-      /* *INDENT-OFF* */
-      foreach_vlib_main ({
-        nrt = vlib_node_get_runtime (this_vlib_main, hw->tx_node_index);
-        nrt->function = node->function;
-	vlib_node_runtime_perf_counter (this_vlib_main, nrt, 0, 0, 0,
-					VLIB_NODE_RUNTIME_PERF_RESET);
-      });
-      /* *INDENT-ON* */
+
+      foreach_vlib_main ()
+	{
+	  nrt = vlib_node_get_runtime (this_vlib_main, hw->tx_node_index);
+	  nrt->function = node->function;
+	  vlib_node_runtime_perf_counter (this_vlib_main, nrt, 0, 0, 0,
+					  VLIB_NODE_RUNTIME_PERF_RESET);
+	}
 
       _vec_len (im->deleted_hw_interface_nodes) -= 1;
     }
@@ -1074,17 +1073,15 @@ vnet_delete_hw_interface (vnet_main_t * vnm, u32 hw_if_index)
       /* Put output/tx nodes into recycle pool */
       vnet_hw_interface_nodes_t *dn;
 
-      /* *INDENT-OFF* */
-      foreach_vlib_main
-	({
+      foreach_vlib_main ()
+	{
 	  vnet_interface_output_runtime_t *rt =
 	    vlib_node_get_runtime_data (this_vlib_main, hw->output_node_index);
 
 	  /* Mark node runtime as deleted so output node (if called)
 	   * will drop packets. */
 	  rt->is_deleted = 1;
-	});
-      /* *INDENT-ON* */
+	}
 
       vlib_node_rename (vm, hw->output_node_index,
 			"interface-%d-output-deleted", hw_if_index);
diff --git a/src/vnet/session/session.c b/src/vnet/session/session.c
index 469ec0ed529..2400a19a351 100644
--- a/src/vnet/session/session.c
+++ b/src/vnet/session/session.c
@@ -1590,16 +1590,14 @@ session_register_transport (transport_proto_t transport_proto,
   vec_validate (smm->session_type_to_next, session_type);
   vec_validate (smm->session_tx_fns, session_type);
 
-  /* *INDENT-OFF* */
   if (output_node != ~0)
     {
-      foreach_vlib_main (({
-          next_index = vlib_node_add_next (this_vlib_main,
-                                           session_queue_node.index,
-                                           output_node);
-      }));
+      foreach_vlib_main ()
+	{
+	  next_index = vlib_node_add_next (
+	    this_vlib_main, session_queue_node.index, output_node);
+	}
     }
-  /* *INDENT-ON* */
 
   smm->session_type_to_next[session_type] = next_index;
   smm->session_tx_fns[session_type] =
@@ -1755,31 +1753,29 @@ session_node_enable_disable (u8 is_en)
   vlib_thread_main_t *vtm = vlib_get_thread_main ();
   u8 have_workers = vtm->n_threads != 0;
 
-  /* *INDENT-OFF* */
-  foreach_vlib_main (({
-    if (have_workers && ii == 0)
-      {
-	if (is_en)
-	  {
-	    vlib_node_set_state (this_vlib_main,
-	                         session_queue_process_node.index, state);
-	    vlib_node_t *n = vlib_get_node (this_vlib_main,
-	                                    session_queue_process_node.index);
-	    vlib_start_process (this_vlib_main, n->runtime_index);
-	  }
-	else
-	  {
-	    vlib_process_signal_event_mt (this_vlib_main,
-	                                  session_queue_process_node.index,
-	                                  SESSION_Q_PROCESS_STOP, 0);
-	  }
-	if (!session_main.poll_main)
-	  continue;
-      }
-    vlib_node_set_state (this_vlib_main, session_queue_node.index,
-                         state);
-  }));
-  /* *INDENT-ON* */
+  foreach_vlib_main ()
+    {
+      if (have_workers && ii == 0)
+	{
+	  if (is_en)
+	    {
+	      vlib_node_set_state (this_vlib_main,
+				   session_queue_process_node.index, state);
+	      vlib_node_t *n = vlib_get_node (
+		this_vlib_main, session_queue_process_node.index);
+	      vlib_start_process (this_vlib_main, n->runtime_index);
+	    }
+	  else
+	    {
+	      vlib_process_signal_event_mt (this_vlib_main,
+					    session_queue_process_node.index,
+					    SESSION_Q_PROCESS_STOP, 0);
+	    }
+	  if (!session_main.poll_main)
+	    continue;
+	}
+      vlib_node_set_state (this_vlib_main, session_queue_node.index, state);
+    }
 }
 
 clib_error_t *
diff --git a/src/vnet/tcp/tcp_syn_filter4.c b/src/vnet/tcp/tcp_syn_filter4.c
index d151eb77c3c..b107899cfb9 100644
--- a/src/vnet/tcp/tcp_syn_filter4.c
+++ b/src/vnet/tcp/tcp_syn_filter4.c
@@ -450,19 +450,19 @@ syn_filter_enable_disable (u32 sw_if_index, int enable_disable)
     {
       syn_filter4_runtime_t *rt;
 
-      /* *INDENT-OFF* */
-      foreach_vlib_main ({
-	rt = vlib_node_get_runtime_data (this_vlib_main, syn_filter4_node.index);
-	vec_validate (rt->syn_counts, 1023);
-	/*
-	 * Given perfect disperson / optimal hashing results:
-	 * Allow 128k (successful) syns/sec. 1024, buckets each of which
-	 * absorb 128 syns before filtering. Reset table once a second.
-	 * Reality bites, lets try resetting once every 100ms.
-	 */
-	rt->reset_interval = 0.1;	/* reset interval in seconds */
-      });
-      /* *INDENT-ON* */
+      foreach_vlib_main ()
+	{
+	  rt = vlib_node_get_runtime_data (this_vlib_main,
+					   syn_filter4_node.index);
+	  vec_validate (rt->syn_counts, 1023);
+	  /*
+	   * Given perfect disperson / optimal hashing results:
+	   * Allow 128k (successful) syns/sec. 1024, buckets each of which
+	   * absorb 128 syns before filtering. Reset table once a second.
+	   * Reality bites, lets try resetting once every 100ms.
+	   */
+	  rt->reset_interval = 0.1; /* reset interval in seconds */
+	}
     }
 
   rv = vnet_feature_enable_disable ("ip4-local", "syn-filter-4",
diff --git a/src/vnet/unix/gdb_funcs.c b/src/vnet/unix/gdb_funcs.c
index ce8db3a8ae3..797cb0eb7d1 100644
--- a/src/vnet/unix/gdb_funcs.c
+++ b/src/vnet/unix/gdb_funcs.c
@@ -238,51 +238,51 @@ gdb_show_traces ()
 
   /* Get active traces from pool. */
 
-  /* *INDENT-OFF* */
-  foreach_vlib_main (
-  ({
-    fmt = "------------------- Start of thread %d %s -------------------\n";
-    s = format (s, fmt, index, vlib_worker_threads[index].name);
-
-    tm = &this_vlib_main->trace_main;
-
-    trace_apply_filter(this_vlib_main);
-
-    traces = 0;
-    pool_foreach (h, tm->trace_buffer_pool)
-     {
-      vec_add1 (traces, h[0]);
+  foreach_vlib_main ()
+    {
+      fmt = "------------------- Start of thread %d %s -------------------\n";
+      s = format (s, fmt, index, vlib_worker_threads[index].name);
+
+      tm = &this_vlib_main->trace_main;
+
+      trace_apply_filter (this_vlib_main);
+
+      traces = 0;
+      pool_foreach (h, tm->trace_buffer_pool)
+	{
+	  vec_add1 (traces, h[0]);
+	}
+
+      if (vec_len (traces) == 0)
+	{
+	  s = format (s, "No packets in trace buffer\n");
+	  goto done;
+	}
+
+      /* Sort them by increasing time. */
+      vec_sort_with_function (traces, trace_cmp);
+
+      for (i = 0; i < vec_len (traces); i++)
+	{
+	  if (i == max)
+	    {
+	      fformat (stderr,
+		       "Limiting display to %d packets."
+		       " To display more specify max.",
+		       max);
+	      goto done;
+	    }
+
+	  s = format (s, "Packet %d\n%U\n\n", i + 1, format_vlib_trace,
+		      vlib_get_first_main (), traces[i]);
+	}
+
+    done:
+      vec_free (traces);
+
+      index++;
     }
 
-    if (vec_len (traces) == 0)
-      {
-        s = format (s, "No packets in trace buffer\n");
-        goto done;
-      }
-
-    /* Sort them by increasing time. */
-    vec_sort_with_function (traces, trace_cmp);
-
-    for (i = 0; i < vec_len (traces); i++)
-      {
-        if (i == max)
-          {
-            fformat (stderr, "Limiting display to %d packets."
-                                 " To display more specify max.", max);
-            goto done;
-          }
-
-	s = format (s, "Packet %d\n%U\n\n", i + 1, format_vlib_trace,
-		    vlib_get_first_main (), traces[i]);
-      }
-
-  done:
-    vec_free (traces);
-
-    index++;
-  }));
-  /* *INDENT-ON* */
-
   fformat (stderr, "%v", s);
   vec_free (s);
 }