1 files changed, 105 insertions, 156 deletions

diff --git a/src/vlib/main.c b/src/vlib/main.c
index 2f219955e70..04b58762646 100644
--- a/src/vlib/main.c
+++ b/src/vlib/main.c
@@ -41,75 +41,17 @@
 #include <vppinfra/format.h>
 #include <vlib/vlib.h>
 #include <vlib/threads.h>
+#include <vlib/stats/stats.h>
 #include <vppinfra/tw_timer_1t_3w_1024sl_ov.h>
 
 #include <vlib/unix/unix.h>
 
-/* Actually allocate a few extra slots of vector data to support
-   speculative vector enqueues which overflow vector data in next frame. */
-#define VLIB_FRAME_SIZE_ALLOC (VLIB_FRAME_SIZE + 4)
-
-always_inline u32
-vlib_frame_bytes (u32 n_scalar_bytes, u32 n_vector_bytes)
-{
-  u32 n_bytes;
-
-  /* Make room for vlib_frame_t plus scalar arguments. */
-  n_bytes = vlib_frame_vector_byte_offset (n_scalar_bytes);
-
-  /* Make room for vector arguments.
-     Allocate a few extra slots of vector data to support
-     speculative vector enqueues which overflow vector data in next frame. */
-#define VLIB_FRAME_SIZE_EXTRA 4
-  n_bytes += (VLIB_FRAME_SIZE + VLIB_FRAME_SIZE_EXTRA) * n_vector_bytes;
-
-  /* Magic number is first 32bit number after vector data.
-     Used to make sure that vector data is never overrun. */
 #define VLIB_FRAME_MAGIC (0xabadc0ed)
-  n_bytes += sizeof (u32);
-
-  /* Pad to cache line. */
-  n_bytes = round_pow2 (n_bytes, CLIB_CACHE_LINE_BYTES);
-
-  return n_bytes;
-}
 
 always_inline u32 *
 vlib_frame_find_magic (vlib_frame_t * f, vlib_node_t * node)
 {
-  void *p = f;
-
-  p += vlib_frame_vector_byte_offset (node->scalar_size);
-
-  p += (VLIB_FRAME_SIZE + VLIB_FRAME_SIZE_EXTRA) * node->vector_size;
-
-  return p;
-}
-
-static inline vlib_frame_size_t *
-get_frame_size_info (vlib_node_main_t * nm,
-		     u32 n_scalar_bytes, u32 n_vector_bytes)
-{
-#ifdef VLIB_SUPPORTS_ARBITRARY_SCALAR_SIZES
-  uword key = (n_scalar_bytes << 16) | n_vector_bytes;
-  uword *p, i;
-
-  p = hash_get (nm->frame_size_hash, key);
-  if (p)
-    i = p[0];
-  else
-    {
-      i = vec_len (nm->frame_sizes);
-      vec_validate (nm->frame_sizes, i);
-      hash_set (nm->frame_size_hash, key, i);
-    }
-
-  return vec_elt_at_index (nm->frame_sizes, i);
-#else
-  ASSERT (vlib_frame_bytes (n_scalar_bytes, n_vector_bytes)
-	  == (vlib_frame_bytes (0, 4)));
-  return vec_elt_at_index (nm->frame_sizes, 0);
-#endif
+  return (void *) f + node->magic_offset;
 }
 
 static vlib_frame_t *
@@ -120,31 +62,35 @@ vlib_frame_alloc_to_node (vlib_main_t * vm, u32 to_node_index,
   vlib_frame_size_t *fs;
   vlib_node_t *to_node;
   vlib_frame_t *f;
-  u32 l, n, scalar_size, vector_size;
+  u32 l, n;
 
   ASSERT (vm == vlib_get_main ());
 
   to_node = vlib_get_node (vm, to_node_index);
 
-  scalar_size = to_node->scalar_size;
-  vector_size = to_node->vector_size;
+  vec_validate (nm->frame_sizes, to_node->frame_size_index);
+  fs = vec_elt_at_index (nm->frame_sizes, to_node->frame_size_index);
 
-  fs = get_frame_size_info (nm, scalar_size, vector_size);
-  n = vlib_frame_bytes (scalar_size, vector_size);
+  if (fs->frame_size == 0)
+    fs->frame_size = to_node->frame_size;
+  else
+    ASSERT (fs->frame_size == to_node->frame_size);
+
+  n = fs->frame_size;
   if ((l = vec_len (fs->free_frames)) > 0)
     {
       /* Allocate from end of free list. */
       f = fs->free_frames[l - 1];
-      _vec_len (fs->free_frames) = l - 1;
+      vec_set_len (fs->free_frames, l - 1);
     }
   else
     {
-      f = clib_mem_alloc_aligned_no_fail (n, VLIB_FRAME_ALIGN);
+      f = clib_mem_alloc_aligned_no_fail (n, CLIB_CACHE_LINE_BYTES);
     }
 
   /* Poison frame when debugging. */
   if (CLIB_DEBUG > 0)
-    clib_memset (f, 0xfe, n);
+    clib_memset_u8 (f, 0xfe, n);
 
   /* Insert magic number. */
   {
@@ -156,9 +102,11 @@ vlib_frame_alloc_to_node (vlib_main_t * vm, u32 to_node_index,
 
   f->frame_flags = VLIB_FRAME_IS_ALLOCATED | frame_flags;
   f->n_vectors = 0;
-  f->scalar_size = scalar_size;
-  f->vector_size = vector_size;
+  f->scalar_offset = to_node->scalar_offset;
+  f->vector_offset = to_node->vector_offset;
+  f->aux_offset = to_node->aux_offset;
   f->flags = 0;
+  f->frame_size_index = to_node->frame_size_index;
 
   fs->n_alloc_frames += 1;
 
@@ -239,17 +187,15 @@ vlib_put_frame_to_node (vlib_main_t * vm, u32 to_node_index, vlib_frame_t * f)
 
 /* Free given frame. */
 void
-vlib_frame_free (vlib_main_t * vm, vlib_node_runtime_t * r, vlib_frame_t * f)
+vlib_frame_free (vlib_main_t *vm, vlib_frame_t *f)
 {
   vlib_node_main_t *nm = &vm->node_main;
-  vlib_node_t *node;
   vlib_frame_size_t *fs;
 
   ASSERT (vm == vlib_get_main ());
   ASSERT (f->frame_flags & VLIB_FRAME_IS_ALLOCATED);
 
-  node = vlib_get_node (vm, r->node_index);
-  fs = get_frame_size_info (nm, node->scalar_size, node->vector_size);
+  fs = vec_elt_at_index (nm->frame_sizes, f->frame_size_index);
 
   ASSERT (f->frame_flags & VLIB_FRAME_IS_ALLOCATED);
 
@@ -261,6 +207,7 @@ vlib_frame_free (vlib_main_t * vm, vlib_node_runtime_t * r, vlib_frame_t * f)
     }
 
   f->frame_flags &= ~(VLIB_FRAME_IS_ALLOCATED | VLIB_FRAME_NO_APPEND);
+  f->flags = 0;
 
   vec_add1 (fs->free_frames, f);
   ASSERT (fs->n_alloc_frames > 0);
@@ -271,30 +218,33 @@ static clib_error_t *
 show_frame_stats (vlib_main_t * vm,
 		  unformat_input_t * input, vlib_cli_command_t * cmd)
 {
-  vlib_node_main_t *nm = &vm->node_main;
   vlib_frame_size_t *fs;
 
-  vlib_cli_output (vm, "%=6s%=12s%=12s", "Size", "# Alloc", "# Free");
-  vec_foreach (fs, nm->frame_sizes)
-  {
-    u32 n_alloc = fs->n_alloc_frames;
-    u32 n_free = vec_len (fs->free_frames);
+  vlib_cli_output (vm, "%=8s%=6s%=12s%=12s", "Thread", "Size", "# Alloc",
+		   "# Free");
+  foreach_vlib_main ()
+    {
+      vlib_node_main_t *nm = &this_vlib_main->node_main;
+      vec_foreach (fs, nm->frame_sizes)
+	{
+	  u32 n_alloc = fs->n_alloc_frames;
+	  u32 n_free = vec_len (fs->free_frames);
 
-    if (n_alloc + n_free > 0)
-      vlib_cli_output (vm, "%=6d%=12d%=12d",
-		       fs - nm->frame_sizes, n_alloc, n_free);
-  }
+	  if (n_alloc + n_free > 0)
+	    vlib_cli_output (vm, "%=8d%=6d%=12d%=12d",
+			     this_vlib_main->thread_index, fs->frame_size,
+			     n_alloc, n_free);
+	}
+    }
 
   return 0;
 }
 
-/* *INDENT-OFF* */
 VLIB_CLI_COMMAND (show_frame_stats_cli, static) = {
   .path = "show vlib frame-allocation",
   .short_help = "Show node dispatch frame statistics",
   .function = show_frame_stats,
 };
-/* *INDENT-ON* */
 
 /* Change ownership of enqueue rights to given next node. */
 static void
@@ -525,12 +475,8 @@ vlib_put_next_frame (vlib_main_t * vm,
       if (!(f->frame_flags & VLIB_FRAME_PENDING))
 	{
 	  __attribute__ ((unused)) vlib_node_t *node;
-	  vlib_node_t *next_node;
-	  vlib_node_runtime_t *next_runtime;
 
 	  node = vlib_get_node (vm, r->node_index);
-	  next_node = vlib_get_next_node (vm, r->node_index, next_index);
-	  next_runtime = vlib_node_get_runtime (vm, next_node->index);
 
 	  vec_add2 (nm->pending_frames, p, 1);
 
@@ -539,18 +485,6 @@ vlib_put_next_frame (vlib_main_t * vm,
 	  p->next_frame_index = nf - nm->next_frames;
 	  nf->flags |= VLIB_FRAME_PENDING;
 	  f->frame_flags |= VLIB_FRAME_PENDING;
-
-	  /*
-	   * If we're going to dispatch this frame on another thread,
-	   * force allocation of a new frame. Otherwise, we create
-	   * a dangling frame reference. Each thread has its own copy of
-	   * the next_frames vector.
-	   */
-	  if (0 && r->thread_index != next_runtime->thread_index)
-	    {
-	      nf->frame = NULL;
-	      nf->flags &= ~(VLIB_FRAME_PENDING | VLIB_FRAME_IS_ALLOCATED);
-	    }
 	}
 
       /* Copy trace flag from next_frame and from runtime. */
@@ -698,13 +632,11 @@ vlib_cli_elog_clear (vlib_main_t * vm,
   return 0;
 }
 
-/* *INDENT-OFF* */
 VLIB_CLI_COMMAND (elog_clear_cli, static) = {
   .path = "event-logger clear",
   .short_help = "Clear the event log",
   .function = vlib_cli_elog_clear,
 };
-/* *INDENT-ON* */
 
 #ifdef CLIB_UNIX
 static clib_error_t *
@@ -753,13 +685,11 @@ vlib_post_mortem_dump (void)
     (vgm->post_mortem_callbacks[i]) ();
 }
 
-/* *INDENT-OFF* */
 VLIB_CLI_COMMAND (elog_save_cli, static) = {
   .path = "event-logger save",
   .short_help = "event-logger save <filename> (saves log in /tmp/<filename>)",
   .function = elog_save_buffer,
 };
-/* *INDENT-ON* */
 
 static clib_error_t *
 elog_stop (vlib_main_t * vm,
@@ -773,13 +703,11 @@ elog_stop (vlib_main_t * vm,
   return 0;
 }
 
-/* *INDENT-OFF* */
 VLIB_CLI_COMMAND (elog_stop_cli, static) = {
   .path = "event-logger stop",
   .short_help = "Stop the event-logger",
   .function = elog_stop,
 };
-/* *INDENT-ON* */
 
 static clib_error_t *
 elog_restart (vlib_main_t * vm,
@@ -793,13 +721,11 @@ elog_restart (vlib_main_t * vm,
   return 0;
 }
 
-/* *INDENT-OFF* */
 VLIB_CLI_COMMAND (elog_restart_cli, static) = {
   .path = "event-logger restart",
   .short_help = "Restart the event-logger",
   .function = elog_restart,
 };
-/* *INDENT-ON* */
 
 static clib_error_t *
 elog_resize_command_fn (vlib_main_t * vm,
@@ -823,13 +749,11 @@ elog_resize_command_fn (vlib_main_t * vm,
   return 0;
 }
 
-/* *INDENT-OFF* */
 VLIB_CLI_COMMAND (elog_resize_cli, static) = {
   .path = "event-logger resize",
   .short_help = "event-logger resize <nnn>",
   .function = elog_resize_command_fn,
 };
-/* *INDENT-ON* */
 
 #endif /* CLIB_UNIX */
 
@@ -882,13 +806,11 @@ elog_show_buffer (vlib_main_t * vm,
   return error;
 }
 
-/* *INDENT-OFF* */
 VLIB_CLI_COMMAND (elog_show_cli, static) = {
   .path = "show event-logger",
   .short_help = "Show event logger info",
   .function = elog_show_buffer,
 };
-/* *INDENT-ON* */
 
 void
 vlib_gdb_show_event_log (void)
@@ -1045,7 +967,6 @@ dispatch_node (vlib_main_t * vm,
      polling mode and vice versa. */
   if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_ADAPTIVE_MODE))
     {
-      /* *INDENT-OFF* */
       ELOG_TYPE_DECLARE (e) =
         {
           .function = (char *) __FUNCTION__,
@@ -1056,7 +977,6 @@ dispatch_node (vlib_main_t * vm,
             "interrupt", "polling",
           },
         };
-      /* *INDENT-ON* */
       struct
       {
 	u32 node_name, vector_length, is_polling;
@@ -1227,13 +1147,14 @@ dispatch_pending_node (vlib_main_t * vm, uword pending_frame_index,
 	  /* no new frame has been assigned to this node, use the saved one */
 	  nf->frame = restore_frame;
 	  f->n_vectors = 0;
+	  f->flags = 0;
 	}
       else
 	{
 	  /* The node has gained a frame, implying packets from the current frame
 	     were re-queued to this same node. we don't need the saved one
 	     anymore */
-	  vlib_frame_free (vm, n, f);
+	  vlib_frame_free (vm, f);
 	}
     }
   else
@@ -1241,7 +1162,7 @@ dispatch_pending_node (vlib_main_t * vm, uword pending_frame_index,
       if (f->frame_flags & VLIB_FRAME_FREE_AFTER_DISPATCH)
 	{
 	  ASSERT (!(n->flags & VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH));
-	  vlib_frame_free (vm, n, f);
+	  vlib_frame_free (vm, f);
 	}
     }
 
@@ -1417,7 +1338,8 @@ vlib_start_process (vlib_main_t * vm, uword process_index)
 {
   vlib_node_main_t *nm = &vm->node_main;
   vlib_process_t *p = vec_elt (nm->processes, process_index);
-  dispatch_process (vm, p, /* frame */ 0, /* cpu_time_now */ 0);
+  u64 cpu_time_now = clib_cpu_time_now ();
+  dispatch_process (vm, p, /* frame */ 0, cpu_time_now);
 }
 
 static u64
@@ -1501,12 +1423,6 @@ dispatch_suspended_process (vlib_main_t * vm,
   return t;
 }
 
-void vl_api_send_pending_rpc_requests (vlib_main_t *) __attribute__ ((weak));
-void
-vl_api_send_pending_rpc_requests (vlib_main_t * vm)
-{
-}
-
 static_always_inline void
 vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
 {
@@ -1522,7 +1438,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
   if (is_main)
     {
       vec_resize (nm->pending_frames, 32);
-      _vec_len (nm->pending_frames) = 0;
+      vec_set_len (nm->pending_frames, 0);
     }
 
   /* Mark time of main loop start. */
@@ -1534,9 +1450,6 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
   else
     cpu_time_now = clib_cpu_time_now ();
 
-  /* Pre-allocate interupt runtime indices and lock. */
-  vec_alloc_aligned (nm->pending_interrupts, 1, CLIB_CACHE_LINE_BYTES);
-
   /* Pre-allocate expired nodes. */
   if (!nm->polling_threshold_vector_length)
     nm->polling_threshold_vector_length = 10;
@@ -1572,7 +1485,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
       if (PREDICT_FALSE (_vec_len (vm->pending_rpc_requests) > 0))
 	{
 	  if (!is_main)
-	    vl_api_send_pending_rpc_requests (vm);
+	    vlib_worker_flush_pending_rpc_requests (vm);
 	}
 
       if (!is_main)
@@ -1581,8 +1494,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
       if (PREDICT_FALSE (vm->check_frame_queues + frame_queue_check_counter))
 	{
 	  u32 processed = 0;
-	  vlib_frame_queue_dequeue_fn_t *fn =
-	    vlib_buffer_func_main.frame_queue_dequeue_fn;
+	  vlib_frame_queue_dequeue_fn_t *fn;
 
 	  if (vm->check_frame_queues)
 	    {
@@ -1591,7 +1503,10 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
 	    }
 
 	  vec_foreach (fqm, tm->frame_queue_mains)
-	    processed += (fn) (vm, fqm);
+	    {
+	      fn = fqm->frame_queue_dequeue_fn;
+	      processed += (fn) (vm, fqm);
+	    }
 
 	  /* No handoff queue work found? */
 	  if (processed)
@@ -1613,6 +1528,22 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
 				      /* frame */ 0,
 				      cpu_time_now);
 
+      if (clib_interrupt_is_any_pending (nm->pre_input_node_interrupts))
+	{
+	  int int_num = -1;
+
+	  while ((int_num = clib_interrupt_get_next_and_clear (
+		    nm->pre_input_node_interrupts, int_num)) != -1)
+	    {
+	      vlib_node_runtime_t *n;
+	      n = vec_elt_at_index (
+		nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT], int_num);
+	      cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_PRE_INPUT,
+					    VLIB_NODE_STATE_INTERRUPT,
+					    /* frame */ 0, cpu_time_now);
+	    }
+	}
+
       /* Next process input nodes. */
       vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_INPUT])
 	cpu_time_now = dispatch_node (vm, n,
@@ -1624,16 +1555,14 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
       if (PREDICT_TRUE (is_main && vm->queue_signal_pending == 0))
 	vm->queue_signal_callback (vm);
 
-      if (__atomic_load_n (nm->pending_interrupts, __ATOMIC_ACQUIRE))
+      if (clib_interrupt_is_any_pending (nm->input_node_interrupts))
 	{
 	  int int_num = -1;
-	  *nm->pending_interrupts = 0;
 
-	  while ((int_num =
-		    clib_interrupt_get_next (nm->interrupts, int_num)) != -1)
+	  while ((int_num = clib_interrupt_get_next_and_clear (
+		    nm->input_node_interrupts, int_num)) != -1)
 	    {
 	      vlib_node_runtime_t *n;
-	      clib_interrupt_clear (nm->interrupts, int_num);
 	      n = vec_elt_at_index (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT],
 				    int_num);
 	      cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_INPUT,
@@ -1648,11 +1577,10 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
       for (i = 0; i < _vec_len (nm->pending_frames); i++)
 	cpu_time_now = dispatch_pending_node (vm, i, cpu_time_now);
       /* Reset pending vector for next iteration. */
-      _vec_len (nm->pending_frames) = 0;
+      vec_set_len (nm->pending_frames, 0);
 
       if (is_main)
 	{
-          /* *INDENT-OFF* */
           ELOG_TYPE_DECLARE (es) =
             {
               .format = "process tw start",
@@ -1663,7 +1591,6 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
               .format = "process tw end: %d",
               .format_args = "i4",
             };
-          /* *INDENT-ON* */
 
 	  struct
 	  {
@@ -1676,10 +1603,8 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
 	  if (PREDICT_FALSE (vm->elog_trace_graph_dispatch))
 	    ed = ELOG_DATA (&vlib_global_main.elog_main, es);
 
-	  nm->data_from_advancing_timing_wheel =
-	    TW (tw_timer_expire_timers_vec)
-	    ((TWT (tw_timer_wheel) *) nm->timing_wheel, vlib_time_now (vm),
-	     nm->data_from_advancing_timing_wheel);
+	  TW (tw_timer_expire_timers)
+	  ((TWT (tw_timer_wheel) *) nm->timing_wheel, vlib_time_now (vm));
 
 	  ASSERT (nm->data_from_advancing_timing_wheel != 0);
 
@@ -1710,6 +1635,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
 			vlib_get_node (vm, te->process_node_index);
 		      vlib_process_t *p =
 			vec_elt (nm->processes, n->runtime_index);
+		      p->stop_timer_handle = ~0;
 		      void *data;
 		      data =
 			vlib_process_signal_event_helper (nm, n, p,
@@ -1734,7 +1660,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
 			dispatch_suspended_process (vm, di, cpu_time_now);
 		    }
 		}
-	      _vec_len (nm->data_from_advancing_timing_wheel) = 0;
+	      vec_set_len (nm->data_from_advancing_timing_wheel, 0);
 	    }
 	}
       vlib_increment_main_loop_counter (vm);
@@ -1875,7 +1801,6 @@ placeholder_queue_signal_callback (vlib_main_t * vm)
 }
 
 #define foreach_weak_reference_stub             \
-_(vlib_map_stat_segment_init)                   \
 _(vpe_api_init)                                 \
 _(vlibmemory_init)                              \
 _(map_api_segment_init)
@@ -1909,6 +1834,23 @@ vl_api_get_elog_trace_api_messages (void)
   return 0;
 }
 
+static void
+process_expired_timer_cb (u32 *expired_timer_handles)
+{
+  vlib_main_t *vm = vlib_get_main ();
+  vlib_node_main_t *nm = &vm->node_main;
+  u32 *handle;
+
+  vec_foreach (handle, expired_timer_handles)
+    {
+      u32 pi = vlib_timing_wheel_data_get_index (*handle);
+      vlib_process_t *p = vec_elt (nm->processes, pi);
+
+      p->stop_timer_handle = ~0;
+    }
+  vec_append (nm->data_from_advancing_timing_wheel, expired_timer_handles);
+}
+
 /* Main function. */
 int
 vlib_main (vlib_main_t * volatile vm, unformat_input_t * input)
@@ -1936,7 +1878,13 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input)
       goto done;
     }
 
-  if ((error = vlib_map_stat_segment_init (vm)))
+  if ((error = vlib_log_init (vm)))
+    {
+      clib_error_report (error);
+      goto done;
+    }
+
+  if ((error = vlib_stats_init (vm)))
     {
       clib_error_report (error);
       goto done;
@@ -2005,18 +1953,18 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input)
 					     CLIB_CACHE_LINE_BYTES);
 
   vec_validate (nm->data_from_advancing_timing_wheel, 10);
-  _vec_len (nm->data_from_advancing_timing_wheel) = 0;
+  vec_set_len (nm->data_from_advancing_timing_wheel, 0);
 
   /* Create the process timing wheel */
-  TW (tw_timer_wheel_init) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
-			    0 /* no callback */ ,
-			    10e-6 /* timer period 10us */ ,
-			    ~0 /* max expirations per call */ );
+  TW (tw_timer_wheel_init)
+  ((TWT (tw_timer_wheel) *) nm->timing_wheel,
+   process_expired_timer_cb /* callback */, 10e-6 /* timer period 10us */,
+   ~0 /* max expirations per call */);
 
   vec_validate (vm->pending_rpc_requests, 0);
-  _vec_len (vm->pending_rpc_requests) = 0;
+  vec_set_len (vm->pending_rpc_requests, 0);
   vec_validate (vm->processing_rpc_requests, 0);
-  _vec_len (vm->processing_rpc_requests) = 0;
+  vec_set_len (vm->processing_rpc_requests, 0);
 
   /* Default params for the buffer allocator fault injector, if configured */
   if (VLIB_BUFFER_ALLOC_FAULT_INJECTOR > 0)
@@ -2066,7 +2014,9 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input)
   vlib_main_loop (vm);
 
 done:
+  /* Stop worker threads, barrier will not be released */
   vlib_worker_thread_barrier_sync (vm);
+
   /* Call all exit functions. */
   {
     clib_error_t *sub_error;
@@ -2074,7 +2024,6 @@ done:
     if (sub_error)
       clib_error_report (sub_error);
   }
-  vlib_worker_thread_barrier_release (vm);
 
   if (error)
     clib_error_report (error);