vlib: process node scheduler rework

This commit allow use od cooperative multitasking with multiple
descheduling reasons (i.e. event wait and suspend) inside the same
process node. In previus code remote node will wake up process node
by sending event evein if process node is waiting in
vlib_process_ssupend().

This change also allowed new vlib_process_yield() API which deschedules
current process and it puts it into the end of queue.

Change-Id: I846e5a99b4ea1809eb80895f6ffe0ef0b2fd21ae
Type: improvement
Signed-off-by: Damjan Marion <damarion@cisco.com>

6 files changed, 244 insertions, 150 deletions

diff --git a/src/plugins/dpdk/main.c b/src/plugins/dpdk/main.c
index 9781d0ed7f0..437cfbd230e 100644
--- a/src/plugins/dpdk/main.c
+++ b/src/plugins/dpdk/main.c
@@ -50,7 +50,7 @@ rte_delay_us_override (unsigned us)
 	{
 	  /* Only suspend for the admin_down_process */
 	  vlib_process_t *proc = vlib_get_current_process (vm);
-	  if (!(proc->flags & VLIB_PROCESS_IS_RUNNING) ||
+	  if (proc->state != VLIB_PROCESS_STATE_RUNNING ||
 	      (proc->node_runtime.node_index !=
 	       admin_up_down_process_node.index))
 	    return 0;
diff --git a/src/vlib/main.c b/src/vlib/main.c
index bf840324b64..a2f833711ab 100644
--- a/src/vlib/main.c
+++ b/src/vlib/main.c
@@ -1243,9 +1243,12 @@ static_always_inline uword
 vlib_process_resume (vlib_main_t * vm, vlib_process_t * p)
 {
   uword r;
-  p->flags &= ~(VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
-		| VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT
-		| VLIB_PROCESS_RESUME_PENDING);
+
+  if (p->state == VLIB_PROCESS_STATE_WAIT_FOR_EVENT ||
+      p->state == VLIB_PROCESS_STATE_WAIT_FOR_EVENT_OR_CLOCK)
+    p->event_resume_pending = 0;
+
+  p->state = VLIB_PROCESS_STATE_RUNNING;
   r = clib_setjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_RETURN);
   if (r == VLIB_PROCESS_RETURN_LONGJMP_RETURN)
     {
@@ -1268,12 +1271,13 @@ dispatch_process (vlib_main_t * vm,
   u64 t;
   uword n_vectors, is_suspend;
 
-  if (node->state != VLIB_NODE_STATE_POLLING
-      || (p->flags & (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
-		      | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT)))
+  if (node->state != VLIB_NODE_STATE_POLLING)
+    return last_time_stamp;
+
+  if (p->state != VLIB_PROCESS_STATE_NOT_STARTED)
     return last_time_stamp;
 
-  p->flags |= VLIB_PROCESS_IS_RUNNING;
+  p->state = VLIB_PROCESS_STATE_RUNNING;
 
   t = last_time_stamp;
   vlib_elog_main_loop_event (vm, node_runtime->node_index, t,
@@ -1305,20 +1309,18 @@ dispatch_process (vlib_main_t * vm,
       p->n_suspends += 1;
       p->suspended_process_frame_index = pf - nm->suspended_process_frames;
 
-      if (p->flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK)
+      if (p->resume_clock_interval)
 	{
-	  TWT (tw_timer_wheel) * tw =
-	    (TWT (tw_timer_wheel) *) nm->timing_wheel;
+	  TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) nm->timing_wheel;
 	  p->stop_timer_handle =
 	    TW (tw_timer_start) (tw,
-				 vlib_timing_wheel_data_set_suspended_process
-				 (node->runtime_index) /* [sic] pool idex */ ,
-				 0 /* timer_id */ ,
-				 p->resume_clock_interval);
+				 vlib_timing_wheel_data_set_suspended_process (
+				   node->runtime_index) /* [sic] pool idex */,
+				 0 /* timer_id */, p->resume_clock_interval);
 	}
     }
   else
-    p->flags &= ~VLIB_PROCESS_IS_RUNNING;
+    p->state = VLIB_PROCESS_STATE_NOT_STARTED;
 
   t = clib_cpu_time_now ();
 
@@ -1346,8 +1348,8 @@ vlib_start_process (vlib_main_t * vm, uword process_index)
 }
 
 static u64
-dispatch_suspended_process (vlib_main_t * vm,
-			    uword process_index, u64 last_time_stamp)
+dispatch_suspended_process (vlib_main_t *vm, vlib_process_restore_t *r,
+			    u64 last_time_stamp)
 {
   vlib_node_main_t *nm = &vm->node_main;
   vlib_node_runtime_t *node_runtime;
@@ -1356,15 +1358,39 @@ dispatch_suspended_process (vlib_main_t * vm,
   vlib_process_t *p;
   vlib_pending_frame_t *pf;
   u64 t, n_vectors, is_suspend;
+  uword process_index = r->runtime_index;
+
+  u8 resume_permissons[VLIB_PROCRSS_N_RESTORE_REASON][VLIB_PROCESS_N_STATES] = {
+      [VLIB_PROCESS_RESTORE_REASON_YIELD] = {
+	[VLIB_PROCESS_STATE_YIELD] = 1,
+      },
+      [VLIB_PROCESS_RESTORE_REASON_CLOCK] = {
+	[VLIB_PROCESS_STATE_WAIT_FOR_CLOCK] = 1,
+	[VLIB_PROCESS_STATE_WAIT_FOR_EVENT_OR_CLOCK] = 1,
+	[VLIB_PROCESS_STATE_SUSPENDED] = 1,
+      },
+      [VLIB_PROCESS_RESTORE_REASON_EVENT] = {
+	[VLIB_PROCESS_STATE_WAIT_FOR_EVENT] = 1,
+	[VLIB_PROCESS_STATE_WAIT_FOR_EVENT_OR_CLOCK] = 1,
+      },
+      [VLIB_PROCESS_RESTORE_REASON_TIMED_EVENT] = {
+	[VLIB_PROCESS_STATE_WAIT_FOR_EVENT] = 1,
+	[VLIB_PROCESS_STATE_WAIT_FOR_EVENT_OR_CLOCK] = 1,
+      },
+  };
 
   t = last_time_stamp;
 
   p = vec_elt (nm->processes, process_index);
-  if (PREDICT_FALSE (!(p->flags & VLIB_PROCESS_IS_RUNNING)))
+
+  if (PREDICT_FALSE (p->state == VLIB_PROCESS_STATE_NOT_STARTED))
     return last_time_stamp;
 
-  ASSERT (p->flags & (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
-		      | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT));
+  if (resume_permissons[r->reason][p->state] == 0)
+    {
+      vec_add1 (nm->process_restore_next, *r);
+      return last_time_stamp;
+    }
 
   pf = pool_elt_at_index (nm->suspended_process_frames,
 			  p->suspended_process_frame_index);
@@ -1393,7 +1419,7 @@ dispatch_suspended_process (vlib_main_t * vm,
       /* Suspend it again. */
       n_vectors = 0;
       p->n_suspends += 1;
-      if (p->flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK)
+      if (p->resume_clock_interval)
 	{
 	  p->stop_timer_handle =
 	    TW (tw_timer_start) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
@@ -1405,7 +1431,7 @@ dispatch_suspended_process (vlib_main_t * vm,
     }
   else
     {
-      p->flags &= ~VLIB_PROCESS_IS_RUNNING;
+      p->state = VLIB_PROCESS_STATE_NOT_STARTED;
       pool_put_index (nm->suspended_process_frames,
 		      p->suspended_process_frame_index);
       p->suspended_process_frame_index = ~0;
@@ -1601,7 +1627,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
 	  } *ed;
 
 	  /* Check if process nodes have expired from timing wheel. */
-	  ASSERT (nm->data_from_advancing_timing_wheel != 0);
+	  ASSERT (nm->process_restore_current != 0);
 
 	  if (PREDICT_FALSE (vm->elog_trace_graph_dispatch))
 	    ed = ELOG_DATA (&vlib_global_main.elog_main, es);
@@ -1609,61 +1635,60 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
 	  TW (tw_timer_expire_timers)
 	  ((TWT (tw_timer_wheel) *) nm->timing_wheel, vlib_time_now (vm));
 
-	  ASSERT (nm->data_from_advancing_timing_wheel != 0);
+	  ASSERT (nm->process_restore_current != 0);
 
 	  if (PREDICT_FALSE (vm->elog_trace_graph_dispatch))
 	    {
 	      ed = ELOG_DATA (&vlib_global_main.elog_main, ee);
-	      ed->nready_procs =
-		_vec_len (nm->data_from_advancing_timing_wheel);
+	      ed->nready_procs = _vec_len (nm->process_restore_current);
 	    }
 
-	  if (PREDICT_FALSE
-	      (_vec_len (nm->data_from_advancing_timing_wheel) > 0))
+	  if (PREDICT_FALSE (_vec_len (nm->process_restore_current) > 0))
 	    {
 	      uword i;
 
-	      for (i = 0; i < _vec_len (nm->data_from_advancing_timing_wheel);
-		   i++)
+	      for (i = 0; i < _vec_len (nm->process_restore_current); i++)
 		{
-		  u32 d = nm->data_from_advancing_timing_wheel[i];
-		  u32 di = vlib_timing_wheel_data_get_index (d);
+		  vlib_process_restore_t *res =
+		    nm->process_restore_current + i;
 
-		  if (vlib_timing_wheel_data_is_timed_event (d))
-		    {
-		      vlib_signal_timed_event_data_t *te =
-			pool_elt_at_index (nm->signal_timed_event_data_pool,
-					   di);
-		      vlib_node_t *n =
-			vlib_get_node (vm, te->process_node_index);
-		      vlib_process_t *p =
-			vec_elt (nm->processes, n->runtime_index);
-		      p->stop_timer_handle = ~0;
-		      void *data;
-		      data =
-			vlib_process_signal_event_helper (nm, n, p,
-							  te->event_type_index,
-							  te->n_data_elts,
-							  te->n_data_elt_bytes);
-		      if (te->n_data_bytes < sizeof (te->inline_event_data))
-			clib_memcpy_fast (data, te->inline_event_data,
+		  if (res->reason == VLIB_PROCESS_RESTORE_REASON_TIMED_EVENT)
+		  {
+		    u32 di = res->timed_event_data_pool_index;
+		    vlib_signal_timed_event_data_t *te =
+		      pool_elt_at_index (nm->signal_timed_event_data_pool, di);
+		    vlib_node_t *n =
+		      vlib_get_node (vm, te->process_node_index);
+		    vlib_process_t *p =
+		      vec_elt (nm->processes, n->runtime_index);
+		    p->stop_timer_handle = ~0;
+		    void *data;
+		    data = vlib_process_signal_event_helper (
+		      nm, n, p, te->event_type_index, te->n_data_elts,
+		      te->n_data_elt_bytes);
+		    if (te->n_data_bytes < sizeof (te->inline_event_data))
+		      clib_memcpy_fast (data, te->inline_event_data,
+					te->n_data_bytes);
+		    else
+		      {
+			clib_memcpy_fast (data, te->event_data_as_vector,
 					  te->n_data_bytes);
-		      else
-			{
-			  clib_memcpy_fast (data, te->event_data_as_vector,
-					    te->n_data_bytes);
-			  vec_free (te->event_data_as_vector);
-			}
-		      pool_put (nm->signal_timed_event_data_pool, te);
-		    }
+			vec_free (te->event_data_as_vector);
+		      }
+		    pool_put (nm->signal_timed_event_data_pool, te);
+		  }
 		  else
 		    {
 		      cpu_time_now = clib_cpu_time_now ();
 		      cpu_time_now =
-			dispatch_suspended_process (vm, di, cpu_time_now);
+			dispatch_suspended_process (vm, res, cpu_time_now);
 		    }
 		}
-	      vec_set_len (nm->data_from_advancing_timing_wheel, 0);
+
+	      vec_set_len (nm->process_restore_current, 0);
+
+	      CLIB_SWAP (nm->process_restore_current,
+			 nm->process_restore_next);
 	    }
 	}
       vlib_increment_main_loop_counter (vm);
@@ -1846,12 +1871,23 @@ process_expired_timer_cb (u32 *expired_timer_handles)
 
   vec_foreach (handle, expired_timer_handles)
     {
-      u32 pi = vlib_timing_wheel_data_get_index (*handle);
-      vlib_process_t *p = vec_elt (nm->processes, pi);
+      u32 index = vlib_timing_wheel_data_get_index (*handle);
+      vlib_process_restore_t restore = {};
 
-      p->stop_timer_handle = ~0;
+      if (vlib_timing_wheel_data_is_timed_event (*handle))
+	{
+	  restore.reason = VLIB_PROCESS_RESTORE_REASON_TIMED_EVENT;
+	  restore.timed_event_data_pool_index = index;
+	}
+      else
+	{
+	  vlib_process_t *p = vec_elt (nm->processes, index);
+	  p->stop_timer_handle = ~0;
+	  restore.reason = VLIB_PROCESS_RESTORE_REASON_CLOCK;
+	  restore.runtime_index = index;
+	}
+      vec_add1 (nm->process_restore_current, restore);
     }
-  vec_append (nm->data_from_advancing_timing_wheel, expired_timer_handles);
 }
 
 /* Main function. */
@@ -1955,8 +1991,10 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input)
   nm->timing_wheel = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)),
 					     CLIB_CACHE_LINE_BYTES);
 
-  vec_validate (nm->data_from_advancing_timing_wheel, 10);
-  vec_set_len (nm->data_from_advancing_timing_wheel, 0);
+  vec_validate (nm->process_restore_current, 10);
+  vec_validate (nm->process_restore_next, 10);
+  vec_set_len (nm->process_restore_current, 0);
+  vec_set_len (nm->process_restore_next, 0);
 
   /* Create the process timing wheel */
   TW (tw_timer_wheel_init)
diff --git a/src/vlib/node.h b/src/vlib/node.h
index 651a39e3119..bb6d8f818a8 100644
--- a/src/vlib/node.h
+++ b/src/vlib/node.h
@@ -543,6 +543,45 @@ typedef struct
   uword opaque;
 } vlib_process_event_type_t;
 
+#define foreach_vlib_process_state                                            \
+  _ (NOT_STARTED, "not started")                                              \
+  _ (RUNNING, "running")                                                      \
+  _ (SUSPENDED, "suspended")                                                  \
+  _ (WAIT_FOR_EVENT, "event wait")                                            \
+  _ (WAIT_FOR_CLOCK, "clock wait")                                            \
+  _ (WAIT_FOR_EVENT_OR_CLOCK, "any wait")                                     \
+  _ (WAIT_FOR_ONE_TIME_EVENT, "one time event wait")                          \
+  _ (YIELD, "yield")
+
+typedef enum
+{
+#define _(n, s) VLIB_PROCESS_STATE_##n,
+  foreach_vlib_process_state VLIB_PROCESS_N_STATES
+#undef _
+} __clib_packed vlib_process_state_t;
+
+STATIC_ASSERT (VLIB_PROCESS_STATE_NOT_STARTED == 0, "");
+
+typedef enum
+{
+  VLIB_PROCESS_RESTORE_REASON_UNKNOWN = 0,
+  VLIB_PROCESS_RESTORE_REASON_EVENT,
+  VLIB_PROCESS_RESTORE_REASON_CLOCK,
+  VLIB_PROCESS_RESTORE_REASON_TIMED_EVENT,
+  VLIB_PROCESS_RESTORE_REASON_YIELD,
+  VLIB_PROCRSS_N_RESTORE_REASON
+} __clib_packed vlib_process_restore_reason_t;
+
+typedef struct
+{
+  vlib_process_restore_reason_t reason;
+  union
+  {
+    u32 runtime_index;
+    u32 timed_event_data_pool_index;
+  };
+} __clib_packed vlib_process_restore_t;
+
 typedef struct
 {
   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
@@ -560,14 +599,11 @@ typedef struct
 #define VLIB_PROCESS_RESUME_LONGJMP_SUSPEND 0
 #define VLIB_PROCESS_RESUME_LONGJMP_RESUME  1
 
-  u16 flags;
-#define VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK (1 << 0)
-#define VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT (1 << 1)
-  /* Set to indicate that this process has been added to resume vector. */
-#define VLIB_PROCESS_RESUME_PENDING (1 << 2)
+  /* Process state. */
+  vlib_process_state_t state;
 
-  /* Process function is currently running. */
-#define VLIB_PROCESS_IS_RUNNING (1 << 3)
+  /* Process is added to resume list due to pending event  */
+  u8 event_resume_pending : 1;
 
   /* Size of process stack. */
   u16 log2_n_stack_bytes;
@@ -711,8 +747,12 @@ typedef struct
 
   vlib_signal_timed_event_data_t *signal_timed_event_data_pool;
 
-  /* Opaque data vector added via timing_wheel_advance. */
-  u32 *data_from_advancing_timing_wheel;
+  /* Vector of process nodes waiting for restore */
+  vlib_process_restore_t *process_restore_current;
+
+  /* Vector of process nodes waiting for restore in next greaph scheduler run
+   */
+  vlib_process_restore_t *process_restore_next;
 
   /* CPU time of next process to be ready on timing wheel. */
   f64 time_next_process_ready;
diff --git a/src/vlib/node_cli.c b/src/vlib/node_cli.c
index d0bdf5b9097..16e904e8433 100644
--- a/src/vlib/node_cli.c
+++ b/src/vlib/node_cli.c
@@ -339,29 +339,16 @@ format_vlib_node_state (u8 * s, va_list * va)
   state = "active";
   if (n->type == VLIB_NODE_TYPE_PROCESS)
     {
+      char *state_str[] = {
+#define _(n, s) [VLIB_PROCESS_STATE_##n] = #s,
+	foreach_vlib_process_state
+      };
       vlib_process_t *p = vlib_get_process_from_node (vm, n);
 
-      switch (p->flags & (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
-			  | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT))
-	{
-	default:
-	  if (!(p->flags & VLIB_PROCESS_IS_RUNNING))
-	    state = "done";
-	  break;
-
-	case VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK:
-	  state = "time wait";
-	  break;
-
-	case VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT:
-	  state = "event wait";
-	  break;
-
-	case (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK):
-	  state =
-	    "any wait";
-	  break;
-	}
+      if (p->state >= ARRAY_LEN (state_str) || state_str[p->state] == 0)
+	state = "unknown";
+      else
+	state = state_str[p->state];
     }
   else if (n->type != VLIB_NODE_TYPE_INTERNAL)
     {
diff --git a/src/vlib/node_funcs.h b/src/vlib/node_funcs.h
index 4c7020d6a42..ffa17ba7bb1 100644
--- a/src/vlib/node_funcs.h
+++ b/src/vlib/node_funcs.h
@@ -187,10 +187,7 @@ vlib_node_set_state (vlib_main_t * vm, u32 node_index,
       vlib_process_t *p = vec_elt (nm->processes, n->runtime_index);
       r = &p->node_runtime;
 
-      /* When disabling make sure flags are cleared. */
-      p->flags &= ~(VLIB_PROCESS_RESUME_PENDING
-		    | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
-		    | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT);
+      p->event_resume_pending = 0;
     }
   else
     r = vec_elt_at_index (nm->nodes_by_type[n->type], n->runtime_index);
@@ -599,7 +596,7 @@ vlib_process_suspend (vlib_main_t * vm, f64 dt)
   if (vlib_process_suspend_time_is_zero (dt))
     return VLIB_PROCESS_RESUME_LONGJMP_RESUME;
 
-  p->flags |= VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK;
+  p->state = VLIB_PROCESS_STATE_SUSPENDED;
   r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND);
   if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND)
     {
@@ -614,6 +611,38 @@ vlib_process_suspend (vlib_main_t * vm, f64 dt)
   return r;
 }
 
+/** Suspend a vlib cooperative multi-tasking thread and put it at the end of
+ * resume queue
+    @param vm - vlib_main_t *
+    @returns VLIB_PROCESS_RESUME_LONGJMP_RESUME, routinely ignored
+*/
+
+always_inline uword
+vlib_process_yield (vlib_main_t *vm)
+{
+  uword r;
+  vlib_node_main_t *nm = &vm->node_main;
+  vlib_process_t *p = vec_elt (nm->processes, nm->current_process_index);
+
+  p->state = VLIB_PROCESS_STATE_YIELD;
+  r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND);
+  if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND)
+    {
+      vlib_process_restore_t r = {
+	.reason = VLIB_PROCESS_RESTORE_REASON_YIELD,
+	.runtime_index = nm->current_process_index,
+      };
+      p->resume_clock_interval = 0;
+      vec_add1 (nm->process_restore_next, r);
+      vlib_process_start_switch_stack (vm, 0);
+      clib_longjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_SUSPEND);
+    }
+  else
+    vlib_process_finish_switch_stack (vm);
+
+  return r;
+}
+
 always_inline void
 vlib_process_free_event_type (vlib_process_t * p, uword t,
 			      uword is_one_time_event)
@@ -773,11 +802,12 @@ vlib_process_wait_for_event (vlib_main_t * vm)
   p = vec_elt (nm->processes, nm->current_process_index);
   if (clib_bitmap_is_zero (p->non_empty_event_type_bitmap))
     {
-      p->flags |= VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT;
+      p->state = VLIB_PROCESS_STATE_WAIT_FOR_EVENT;
       r =
 	clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND);
       if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND)
 	{
+	  p->resume_clock_interval = 0;
 	  vlib_process_start_switch_stack (vm, 0);
 	  clib_longjmp (&p->return_longjmp,
 			VLIB_PROCESS_RETURN_LONGJMP_SUSPEND);
@@ -802,11 +832,12 @@ vlib_process_wait_for_one_time_event (vlib_main_t * vm,
   ASSERT (!pool_is_free_index (p->event_type_pool, with_type_index));
   while (!clib_bitmap_get (p->non_empty_event_type_bitmap, with_type_index))
     {
-      p->flags |= VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT;
+      p->state = VLIB_PROCESS_STATE_WAIT_FOR_ONE_TIME_EVENT;
       r =
 	clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND);
       if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND)
 	{
+	  p->resume_clock_interval = 0;
 	  vlib_process_start_switch_stack (vm, 0);
 	  clib_longjmp (&p->return_longjmp,
 			VLIB_PROCESS_RETURN_LONGJMP_SUSPEND);
@@ -831,11 +862,12 @@ vlib_process_wait_for_event_with_type (vlib_main_t * vm,
   h = hash_get (p->event_type_index_by_type_opaque, with_type_opaque);
   while (!h || !clib_bitmap_get (p->non_empty_event_type_bitmap, h[0]))
     {
-      p->flags |= VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT;
+      p->state = VLIB_PROCESS_STATE_WAIT_FOR_EVENT;
       r =
 	clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND);
       if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND)
 	{
+	  p->resume_clock_interval = 0;
 	  vlib_process_start_switch_stack (vm, 0);
 	  clib_longjmp (&p->return_longjmp,
 			VLIB_PROCESS_RETURN_LONGJMP_SUSPEND);
@@ -875,8 +907,7 @@ vlib_process_wait_for_event_or_clock (vlib_main_t * vm, f64 dt)
   wakeup_time = vlib_time_now (vm) + dt;
 
   /* Suspend waiting for both clock and event to occur. */
-  p->flags |= (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT
-	       | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK);
+  p->state = VLIB_PROCESS_STATE_WAIT_FOR_EVENT_OR_CLOCK;
 
   r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND);
   if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND)
@@ -938,7 +969,7 @@ vlib_process_signal_event_helper (vlib_node_main_t * nm,
 				  uword t,
 				  uword n_data_elts, uword n_data_elt_bytes)
 {
-  uword p_flags, add_to_pending, delete_from_wheel;
+  uword add_to_pending = 0, delete_from_wheel = 0;
   u8 *data_to_be_written_by_caller;
   vec_attr_t va = { .elt_sz = n_data_elt_bytes };
 
@@ -970,47 +1001,44 @@ vlib_process_signal_event_helper (vlib_node_main_t * nm,
   p->non_empty_event_type_bitmap =
     clib_bitmap_ori (p->non_empty_event_type_bitmap, t);
 
-  p_flags = p->flags;
+  switch (p->state)
+    {
+    case VLIB_PROCESS_STATE_WAIT_FOR_EVENT:
+      add_to_pending = 1;
+      break;
 
-  /* Event was already signalled? */
-  add_to_pending = (p_flags & VLIB_PROCESS_RESUME_PENDING) == 0;
+    case VLIB_PROCESS_STATE_WAIT_FOR_EVENT_OR_CLOCK:
+      add_to_pending = 1;
+      delete_from_wheel = 1;
+      break;
 
-  /* Process will resume when suspend time elapses? */
-  delete_from_wheel = 0;
-  if (p_flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK)
-    {
-      /* Waiting for both event and clock? */
-      if (p_flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT)
-	{
-	  if (!TW (tw_timer_handle_is_free)
-	      ((TWT (tw_timer_wheel) *) nm->timing_wheel,
-	       p->stop_timer_handle))
-	    delete_from_wheel = 1;
-	  else
-	    /* timer just popped so process should already be on the list */
-	    add_to_pending = 0;
-	}
-      else
-	/* Waiting only for clock.  Event will be queue and may be
-	   handled when timer expires. */
-	add_to_pending = 0;
+    default:
+      break;
     }
 
+  if (TW (tw_timer_handle_is_free) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
+				    p->stop_timer_handle))
+    delete_from_wheel = 0;
+
   /* Never add current process to pending vector since current process is
      already running. */
   add_to_pending &= nm->current_process_index != n->runtime_index;
 
-  if (add_to_pending)
+  if (add_to_pending && p->event_resume_pending == 0)
     {
-      u32 x = vlib_timing_wheel_data_set_suspended_process (n->runtime_index);
-      p->flags = p_flags | VLIB_PROCESS_RESUME_PENDING;
-      vec_add1 (nm->data_from_advancing_timing_wheel, x);
-      if (delete_from_wheel)
-	{
-	  TW (tw_timer_stop)
-	  ((TWT (tw_timer_wheel) *) nm->timing_wheel, p->stop_timer_handle);
-	  p->stop_timer_handle = ~0;
-	}
+      vlib_process_restore_t restore = {
+	.runtime_index = n->runtime_index,
+	.reason = VLIB_PROCESS_RESTORE_REASON_EVENT,
+      };
+      p->event_resume_pending = 1;
+      vec_add1 (nm->process_restore_current, restore);
+    }
+
+  if (delete_from_wheel)
+    {
+      TW (tw_timer_stop)
+      ((TWT (tw_timer_wheel) *) nm->timing_wheel, p->stop_timer_handle);
+      p->stop_timer_handle = ~0;
     }
 
   return data_to_be_written_by_caller;
diff --git a/src/vlibapi/node_serialize.c b/src/vlibapi/node_serialize.c
index 09b59247eab..3f921fb0450 100644
--- a/src/vlibapi/node_serialize.c
+++ b/src/vlibapi/node_serialize.c
@@ -84,26 +84,27 @@ vlib_node_serialize (vlib_main_t * vm, vlib_node_t *** node_dups, u8 * vector,
 	    {
 	      vlib_process_t *p = vlib_get_process_from_node (vm, n);
 
-	      switch (p->flags
-		      & (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
-			 | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT))
+	      switch (p->state)
 		{
 		default:
-		  if (!(p->flags & VLIB_PROCESS_IS_RUNNING))
-		    state_code = STATE_DONE;
 		  break;
 
-		case VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK:
+		case VLIB_PROCESS_STATE_WAIT_FOR_CLOCK:
+		case VLIB_PROCESS_STATE_SUSPENDED:
 		  state_code = STATE_TIME_WAIT;
 		  break;
 
-		case VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT:
+		case VLIB_PROCESS_STATE_WAIT_FOR_EVENT:
+		case VLIB_PROCESS_STATE_WAIT_FOR_ONE_TIME_EVENT:
 		  state_code = STATE_EVENT_WAIT;
 		  break;
 
-		case (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK):
-		  state_code =
-		    STATE_ANY_WAIT;
+		case VLIB_PROCESS_STATE_WAIT_FOR_EVENT_OR_CLOCK:
+		  state_code = STATE_ANY_WAIT;
+		  break;
+
+		case VLIB_PROCESS_STATE_NOT_STARTED:
+		  state_code = STATE_DONE;
 		  break;
 		}
 	    }