4 files changed, 232 insertions, 139 deletions

diff --git a/src/vlib/main.c b/src/vlib/main.c
index bf840324b64..a2f833711ab 100644
--- a/src/vlib/main.c
+++ b/src/vlib/main.c
@@ -1243,9 +1243,12 @@ static_always_inline uword
 vlib_process_resume (vlib_main_t * vm, vlib_process_t * p)
 {
   uword r;
-  p->flags &= ~(VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
-		| VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT
-		| VLIB_PROCESS_RESUME_PENDING);
+
+  if (p->state == VLIB_PROCESS_STATE_WAIT_FOR_EVENT ||
+      p->state == VLIB_PROCESS_STATE_WAIT_FOR_EVENT_OR_CLOCK)
+    p->event_resume_pending = 0;
+
+  p->state = VLIB_PROCESS_STATE_RUNNING;
   r = clib_setjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_RETURN);
   if (r == VLIB_PROCESS_RETURN_LONGJMP_RETURN)
     {
@@ -1268,12 +1271,13 @@ dispatch_process (vlib_main_t * vm,
   u64 t;
   uword n_vectors, is_suspend;
 
-  if (node->state != VLIB_NODE_STATE_POLLING
-      || (p->flags & (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
-		      | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT)))
+  if (node->state != VLIB_NODE_STATE_POLLING)
+    return last_time_stamp;
+
+  if (p->state != VLIB_PROCESS_STATE_NOT_STARTED)
     return last_time_stamp;
 
-  p->flags |= VLIB_PROCESS_IS_RUNNING;
+  p->state = VLIB_PROCESS_STATE_RUNNING;
 
   t = last_time_stamp;
   vlib_elog_main_loop_event (vm, node_runtime->node_index, t,
@@ -1305,20 +1309,18 @@ dispatch_process (vlib_main_t * vm,
       p->n_suspends += 1;
       p->suspended_process_frame_index = pf - nm->suspended_process_frames;
 
-      if (p->flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK)
+      if (p->resume_clock_interval)
 	{
-	  TWT (tw_timer_wheel) * tw =
-	    (TWT (tw_timer_wheel) *) nm->timing_wheel;
+	  TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) nm->timing_wheel;
 	  p->stop_timer_handle =
 	    TW (tw_timer_start) (tw,
-				 vlib_timing_wheel_data_set_suspended_process
-				 (node->runtime_index) /* [sic] pool idex */ ,
-				 0 /* timer_id */ ,
-				 p->resume_clock_interval);
+				 vlib_timing_wheel_data_set_suspended_process (
+				   node->runtime_index) /* [sic] pool idex */,
+				 0 /* timer_id */, p->resume_clock_interval);
 	}
     }
   else
-    p->flags &= ~VLIB_PROCESS_IS_RUNNING;
+    p->state = VLIB_PROCESS_STATE_NOT_STARTED;
 
   t = clib_cpu_time_now ();
 
@@ -1346,8 +1348,8 @@ vlib_start_process (vlib_main_t * vm, uword process_index)
 }
 
 static u64
-dispatch_suspended_process (vlib_main_t * vm,
-			    uword process_index, u64 last_time_stamp)
+dispatch_suspended_process (vlib_main_t *vm, vlib_process_restore_t *r,
+			    u64 last_time_stamp)
 {
   vlib_node_main_t *nm = &vm->node_main;
   vlib_node_runtime_t *node_runtime;
@@ -1356,15 +1358,39 @@ dispatch_suspended_process (vlib_main_t * vm,
   vlib_process_t *p;
   vlib_pending_frame_t *pf;
   u64 t, n_vectors, is_suspend;
+  uword process_index = r->runtime_index;
+
+  u8 resume_permissons[VLIB_PROCRSS_N_RESTORE_REASON][VLIB_PROCESS_N_STATES] = {
+      [VLIB_PROCESS_RESTORE_REASON_YIELD] = {
+	[VLIB_PROCESS_STATE_YIELD] = 1,
+      },
+      [VLIB_PROCESS_RESTORE_REASON_CLOCK] = {
+	[VLIB_PROCESS_STATE_WAIT_FOR_CLOCK] = 1,
+	[VLIB_PROCESS_STATE_WAIT_FOR_EVENT_OR_CLOCK] = 1,
+	[VLIB_PROCESS_STATE_SUSPENDED] = 1,
+      },
+      [VLIB_PROCESS_RESTORE_REASON_EVENT] = {
+	[VLIB_PROCESS_STATE_WAIT_FOR_EVENT] = 1,
+	[VLIB_PROCESS_STATE_WAIT_FOR_EVENT_OR_CLOCK] = 1,
+      },
+      [VLIB_PROCESS_RESTORE_REASON_TIMED_EVENT] = {
+	[VLIB_PROCESS_STATE_WAIT_FOR_EVENT] = 1,
+	[VLIB_PROCESS_STATE_WAIT_FOR_EVENT_OR_CLOCK] = 1,
+      },
+  };
 
   t = last_time_stamp;
 
   p = vec_elt (nm->processes, process_index);
-  if (PREDICT_FALSE (!(p->flags & VLIB_PROCESS_IS_RUNNING)))
+
+  if (PREDICT_FALSE (p->state == VLIB_PROCESS_STATE_NOT_STARTED))
     return last_time_stamp;
 
-  ASSERT (p->flags & (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
-		      | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT));
+  if (resume_permissons[r->reason][p->state] == 0)
+    {
+      vec_add1 (nm->process_restore_next, *r);
+      return last_time_stamp;
+    }
 
   pf = pool_elt_at_index (nm->suspended_process_frames,
 			  p->suspended_process_frame_index);
@@ -1393,7 +1419,7 @@ dispatch_suspended_process (vlib_main_t * vm,
       /* Suspend it again. */
       n_vectors = 0;
       p->n_suspends += 1;
-      if (p->flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK)
+      if (p->resume_clock_interval)
 	{
 	  p->stop_timer_handle =
 	    TW (tw_timer_start) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
@@ -1405,7 +1431,7 @@ dispatch_suspended_process (vlib_main_t * vm,
     }
   else
     {
-      p->flags &= ~VLIB_PROCESS_IS_RUNNING;
+      p->state = VLIB_PROCESS_STATE_NOT_STARTED;
       pool_put_index (nm->suspended_process_frames,
 		      p->suspended_process_frame_index);
       p->suspended_process_frame_index = ~0;
@@ -1601,7 +1627,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
 	  } *ed;
 
 	  /* Check if process nodes have expired from timing wheel. */
-	  ASSERT (nm->data_from_advancing_timing_wheel != 0);
+	  ASSERT (nm->process_restore_current != 0);
 
 	  if (PREDICT_FALSE (vm->elog_trace_graph_dispatch))
 	    ed = ELOG_DATA (&vlib_global_main.elog_main, es);
@@ -1609,61 +1635,60 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
 	  TW (tw_timer_expire_timers)
 	  ((TWT (tw_timer_wheel) *) nm->timing_wheel, vlib_time_now (vm));
 
-	  ASSERT (nm->data_from_advancing_timing_wheel != 0);
+	  ASSERT (nm->process_restore_current != 0);
 
 	  if (PREDICT_FALSE (vm->elog_trace_graph_dispatch))
 	    {
 	      ed = ELOG_DATA (&vlib_global_main.elog_main, ee);
-	      ed->nready_procs =
-		_vec_len (nm->data_from_advancing_timing_wheel);
+	      ed->nready_procs = _vec_len (nm->process_restore_current);
 	    }
 
-	  if (PREDICT_FALSE
-	      (_vec_len (nm->data_from_advancing_timing_wheel) > 0))
+	  if (PREDICT_FALSE (_vec_len (nm->process_restore_current) > 0))
 	    {
 	      uword i;
 
-	      for (i = 0; i < _vec_len (nm->data_from_advancing_timing_wheel);
-		   i++)
+	      for (i = 0; i < _vec_len (nm->process_restore_current); i++)
 		{
-		  u32 d = nm->data_from_advancing_timing_wheel[i];
-		  u32 di = vlib_timing_wheel_data_get_index (d);
+		  vlib_process_restore_t *res =
+		    nm->process_restore_current + i;
 
-		  if (vlib_timing_wheel_data_is_timed_event (d))
-		    {
-		      vlib_signal_timed_event_data_t *te =
-			pool_elt_at_index (nm->signal_timed_event_data_pool,
-					   di);
-		      vlib_node_t *n =
-			vlib_get_node (vm, te->process_node_index);
-		      vlib_process_t *p =
-			vec_elt (nm->processes, n->runtime_index);
-		      p->stop_timer_handle = ~0;
-		      void *data;
-		      data =
-			vlib_process_signal_event_helper (nm, n, p,
-							  te->event_type_index,
-							  te->n_data_elts,
-							  te->n_data_elt_bytes);
-		      if (te->n_data_bytes < sizeof (te->inline_event_data))
-			clib_memcpy_fast (data, te->inline_event_data,
+		  if (res->reason == VLIB_PROCESS_RESTORE_REASON_TIMED_EVENT)
+		  {
+		    u32 di = res->timed_event_data_pool_index;
+		    vlib_signal_timed_event_data_t *te =
+		      pool_elt_at_index (nm->signal_timed_event_data_pool, di);
+		    vlib_node_t *n =
+		      vlib_get_node (vm, te->process_node_index);
+		    vlib_process_t *p =
+		      vec_elt (nm->processes, n->runtime_index);
+		    p->stop_timer_handle = ~0;
+		    void *data;
+		    data = vlib_process_signal_event_helper (
+		      nm, n, p, te->event_type_index, te->n_data_elts,
+		      te->n_data_elt_bytes);
+		    if (te->n_data_bytes < sizeof (te->inline_event_data))
+		      clib_memcpy_fast (data, te->inline_event_data,
+					te->n_data_bytes);
+		    else
+		      {
+			clib_memcpy_fast (data, te->event_data_as_vector,
 					  te->n_data_bytes);
-		      else
-			{
-			  clib_memcpy_fast (data, te->event_data_as_vector,
-					    te->n_data_bytes);
-			  vec_free (te->event_data_as_vector);
-			}
-		      pool_put (nm->signal_timed_event_data_pool, te);
-		    }
+			vec_free (te->event_data_as_vector);
+		      }
+		    pool_put (nm->signal_timed_event_data_pool, te);
+		  }
 		  else
 		    {
 		      cpu_time_now = clib_cpu_time_now ();
 		      cpu_time_now =
-			dispatch_suspended_process (vm, di, cpu_time_now);
+			dispatch_suspended_process (vm, res, cpu_time_now);
 		    }
 		}
-	      vec_set_len (nm->data_from_advancing_timing_wheel, 0);
+
+	      vec_set_len (nm->process_restore_current, 0);
+
+	      CLIB_SWAP (nm->process_restore_current,
+			 nm->process_restore_next);
 	    }
 	}
       vlib_increment_main_loop_counter (vm);
@@ -1846,12 +1871,23 @@ process_expired_timer_cb (u32 *expired_timer_handles)
 
   vec_foreach (handle, expired_timer_handles)
     {
-      u32 pi = vlib_timing_wheel_data_get_index (*handle);
-      vlib_process_t *p = vec_elt (nm->processes, pi);
+      u32 index = vlib_timing_wheel_data_get_index (*handle);
+      vlib_process_restore_t restore = {};
 
-      p->stop_timer_handle = ~0;
+      if (vlib_timing_wheel_data_is_timed_event (*handle))
+	{
+	  restore.reason = VLIB_PROCESS_RESTORE_REASON_TIMED_EVENT;
+	  restore.timed_event_data_pool_index = index;
+	}
+      else
+	{
+	  vlib_process_t *p = vec_elt (nm->processes, index);
+	  p->stop_timer_handle = ~0;
+	  restore.reason = VLIB_PROCESS_RESTORE_REASON_CLOCK;
+	  restore.runtime_index = index;
+	}
+      vec_add1 (nm->process_restore_current, restore);
     }
-  vec_append (nm->data_from_advancing_timing_wheel, expired_timer_handles);
 }
 
 /* Main function. */
@@ -1955,8 +1991,10 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input)
   nm->timing_wheel = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)),
 					     CLIB_CACHE_LINE_BYTES);
 
-  vec_validate (nm->data_from_advancing_timing_wheel, 10);
-  vec_set_len (nm->data_from_advancing_timing_wheel, 0);
+  vec_validate (nm->process_restore_current, 10);
+  vec_validate (nm->process_restore_next, 10);
+  vec_set_len (nm->process_restore_current, 0);
+  vec_set_len (nm->process_restore_next, 0);
 
   /* Create the process timing wheel */
   TW (tw_timer_wheel_init)
diff --git a/src/vlib/node.h b/src/vlib/node.h
index 651a39e3119..bb6d8f818a8 100644
--- a/src/vlib/node.h
+++ b/src/vlib/node.h
@@ -543,6 +543,45 @@ typedef struct
   uword opaque;
 } vlib_process_event_type_t;
 
+#define foreach_vlib_process_state                                            \
+  _ (NOT_STARTED, "not started")                                              \
+  _ (RUNNING, "running")                                                      \
+  _ (SUSPENDED, "suspended")                                                  \
+  _ (WAIT_FOR_EVENT, "event wait")                                            \
+  _ (WAIT_FOR_CLOCK, "clock wait")                                            \
+  _ (WAIT_FOR_EVENT_OR_CLOCK, "any wait")                                     \
+  _ (WAIT_FOR_ONE_TIME_EVENT, "one time event wait")                          \
+  _ (YIELD, "yield")
+
+typedef enum
+{
+#define _(n, s) VLIB_PROCESS_STATE_##n,
+  foreach_vlib_process_state VLIB_PROCESS_N_STATES
+#undef _
+} __clib_packed vlib_process_state_t;
+
+STATIC_ASSERT (VLIB_PROCESS_STATE_NOT_STARTED == 0, "");
+
+typedef enum
+{
+  VLIB_PROCESS_RESTORE_REASON_UNKNOWN = 0,
+  VLIB_PROCESS_RESTORE_REASON_EVENT,
+  VLIB_PROCESS_RESTORE_REASON_CLOCK,
+  VLIB_PROCESS_RESTORE_REASON_TIMED_EVENT,
+  VLIB_PROCESS_RESTORE_REASON_YIELD,
+  VLIB_PROCRSS_N_RESTORE_REASON
+} __clib_packed vlib_process_restore_reason_t;
+
+typedef struct
+{
+  vlib_process_restore_reason_t reason;
+  union
+  {
+    u32 runtime_index;
+    u32 timed_event_data_pool_index;
+  };
+} __clib_packed vlib_process_restore_t;
+
 typedef struct
 {
   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
@@ -560,14 +599,11 @@ typedef struct
 #define VLIB_PROCESS_RESUME_LONGJMP_SUSPEND 0
 #define VLIB_PROCESS_RESUME_LONGJMP_RESUME  1
 
-  u16 flags;
-#define VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK (1 << 0)
-#define VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT (1 << 1)
-  /* Set to indicate that this process has been added to resume vector. */
-#define VLIB_PROCESS_RESUME_PENDING (1 << 2)
+  /* Process state. */
+  vlib_process_state_t state;
 
-  /* Process function is currently running. */
-#define VLIB_PROCESS_IS_RUNNING (1 << 3)
+  /* Process is added to resume list due to pending event  */
+  u8 event_resume_pending : 1;
 
   /* Size of process stack. */
   u16 log2_n_stack_bytes;
@@ -711,8 +747,12 @@ typedef struct
 
   vlib_signal_timed_event_data_t *signal_timed_event_data_pool;
 
-  /* Opaque data vector added via timing_wheel_advance. */
-  u32 *data_from_advancing_timing_wheel;
+  /* Vector of process nodes waiting for restore */
+  vlib_process_restore_t *process_restore_current;
+
+  /* Vector of process nodes waiting for restore in next greaph scheduler run
+   */
+  vlib_process_restore_t *process_restore_next;
 
   /* CPU time of next process to be ready on timing wheel. */
   f64 time_next_process_ready;
diff --git a/src/vlib/node_cli.c b/src/vlib/node_cli.c
index d0bdf5b9097..16e904e8433 100644
--- a/src/vlib/node_cli.c
+++ b/src/vlib/node_cli.c
@@ -339,29 +339,16 @@ format_vlib_node_state (u8 * s, va_list * va)
   state = "active";
   if (n->type == VLIB_NODE_TYPE_PROCESS)
     {
+      char *state_str[] = {
+#define _(n, s) [VLIB_PROCESS_STATE_##n] = #s,
+	foreach_vlib_process_state
+      };
       vlib_process_t *p = vlib_get_process_from_node (vm, n);
 
-      switch (p->flags & (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
-			  | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT))
-	{
-	default:
-	  if (!(p->flags & VLIB_PROCESS_IS_RUNNING))
-	    state = "done";
-	  break;
-
-	case VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK:
-	  state = "time wait";
-	  break;
-
-	case VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT:
-	  state = "event wait";
-	  break;
-
-	case (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK):
-	  state =
-	    "any wait";
-	  break;
-	}
+      if (p->state >= ARRAY_LEN (state_str) || state_str[p->state] == 0)
+	state = "unknown";
+      else
+	state = state_str[p->state];
     }
   else if (n->type != VLIB_NODE_TYPE_INTERNAL)
     {
diff --git a/src/vlib/node_funcs.h b/src/vlib/node_funcs.h
index 4c7020d6a42..ffa17ba7bb1 100644
--- a/src/vlib/node_funcs.h
+++ b/src/vlib/node_funcs.h
@@ -187,10 +187,7 @@ vlib_node_set_state (vlib_main_t * vm, u32 node_index,
       vlib_process_t *p = vec_elt (nm->processes, n->runtime_index);
       r = &p->node_runtime;
 
-      /* When disabling make sure flags are cleared. */
-      p->flags &= ~(VLIB_PROCESS_RESUME_PENDING
-		    | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
-		    | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT);
+      p->event_resume_pending = 0;
     }
   else
     r = vec_elt_at_index (nm->nodes_by_type[n->type], n->runtime_index);
@@ -599,7 +596,7 @@ vlib_process_suspend (vlib_main_t * vm, f64 dt)
   if (vlib_process_suspend_time_is_zero (dt))
     return VLIB_PROCESS_RESUME_LONGJMP_RESUME;
 
-  p->flags |= VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK;
+  p->state = VLIB_PROCESS_STATE_SUSPENDED;
   r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND);
   if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND)
     {
@@ -614,6 +611,38 @@ vlib_process_suspend (vlib_main_t * vm, f64 dt)
   return r;
 }
 
+/** Suspend a vlib cooperative multi-tasking thread and put it at the end of
+ * resume queue
+    @param vm - vlib_main_t *
+    @returns VLIB_PROCESS_RESUME_LONGJMP_RESUME, routinely ignored
+*/
+
+always_inline uword
+vlib_process_yield (vlib_main_t *vm)
+{
+  uword r;
+  vlib_node_main_t *nm = &vm->node_main;
+  vlib_process_t *p = vec_elt (nm->processes, nm->current_process_index);
+
+  p->state = VLIB_PROCESS_STATE_YIELD;
+  r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND);
+  if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND)
+    {
+      vlib_process_restore_t r = {
+	.reason = VLIB_PROCESS_RESTORE_REASON_YIELD,
+	.runtime_index = nm->current_process_index,
+      };
+      p->resume_clock_interval = 0;
+      vec_add1 (nm->process_restore_next, r);
+      vlib_process_start_switch_stack (vm, 0);
+      clib_longjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_SUSPEND);
+    }
+  else
+    vlib_process_finish_switch_stack (vm);
+
+  return r;
+}
+
 always_inline void
 vlib_process_free_event_type (vlib_process_t * p, uword t,
 			      uword is_one_time_event)
@@ -773,11 +802,12 @@ vlib_process_wait_for_event (vlib_main_t * vm)
   p = vec_elt (nm->processes, nm->current_process_index);
   if (clib_bitmap_is_zero (p->non_empty_event_type_bitmap))
     {
-      p->flags |= VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT;
+      p->state = VLIB_PROCESS_STATE_WAIT_FOR_EVENT;
       r =
 	clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND);
       if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND)
 	{
+	  p->resume_clock_interval = 0;
 	  vlib_process_start_switch_stack (vm, 0);
 	  clib_longjmp (&p->return_longjmp,
 			VLIB_PROCESS_RETURN_LONGJMP_SUSPEND);
@@ -802,11 +832,12 @@ vlib_process_wait_for_one_time_event (vlib_main_t * vm,
   ASSERT (!pool_is_free_index (p->event_type_pool, with_type_index));
   while (!clib_bitmap_get (p->non_empty_event_type_bitmap, with_type_index))
     {
-      p->flags |= VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT;
+      p->state = VLIB_PROCESS_STATE_WAIT_FOR_ONE_TIME_EVENT;
       r =
 	clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND);
       if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND)
 	{
+	  p->resume_clock_interval = 0;
 	  vlib_process_start_switch_stack (vm, 0);
 	  clib_longjmp (&p->return_longjmp,
 			VLIB_PROCESS_RETURN_LONGJMP_SUSPEND);
@@ -831,11 +862,12 @@ vlib_process_wait_for_event_with_type (vlib_main_t * vm,
   h = hash_get (p->event_type_index_by_type_opaque, with_type_opaque);
   while (!h || !clib_bitmap_get (p->non_empty_event_type_bitmap, h[0]))
     {
-      p->flags |= VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT;
+      p->state = VLIB_PROCESS_STATE_WAIT_FOR_EVENT;
       r =
 	clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND);
       if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND)
 	{
+	  p->resume_clock_interval = 0;
 	  vlib_process_start_switch_stack (vm, 0);
 	  clib_longjmp (&p->return_longjmp,
 			VLIB_PROCESS_RETURN_LONGJMP_SUSPEND);
@@ -875,8 +907,7 @@ vlib_process_wait_for_event_or_clock (vlib_main_t * vm, f64 dt)
   wakeup_time = vlib_time_now (vm) + dt;
 
   /* Suspend waiting for both clock and event to occur. */
-  p->flags |= (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT
-	       | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK);
+  p->state = VLIB_PROCESS_STATE_WAIT_FOR_EVENT_OR_CLOCK;
 
   r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND);
   if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND)
@@ -938,7 +969,7 @@ vlib_process_signal_event_helper (vlib_node_main_t * nm,
 				  uword t,
 				  uword n_data_elts, uword n_data_elt_bytes)
 {
-  uword p_flags, add_to_pending, delete_from_wheel;
+  uword add_to_pending = 0, delete_from_wheel = 0;
   u8 *data_to_be_written_by_caller;
   vec_attr_t va = { .elt_sz = n_data_elt_bytes };
 
@@ -970,47 +1001,44 @@ vlib_process_signal_event_helper (vlib_node_main_t * nm,
   p->non_empty_event_type_bitmap =
     clib_bitmap_ori (p->non_empty_event_type_bitmap, t);
 
-  p_flags = p->flags;
+  switch (p->state)
+    {
+    case VLIB_PROCESS_STATE_WAIT_FOR_EVENT:
+      add_to_pending = 1;
+      break;
 
-  /* Event was already signalled? */
-  add_to_pending = (p_flags & VLIB_PROCESS_RESUME_PENDING) == 0;
+    case VLIB_PROCESS_STATE_WAIT_FOR_EVENT_OR_CLOCK:
+      add_to_pending = 1;
+      delete_from_wheel = 1;
+      break;
 
-  /* Process will resume when suspend time elapses? */
-  delete_from_wheel = 0;
-  if (p_flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK)
-    {
-      /* Waiting for both event and clock? */
-      if (p_flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT)
-	{
-	  if (!TW (tw_timer_handle_is_free)
-	      ((TWT (tw_timer_wheel) *) nm->timing_wheel,
-	       p->stop_timer_handle))
-	    delete_from_wheel = 1;
-	  else
-	    /* timer just popped so process should already be on the list */
-	    add_to_pending = 0;
-	}
-      else
-	/* Waiting only for clock.  Event will be queue and may be
-	   handled when timer expires. */
-	add_to_pending = 0;
+    default:
+      break;
     }
 
+  if (TW (tw_timer_handle_is_free) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
+				    p->stop_timer_handle))
+    delete_from_wheel = 0;
+
   /* Never add current process to pending vector since current process is
      already running. */
   add_to_pending &= nm->current_process_index != n->runtime_index;
 
-  if (add_to_pending)
+  if (add_to_pending && p->event_resume_pending == 0)
     {
-      u32 x = vlib_timing_wheel_data_set_suspended_process (n->runtime_index);
-      p->flags = p_flags | VLIB_PROCESS_RESUME_PENDING;
-      vec_add1 (nm->data_from_advancing_timing_wheel, x);
-      if (delete_from_wheel)
-	{
-	  TW (tw_timer_stop)
-	  ((TWT (tw_timer_wheel) *) nm->timing_wheel, p->stop_timer_handle);
-	  p->stop_timer_handle = ~0;
-	}
+      vlib_process_restore_t restore = {
+	.runtime_index = n->runtime_index,
+	.reason = VLIB_PROCESS_RESTORE_REASON_EVENT,
+      };
+      p->event_resume_pending = 1;
+      vec_add1 (nm->process_restore_current, restore);
+    }
+
+  if (delete_from_wheel)
+    {
+      TW (tw_timer_stop)
+      ((TWT (tw_timer_wheel) *) nm->timing_wheel, p->stop_timer_handle);
+      p->stop_timer_handle = ~0;
     }
 
   return data_to_be_written_by_caller;