vcl: register workers in order

Change-Id: Ibc74e7f7587f8b17fc0dcec20cc4530b9dd4c3ca
Signed-off-by: Florin Coras <fcoras@cisco.com>

4 files changed, 19 insertions, 5 deletions

diff --git a/src/vcl/vcl_private.c b/src/vcl/vcl_private.c
index f997c23aee2..ae4498ef4b5 100644
--- a/src/vcl/vcl_private.c
+++ b/src/vcl/vcl_private.c
@@ -224,7 +224,6 @@ static void
 vcl_worker_cleanup (void *arg)
 {
   vcl_worker_t *wrk = vcl_worker_get_current ();
-
   VDBG (0, "cleaning up worker %u", wrk->wrk_index);
   vcl_send_app_worker_add_del (0 /* is_add */ );
   close (wrk->mqs_epfd);
@@ -246,6 +245,13 @@ vcl_worker_alloc_and_init ()
   if (vcl_get_worker_index () != ~0)
     return 0;
 
+  if (pool_elts (vcm->workers) == vcm->cfg.max_workers)
+    {
+      VDBG (0, "max-workers %u limit reached", vcm->cfg.max_workers);
+      return 0;
+    }
+
+  clib_spinlock_lock (&vcm->workers_lock);
   wrk = vcl_worker_alloc ();
   vcl_set_worker_index (wrk->wrk_index);
 
@@ -269,10 +275,11 @@ vcl_worker_alloc_and_init ()
   vec_reset_length (wrk->mq_msg_vector);
 
   if (wrk->wrk_index == 0)
-    return wrk;
+    {
+      clib_spinlock_unlock (&vcm->workers_lock);
+      return wrk;
+    }
 
-  while (vcm->app_state == STATE_APP_ADDING_WORKER)
-    ;
   vcm->app_state = STATE_APP_ADDING_WORKER;
   vcl_send_app_worker_add_del (1 /* is_add */ );
   if (vcl_wait_for_app_state_change (STATE_APP_READY))
@@ -284,6 +291,8 @@ vcl_worker_alloc_and_init ()
   if (pthread_key_create (&vcl_worker_stop_key, vcl_worker_cleanup))
     clib_warning ("failed to add pthread cleanup function");
 
+  clib_spinlock_unlock (&vcm->workers_lock);
+
   VDBG (0, "added worker %u", wrk->wrk_index);
 
   return wrk;
diff --git a/src/vcl/vcl_private.h b/src/vcl/vcl_private.h
index ba9094cd96b..75f16ca74c2 100644
--- a/src/vcl/vcl_private.h
+++ b/src/vcl/vcl_private.h
@@ -283,6 +283,9 @@ typedef struct vppcom_main_t_
   /** Workers */
   vcl_worker_t *workers;
 
+  /** Lock to protect worker registrations */
+  clib_spinlock_t workers_lock;
+
 #ifdef VCL_ELOG
   /* VPP Event-logger */
   elog_main_t elog_main;
diff --git a/src/vcl/vcl_test_server.c b/src/vcl/vcl_test_server.c
index 8d97d0cfaa6..2fdd7ec157d 100644
--- a/src/vcl/vcl_test_server.c
+++ b/src/vcl/vcl_test_server.c
@@ -467,7 +467,8 @@ vts_worker_init (vcl_test_server_worker_t * wrk)
 
   conn_pool_expand (wrk, SOCK_SERVER_MAX_TEST_CONN + 1);
   if (wrk->wrk_index)
-    vppcom_worker_register ();
+    if (vppcom_worker_register ())
+      vtfail ("vppcom_worker_register()", 1);
 
   wrk->listen_fd = vppcom_session_create (ssm->cfg.proto,
 					  0 /* is_nonblocking */ );
diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c
index 15622014dcd..645550812b1 100644
--- a/src/vcl/vppcom.c
+++ b/src/vcl/vppcom.c
@@ -700,6 +700,7 @@ vppcom_app_create (char *app_name)
       svm_fifo_segment_main_init (vcl_cfg->segment_baseva,
 				  20 /* timeout in secs */ );
       pool_init_fixed (vcm->workers, vcl_cfg->max_workers);
+      clib_spinlock_init (&vcm->workers_lock);
       vcl_worker_alloc_and_init ();
     }