vcl: register workers when reattaching to vpp

Type: improvement
Signed-off-by: Maros Ondrejicka <maros.ondrejicka@pantheon.tech>
Change-Id: I82a286e2872338974c1930138c30db78103ae499

3 files changed, 51 insertions, 2 deletions

diff --git a/src/vcl/vcl_private.h b/src/vcl/vcl_private.h
index 846875df8a9..3df8b4bb473 100644
--- a/src/vcl/vcl_private.h
+++ b/src/vcl/vcl_private.h
@@ -341,6 +341,10 @@ typedef struct vppcom_main_t_
   /** Lock to protect worker registrations */
   clib_spinlock_t workers_lock;
 
+  /** Counter to determine order of execution of `vcl_api_retry_attach`
+   * function by multiple workers */
+  int reattach_count;
+
   /** Lock to protect segment hash table */
   clib_rwlock_t segment_table_lock;
 
diff --git a/src/vcl/vcl_sapi.c b/src/vcl/vcl_sapi.c
index 7d677e9dc1a..3a97fa25c0a 100644
--- a/src/vcl/vcl_sapi.c
+++ b/src/vcl/vcl_sapi.c
@@ -59,6 +59,8 @@ vcl_api_attach_reply_handler (app_sapi_attach_reply_msg_t * mp, int *fds)
     }
 
   wrk->api_client_handle = mp->api_client_handle;
+  /* reattaching via `vcl_api_retry_attach` wants wrk->vpp_wrk_index to be 0 */
+  wrk->vpp_wrk_index = 0;
   segment_handle = mp->segment_handle;
   if (segment_handle == VCL_INVALID_SEGMENT_HANDLE)
     {
diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c
index ae45be3b359..3538a36f508 100644
--- a/src/vcl/vppcom.c
+++ b/src/vcl/vppcom.c
@@ -1268,13 +1268,56 @@ vcl_api_attach (void)
   return vcl_bapi_attach ();
 }
 
+int
+vcl_is_first_reattach_to_execute ()
+{
+  if (vcm->reattach_count == 0)
+    return 1;
+
+  return 0;
+}
+
+void
+vcl_set_reattach_counter ()
+{
+  ++vcm->reattach_count;
+
+  if (vcm->reattach_count == vec_len (vcm->workers))
+    vcm->reattach_count = 0;
+}
+
+/**
+ * Reattach vcl to vpp after it has previously been disconnected.
+ *
+ * The logic should be:
+ * - first worker to hit `vcl_api_retry_attach` should attach to vpp,
+ *   to reproduce the `vcl_api_attach` in `vppcom_app_create`.
+ * - the rest of the workers should `reproduce vcl_worker_register_with_vpp`
+ *   from `vppcom_worker_register` since they were already allocated.
+ */
+
 static void
 vcl_api_retry_attach (vcl_worker_t *wrk)
 {
   vcl_session_t *s;
 
-  if (vcl_api_attach ())
-    return;
+  clib_spinlock_lock (&vcm->workers_lock);
+  if (vcl_is_first_reattach_to_execute ())
+    {
+      if (vcl_api_attach ())
+	{
+	  clib_spinlock_unlock (&vcm->workers_lock);
+	  return;
+	}
+      vcl_set_reattach_counter ();
+      clib_spinlock_unlock (&vcm->workers_lock);
+    }
+  else
+    {
+      vcl_set_reattach_counter ();
+      clib_spinlock_unlock (&vcm->workers_lock);
+      vcl_worker_register_with_vpp ();
+    }
 
   /* Treat listeners as configuration that needs to be re-added to vpp */
   pool_foreach (s, wrk->sessions)