vppinfra: Expose function setting __os_thread_index

Type: feature

This is needed when creating pthreads in client applications,
they need a way to set __os_thread_index per thread
that does not conflict with the binary API thread index.

If __os_thread_index is left to 0 in two client pthreads and
they call vl_msg_api_alloc and vec_resize at the same time
it can fail due to them sharing (and push/poping) the same
clib_per_cpu_mheaps slot.

Change-Id: I85d4248a39b641a4d3ad5a1c1bd6e0db5875fab6
Signed-off-by: Nathan Skrzypczak <nathan.skrzypczak@gmail.com>

3 files changed, 28 insertions, 20 deletions

diff --git a/src/vlibmemory/memory_client.c b/src/vlibmemory/memory_client.c
index d002845a00c..f032ae77d19 100644
--- a/src/vlibmemory/memory_client.c
+++ b/src/vlibmemory/memory_client.c
@@ -63,7 +63,6 @@ rx_thread_fn (void *arg)
   svm_queue_t *q;
   memory_client_main_t *mm = &memory_client_main;
   api_main_t *am = &api_main;
-  int i;
 
   q = am->vl_input_queue;
 
@@ -71,25 +70,7 @@ rx_thread_fn (void *arg)
   if (setjmp (mm->rx_thread_jmpbuf) == 0)
     {
       mm->rx_thread_jmpbuf_valid = 1;
-      /*
-       * Find an unused slot in the per-cpu-mheaps array,
-       * and grab it for this thread. We need to be able to
-       * push/pop the thread heap without affecting other thread(s).
-       */
-      if (__os_thread_index == 0)
-	{
-	  for (i = 0; i < ARRAY_LEN (clib_per_cpu_mheaps); i++)
-	    {
-	      if (clib_per_cpu_mheaps[i] == 0)
-		{
-		  /* Copy the main thread mheap pointer */
-		  clib_per_cpu_mheaps[i] = clib_per_cpu_mheaps[0];
-		  __os_thread_index = i;
-		  break;
-		}
-	    }
-	  ASSERT (__os_thread_index > 0);
-	}
+      clib_mem_set_thread_index ();
       while (1)
 	vl_msg_api_queue_handler (q);
     }
diff --git a/src/vppinfra/mem.h b/src/vppinfra/mem.h
index e31ec82e7c5..f2930a11253 100644
--- a/src/vppinfra/mem.h
+++ b/src/vppinfra/mem.h
@@ -60,6 +60,27 @@
 /* Per CPU heaps. */
 extern void *clib_per_cpu_mheaps[CLIB_MAX_MHEAPS];
 
+always_inline void
+clib_mem_set_thread_index (void)
+{
+  /*
+   * Find an unused slot in the per-cpu-mheaps array,
+   * and grab it for this thread. We need to be able to
+   * push/pop the thread heap without affecting other thread(s).
+   */
+  int i;
+  if (__os_thread_index != 0)
+    return;
+  for (i = 0; i < ARRAY_LEN (clib_per_cpu_mheaps); i++)
+    if (clib_atomic_bool_cmp_and_swap (&clib_per_cpu_mheaps[i],
+				       0, clib_per_cpu_mheaps[0]))
+      {
+	os_set_thread_index (i);
+	break;
+      }
+  ASSERT (__os_thread_index > 0);
+}
+
 always_inline void *
 clib_mem_get_per_cpu_heap (void)
 {
diff --git a/src/vppinfra/os.h b/src/vppinfra/os.h
index 3ec214e29b1..50a4ad97c93 100644
--- a/src/vppinfra/os.h
+++ b/src/vppinfra/os.h
@@ -64,6 +64,12 @@ os_get_thread_index (void)
   return __os_thread_index;
 }
 
+static_always_inline void
+os_set_thread_index (uword thread_index)
+{
+  __os_thread_index = thread_index;
+}
+
 static_always_inline uword
 os_get_cpu_number (void) __attribute__ ((deprecated));