From 7752b39af3b4d99e6ff649ec7c294ad56665b1f9 Mon Sep 17 00:00:00 2001
From: Damjan Marion <damarion@cisco.com>
Date: Tue, 3 Jul 2018 17:24:45 +0200
Subject: Set main thread affinity before main heap is allocated

Change-Id: I524909570fc1736f51fd437d6d30566c461139bd
Signed-off-by: Damjan Marion <damarion@cisco.com>
---
 src/vlib/threads.c  | 14 +++++++-------
 src/vpp/vnet/main.c | 21 +++++++++++++++++++++
 2 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/src/vlib/threads.c b/src/vlib/threads.c
index 487c501db5f..4e4f13eae86 100644
--- a/src/vlib/threads.c
+++ b/src/vlib/threads.c
@@ -337,6 +337,12 @@ vlib_thread_init (vlib_main_t * vm)
 
   avail_cpu = clib_bitmap_dup (tm->cpu_core_bitmap);
 
+  /* by default we skip core 0, unless it is the only one available */
+  if (tm->skip_cores == ~0 && clib_bitmap_count_set_bits (avail_cpu) == 1)
+    tm->skip_cores = 0;
+  else
+    tm->skip_cores = 1;
+
   /* skip cores */
   for (i = 0; i < tm->skip_cores; i++)
     {
@@ -372,13 +378,6 @@ vlib_thread_init (vlib_main_t * vm)
     {
       tm->cb.vlib_thread_set_lcore_cb (0, tm->main_lcore);
     }
-  else
-    {
-      cpu_set_t cpuset;
-      CPU_ZERO (&cpuset);
-      CPU_SET (tm->main_lcore, &cpuset);
-      pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t), &cpuset);
-    }
 
   /* as many threads as stacks... */
   vec_validate_aligned (vlib_worker_threads, vec_len (vlib_thread_stacks) - 1,
@@ -1262,6 +1261,7 @@ cpu_config (vlib_main_t * vm, unformat_input_t * input)
   tm->n_thread_stacks = 1;	/* account for main thread */
   tm->sched_policy = ~0;
   tm->sched_priority = ~0;
+  tm->skip_cores = ~0;
 
   tr = tm->next;
 
diff --git a/src/vpp/vnet/main.c b/src/vpp/vnet/main.c
index 6e136e19201..9a804eb3641 100644
--- a/src/vpp/vnet/main.c
+++ b/src/vpp/vnet/main.c
@@ -13,6 +13,10 @@
  * limitations under the License.
  */
 
+#define _GNU_SOURCE
+#include <pthread.h>
+#include <sched.h>
+
 #include <vppinfra/cpu.h>
 #include <vlib/vlib.h>
 #include <vlib/unix/unix.h>
@@ -103,6 +107,8 @@ main (int argc, char *argv[])
   uword main_heap_size = (1ULL << 30);
   u8 *sizep;
   u32 size;
+  int main_core = 1;
+  cpu_set_t cpuset;
 
 #if __x86_64__
   CLIB_UNUSED (const char *msg)
@@ -234,10 +240,25 @@ main (int argc, char *argv[])
 	  else if (*sizep == 'm' || *sizep == 'M')
 	    main_heap_size <<= 20;
 	}
+      else if (!strncmp (argv[i], "main-core", 9))
+	{
+	  if (i < (argc - 1))
+	    {
+	      errno = 0;
+	      unsigned long x = strtol (argv[++i], 0, 0);
+	      if (errno == 0)
+		main_core = x;
+	    }
+	}
     }
 
 defaulted:
 
+  /* set process affinity for main thread */
+  CPU_ZERO (&cpuset);
+  CPU_SET (main_core, &cpuset);
+  pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t), &cpuset);
+
   /* Set up the plugin message ID allocator right now... */
   vl_msg_api_set_first_available_msg_id (VL_MSG_FIRST_AVAILABLE);
 
-- 
cgit 1.2.3-korg