vlib: remove external thread management support

Now DPDK have API to register external threads so we can remove this
mess...

Type: improvement
Change-Id: I71a21f0cd94bd668aa406710c75a0bcc63fdc840
Signed-off-by: Damjan Marion <damarion@cisco.com>

8 files changed, 19 insertions, 232 deletions

diff --git a/src/plugins/dpdk/CMakeLists.txt b/src/plugins/dpdk/CMakeLists.txt
index 5de75e76289..b53f4ad2375 100644
--- a/src/plugins/dpdk/CMakeLists.txt
+++ b/src/plugins/dpdk/CMakeLists.txt
@@ -131,7 +131,6 @@ add_vpp_plugin(dpdk
   SOURCES
   buffer.c
   main.c
-  thread.c
   device/cli.c
   device/common.c
   device/device.c
diff --git a/src/plugins/dpdk/device/cli.c b/src/plugins/dpdk/device/cli.c
index 0f771c6ba77..ba922e29a54 100644
--- a/src/plugins/dpdk/device/cli.c
+++ b/src/plugins/dpdk/device/cli.c
@@ -373,7 +373,8 @@ show_dpdk_version_command_fn (vlib_main_t * vm,
  * Example of how to display how many DPDK buffer test command has allocated:
  * @cliexstart{show dpdk version}
  * DPDK Version:        DPDK 16.11.0
- * DPDK EAL init args:  -c 1 -n 4 --huge-dir /run/vpp/hugepages --file-prefix vpp -w 0000:00:08.0 -w 0000:00:09.0 --master-lcore 0 --socket-mem 256
+ * DPDK EAL init args:  --in-memory --no-telemetry --file-prefix vpp
+ *  -w 0000:00:08.0 -w 0000:00:09.0
  * @cliexend
 ?*/
 /* *INDENT-OFF* */
diff --git a/src/plugins/dpdk/device/dpdk.h b/src/plugins/dpdk/device/dpdk.h
index 51f80a80462..cc7305dbdc4 100644
--- a/src/plugins/dpdk/device/dpdk.h
+++ b/src/plugins/dpdk/device/dpdk.h
@@ -22,27 +22,12 @@
 
 #include <rte_config.h>
 
-#include <rte_common.h>
-#include <rte_dev.h>
-#include <rte_memory.h>
 #include <rte_eal.h>
-#include <rte_per_lcore.h>
-#include <rte_cycles.h>
-#include <rte_lcore.h>
-#include <rte_per_lcore.h>
-#include <rte_interrupts.h>
-#include <rte_pci.h>
+#include <rte_bus_pci.h>
 #include <rte_bus_vmbus.h>
-#include <rte_ether.h>
 #include <rte_ethdev.h>
-#include <rte_ring.h>
-#include <rte_mempool.h>
-#include <rte_mbuf.h>
 #include <rte_version.h>
-#include <rte_sched.h>
 #include <rte_net.h>
-#include <rte_bus_pci.h>
-#include <rte_flow.h>
 
 #include <vnet/devices/devices.h>
 
@@ -313,13 +298,6 @@ typedef struct
 #define DPDK_MAX_SIMD_BITWIDTH_256     256
 #define DPDK_MAX_SIMD_BITWIDTH_512     512
 
-  /* Required config parameters */
-  u8 coremask_set_manually;
-  u8 nchannels_set_manually;
-  u32 coremask;
-  u32 nchannels;
-  u32 num_crypto_mbufs;
-
   /*
    * format interface names ala xxxEthernet%d/%d/%d instead of
    * xxxEthernet%x/%x/%x.
diff --git a/src/plugins/dpdk/device/dpdk_priv.h b/src/plugins/dpdk/device/dpdk_priv.h
index 7f595ae8e19..267c0b55aa3 100644
--- a/src/plugins/dpdk/device/dpdk_priv.h
+++ b/src/plugins/dpdk/device/dpdk_priv.h
@@ -33,10 +33,6 @@ _(no-hpet)                                      \
 _(no-huge)                                      \
 _(vmware-tsc-map)
 
-#define foreach_eal_single_hyphen_mandatory_arg \
-_(coremask, c)                                  \
-_(nchannels, n)                                 \
-
 #define foreach_eal_single_hyphen_arg           \
 _(mem-alloc-request, m)                         \
 _(force-ranks, r)
diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c
index 3808a5b1e4d..04d6f89ac7b 100644
--- a/src/plugins/dpdk/device/init.c
+++ b/src/plugins/dpdk/device/init.c
@@ -1504,11 +1504,6 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input)
 
 	  num_whitelisted++;
 	}
-      else if (unformat (input, "num-mem-channels %d", &conf->nchannels))
-	conf->nchannels_set_manually = 0;
-      else if (unformat (input, "num-crypto-mbufs %d",
-			 &conf->num_crypto_mbufs))
-	;
       else if (unformat (input, "uio-driver %s", &conf->uio_driver_name))
 	;
       else if (unformat (input, "socket-mem %s", &socket_mem))
@@ -1585,17 +1580,6 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input)
 	    }
 	foreach_eal_single_hyphen_arg
 #undef _
-#define _(a,b)						\
-	    else if (unformat(input, #a " %s", &s))	\
-	      {						\
-		tmp = format (0, "-%s%c", #b, 0);	\
-		vec_add1 (conf->eal_init_args, tmp);	\
-		vec_add1 (s, 0);			\
-		vec_add1 (conf->eal_init_args, s);	\
-		conf->a##_set_manually = 1;		\
-	      }
-	foreach_eal_single_hyphen_mandatory_arg
-#undef _
 	else if (unformat (input, "default"))
 	;
 
@@ -1651,38 +1635,6 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input)
   if (error)
     return error;
 
-  /* I'll bet that -c and -n must be the first and second args... */
-  if (!conf->coremask_set_manually)
-    {
-      vlib_thread_registration_t *tr;
-      uword *coremask = 0;
-      int i;
-
-      /* main thread core */
-      coremask = clib_bitmap_set (coremask, tm->main_lcore, 1);
-
-      for (i = 0; i < vec_len (tm->registrations); i++)
-	{
-	  tr = tm->registrations[i];
-	  coremask = clib_bitmap_or (coremask, tr->coremask);
-	}
-
-      vec_insert (conf->eal_init_args, 2, 1);
-      conf->eal_init_args[1] = (u8 *) "-c";
-      tmp = format (0, "%U%c", format_bitmap_hex, coremask, 0);
-      conf->eal_init_args[2] = tmp;
-      clib_bitmap_free (coremask);
-    }
-
-  if (!conf->nchannels_set_manually)
-    {
-      vec_insert (conf->eal_init_args, 2, 3);
-      conf->eal_init_args[3] = (u8 *) "-n";
-      tmp = format (0, "%d", conf->nchannels);
-      vec_terminate_c_string (tmp);
-      conf->eal_init_args[4] = tmp;
-    }
-
   if (no_pci == 0 && geteuid () == 0)
     dpdk_bind_devices_to_uio (conf);
 
@@ -1693,7 +1645,6 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input)
     if (devconf->x == 0 && conf->default_devconf.x > 0) \
       devconf->x = conf->default_devconf.x ;
 
-  /* *INDENT-OFF* */
   pool_foreach (devconf, conf->dev_confs)  {
 
     /* default per-device config items */
@@ -1737,17 +1688,9 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input)
 	  vec_add1 (conf->eal_init_args, tmp);
     }
   }
-  /* *INDENT-ON* */
 
 #undef _
 
-  /* set master-lcore */
-  tmp = format (0, "--main-lcore%c", 0);
-  vec_add1 (conf->eal_init_args, tmp);
-  tmp = format (0, "%u%c", tm->main_lcore, 0);
-  vec_add1 (conf->eal_init_args, tmp);
-
-
   if (socket_mem)
     clib_warning ("socket-mem argument is deprecated");
 
@@ -1999,7 +1942,6 @@ dpdk_init (vlib_main_t * vm)
   dm->vnet_main = vnet_get_main ();
   dm->conf = &dpdk_config_main;
 
-  dm->conf->nchannels = 4;
   vec_add1 (dm->conf->eal_init_args, (u8 *) "vnet");
 
   /* Default vlib_buffer_t flags, DISABLES tcp/udp checksumming... */
@@ -2020,10 +1962,13 @@ dpdk_init (vlib_main_t * vm)
 
 VLIB_INIT_FUNCTION (dpdk_init);
 
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
+static clib_error_t *
+dpdk_worker_thread_init (vlib_main_t *vm)
+{
+  if (rte_thread_register () < 0)
+    clib_panic ("dpdk: cannot register thread %u - %s", vm->thread_index,
+		rte_strerror (rte_errno));
+  return 0;
+}
+
+VLIB_WORKER_INIT_FUNCTION (dpdk_worker_thread_init);
diff --git a/src/plugins/dpdk/thread.c b/src/plugins/dpdk/thread.c
deleted file mode 100644
index 3a3fcc6cea6..00000000000
--- a/src/plugins/dpdk/thread.c
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Copyright (c) 2017 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <rte_config.h>
-
-#include <rte_common.h>
-#include <rte_log.h>
-#include <rte_memory.h>
-#include <rte_memzone.h>
-#include <rte_tailq.h>
-#include <rte_eal.h>
-#include <rte_per_lcore.h>
-#include <rte_launch.h>
-#include <rte_atomic.h>
-#include <rte_cycles.h>
-#include <rte_prefetch.h>
-#include <rte_lcore.h>
-#include <rte_per_lcore.h>
-#include <rte_branch_prediction.h>
-#include <rte_interrupts.h>
-#include <rte_pci.h>
-#include <rte_random.h>
-#include <rte_debug.h>
-#include <rte_ether.h>
-#include <rte_ethdev.h>
-#include <rte_ring.h>
-#include <rte_mempool.h>
-#include <rte_mbuf.h>
-#include <rte_version.h>
-
-#include <vlib/vlib.h>
-#include <vnet/vnet.h>
-#include <dpdk/device/dpdk.h>
-#include <dpdk/device/dpdk_priv.h>
-
-static clib_error_t *
-dpdk_launch_thread (void *fp, vlib_worker_thread_t * w, unsigned lcore_id)
-{
-  int r;
-  r = rte_eal_remote_launch (fp, (void *) w, lcore_id);
-  if (r)
-    return clib_error_return (0, "Failed to launch thread %u", lcore_id);
-  return 0;
-}
-
-static clib_error_t *
-dpdk_thread_set_lcore (u32 thread, u16 lcore)
-{
-  return 0;
-}
-
-static vlib_thread_callbacks_t callbacks = {
-  .vlib_launch_thread_cb = &dpdk_launch_thread,
-  .vlib_thread_set_lcore_cb = &dpdk_thread_set_lcore,
-};
-
-static clib_error_t *
-dpdk_thread_init (vlib_main_t * vm)
-{
-  vlib_thread_cb_register (vm, &callbacks);
-  return 0;
-}
-
-VLIB_INIT_FUNCTION (dpdk_thread_init);
-
-/** @endcond */
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vlib/threads.c b/src/vlib/threads.c
index 760aa8b5776..f4d95c45059 100644
--- a/src/vlib/threads.c
+++ b/src/vlib/threads.c
@@ -175,6 +175,7 @@ vlib_thread_init (vlib_main_t * vm)
   vlib_thread_main_t *tm = &vlib_thread_main;
   vlib_worker_thread_t *w;
   vlib_thread_registration_t *tr;
+  cpu_set_t cpuset;
   u32 n_vlib_mains = 1;
   u32 first_index = 1;
   u32 i;
@@ -223,17 +224,9 @@ vlib_thread_init (vlib_main_t * vm)
     tm->cpu_socket_bitmap = clib_bitmap_set (0, 0, 1);
 
   /* pin main thread to main_lcore  */
-  if (tm->cb.vlib_thread_set_lcore_cb)
-    {
-      tm->cb.vlib_thread_set_lcore_cb (0, tm->main_lcore);
-    }
-  else
-    {
-      cpu_set_t cpuset;
-      CPU_ZERO (&cpuset);
-      CPU_SET (tm->main_lcore, &cpuset);
-      pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t), &cpuset);
-    }
+  CPU_ZERO (&cpuset);
+  CPU_SET (tm->main_lcore, &cpuset);
+  pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t), &cpuset);
 
   /* Set up thread 0 */
   vec_validate_aligned (vlib_worker_threads, 0, CLIB_CACHE_LINE_BYTES);
@@ -463,6 +456,8 @@ vlib_launch_thread_int (void *fp, vlib_worker_thread_t * w, unsigned cpu_id)
 {
   clib_mem_main_t *mm = &clib_mem_main;
   vlib_thread_main_t *tm = &vlib_thread_main;
+  pthread_t worker;
+  cpu_set_t cpuset;
   void *(*fp_arg) (void *) = fp;
   void *numa_heap;
 
@@ -489,12 +484,6 @@ vlib_launch_thread_int (void *fp, vlib_worker_thread_t * w, unsigned cpu_id)
 	}
     }
 
-  if (tm->cb.vlib_launch_thread_cb && !w->registration->use_pthreads)
-    return tm->cb.vlib_launch_thread_cb (fp, (void *) w, cpu_id);
-  else
-    {
-      pthread_t worker;
-      cpu_set_t cpuset;
       CPU_ZERO (&cpuset);
       CPU_SET (cpu_id, &cpuset);
 
@@ -505,7 +494,6 @@ vlib_launch_thread_int (void *fp, vlib_worker_thread_t * w, unsigned cpu_id)
 	return clib_error_return_unix (0, "pthread_setaffinity_np");
 
       return 0;
-    }
 }
 
 static clib_error_t *
@@ -1520,7 +1508,6 @@ vlib_worker_thread_fn (void *arg)
 {
   vlib_global_main_t *vgm = vlib_get_global_main ();
   vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg;
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
   vlib_main_t *vm = vlib_get_main ();
   clib_error_t *e;
 
@@ -1540,10 +1527,6 @@ vlib_worker_thread_fn (void *arg)
   if (e)
     clib_error_report (e);
 
-  /* Wait until the dpdk init sequence is complete */
-  while (tm->extern_thread_mgmt && tm->worker_thread_release == 0)
-    vlib_worker_thread_barrier_check ();
-
   vlib_worker_loop (vm);
 }
 
@@ -1586,19 +1569,6 @@ vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts)
   return (fqm - tm->frame_queue_mains);
 }
 
-int
-vlib_thread_cb_register (struct vlib_main_t *vm, vlib_thread_callbacks_t * cb)
-{
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
-
-  if (tm->extern_thread_mgmt)
-    return -1;
-
-  tm->cb.vlib_launch_thread_cb = cb->vlib_launch_thread_cb;
-  tm->extern_thread_mgmt = 1;
-  return 0;
-}
-
 void
 vlib_process_signal_event_mt_helper (vlib_process_signal_event_mt_args_t *
 				     args)
diff --git a/src/vlib/threads.h b/src/vlib/threads.h
index 91727bacc23..e406dde5b07 100644
--- a/src/vlib/threads.h
+++ b/src/vlib/threads.h
@@ -235,13 +235,6 @@ typedef enum
 
 typedef struct
 {
-  clib_error_t *(*vlib_launch_thread_cb) (void *fp, vlib_worker_thread_t * w,
-					  unsigned cpu_id);
-  clib_error_t *(*vlib_thread_set_lcore_cb) (u32 thread, u16 cpu);
-} vlib_thread_callbacks_t;
-
-typedef struct
-{
   /* Link list of registrations, built by constructors */
   vlib_thread_registration_t *next;
 
@@ -252,10 +245,6 @@ typedef struct
 
   vlib_worker_thread_t *worker_threads;
 
-  /*
-   * Launch all threads as pthreads,
-   * not eal_rte_launch (strict affinity) threads
-   */
   int use_pthreads;
 
   /* Number of vlib_main / vnet_main clones */
@@ -297,10 +286,6 @@ typedef struct
   /* scheduling policy priority */
   u32 sched_priority;
 
-  /* callbacks */
-  vlib_thread_callbacks_t cb;
-  int extern_thread_mgmt;
-
   /* NUMA-bound heap size */
   uword numa_heap_size;
 
@@ -490,8 +475,6 @@ vlib_thread_is_main_w_barrier (void)
 }
 
 u8 *vlib_thread_stack_init (uword thread_index);
-int vlib_thread_cb_register (struct vlib_main_t *vm,
-			     vlib_thread_callbacks_t * cb);
 extern void *rpc_call_main_thread_cb_fn;
 
 void