vlib: make runtime_data thread-local

Change-Id: I4aa3e7e42fb81211de1aed07dc7befee87a1e18b
Signed-off-by: Damjan Marion <damarion@cisco.com>

5 files changed, 103 insertions, 45 deletions

diff --git a/src/vlib/init.h b/src/vlib/init.h
index 4fa5b304590..12db3f90b96 100644
--- a/src/vlib/init.h
+++ b/src/vlib/init.h
@@ -109,6 +109,7 @@ static void __vlib_add_##tag##_function_##x (void)              \
 }
 
 #define VLIB_INIT_FUNCTION(x) VLIB_DECLARE_INIT_FUNCTION(x,init)
+#define VLIB_WORKER_INIT_FUNCTION(x) VLIB_DECLARE_INIT_FUNCTION(x,worker_init)
 
 #define VLIB_MAIN_LOOP_ENTER_FUNCTION(x) \
   VLIB_DECLARE_INIT_FUNCTION(x,main_loop_enter)
diff --git a/src/vlib/main.h b/src/vlib/main.h
index a6d50b39dac..98bc823d874 100644
--- a/src/vlib/main.h
+++ b/src/vlib/main.h
@@ -162,6 +162,7 @@ typedef struct vlib_main_t
 
   /* List of init functions to call, setup by constructors */
   _vlib_init_function_list_elt_t *init_function_registrations;
+  _vlib_init_function_list_elt_t *worker_init_function_registrations;
   _vlib_init_function_list_elt_t *main_loop_enter_function_registrations;
   _vlib_init_function_list_elt_t *main_loop_exit_function_registrations;
   _vlib_init_function_list_elt_t *api_init_function_registrations;
diff --git a/src/vlib/node.c b/src/vlib/node.c
index c419a13a487..dc0a4de5082 100644
--- a/src/vlib/node.c
+++ b/src/vlib/node.c
@@ -434,9 +434,7 @@ register_node (vlib_main_t * vm, vlib_node_registration_t * r)
       rt->errors[i] = vlib_error_set (n->index, i);
 
     STATIC_ASSERT_SIZEOF (vlib_node_runtime_t, 128);
-    ASSERT (vec_len (n->runtime_data) <=
-	    sizeof (vlib_node_runtime_t) -
-	    STRUCT_OFFSET_OF (vlib_node_runtime_t, runtime_data));
+    ASSERT (vec_len (n->runtime_data) <= VLIB_NODE_RUNTIME_DATA_SIZE);
 
     if (vec_len (n->runtime_data) > 0)
       clib_memcpy (rt->runtime_data, n->runtime_data,
diff --git a/src/vlib/node.h b/src/vlib/node.h
index b624e9d636d..2a532cc3988 100644
--- a/src/vlib/node.h
+++ b/src/vlib/node.h
@@ -411,65 +411,68 @@ typedef struct
 
 typedef struct vlib_node_runtime_t
 {
-  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
-  /* Node function to call. */
-  vlib_node_function_t *function;
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);	/**< cacheline mark */
 
-  /* Vector of errors for this node. */
-  vlib_error_t *errors;
+  vlib_node_function_t *function;	/**< Node function to call. */
 
-  /* Number of clock cycles. */
-  u32 clocks_since_last_overflow;
+  vlib_error_t *errors;			/**< Vector of errors for this node. */
 
-  /* Maximum clock cycle for an invocation. */
-  u32 max_clock;
+  u32 clocks_since_last_overflow;	/**< Number of clock cycles. */
 
-  /* Number of vectors in the recorded max_clock. */
-  u32 max_clock_n;
+  u32 max_clock;			/**< Maximum clock cycle for an
+					  invocation. */
 
-  /* Number of calls. */
-  u32 calls_since_last_overflow;
+  u32 max_clock_n;			/**< Number of vectors in the recorded
+					  max_clock. */
 
-  /* Number of vector elements processed by this node. */
-  u32 vectors_since_last_overflow;
+  u32 calls_since_last_overflow;	/**< Number of calls. */
 
-  /* Start of next frames for this node. */
-  u32 next_frame_index;
+  u32 vectors_since_last_overflow;	/**< Number of vector elements
+					  processed by this node. */
 
-  /* Node index. */
-  u32 node_index;
+  u32 next_frame_index;			/**< Start of next frames for this
+					  node. */
 
-  /* For input nodes: decremented on each main loop interation until it reaches zero
-     and function is called.  Allows some input nodes to be called
-     more than others. */
-  u32 input_main_loops_per_call;
+  u32 node_index;			/**< Node index. */
 
-  /* Saved main loop counter of last dispatch of this node. */
-  u32 main_loop_count_last_dispatch;
+  u32 input_main_loops_per_call;	/**< For input nodes: decremented
+					  on each main loop interation until
+					  it reaches zero and function is
+					  called.  Allows some input nodes to
+					  be called more than others. */
+
+  u32 main_loop_count_last_dispatch;	/**< Saved main loop counter of last
+					  dispatch of this node. */
 
   u32 main_loop_vector_stats[2];
 
-  /* Copy of main node flags. */
-  u16 flags;
+  u16 flags;				/**< Copy of main node flags. */
 
-  /* Input node state. */
-  u16 state;
+  u16 state;				/**< Input node state. */
 
   u16 n_next_nodes;
 
-  /* Next frame index that vector arguments were last enqueued to
-     last time this node ran.  Set to zero before first run
-     of this node. */
-  u16 cached_next_index;
-
-  /* CPU this node runs on */
-  u16 cpu_index;
-
-  /* Function dependent node-runtime. */
-  u8 runtime_data[0];
+  u16 cached_next_index;		/**< Next frame index that vector
+					  arguments were last enqueued to
+					  last time this node ran. Set to
+					  zero before first run of this
+					  node. */
+
+  u16 cpu_index;			/**< CPU this node runs on */
+
+  u8 runtime_data[0];			/**< Function dependent
+					  node-runtime data. This data is
+					  thread local, and it is not
+					  cloned from main thread. It needs
+					  to be initialized for each thread
+					  before it is used unless
+					  runtime_data template exists in
+					  vlib_node_t. */
 }
 vlib_node_runtime_t;
 
+#define VLIB_NODE_RUNTIME_DATA_SIZE	(sizeof (vlib_node_runtime_t) - STRUCT_OFFSET_OF (vlib_node_runtime_t, runtime_data))
+
 typedef struct
 {
   /* Number of allocated frames for this scalar/vector size. */
diff --git a/src/vlib/threads.c b/src/vlib/threads.c
index 07dbff33dc2..3756c3fa3ee 100644
--- a/src/vlib/threads.c
+++ b/src/vlib/threads.c
@@ -633,6 +633,8 @@ start_workers (vlib_main_t * vm)
 	      vm_clone->cpu_index = worker_thread_index;
 	      vm_clone->heap_base = w->thread_mheap;
 	      vm_clone->mbuf_alloc_list = 0;
+	      vm_clone->init_functions_called =
+		hash_create (0, /* value bytes */ 0);
 	      memset (&vm_clone->random_buffer, 0,
 		      sizeof (vm_clone->random_buffer));
 
@@ -674,11 +676,33 @@ start_workers (vlib_main_t * vm)
 		}
 	      nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL] =
 		vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]);
+	      vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])
+	      {
+		vlib_node_t *n = vlib_get_node (vm, rt->node_index);
+		rt->cpu_index = vm_clone->cpu_index;
+		/* copy initial runtime_data from node */
+		if (n->runtime_data_bytes > 0)
+		  clib_memcpy (rt->runtime_data, n->runtime_data,
+			       VLIB_NODE_RUNTIME_DATA_SIZE);
+		else if (CLIB_DEBUG > 0)
+		  memset (rt->runtime_data, 0xfe,
+			  VLIB_NODE_RUNTIME_DATA_SIZE);
+	      }
 
 	      nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] =
 		vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT]);
 	      vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])
+	      {
+		vlib_node_t *n = vlib_get_node (vm, rt->node_index);
 		rt->cpu_index = vm_clone->cpu_index;
+		/* copy initial runtime_data from node */
+		if (n->runtime_data_bytes > 0)
+		  clib_memcpy (rt->runtime_data, n->runtime_data,
+			       VLIB_NODE_RUNTIME_DATA_SIZE);
+		else if (CLIB_DEBUG > 0)
+		  memset (rt->runtime_data, 0xfe,
+			  VLIB_NODE_RUNTIME_DATA_SIZE);
+	      }
 
 	      nm_clone->processes = vec_dup (nm->processes);
 
@@ -926,26 +950,51 @@ vlib_worker_thread_node_runtime_update (void)
 	clib_mem_free (old_nodes_clone[j]);
       vec_free (old_nodes_clone);
 
-      vec_free (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]);
 
+      /* re-clone internal nodes */
+      old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL];
       nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL] =
 	vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]);
 
-      /* clone input node runtime */
-      old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT];
+      vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL])
+      {
+	vlib_node_t *n = vlib_get_node (vm, rt->node_index);
+	rt->cpu_index = vm_clone->cpu_index;
+	/* copy runtime_data, will be overwritten later for existing rt */
+	clib_memcpy (rt->runtime_data, n->runtime_data,
+		     VLIB_NODE_RUNTIME_DATA_SIZE);
+      }
+
+      for (j = 0; j < vec_len (old_rt); j++)
+	{
+	  rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
+	  rt->state = old_rt[j].state;
+	  clib_memcpy (rt->runtime_data, old_rt[j].runtime_data,
+		       VLIB_NODE_RUNTIME_DATA_SIZE);
+	}
 
+      vec_free (old_rt);
+
+      /* re-clone input nodes */
+      old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT];
       nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] =
 	vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT]);
 
       vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])
       {
+	vlib_node_t *n = vlib_get_node (vm, rt->node_index);
 	rt->cpu_index = vm_clone->cpu_index;
+	/* copy runtime_data, will be overwritten later for existing rt */
+	clib_memcpy (rt->runtime_data, n->runtime_data,
+		     VLIB_NODE_RUNTIME_DATA_SIZE);
       }
 
       for (j = 0; j < vec_len (old_rt); j++)
 	{
 	  rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
 	  rt->state = old_rt[j].state;
+	  clib_memcpy (rt->runtime_data, old_rt[j].runtime_data,
+		       VLIB_NODE_RUNTIME_DATA_SIZE);
 	}
 
       vec_free (old_rt);
@@ -1342,6 +1391,7 @@ vlib_worker_thread_fn (void *arg)
   vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg;
   vlib_thread_main_t *tm = vlib_get_thread_main ();
   vlib_main_t *vm = vlib_get_main ();
+  clib_error_t *e;
 
   ASSERT (vm->cpu_index == os_get_cpu_number ());
 
@@ -1349,6 +1399,11 @@ vlib_worker_thread_fn (void *arg)
   clib_time_init (&vm->clib_time);
   clib_mem_set_heap (w->thread_mheap);
 
+  e = vlib_call_init_exit_functions
+    (vm, vm->worker_init_function_registrations, 1 /* call_once */ );
+  if (e)
+    clib_error_report (e);
+
   /* Wait until the dpdk init sequence is complete */
   while (tm->extern_thread_mgmt && tm->worker_thread_release == 0)
     vlib_worker_thread_barrier_check ();