1 files changed, 128 insertions, 53 deletions

diff --git a/src/vlib/node.c b/src/vlib/node.c
index f4329e7c503..8f6c852188b 100644
--- a/src/vlib/node.c
+++ b/src/vlib/node.c
@@ -130,12 +130,10 @@ vlib_node_runtime_update (vlib_main_t * vm, u32 node_index, u32 next_index)
 	    && pf->next_frame_index >= i)
 	  pf->next_frame_index += n_insert;
       }
-      /* *INDENT-OFF* */
       pool_foreach (pf, nm->suspended_process_frames)  {
 	  if (pf->next_frame_index != ~0 && pf->next_frame_index >= i)
 	    pf->next_frame_index += n_insert;
       }
-      /* *INDENT-ON* */
 
       r->n_next_nodes = vec_len (node->next_nodes);
     }
@@ -223,7 +221,6 @@ vlib_node_add_next_with_slot (vlib_main_t * vm,
   {
     uword sib_node_index, sib_slot;
     vlib_node_t *sib_node;
-    /* *INDENT-OFF* */
     clib_bitmap_foreach (sib_node_index, node->sibling_bitmap)  {
       sib_node = vec_elt (nm->nodes, sib_node_index);
       if (sib_node != node)
@@ -232,7 +229,6 @@ vlib_node_add_next_with_slot (vlib_main_t * vm,
 	  ASSERT (sib_slot == slot);
 	}
     }
-    /* *INDENT-ON* */
   }
 
   vlib_worker_thread_barrier_release (vm);
@@ -329,12 +325,54 @@ vlib_node_get_preferred_node_fn_variant (vlib_main_t *vm,
 }
 
 static void
-register_node (vlib_main_t * vm, vlib_node_registration_t * r)
+vlib_node_add_to_sibling_bitmap (vlib_main_t *vm, vlib_node_t *n,
+				 vlib_node_t *sib)
 {
   vlib_node_main_t *nm = &vm->node_main;
-  vlib_node_t *n;
+  u32 si;
+
+  clib_bitmap_foreach (si, sib->sibling_bitmap)
+    {
+      vlib_node_t *m = vec_elt (nm->nodes, si);
+
+      /* Connect all of sibling's siblings to us. */
+      m->sibling_bitmap = clib_bitmap_ori (m->sibling_bitmap, n->index);
+
+      /* Connect us to all of sibling's siblings. */
+      n->sibling_bitmap = clib_bitmap_ori (n->sibling_bitmap, si);
+    }
+
+  /* Connect sibling to us. */
+  sib->sibling_bitmap = clib_bitmap_ori (sib->sibling_bitmap, n->index);
+
+  /* Connect us to sibling. */
+  n->sibling_bitmap = clib_bitmap_ori (n->sibling_bitmap, sib->index);
+}
+
+u32
+vlib_register_node (vlib_main_t *vm, vlib_node_registration_t *r, char *fmt,
+		    ...)
+{
+  vlib_node_main_t *nm = &vm->node_main;
+  vlib_node_t *n, *sib = 0;
+  va_list va;
+  u32 size;
   int i;
 
+  if (r->sibling_of)
+    {
+      if (r->n_next_nodes > 0)
+	clib_error ("sibling node should not have any next nodes `%v'",
+		    r->name);
+      if (nm->flags & VLIB_NODE_MAIN_RUNTIME_STARTED)
+	{
+	  sib = vlib_get_node_by_name (vm, (u8 *) r->sibling_of);
+
+	  if (sib == 0)
+	    clib_error ("unknown sibling node '%s'", r->sibling_of);
+	}
+    }
+
   if (CLIB_DEBUG > 0)
     {
       /* Default (0) type should match INTERNAL. */
@@ -362,11 +400,9 @@ register_node (vlib_main_t * vm, vlib_node_registration_t * r)
 
   vec_add1 (nm->nodes, n);
 
-  /* Name is always a vector so it can be formatted with %v. */
-  if (clib_mem_is_heap_object (vec_header (r->name, 0)))
-    n->name = vec_dup ((u8 *) r->name);
-  else
-    n->name = format (0, "%s", r->name);
+  va_start (va, fmt);
+  n->name = va_format (0, fmt, &va);
+  va_end (va);
 
   if (!nm->node_by_name)
     nm->node_by_name = hash_create_vec ( /* size */ 32,
@@ -387,11 +423,6 @@ register_node (vlib_main_t * vm, vlib_node_registration_t * r)
   r->index = n->index;		/* save index in registration */
   n->function = r->function;
 
-  /* Node index of next sibling will be filled in by vlib_node_main_init. */
-  n->sibling_of = r->sibling_of;
-  if (r->sibling_of && r->n_next_nodes > 0)
-    clib_error ("sibling node should not have any next nodes `%v'", n->name);
-
   if (r->type == VLIB_NODE_TYPE_INTERNAL)
     ASSERT (r->vector_size > 0);
 
@@ -400,13 +431,66 @@ register_node (vlib_main_t * vm, vlib_node_registration_t * r)
   _(type);
   _(flags);
   _(state);
-  _(scalar_size);
-  _(vector_size);
   _(format_buffer);
   _(unformat_buffer);
   _(format_trace);
   _(validate_frame);
 
+  size = round_pow2 (sizeof (vlib_frame_t), VLIB_FRAME_DATA_ALIGN);
+
+  /* scalar data size */
+  if (r->scalar_size)
+    {
+      n->scalar_offset = size;
+      size += round_pow2 (r->scalar_size, VLIB_FRAME_DATA_ALIGN);
+    }
+  else
+    n->scalar_offset = 0;
+
+  /* Vecor data size */
+  n->vector_offset = size;
+  size += r->vector_size * VLIB_FRAME_SIZE;
+
+  /* Allocate a few extra slots of vector data to support
+     speculative vector enqueues which overflow vector data in next frame. */
+  size += r->vector_size * VLIB_FRAME_SIZE_EXTRA;
+
+  /* space for VLIB_FRAME_MAGIC */
+  n->magic_offset = size;
+  size += sizeof (u32);
+
+  /* round size to VLIB_FRAME_DATA_ALIGN */
+  size = round_pow2 (size, VLIB_FRAME_DATA_ALIGN);
+
+  if (r->aux_size)
+    {
+      n->aux_offset = size;
+      size += r->aux_size * VLIB_FRAME_SIZE;
+    }
+  else
+    n->aux_offset = 0;
+
+  /* final size */
+  n->frame_size = size = round_pow2 (size, CLIB_CACHE_LINE_BYTES);
+  ASSERT (size <= __UINT16_MAX__);
+
+  vlib_frame_size_t *fs = 0;
+
+  n->frame_size_index = (u16) ~0;
+  vec_foreach (fs, nm->frame_sizes)
+    if (fs->frame_size == size)
+      {
+	n->frame_size_index = fs - nm->frame_sizes;
+	break;
+      }
+
+  if (n->frame_size_index == (u16) ~0)
+    {
+      vec_add2 (nm->frame_sizes, fs, 1);
+      fs->frame_size = size;
+      n->frame_size_index = fs - nm->frame_sizes;
+    }
+
   /* Register error counters. */
   vlib_register_errors (vm, n->index, r->n_errors, r->error_strings,
 			r->error_counters);
@@ -476,7 +560,10 @@ register_node (vlib_main_t * vm, vlib_node_registration_t * r)
 	vec_add2_aligned (nm->nodes_by_type[n->type], rt, 1,
 			  /* align */ CLIB_CACHE_LINE_BYTES);
 	if (n->type == VLIB_NODE_TYPE_INPUT)
-	  clib_interrupt_resize (&nm->interrupts,
+	  clib_interrupt_resize (&nm->input_node_interrupts,
+				 vec_len (nm->nodes_by_type[n->type]));
+	else if (n->type == VLIB_NODE_TYPE_PRE_INPUT)
+	  clib_interrupt_resize (&nm->pre_input_node_interrupts,
 				 vec_len (nm->nodes_by_type[n->type]));
 	n->runtime_index = rt - nm->nodes_by_type[n->type];
       }
@@ -512,13 +599,24 @@ register_node (vlib_main_t * vm, vlib_node_registration_t * r)
     vec_free (n->runtime_data);
   }
 #undef _
-}
 
-/* Register new packet processing node. */
-u32
-vlib_register_node (vlib_main_t * vm, vlib_node_registration_t * r)
-{
-  register_node (vm, r);
+  if (sib)
+    {
+      u32 slot, i;
+
+      vec_foreach_index (i, sib->next_nodes)
+	{
+	  slot =
+	    vlib_node_add_next_with_slot (vm, n->index, sib->next_nodes[i], i);
+	  ASSERT (slot == i);
+	}
+
+      vlib_node_add_to_sibling_bitmap (vm, n, sib);
+
+      r->n_next_nodes = vec_len (n->next_nodes);
+    }
+  n->sibling_of = r->sibling_of;
+
   return r->index;
 }
 
@@ -530,7 +628,7 @@ null_node_fn (vlib_main_t * vm,
 
   vlib_node_increment_counter (vm, node->node_index, 0, n_vectors);
   vlib_buffer_free (vm, vlib_frame_vector_args (frame), n_vectors);
-  vlib_frame_free (vm, node, frame);
+  vlib_frame_free (vm, frame);
 
   return n_vectors;
 }
@@ -582,19 +680,18 @@ vlib_register_all_static_nodes (vlib_main_t * vm)
   static vlib_node_registration_t null_node_reg = {
     .function = null_node_fn,
     .vector_size = sizeof (u32),
-    .name = "null-node",
     .n_errors = 1,
     .error_strings = null_node_error_strings,
   };
 
   /* make sure that node index 0 is not used by
      real node */
-  register_node (vm, &null_node_reg);
+  vlib_register_node (vm, &null_node_reg, "null-node");
 
   r = vgm->node_registrations;
   while (r)
     {
-      register_node (vm, r);
+      vlib_register_node (vm, r, "%s", r->name);
       r = r->next_registration;
     }
 }
@@ -669,16 +766,11 @@ vlib_node_main_init (vlib_main_t * vm)
   vlib_node_t *n;
   uword ni;
 
-  nm->frame_sizes = vec_new (vlib_frame_size_t, 1);
-#ifdef VLIB_SUPPORTS_ARBITRARY_SCALAR_SIZES
-  nm->frame_size_hash = hash_create (0, sizeof (uword));
-#endif
   nm->flags |= VLIB_NODE_MAIN_RUNTIME_STARTED;
 
   /* Generate sibling relationships */
   {
     vlib_node_t *n, *sib;
-    uword si;
 
     for (ni = 0; ni < vec_len (nm->nodes); ni++)
       {
@@ -695,23 +787,7 @@ vlib_node_main_init (vlib_main_t * vm)
 	    goto done;
 	  }
 
-        /* *INDENT-OFF* */
-	clib_bitmap_foreach (si, sib->sibling_bitmap)  {
-	      vlib_node_t * m = vec_elt (nm->nodes, si);
-
-	      /* Connect all of sibling's siblings to us. */
-	      m->sibling_bitmap = clib_bitmap_ori (m->sibling_bitmap, n->index);
-
-	      /* Connect us to all of sibling's siblings. */
-	      n->sibling_bitmap = clib_bitmap_ori (n->sibling_bitmap, si);
-	    }
-        /* *INDENT-ON* */
-
-	/* Connect sibling to us. */
-	sib->sibling_bitmap = clib_bitmap_ori (sib->sibling_bitmap, n->index);
-
-	/* Connect us to sibling. */
-	n->sibling_bitmap = clib_bitmap_ori (n->sibling_bitmap, sib->index);
+	vlib_node_add_to_sibling_bitmap (vm, n, sib);
       }
   }
 
@@ -800,14 +876,13 @@ vlib_process_create (vlib_main_t * vm, char *name,
 
   memset (&r, 0, sizeof (r));
 
-  r.name = (char *) format (0, "%s", name, 0);
   r.function = f;
   r.process_log2_n_stack_bytes = log2_n_stack_bytes;
   r.type = VLIB_NODE_TYPE_PROCESS;
 
   vlib_worker_thread_barrier_sync (vm);
 
-  vlib_register_node (vm, &r);
+  vlib_register_node (vm, &r, "%s", name);
   vec_free (r.name);
 
   vlib_worker_thread_node_runtime_update ();