feature: store arc index in vlib_buffer

Previos implementation imposed a limit that node can be member of
only one feature arc as feature arc index was stored in node runtime.
Now start node stores index into vlib_buffer_t and feature
nodes are taking it from there.

Change-Id: I252841083bc292af7ed36bf1ac652f2b8c16bfb8
Signed-off-by: Damjan Marion <damarion@cisco.com>

11 files changed, 118 insertions, 71 deletions

diff --git a/vlib/vlib/buffer.h b/vlib/vlib/buffer.h
index 314ed193107..4ede684aa4b 100644
--- a/vlib/vlib/buffer.h
+++ b/vlib/vlib/buffer.h
@@ -125,10 +125,14 @@ typedef struct
                                visit enabled feature nodes
                             */
 
-  u32 dont_waste_me; /**< Available space in the (precious)
-                        first 32 octets of buffer metadata
-                        Before allocating any of it, discussion required!
-                     */
+  u8 feature_arc_index;	/**< Used to identify feature arcs by intermediate
+                           feature node
+                        */
+
+  u8 dont_waste_me[3]; /**< Available space in the (precious)
+                          first 32 octets of buffer metadata
+                          Before allocating any of it, discussion required!
+                       */
 
   u32 opaque[8]; /**< Opaque data used by sub-graphs for their own purposes.
                     See .../vnet/vnet/buffer.h
diff --git a/vlib/vlib/node.c b/vlib/vlib/node.c
index 118fabe749d..23f7ea0239e 100644
--- a/vlib/vlib/node.c
+++ b/vlib/vlib/node.c
@@ -424,7 +424,6 @@ register_node (vlib_main_t * vm, vlib_node_registration_t * r)
 
     rt->n_next_nodes = r->n_next_nodes;
     rt->next_frame_index = vec_len (nm->next_frames);
-    rt->feature_arc_index = ~0;
 
     vec_resize (nm->next_frames, rt->n_next_nodes);
     for (i = 0; i < rt->n_next_nodes; i++)
diff --git a/vlib/vlib/node.h b/vlib/vlib/node.h
index 251ee606d20..b624e9d636d 100644
--- a/vlib/vlib/node.h
+++ b/vlib/vlib/node.h
@@ -465,9 +465,6 @@ typedef struct vlib_node_runtime_t
   /* CPU this node runs on */
   u16 cpu_index;
 
-  /* Index of feature arc in which the node participates */
-  u16 feature_arc_index;
-
   /* Function dependent node-runtime. */
   u8 runtime_data[0];
 }
diff --git a/vnet/vnet/devices/af_packet/node.c b/vnet/vnet/devices/af_packet/node.c
index 8c0d645596a..29fdc7142e8 100644
--- a/vnet/vnet/devices/af_packet/node.c
+++ b/vnet/vnet/devices/af_packet/node.c
@@ -233,8 +233,8 @@ af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 	    }
 
 	  /* redirect if feature path enabled */
-	  vnet_feature_device_input_redirect_x1 (node, apif->sw_if_index,
-						 &next0, b0, 0);
+	  vnet_feature_start_device_input_x1 (apif->sw_if_index, &next0, b0,
+					      0);
 
 	  /* enque and take next packet */
 	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
diff --git a/vnet/vnet/devices/dpdk/node.c b/vnet/vnet/devices/dpdk/node.c
index bcb4feac315..bd2355b2a6a 100644
--- a/vnet/vnet/devices/dpdk/node.c
+++ b/vnet/vnet/devices/dpdk/node.c
@@ -533,7 +533,7 @@ dpdk_device_input (dpdk_main_t * dm,
 	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
 
           /* Do we have any driver RX features configured on the interface? */
-	  vnet_feature_device_input_redirect_x1 (node, xd->vlib_sw_if_index, &next0, b0, l3_offset0);
+	  vnet_feature_start_device_input_x1 (xd->vlib_sw_if_index, &next0, b0, l3_offset0);
 
 	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 					   to_next, n_left_to_next,
diff --git a/vnet/vnet/devices/netmap/node.c b/vnet/vnet/devices/netmap/node.c
index fb60fbd6cce..1e2a6a2792f 100644
--- a/vnet/vnet/devices/netmap/node.c
+++ b/vnet/vnet/devices/netmap/node.c
@@ -232,8 +232,8 @@ netmap_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 		}
 
 	      /* redirect if feature path enabled */
-	      vnet_feature_device_input_redirect_x1 (node, nif->sw_if_index,
-						     &next0, b0, 0);
+	      vnet_feature_start_device_input_x1 (nif->sw_if_index, &next0,
+						  b0, 0);
 
 	      /* enque and take next packet */
 	      vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
diff --git a/vnet/vnet/devices/virtio/vhost-user.c b/vnet/vnet/devices/virtio/vhost-user.c
index 7137a22a2d9..a3f2a31a934 100644
--- a/vnet/vnet/devices/virtio/vhost-user.c
+++ b/vnet/vnet/devices/virtio/vhost-user.c
@@ -1609,8 +1609,8 @@ vhost_user_if_input (vlib_main_t * vm,
 	  n_left_to_next--;
 
 	  /* redirect if feature path enabled */
-	  vnet_feature_device_input_redirect_x1 (node, vui->sw_if_index,
-						 &next0, b_head, 0);
+	  vnet_feature_start_device_input_x1 (vui->sw_if_index, &next0,
+					      b_head, 0);
 
 	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 					   to_next, n_left_to_next,
diff --git a/vnet/vnet/feature/feature.c b/vnet/vnet/feature/feature.c
index 4cfe1faa20f..c19b3e0ebbb 100644
--- a/vnet/vnet/feature/feature.c
+++ b/vnet/vnet/feature/feature.c
@@ -40,17 +40,6 @@ vnet_feature_init (vlib_main_t * vm)
       /* process start nodes */
       while ((s = areg->start_nodes[i]))
 	{
-	  vlib_node_t *n;
-	  vlib_node_runtime_t *rt;
-	  n = vlib_get_node_by_name (vm, (u8 *) s);
-
-	  if (n == 0)
-	    return clib_error_return (0,
-				      "Unknown start node '%s' on feature arc '%s'",
-				      s, areg->arc_name);
-
-	  rt = vlib_node_get_runtime (vm, n->index);
-	  rt->feature_arc_index = arc_index;
 	  i++;
 	}
       areg->n_start_nodes = i;
@@ -70,8 +59,6 @@ vnet_feature_init (vlib_main_t * vm)
   freg = fm->next_feature;
   while (freg)
     {
-      vlib_node_t *n;
-      vlib_node_runtime_t *rt;
       uword *p = hash_get_mem (fm->arc_index_by_name, freg->arc_name);
       if (p == 0)
 	return clib_error_return (0, "Unknown feature arc '%s'",
@@ -80,13 +67,6 @@ vnet_feature_init (vlib_main_t * vm)
       areg = uword_to_pointer (p[0], vnet_feature_arc_registration_t *);
       arc_index = areg->feature_arc_index;
 
-      /* set feature arc index in node runtime */
-      n = vlib_get_node_by_name (vm, (u8 *) freg->node_name);
-      if (n == 0)
-	return clib_error_return (0, "Unknown node '%s', freg->node_name");
-      rt = vlib_node_get_runtime (vm, n->index);
-      rt->feature_arc_index = arc_index;
-
       vec_add1 (fm->next_feature_by_arc[arc_index], *freg);
 
       /* next */
@@ -127,13 +107,15 @@ vnet_feature_init (vlib_main_t * vm)
       arc_index++;
     }
 
+  fm->device_input_feature_arc_index =
+    vnet_get_feature_arc_index ("device-input");
   return 0;
 }
 
 VLIB_INIT_FUNCTION (vnet_feature_init);
 
 void
-vnet_config_update_feature_count (vnet_feature_main_t * fm, u16 arc,
+vnet_config_update_feature_count (vnet_feature_main_t * fm, u8 arc,
 				  u32 sw_if_index, int is_add)
 {
   uword bit_value;
@@ -151,8 +133,8 @@ vnet_config_update_feature_count (vnet_feature_main_t * fm, u16 arc,
 		     bit_value);
 }
 
-u16
-vnet_feature_arc_index_from_node_name (const char *s)
+u8
+vnet_get_feature_arc_index (const char *s)
 {
   vnet_feature_main_t *fm = &feature_main;
   vnet_feature_arc_registration_t *reg;
@@ -167,7 +149,7 @@ vnet_feature_arc_index_from_node_name (const char *s)
 }
 
 u32
-vnet_feature_index_from_node_name (u16 arc, const char *s)
+vnet_get_feature_index (u8 arc, const char *s)
 {
   vnet_feature_main_t *fm = &feature_main;
   vnet_feature_registration_t *reg;
@@ -189,16 +171,16 @@ vnet_feature_enable_disable (const char *arc_name, const char *node_name,
   vnet_feature_main_t *fm = &feature_main;
   vnet_feature_config_main_t *cm;
   u32 feature_index, ci;
-  u16 arc_index;
+  u8 arc_index;
 
-  arc_index = vnet_feature_arc_index_from_node_name (arc_name);
+  arc_index = vnet_get_feature_arc_index (arc_name);
 
   if (arc_index == ~0)
     return;
 
   cm = &fm->feature_config_mains[arc_index];
   vec_validate_init_empty (cm->config_index_by_sw_if_index, sw_if_index, ~0);
-  feature_index = vnet_feature_index_from_node_name (arc_index, node_name);
+  feature_index = vnet_get_feature_index (arc_index, node_name);
   if (feature_index == ~0)
     return;
   ci = cm->config_index_by_sw_if_index[sw_if_index];
@@ -297,7 +279,8 @@ vnet_interface_features_show (vlib_main_t * vm, u32 sw_if_index)
       areg = areg->next;
 
       if (NULL == cm[feature_arc].config_index_by_sw_if_index ||
-	  vec_len (cm[feature_arc].config_index_by_sw_if_index) < sw_if_index)
+	  vec_len (cm[feature_arc].config_index_by_sw_if_index) <=
+	  sw_if_index)
 	{
 	  vlib_cli_output (vm, "  none configured");
 	  continue;
diff --git a/vnet/vnet/feature/feature.h b/vnet/vnet/feature/feature.h
index 1706dec5456..ad4317afd6f 100644
--- a/vnet/vnet/feature/feature.h
+++ b/vnet/vnet/feature/feature.h
@@ -29,7 +29,7 @@ typedef struct _vnet_feature_arc_registration
   char **start_nodes;
   int n_start_nodes;
   /* Feature arc index, assigned by init function */
-  u16 feature_arc_index;
+  u8 feature_arc_index;
 } vnet_feature_arc_registration_t;
 
 /** feature registration object */
@@ -79,6 +79,9 @@ typedef struct
   /** feature reference counts by interface */
   i16 **feature_count_by_sw_if_index;
 
+  /** Feature arc index for device-input */
+  u8 device_input_feature_arc_index;
+
   /** convenience */
   vlib_main_t *vlib_main;
   vnet_main_t *vnet_main;
@@ -111,10 +114,11 @@ static void __vnet_add_feature_registration_##x (void)		\
 __VA_ARGS__ vnet_feature_registration_t vnet_feat_##x
 
 void
-vnet_config_update_feature_count (vnet_feature_main_t * fm, u16 arc,
+vnet_config_update_feature_count (vnet_feature_main_t * fm, u8 arc,
 				  u32 sw_if_index, int is_add);
 
-u32 vnet_feature_index_from_node_name (u16 type, const char *s);
+u32 vnet_get_feature_index (u8 arc, const char *s);
+u8 vnet_get_feature_arc_index (const char *s);
 
 void
 vnet_feature_enable_disable (const char *arc_name, const char *node_name,
@@ -124,29 +128,86 @@ vnet_feature_enable_disable (const char *arc_name, const char *node_name,
 
 
 static_always_inline int
-vnet_have_features (u32 arc, u32 sw_if_index)
+vnet_have_features (u8 arc, u32 sw_if_index)
 {
   vnet_feature_main_t *fm = &feature_main;
   return clib_bitmap_get (fm->sw_if_index_has_features[arc], sw_if_index);
 }
 
 static_always_inline u32
-vnet_feature_get_config_index (u16 arc, u32 sw_if_index)
+vnet_get_feature_config_index (u8 arc, u32 sw_if_index)
 {
   vnet_feature_main_t *fm = &feature_main;
   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc];
   return vec_elt (cm->config_index_by_sw_if_index, sw_if_index);
 }
 
+static_always_inline void *
+vnet_feature_arc_start_with_data (u8 arc, u32 sw_if_index, u32 * next,
+				  vlib_buffer_t * b, u32 n_data_bytes)
+{
+  vnet_feature_main_t *fm = &feature_main;
+  vnet_feature_config_main_t *cm;
+  cm = &fm->feature_config_mains[arc];
+
+  if (PREDICT_FALSE (vnet_have_features (arc, sw_if_index)))
+    {
+      b->feature_arc_index = arc;
+      b->current_config_index =
+	vec_elt (cm->config_index_by_sw_if_index, sw_if_index);
+      return vnet_get_config_data (&cm->config_main, &b->current_config_index,
+				   next, n_data_bytes);
+    }
+}
+
 static_always_inline void
-vnet_feature_redirect (u16 arc, u32 sw_if_index, u32 * next0,
-		       vlib_buffer_t * b0)
+vnet_feature_arc_start (u8 arc, u32 sw_if_index, u32 * next0,
+			vlib_buffer_t * b0)
+{
+  vnet_feature_arc_start_with_data (arc, sw_if_index, next0, b0, 0);
+}
+
+static_always_inline void *
+vnet_feature_next_with_data (u32 sw_if_index, u32 * next0,
+			     vlib_buffer_t * b0, u32 n_data_bytes)
 {
   vnet_feature_main_t *fm = &feature_main;
+  u8 arc = b0->feature_arc_index;
   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc];
 
-  if (PREDICT_FALSE (vnet_have_features (arc, sw_if_index)))
+  return vnet_get_config_data (&cm->config_main,
+			       &b0->current_config_index, next0,
+			       n_data_bytes);
+}
+
+static_always_inline void
+vnet_feature_next (u32 sw_if_index, u32 * next0, vlib_buffer_t * b0)
+{
+  vnet_feature_next_with_data (sw_if_index, next0, b0, 0);
+}
+
+static_always_inline void
+vnet_feature_start_device_input_x1 (u32 sw_if_index, u32 * next0,
+				    vlib_buffer_t * b0, u16 buffer_advanced0)
+{
+  vnet_feature_main_t *fm = &feature_main;
+  vnet_feature_config_main_t *cm;
+  u8 feature_arc_index = fm->device_input_feature_arc_index;
+  cm = &fm->feature_config_mains[feature_arc_index];
+
+  if (PREDICT_FALSE
+      (clib_bitmap_get
+       (fm->sw_if_index_has_features[feature_arc_index], sw_if_index)))
     {
+      /*
+       * Save next0 so that the last feature in the chain
+       * can skip ethernet-input if indicated...
+       */
+      vnet_buffer (b0)->device_input_feat.saved_next_index = *next0;
+      vnet_buffer (b0)->device_input_feat.buffer_advance = buffer_advanced0;
+      vlib_buffer_advance (b0, -buffer_advanced0);
+
+      b0->feature_arc_index = feature_arc_index;
       b0->current_config_index =
 	vec_elt (cm->config_index_by_sw_if_index, sw_if_index);
       vnet_get_config_data (&cm->config_main, &b0->current_config_index,
@@ -155,36 +216,47 @@ vnet_feature_redirect (u16 arc, u32 sw_if_index, u32 * next0,
 }
 
 static_always_inline void
-vnet_feature_device_input_redirect_x1 (vlib_node_runtime_t * node,
-				       u32 sw_if_index, u32 * next0,
-				       vlib_buffer_t * b0,
-				       u16 buffer_advanced0)
+vnet_feature_start_device_input_x2 (u32 sw_if_index,
+				    u32 * next0,
+				    u32 * next1,
+				    vlib_buffer_t * b0,
+				    vlib_buffer_t * b1,
+				    u16 buffer_advanced0,
+				    u16 buffer_advanced1)
 {
   vnet_feature_main_t *fm = &feature_main;
   vnet_feature_config_main_t *cm;
-
-  ASSERT (node->feature_arc_index != ~(u16) 0);
-  cm = &fm->feature_config_mains[node->feature_arc_index];
+  u8 feature_arc_index = fm->device_input_feature_arc_index;
+  cm = &fm->feature_config_mains[feature_arc_index];
 
   if (PREDICT_FALSE
       (clib_bitmap_get
-       (fm->sw_if_index_has_features[node->feature_arc_index], sw_if_index)))
+       (fm->sw_if_index_has_features[feature_arc_index], sw_if_index)))
     {
       /*
        * Save next0 so that the last feature in the chain
        * can skip ethernet-input if indicated...
        */
       vnet_buffer (b0)->device_input_feat.saved_next_index = *next0;
+      vnet_buffer (b1)->device_input_feat.saved_next_index = *next1;
       vnet_buffer (b0)->device_input_feat.buffer_advance = buffer_advanced0;
+      vnet_buffer (b1)->device_input_feat.buffer_advance = buffer_advanced1;
       vlib_buffer_advance (b0, -buffer_advanced0);
+      vlib_buffer_advance (b1, -buffer_advanced1);
 
+      b0->feature_arc_index = feature_arc_index;
+      b1->feature_arc_index = feature_arc_index;
       b0->current_config_index =
 	vec_elt (cm->config_index_by_sw_if_index, sw_if_index);
+      b1->current_config_index = b0->current_config_index;
       vnet_get_config_data (&cm->config_main, &b0->current_config_index,
 			    next0, /* # bytes of config data */ 0);
+      vnet_get_config_data (&cm->config_main, &b1->current_config_index,
+			    next1, /* # bytes of config data */ 0);
     }
 }
 
+
 #define ORDER_CONSTRAINTS (char*[])
 #define VNET_FEATURES(...)  (char*[]) { __VA_ARGS__, 0}
 
diff --git a/vnet/vnet/feature/registration.c b/vnet/vnet/feature/registration.c
index 5f59eed03a6..e6b0968e53c 100644
--- a/vnet/vnet/feature/registration.c
+++ b/vnet/vnet/feature/registration.c
@@ -52,7 +52,7 @@
     {
       .arch_name = "ip4-unicast",
       .node_name = "my-ip4-unicast-feature",
-      .runs_before = VLIB_FEATURES ("ip4-lookup", 0)
+      .runs_before = VLIB_FEATURES ("ip4-lookup")
     };
     </PRE></CODE>
 
@@ -70,21 +70,13 @@
     processing code, aka in the implementation of @c my-ip4-unicast-feature:
 
     <CODE><PRE>
-    vnet_feature_main_t *fm = &feature_main;
-    vnet_feature_config_main_t * cm = &fm->feature_config_mains[VNET_FEAT_IP4_UNICAST];
+    vnet_feature_next (sw_if_index0, &next0, b0);
 
-    Call @c vnet_get_config_data to set next0, and to advance
-    @c b0->current_config_index:
-
-    config_data0 = vnet_get_config_data (&cm->config_main,
-                                         &b0->current_config_index,
-                                         &next0,
-                                         0 / * sizeof config data * /);
     </PRE></CODE>
 
     Nodes are free to drop or otherwise redirect packets. Packets
     which "pass" should be enqueued via the next0 arc computed by
-    vnet_get_config_data.
+    vnet_feature_next.
 */
 
 static const char *vnet_cast_names[] = VNET_CAST_NAMES;
diff --git a/vnet/vnet/unix/tuntap.c b/vnet/vnet/unix/tuntap.c
index 13520d2781a..43ff06aa9ff 100644
--- a/vnet/vnet/unix/tuntap.c
+++ b/vnet/vnet/unix/tuntap.c
@@ -389,7 +389,7 @@ tuntap_rx (vlib_main_t * vm,
           next_index = VNET_DEVICE_INPUT_NEXT_DROP;
       }
 
-    vnet_feature_device_input_redirect_x1 (node, tm->hw_if_index, &next_index, b, 0);
+    vnet_feature_start_device_input_x1 (tm->sw_if_index, &next_index, b, 0);
 
     vlib_set_next_frame_buffer (vm, node, next_index, bi);