policer: add thread handoff for device input

Add worker thread handoff for policers on the device input feature arc
on an interface.

Type: improvement
Signed-off-by: Brian Russell <brian@graphiant.com>
Change-Id: Ib795457a09a5b3be3c4e6422c91e33100192b8e2

4 files changed, 71 insertions, 13 deletions

diff --git a/src/vnet/policer/node_funcs.c b/src/vnet/policer/node_funcs.c
index be8ce58799a..c7b6e8c7585 100644
--- a/src/vnet/policer/node_funcs.c
+++ b/src/vnet/policer/node_funcs.c
@@ -127,28 +127,38 @@ vnet_policer_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
 	  pi1 = pm->policer_index_by_sw_if_index[sw_if_index1];
 
 	  act0 = vnet_policer_police (vm, b0, pi0, time_in_policer_periods,
-				      POLICE_CONFORM /* no chaining */, false);
+				      POLICE_CONFORM /* no chaining */, true);
 
 	  act1 = vnet_policer_police (vm, b1, pi1, time_in_policer_periods,
-				      POLICE_CONFORM /* no chaining */, false);
+				      POLICE_CONFORM /* no chaining */, true);
 
-	  if (PREDICT_FALSE (act0 == QOS_ACTION_DROP)) /* drop action */
+	  if (PREDICT_FALSE (act0 == QOS_ACTION_HANDOFF))
+	    {
+	      next0 = VNET_POLICER_NEXT_HANDOFF;
+	      vnet_buffer (b0)->policer.index = pi0;
+	    }
+	  else if (PREDICT_FALSE (act0 == QOS_ACTION_DROP))
 	    {
 	      next0 = VNET_POLICER_NEXT_DROP;
 	      b0->error = node->errors[VNET_POLICER_ERROR_DROP];
 	    }
-	  else			/* transmit or mark-and-transmit action */
+	  else /* transmit or mark-and-transmit action */
 	    {
 	      transmitted++;
 	      vnet_feature_next (&next0, b0);
 	    }
 
-	  if (PREDICT_FALSE (act1 == QOS_ACTION_DROP)) /* drop action */
+	  if (PREDICT_FALSE (act1 == QOS_ACTION_HANDOFF))
+	    {
+	      next1 = VNET_POLICER_NEXT_HANDOFF;
+	      vnet_buffer (b1)->policer.index = pi1;
+	    }
+	  else if (PREDICT_FALSE (act1 == QOS_ACTION_DROP)) /* drop action */
 	    {
 	      next1 = VNET_POLICER_NEXT_DROP;
 	      b1->error = node->errors[VNET_POLICER_ERROR_DROP];
 	    }
-	  else			/* transmit or mark-and-transmit action */
+	  else /* transmit or mark-and-transmit action */
 	    {
 	      transmitted++;
 	      vnet_feature_next (&next1, b1);
@@ -201,14 +211,19 @@ vnet_policer_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
 	  pi0 = pm->policer_index_by_sw_if_index[sw_if_index0];
 
 	  act0 = vnet_policer_police (vm, b0, pi0, time_in_policer_periods,
-				      POLICE_CONFORM /* no chaining */, false);
+				      POLICE_CONFORM /* no chaining */, true);
 
-	  if (PREDICT_FALSE (act0 == QOS_ACTION_DROP)) /* drop action */
+	  if (PREDICT_FALSE (act0 == QOS_ACTION_HANDOFF))
+	    {
+	      next0 = VNET_POLICER_NEXT_HANDOFF;
+	      vnet_buffer (b0)->policer.index = pi0;
+	    }
+	  else if (PREDICT_FALSE (act0 == QOS_ACTION_DROP))
 	    {
 	      next0 = VNET_POLICER_NEXT_DROP;
 	      b0->error = node->errors[VNET_POLICER_ERROR_DROP];
 	    }
-	  else			/* transmit or mark-and-transmit action */
+	  else /* transmit or mark-and-transmit action */
 	    {
 	      transmitted++;
 	      vnet_feature_next (&next0, b0);
@@ -254,6 +269,7 @@ VLIB_REGISTER_NODE (policer_input_node) = {
   .n_next_nodes = VNET_POLICER_N_NEXT,
   .next_nodes = {
 		 [VNET_POLICER_NEXT_DROP] = "error-drop",
+		 [VNET_POLICER_NEXT_HANDOFF] = "policer-input-handoff",
 		 },
 };
 
@@ -263,6 +279,28 @@ VNET_FEATURE_INIT (policer_input_node, static) = {
   .runs_before = VNET_FEATURES ("ethernet-input"),
 };
 
+static char *policer_input_handoff_error_strings[] = { "congestion drop" };
+
+VLIB_NODE_FN (policer_input_handoff_node)
+(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
+{
+  return policer_handoff (vm, node, frame, vnet_policer_main.fq_index, ~0);
+}
+
+VLIB_REGISTER_NODE (policer_input_handoff_node) = {
+  .name = "policer-input-handoff",
+  .vector_size = sizeof (u32),
+  .format_trace = format_policer_handoff_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = ARRAY_LEN(policer_input_handoff_error_strings),
+  .error_strings = policer_input_handoff_error_strings,
+
+  .n_next_nodes = 1,
+  .next_nodes = {
+    [0] = "error-drop",
+  },
+};
+
 typedef struct
 {
   u32 sw_if_index;
diff --git a/src/vnet/policer/police_inlines.h b/src/vnet/policer/police_inlines.h
index 9f56e1134d6..59afb562509 100644
--- a/src/vnet/policer/police_inlines.h
+++ b/src/vnet/policer/police_inlines.h
@@ -124,10 +124,14 @@ policer_handoff (vlib_main_t *vm, vlib_node_runtime_t *node,
   vnet_policer_main_t *pm;
   policer_t *policer;
   u32 this_thread, policer_thread;
+  bool single_policer_node = (policer_index != ~0);
 
   pm = &vnet_policer_main;
-  policer = &pm->policers[policer_index];
-  policer_thread = policer->thread_index;
+  if (single_policer_node)
+    {
+      policer = &pm->policers[policer_index];
+      policer_thread = policer->thread_index;
+    }
 
   this_thread = vm->thread_index;
   from = vlib_frame_vector_args (frame);
@@ -139,7 +143,16 @@ policer_handoff (vlib_main_t *vm, vlib_node_runtime_t *node,
 
   while (n_left_from > 0)
     {
-      ti[0] = policer_thread;
+      if (!single_policer_node)
+	{
+	  policer_index = vnet_buffer (b[0])->policer.index;
+	  policer = &pm->policers[policer_index];
+	  ti[0] = policer->thread_index;
+	}
+      else
+	{
+	  ti[0] = policer_thread;
+	}
 
       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) &&
 			 b[0]->flags & VLIB_BUFFER_IS_TRACED))
@@ -147,7 +160,7 @@ policer_handoff (vlib_main_t *vm, vlib_node_runtime_t *node,
 	  policer_handoff_trace_t *t =
 	    vlib_add_trace (vm, node, b[0], sizeof (*t));
 	  t->current_worker_index = this_thread;
-	  t->next_worker_index = policer_thread;
+	  t->next_worker_index = ti[0];
 	  t->policer_index = policer_index;
 	}
 
diff --git a/src/vnet/policer/policer.c b/src/vnet/policer/policer.c
index 678e5884cdd..4aaf9e01e04 100644
--- a/src/vnet/policer/policer.c
+++ b/src/vnet/policer/policer.c
@@ -628,6 +628,7 @@ policer_init (vlib_main_t * vm)
   pm->vlib_main = vm;
   pm->vnet_main = vnet_get_main ();
   pm->log_class = vlib_log_register_class ("policer", 0);
+  pm->fq_index = vlib_frame_queue_main_init (policer_input_node.index, 0);
 
   pm->policer_config_by_name = hash_create_string (0, sizeof (uword));
   pm->policer_index_by_name = hash_create_string (0, sizeof (uword));
diff --git a/src/vnet/policer/policer.h b/src/vnet/policer/policer.h
index 9f090b1bb74..d9c4ed2a451 100644
--- a/src/vnet/policer/policer.h
+++ b/src/vnet/policer/policer.h
@@ -46,15 +46,21 @@ typedef struct
   vnet_main_t *vnet_main;
 
   vlib_log_class_t log_class;
+
+  /* frame queue for thread handoff */
+  u32 fq_index;
 } vnet_policer_main_t;
 
 extern vnet_policer_main_t vnet_policer_main;
 
 extern vlib_combined_counter_main_t policer_counters[];
 
+extern vlib_node_registration_t policer_input_node;
+
 typedef enum
 {
   VNET_POLICER_NEXT_DROP,
+  VNET_POLICER_NEXT_HANDOFF,
   VNET_POLICER_N_NEXT,
 } vnet_policer_next_t;