Convergence Improvements

addressing convergence times when interface is shut.
1) prioritise the registered callback handlers. Add FIB convergence handler as high priority
2) hook the FIB convergence call-back into HW link down.
3) don't schedule a walk of a FIB node if it has no children
4) Checks at fib_path_t to prevent unnecessary walks, that it prevent the same information propagting the graph multiple times.

Change-Id: I406966b50f31d77c221821b8649776d66655194c
Signed-off-by: Neale Ranns <nranns@cisco.com>

12 files changed, 380 insertions, 37 deletions

diff --git a/vnet/vnet/adj/adj.c b/vnet/vnet/adj/adj.c
index 0bdecc6affe..24f7662d943 100644
--- a/vnet/vnet/adj/adj.c
+++ b/vnet/vnet/adj/adj.c
@@ -387,8 +387,11 @@ adj_show (vlib_main_t * vm,
 	pool_foreach_index(ai, adj_pool,
 	({
 	    if (~0 != sw_if_index &&
-		sw_if_index == adj_get_sw_if_index(ai))
+		sw_if_index != adj_get_sw_if_index(ai))
 	    {
+            }
+            else
+            {
 		vlib_cli_output (vm, "[@%d] %U",
 				 ai,
 				 format_ip_adjacency, ai,
diff --git a/vnet/vnet/adj/adj_glean.c b/vnet/vnet/adj/adj_glean.c
index d48cf21d24b..8d86e2a9f00 100644
--- a/vnet/vnet/adj/adj_glean.c
+++ b/vnet/vnet/adj/adj_glean.c
@@ -131,6 +131,45 @@ adj_glean_interface_state_change (vnet_main_t * vnm,
 
 VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION(adj_glean_interface_state_change);
 
+/**
+ * @brief Invoked on each SW interface of a HW interface when the
+ * HW interface state changes
+ */
+static void
+adj_nbr_hw_sw_interface_state_change (vnet_main_t * vnm,
+                                      u32 sw_if_index,
+                                      void *arg)
+{
+    adj_glean_interface_state_change(vnm, sw_if_index, (uword) arg);
+}
+
+/**
+ * @brief Registered callback for HW interface state changes
+ */
+static clib_error_t *
+adj_glean_hw_interface_state_change (vnet_main_t * vnm,
+                                     u32 hw_if_index,
+                                     u32 flags)
+{
+    /*
+     * walk SW interfaces on the HW
+     */
+    uword sw_flags;
+
+    sw_flags = ((flags & VNET_HW_INTERFACE_FLAG_LINK_UP) ?
+                VNET_SW_INTERFACE_FLAG_ADMIN_UP :
+                0);
+
+    vnet_hw_interface_walk_sw(vnm, hw_if_index,
+                              adj_nbr_hw_sw_interface_state_change,
+                              (void*) sw_flags);
+
+    return (NULL);
+}
+
+VNET_HW_INTERFACE_LINK_UP_DOWN_FUNCTION(
+    adj_glean_hw_interface_state_change);
+
 static clib_error_t *
 adj_glean_interface_delete (vnet_main_t * vnm,
 			    u32 sw_if_index,
diff --git a/vnet/vnet/adj/adj_nbr.c b/vnet/vnet/adj/adj_nbr.c
index 4245990585e..1a78ecbc49f 100644
--- a/vnet/vnet/adj/adj_nbr.c
+++ b/vnet/vnet/adj/adj_nbr.c
@@ -604,30 +604,47 @@ adj_nbr_walk_nh (u32 sw_if_index,
 }
 
 /**
+ * Flags associated with the interface state walks
+ */
+typedef enum adj_nbr_interface_flags_t_
+{
+    ADJ_NBR_INTERFACE_UP = (1 << 0),
+} adj_nbr_interface_flags_t;
+
+/**
  * Context for the state change walk of the DB
  */
 typedef struct adj_nbr_interface_state_change_ctx_t_
 {
     /**
-     * Flags passed from the vnet notifiy function
+     * Flags on the interface
      */
-    int flags;
+    adj_nbr_interface_flags_t flags;
 } adj_nbr_interface_state_change_ctx_t;
 
 static adj_walk_rc_t
 adj_nbr_interface_state_change_one (adj_index_t ai,
-				    void *arg)
+                                    void *arg)
 {
     /*
      * Back walk the graph to inform the forwarding entries
-     * that this interface state has changed.
+     * that this interface state has changed. Do this synchronously
+     * since this is the walk that provides convergence
      */
     adj_nbr_interface_state_change_ctx_t *ctx = arg;
 
     fib_node_back_walk_ctx_t bw_ctx = {
-	.fnbw_reason = (ctx->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP ?
-			FIB_NODE_BW_REASON_FLAG_INTERFACE_UP :
-			FIB_NODE_BW_REASON_FLAG_INTERFACE_DOWN),
+	.fnbw_reason = ((ctx->flags & ADJ_NBR_INTERFACE_UP) ?
+                        FIB_NODE_BW_REASON_FLAG_INTERFACE_UP :
+                        FIB_NODE_BW_REASON_FLAG_INTERFACE_DOWN),
+        /*
+         * the force sync applies only as far as the first fib_entry.
+         * And it's the fib_entry's we need to converge away from
+         * the adjacencies on the now down link
+         */
+        .fnbw_flags = (!(ctx->flags & ADJ_NBR_INTERFACE_UP) ?
+                       FIB_NODE_BW_FLAG_FORCE_SYNC :
+                       0),
     };
 
     fib_walk_sync(FIB_NODE_TYPE_ADJ, ai, &bw_ctx);
@@ -635,10 +652,13 @@ adj_nbr_interface_state_change_one (adj_index_t ai,
     return (ADJ_WALK_RC_CONTINUE);
 }
 
+/**
+ * @brief Registered function for SW interface state changes
+ */
 static clib_error_t *
-adj_nbr_interface_state_change (vnet_main_t * vnm,
-				u32 sw_if_index,
-				u32 flags)
+adj_nbr_sw_interface_state_change (vnet_main_t * vnm,
+                                   u32 sw_if_index,
+                                   u32 flags)
 {
     fib_protocol_t proto;
 
@@ -648,7 +668,9 @@ adj_nbr_interface_state_change (vnet_main_t * vnm,
     for (proto = FIB_PROTOCOL_IP4; proto <= FIB_PROTOCOL_IP6; proto++)
     {
 	adj_nbr_interface_state_change_ctx_t ctx = {
-	    .flags = flags,
+	    .flags = ((flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) ?
+                      ADJ_NBR_INTERFACE_UP :
+                      0),
 	};
 
 	adj_nbr_walk(sw_if_index, proto,
@@ -659,7 +681,60 @@ adj_nbr_interface_state_change (vnet_main_t * vnm,
     return (NULL);
 }
 
-VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION(adj_nbr_interface_state_change);
+VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION_PRIO(
+    adj_nbr_sw_interface_state_change,
+    VNET_ITF_FUNC_PRIORITY_HIGH);
+
+/**
+ * @brief Invoked on each SW interface of a HW interface when the
+ * HW interface state changes
+ */
+static void
+adj_nbr_hw_sw_interface_state_change (vnet_main_t * vnm,
+                                      u32 sw_if_index,
+                                      void *arg)
+{
+    adj_nbr_interface_state_change_ctx_t *ctx = arg;
+    fib_protocol_t proto;
+
+    /*
+     * walk each adj on the interface and trigger a walk from that adj
+     */
+    for (proto = FIB_PROTOCOL_IP4; proto <= FIB_PROTOCOL_IP6; proto++)
+    {
+	adj_nbr_walk(sw_if_index, proto,
+		     adj_nbr_interface_state_change_one,
+		     ctx);
+    }
+}
+
+/**
+ * @brief Registered callback for HW interface state changes
+ */
+static clib_error_t *
+adj_nbr_hw_interface_state_change (vnet_main_t * vnm,
+                                   u32 hw_if_index,
+                                   u32 flags)
+{
+    /*
+     * walk SW interface on the HW
+     */
+    adj_nbr_interface_state_change_ctx_t ctx = {
+        .flags = ((flags & VNET_HW_INTERFACE_FLAG_LINK_UP) ?
+                  ADJ_NBR_INTERFACE_UP :
+                  0),
+    };
+
+    vnet_hw_interface_walk_sw(vnm, hw_if_index,
+                              adj_nbr_hw_sw_interface_state_change,
+                              &ctx);
+
+    return (NULL);
+}
+
+VNET_HW_INTERFACE_LINK_UP_DOWN_FUNCTION_PRIO(
+    adj_nbr_hw_interface_state_change,
+    VNET_ITF_FUNC_PRIORITY_HIGH);
 
 static adj_walk_rc_t
 adj_nbr_interface_delete_one (adj_index_t ai,
diff --git a/vnet/vnet/fib/fib_node.c b/vnet/vnet/fib/fib_node.c
index 5ecc9a7fb2f..35dc874effb 100644
--- a/vnet/vnet/fib/fib_node.c
+++ b/vnet/vnet/fib/fib_node.c
@@ -117,6 +117,22 @@ fib_node_child_add (fib_node_type_t parent_type,
                                      index));
 }
 
+u32
+fib_node_child_get_n_children (fib_node_type_t parent_type,
+                               fib_node_index_t parent_index)
+{
+    fib_node_t *parent;
+
+    parent = fn_vfts[parent_type].fnv_get(parent_index);
+
+    if (FIB_NODE_INDEX_INVALID == parent->fn_children)
+    {
+        return (0);
+    }
+
+    return (fib_node_list_get_size(parent->fn_children));
+}
+
 void
 fib_node_child_remove (fib_node_type_t parent_type,
                        fib_node_index_t parent_index,
diff --git a/vnet/vnet/fib/fib_node.h b/vnet/vnet/fib/fib_node.h
index 33f2203e170..4aabc64e288 100644
--- a/vnet/vnet/fib/fib_node.h
+++ b/vnet/vnet/fib/fib_node.h
@@ -341,6 +341,8 @@ extern void fib_node_deinit(fib_node_t *node);
 extern void fib_node_lock(fib_node_t *node);
 extern void fib_node_unlock(fib_node_t *node);
 
+extern u32 fib_node_child_get_n_children(fib_node_type_t parent_type,
+                                         fib_node_index_t parent_index);
 extern u32 fib_node_child_add(fib_node_type_t parent_type,
 			      fib_node_index_t parent_index,
 			      fib_node_type_t child_type,
diff --git a/vnet/vnet/fib/fib_path.c b/vnet/vnet/fib/fib_path.c
index 5796539b19a..988f689b102 100644
--- a/vnet/vnet/fib/fib_path.c
+++ b/vnet/vnet/fib/fib_path.c
@@ -787,10 +787,24 @@ FIXME comment
 	 */
 	if (FIB_NODE_BW_REASON_FLAG_INTERFACE_UP & ctx->fnbw_reason)
 	{
+            if (path->fp_oper_flags & FIB_PATH_OPER_FLAG_RESOLVED)
+            {
+                /*
+                 * alreday resolved. no need to walk back again
+                 */
+                return (FIB_NODE_BACK_WALK_CONTINUE);
+            }
 	    path->fp_oper_flags |= FIB_PATH_OPER_FLAG_RESOLVED;
 	}
 	if (FIB_NODE_BW_REASON_FLAG_INTERFACE_DOWN & ctx->fnbw_reason)
 	{
+            if (!(path->fp_oper_flags & FIB_PATH_OPER_FLAG_RESOLVED))
+            {
+                /*
+                 * alreday unresolved. no need to walk back again
+                 */
+                return (FIB_NODE_BACK_WALK_CONTINUE);
+            }
 	    path->fp_oper_flags &= ~FIB_PATH_OPER_FLAG_RESOLVED;
 	}
 	if (FIB_NODE_BW_REASON_FLAG_INTERFACE_DELETE & ctx->fnbw_reason)
@@ -809,10 +823,14 @@ FIXME comment
             /*
              * restack the DPO to pick up the correct DPO sub-type
              */
+            uword if_is_up;
             adj_index_t ai;
 
-            if (vnet_sw_interface_is_admin_up(vnet_get_main(),
-                                              path->attached_next_hop.fp_interface))
+            if_is_up = vnet_sw_interface_is_admin_up(
+                           vnet_get_main(),
+                           path->attached_next_hop.fp_interface);
+
+            if (if_is_up)
             {
                 path->fp_oper_flags |= FIB_PATH_OPER_FLAG_RESOLVED;
             }
@@ -825,9 +843,28 @@ FIXME comment
                     fib_proto_to_dpo(path->fp_nh_proto),
                     ai);
             adj_unlock(ai);
+
+            if (!if_is_up)
+            {
+                /*
+                 * If the interface is not up there is no reason to walk
+                 * back to children. if we did they would only evalute
+                 * that this path is unresolved and hence it would
+                 * not contribute the adjacency - so it would be wasted
+                 * CPU time.
+                 */
+                return (FIB_NODE_BACK_WALK_CONTINUE);
+            }
         }
         if (FIB_NODE_BW_REASON_FLAG_ADJ_DOWN & ctx->fnbw_reason)
 	{
+            if (!(path->fp_oper_flags & FIB_PATH_OPER_FLAG_RESOLVED))
+            {
+                /*
+                 * alreday unresolved. no need to walk back again
+                 */
+                return (FIB_NODE_BACK_WALK_CONTINUE);
+            }
             /*
              * the adj has gone down. the path is no longer resolved.
              */
diff --git a/vnet/vnet/fib/fib_test.c b/vnet/vnet/fib/fib_test.c
index d2216df56d7..eb5253d06da 100644
--- a/vnet/vnet/fib/fib_test.c
+++ b/vnet/vnet/fib/fib_test.c
@@ -83,10 +83,22 @@ static uword dummy_interface_tx (vlib_main_t * vm,
   return frame->n_vectors;
 }
 
+static clib_error_t *
+test_interface_admin_up_down (vnet_main_t * vnm,
+                              u32 hw_if_index,
+                              u32 flags)
+{
+  u32 hw_flags = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) ?
+    VNET_HW_INTERFACE_FLAG_LINK_UP : 0;
+  vnet_hw_interface_set_flags (vnm, hw_if_index, hw_flags);
+  return 0;
+}
+
 VNET_DEVICE_CLASS (test_interface_device_class,static) = {
   .name = "Test interface",
   .format_device_name = format_test_interface_name,
   .tx_function = dummy_interface_tx,
+  .admin_up_down_function = test_interface_admin_up_down,
 };
 
 static u8 *hw_address;
@@ -112,7 +124,7 @@ fib_test_mk_intf (u32 ninterfaces)
 	hw_address[5] = i;
 
 	error = ethernet_register_interface(vnet_get_main(),
-					    ethernet_hw_interface_class.index,
+                                            test_interface_device_class.index,
 					    i /* instance */,
 					    hw_address,
 					    &tm->hw_if_indicies[i], 
@@ -120,12 +132,18 @@ fib_test_mk_intf (u32 ninterfaces)
 
 	FIB_TEST((NULL == error), "ADD interface %d", i);
       
-	tm->hw[i] = vnet_get_hw_interface(vnet_get_main(),
+        error = vnet_hw_interface_set_flags(vnet_get_main(),
+                                            tm->hw_if_indicies[i],
+                                            VNET_HW_INTERFACE_FLAG_LINK_UP);
+        tm->hw[i] = vnet_get_hw_interface(vnet_get_main(),
 					  tm->hw_if_indicies[i]);
-	vec_validate (ip4_main.fib_index_by_sw_if_index, tm->hw[i]->sw_if_index);
-	vec_validate (ip6_main.fib_index_by_sw_if_index, tm->hw[i]->sw_if_index);
+	vec_validate (ip4_main.fib_index_by_sw_if_index,
+                      tm->hw[i]->sw_if_index);
+	vec_validate (ip6_main.fib_index_by_sw_if_index,
+                      tm->hw[i]->sw_if_index);
 	ip4_main.fib_index_by_sw_if_index[tm->hw[i]->sw_if_index] = 0;
 	ip6_main.fib_index_by_sw_if_index[tm->hw[i]->sw_if_index] = 0;
+
 	error = vnet_sw_interface_set_flags(vnet_get_main(),
 					    tm->hw[i]->sw_if_index,
 					    VNET_SW_INTERFACE_FLAG_ADMIN_UP);
@@ -4217,6 +4235,64 @@ fib_test_v6 (void)
 	     "attached interface adj is glean");
 
     /*
+     * Same test as above, but this time the HW interface goes down
+     */
+    error = vnet_hw_interface_set_flags(vnet_get_main(),
+					tm->hw_if_indicies[0],
+					~VNET_HW_INTERFACE_FLAG_LINK_UP);
+    FIB_TEST((NULL == error), "Interface shutdown OK");
+
+    fei = fib_table_lookup_exact_match(fib_index, &pfx_2001_b_s_64);
+    dpo = fib_entry_contribute_ip_forwarding(fei);
+    FIB_TEST(!dpo_cmp(dpo_drop, load_balance_get_bucket(dpo->dpoi_index, 0)),
+	     "2001::b/64 resolves via drop");
+    fei = fib_table_lookup_exact_match(fib_index, &pfx_2001_a_s_64);
+    dpo = fib_entry_contribute_ip_forwarding(fei);
+    FIB_TEST(!dpo_cmp(dpo_drop, load_balance_get_bucket(dpo->dpoi_index, 0)),
+	     "2001::a/64 resolves via drop");
+    fei = fib_table_lookup_exact_match(fib_index, &pfx_2001_1_3_s_128);
+    dpo = fib_entry_contribute_ip_forwarding(fei);
+    FIB_TEST(!dpo_cmp(dpo_drop, load_balance_get_bucket(dpo->dpoi_index, 0)),
+	     "2001:0:0:1::3/128 resolves via drop");
+    fei = fib_table_lookup_exact_match(fib_index, &pfx_2001_1_2_s_128);
+    dpo = fib_entry_contribute_ip_forwarding(fei);
+    FIB_TEST(!dpo_cmp(dpo_drop, load_balance_get_bucket(dpo->dpoi_index, 0)),
+	     "2001:0:0:1::2/128 resolves via drop");
+    local_pfx.fp_len = 128;
+    fei = fib_table_lookup_exact_match(fib_index, &local_pfx);
+    dpo = fib_entry_contribute_ip_forwarding(fei);
+    FIB_TEST(dpo_cmp(dpo_drop, load_balance_get_bucket(dpo->dpoi_index, 0)),
+	     "2001:0:0:1::1/128 not drop");
+    local_pfx.fp_len = 64;
+    fei = fib_table_lookup_exact_match(fib_index, &local_pfx);
+    dpo = fib_entry_contribute_ip_forwarding(fei);
+    FIB_TEST(!dpo_cmp(dpo_drop, load_balance_get_bucket(dpo->dpoi_index, 0)),
+	     "2001:0:0:1/64 resolves via drop");
+
+    error = vnet_hw_interface_set_flags(vnet_get_main(),
+					tm->hw_if_indicies[0],
+					VNET_HW_INTERFACE_FLAG_LINK_UP);
+    FIB_TEST((NULL == error), "Interface bring-up OK");
+    fei = fib_table_lookup_exact_match(fib_index, &pfx_2001_a_s_64);
+    ai = fib_entry_get_adj(fei);
+    FIB_TEST((ai_01 == ai), "2001::a/64 resolves via 2001:0:0:1::1");
+    fei = fib_table_lookup_exact_match(fib_index, &pfx_2001_b_s_64);
+    ai = fib_entry_get_adj(fei);
+    FIB_TEST((ai_01 == ai), "2001::b/64 resolves via 2001:0:0:1::1");
+    fei = fib_table_lookup_exact_match(fib_index, &pfx_2001_1_3_s_128);
+    ai = fib_entry_get_adj(fei);
+    FIB_TEST((ai_02 == ai), "ADJ-FIB resolves via adj");
+    fei = fib_table_lookup_exact_match(fib_index, &pfx_2001_1_2_s_128);
+    ai = fib_entry_get_adj(fei);
+    FIB_TEST((ai_01 == ai), "ADJ-FIB resolves via adj");
+    local_pfx.fp_len = 64;
+    fei = fib_table_lookup_exact_match(fib_index, &local_pfx);
+    ai = fib_entry_get_adj(fei);
+    adj = adj_get(ai);
+    FIB_TEST((IP_LOOKUP_NEXT_GLEAN == adj->lookup_next_index),
+	     "attached interface adj is glean");
+
+    /*
      * Delete the interface that the routes reolve through.
      * Again no routes are removed. They all point to drop.
      *
diff --git a/vnet/vnet/fib/fib_walk.c b/vnet/vnet/fib/fib_walk.c
index 215d21dc273..e5f9b87f927 100644
--- a/vnet/vnet/fib/fib_walk.c
+++ b/vnet/vnet/fib/fib_walk.c
@@ -181,6 +181,7 @@ static u32 history_last_walk_pos;
 typedef struct fib_walk_history_t_ {
     u32 fwh_n_visits;
     f64 fwh_duration;
+    f64 fwh_completed;
     fib_node_ptr_t fwh_parent;
     fib_walk_flags_t fwh_flags;
     fib_node_bw_reason_flag_t fwh_reason;
@@ -266,8 +267,11 @@ fib_walk_destroy (fib_walk_t *fwalk)
 
     fib_walk_history[history_last_walk_pos].fwh_n_visits =
 	fwalk->fw_n_visits;
+    fib_walk_history[history_last_walk_pos].fwh_completed =
+	vlib_time_now(vlib_get_main());
     fib_walk_history[history_last_walk_pos].fwh_duration =
-	vlib_time_now(vlib_get_main()) - fwalk->fw_start_time;
+	fib_walk_history[history_last_walk_pos].fwh_completed -
+        fwalk->fw_start_time;
     fib_walk_history[history_last_walk_pos].fwh_parent =
 	fwalk->fw_parent;
     fib_walk_history[history_last_walk_pos].fwh_flags =
@@ -615,6 +619,14 @@ fib_walk_async (fib_node_type_t parent_type,
 	 */
 	return;
     }
+    if (0 == fib_node_child_get_n_children(parent_type,
+                                           parent_index))
+    {
+        /*
+         * no children to walk - quit now
+         */
+        return;
+    }
     if (ctx->fnbw_flags & FIB_NODE_BW_FLAG_FORCE_SYNC)
     {
         /*
@@ -662,6 +674,14 @@ fib_walk_sync (fib_node_type_t parent_type,
 	 */
 	return;
     }
+    if (0 == fib_node_child_get_n_children(parent_type,
+                                           parent_index))
+    {
+        /*
+         * no children to walk - quit now
+         */
+        return;
+    }
 
     fwalk = fib_walk_alloc(parent_type,
 			   parent_index,
@@ -959,16 +979,17 @@ fib_walk_show (vlib_main_t * vm,
 
     while (ii != history_last_walk_pos)
     {
-	if (0 != fib_walk_history[ii].fwh_n_visits)
+	if (0 != fib_walk_history[ii].fwh_reason)
 	{
             fib_node_back_walk_reason_t reason;
             u8 *s = NULL;
 
-	    s = format(s, " %s:%d visits:%d duration:%.2f ",
-                       fib_node_type_get_name(fib_walk_history[ii].fwh_parent.fnp_type),
+	    s = format(s, "[@%d]: %s:%d visits:%d duration:%.2f completed:%.2f ",
+                       ii, fib_node_type_get_name(fib_walk_history[ii].fwh_parent.fnp_type),
                        fib_walk_history[ii].fwh_parent.fnp_index,
                        fib_walk_history[ii].fwh_n_visits,
-                       fib_walk_history[ii].fwh_duration);
+                       fib_walk_history[ii].fwh_duration,
+                       fib_walk_history[ii].fwh_completed);
             if (FIB_WALK_FLAG_SYNC & fib_walk_history[ii].fwh_flags)
                 s = format(s, "sync, ");
             if (FIB_WALK_FLAG_ASYNC & fib_walk_history[ii].fwh_flags)
diff --git a/vnet/vnet/interface.c b/vnet/vnet/interface.c
index 97855d53d73..33827e2b673 100644
--- a/vnet/vnet/interface.c
+++ b/vnet/vnet/interface.c
@@ -240,17 +240,25 @@ unserialize_vnet_interface_state (serialize_main_t * m, va_list * va)
 static clib_error_t *
 call_elf_section_interface_callbacks (vnet_main_t * vnm, u32 if_index,
 				      u32 flags,
-				      _vnet_interface_function_list_elt_t *
-				      elt)
+				      _vnet_interface_function_list_elt_t **
+				      elts)
 {
+  _vnet_interface_function_list_elt_t *elt;
+  vnet_interface_function_priority_t prio;
   clib_error_t *error = 0;
 
-  while (elt)
+  for (prio = VNET_ITF_FUNC_PRIORITY_LOW;
+       prio <= VNET_ITF_FUNC_PRIORITY_HIGH; prio++)
     {
-      error = elt->fp (vnm, if_index, flags);
-      if (error)
-	return error;
-      elt = elt->next_interface_function;
+      elt = elts[prio];
+
+      while (elt)
+	{
+	  error = elt->fp (vnm, if_index, flags);
+	  if (error)
+	    return error;
+	  elt = elt->next_interface_function;
+	}
     }
   return error;
 }
@@ -888,6 +896,27 @@ vnet_delete_hw_interface (vnet_main_t * vnm, u32 hw_if_index)
   pool_put (im->hw_interfaces, hw);
 }
 
+void
+vnet_hw_interface_walk_sw (vnet_main_t * vnm,
+			   u32 hw_if_index,
+			   vnet_hw_sw_interface_walk_t fn, void *ctx)
+{
+  vnet_hw_interface_t *hi;
+  u32 id, sw_if_index;
+
+  hi = vnet_get_hw_interface (vnm, hw_if_index);
+  /* the super first, then the and sub interfaces */
+  fn (vnm, hi->sw_if_index, ctx);
+
+  /* *INDENT-OFF* */
+  hash_foreach (id, sw_if_index,
+                hi->sub_interface_sw_if_index_by_id,
+  ({
+    fn (vnm, sw_if_index, ctx);
+  }));
+  /* *INDENT-ON* */
+}
+
 static void
 serialize_vnet_hw_interface_set_class (serialize_main_t * m, va_list * va)
 {
diff --git a/vnet/vnet/interface.h b/vnet/vnet/interface.h
index 4d7af374c95..5ac7fbd5bc1 100644
--- a/vnet/vnet/interface.h
+++ b/vnet/vnet/interface.h
@@ -61,6 +61,13 @@ typedef clib_error_t *(vnet_subif_add_del_function_t)
 typedef clib_error_t *(vnet_interface_set_mac_address_function_t)
   (struct vnet_hw_interface_t * hi, char *address);
 
+typedef enum vnet_interface_function_priority_t_
+{
+  VNET_ITF_FUNC_PRIORITY_LOW,
+  VNET_ITF_FUNC_PRIORITY_HIGH,
+} vnet_interface_function_priority_t;
+#define VNET_ITF_FUNC_N_PRIO ((vnet_interface_function_priority_t)VNET_ITF_FUNC_PRIORITY_HIGH+1)
+
 typedef struct _vnet_interface_function_list_elt
 {
   struct _vnet_interface_function_list_elt *next_interface_function;
@@ -76,8 +83,23 @@ static void __vnet_interface_function_init_##tag##_##f (void)           \
 {                                                                       \
  vnet_main_t * vnm = vnet_get_main();                                   \
  static _vnet_interface_function_list_elt_t init_function;              \
- init_function.next_interface_function = vnm->tag##_functions;          \
- vnm->tag##_functions = &init_function;                                 \
+ init_function.next_interface_function =                                \
+   vnm->tag##_functions[VNET_ITF_FUNC_PRIORITY_LOW];                    \
+ vnm->tag##_functions[VNET_ITF_FUNC_PRIORITY_LOW] = &init_function;     \
+ init_function.fp = (void *) &f;                                        \
+}
+
+#define _VNET_INTERFACE_FUNCTION_DECL_PRIO(f,tag,p)                    \
+                                                                        \
+static void __vnet_interface_function_init_##tag##_##f (void)           \
+    __attribute__((__constructor__)) ;                                  \
+                                                                        \
+static void __vnet_interface_function_init_##tag##_##f (void)           \
+{                                                                       \
+ vnet_main_t * vnm = vnet_get_main();                                   \
+ static _vnet_interface_function_list_elt_t init_function;              \
+ init_function.next_interface_function = vnm->tag##_functions[p];       \
+ vnm->tag##_functions[p] = &init_function;                              \
  init_function.fp = (void *) &f;                                        \
 }
 
@@ -85,10 +107,14 @@ static void __vnet_interface_function_init_##tag##_##f (void)           \
   _VNET_INTERFACE_FUNCTION_DECL(f,hw_interface_add_del)
 #define VNET_HW_INTERFACE_LINK_UP_DOWN_FUNCTION(f)		\
   _VNET_INTERFACE_FUNCTION_DECL(f,hw_interface_link_up_down)
+#define VNET_HW_INTERFACE_LINK_UP_DOWN_FUNCTION_PRIO(f,p)        \
+  _VNET_INTERFACE_FUNCTION_DECL_PRIO(f,hw_interface_link_up_down,p)
 #define VNET_SW_INTERFACE_ADD_DEL_FUNCTION(f)			\
   _VNET_INTERFACE_FUNCTION_DECL(f,sw_interface_add_del)
 #define VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION(f)		\
   _VNET_INTERFACE_FUNCTION_DECL(f,sw_interface_admin_up_down)
+#define VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION_PRIO(f,p)     	\
+  _VNET_INTERFACE_FUNCTION_DECL_PRIO(f,sw_interface_admin_up_down, p)
 
 /* A class of hardware interface devices. */
 typedef struct _vnet_device_class
diff --git a/vnet/vnet/interface_funcs.h b/vnet/vnet/interface_funcs.h
index 076e62eca64..17c677f07b6 100644
--- a/vnet/vnet/interface_funcs.h
+++ b/vnet/vnet/interface_funcs.h
@@ -92,6 +92,21 @@ vnet_get_device_class (vnet_main_t * vnm, u32 dev_class_index)
 			   dev_class_index);
 }
 
+/**
+ * Call back walk type for walking SW indices on a HW interface
+ */
+typedef void (*vnet_hw_sw_interface_walk_t) (vnet_main_t * vnm,
+					     u32 sw_if_index, void *ctx);
+
+/**
+ * @brief
+ * Walk the SW interfaces on a HW interface - this is the super
+ * interface and any sub-interfaces.
+ */
+void vnet_hw_interface_walk_sw (vnet_main_t * vnm,
+				u32 hw_if_index,
+				vnet_hw_sw_interface_walk_t fn, void *ctx);
+
 /* Register a hardware interface instance. */
 u32 vnet_register_interface (vnet_main_t * vnm,
 			     u32 dev_class_index,
diff --git a/vnet/vnet/vnet.h b/vnet/vnet/vnet.h
index 98b29e9c0d3..36cdddd6ba4 100644
--- a/vnet/vnet/vnet.h
+++ b/vnet/vnet/vnet.h
@@ -59,10 +59,14 @@ typedef struct vnet_main_t
   /* set up by constructors */
   vnet_device_class_t *device_class_registrations;
   vnet_hw_interface_class_t *hw_interface_class_registrations;
-  _vnet_interface_function_list_elt_t *hw_interface_add_del_functions;
-  _vnet_interface_function_list_elt_t *hw_interface_link_up_down_functions;
-  _vnet_interface_function_list_elt_t *sw_interface_add_del_functions;
-  _vnet_interface_function_list_elt_t *sw_interface_admin_up_down_functions;
+    _vnet_interface_function_list_elt_t
+    * hw_interface_add_del_functions[VNET_ITF_FUNC_N_PRIO];
+    _vnet_interface_function_list_elt_t
+    * hw_interface_link_up_down_functions[VNET_ITF_FUNC_N_PRIO];
+    _vnet_interface_function_list_elt_t
+    * sw_interface_add_del_functions[VNET_ITF_FUNC_N_PRIO];
+    _vnet_interface_function_list_elt_t
+    * sw_interface_admin_up_down_functions[VNET_ITF_FUNC_N_PRIO];
 
   /*
    * Last "api" error, preserved so we can issue reasonable diagnostics