fib: fib api updates

Enhance the route add/del APIs to take a set of paths rather than just one.
Most unicast routing protocols calcualte all the available paths in one
run of the algorithm so updating all the paths at once is beneficial for the client.
two knobs control the behaviour:
  is_multipath - if set the the set of paths passed will be added to those
                 that already exist, otherwise the set will replace them.
  is_add - add or remove the set

is_add=0, is_multipath=1 and an empty set, results in deleting the route.

It is also considerably faster to add multiple paths at once, than one at a time:

vat# ip_add_del_route 1.1.1.1/32 count 100000 multipath via 10.10.10.11
100000 routes in .572240 secs, 174751.80 routes/sec
vat# ip_add_del_route 1.1.1.1/32 count 100000 multipath via 10.10.10.12
100000 routes in .528383 secs, 189256.54 routes/sec
vat# ip_add_del_route 1.1.1.1/32 count 100000 multipath via 10.10.10.13
100000 routes in .757131 secs, 132077.52 routes/sec
vat# ip_add_del_route 1.1.1.1/32 count 100000 multipath via 10.10.10.14
100000 routes in .878317 secs, 113854.12 routes/sec

vat# ip_route_add_del 1.1.1.1/32 count 100000 multipath via 10.10.10.11 via 10.10.10.12 via 10.10.10.13 via 10.10.10.14
100000 routes in .900212 secs, 111084.93 routes/sec

Change-Id: I416b93f7684745099c1adb0b33edac58c9339c1a
Signed-off-by: Neale Ranns <neale.ranns@cisco.com>
Signed-off-by: Ole Troan <ot@cisco.com>
Signed-off-by: Paul Vinciguerra <pvinci@vinciconsulting.com>

10 files changed, 170 insertions, 158 deletions

diff --git a/src/plugins/abf/abf.api b/src/plugins/abf/abf.api
index 6716dce562f..03044ad5a6b 100644
--- a/src/plugins/abf/abf.api
+++ b/src/plugins/abf/abf.api
@@ -20,6 +20,7 @@
 */
 
 option version = "1.0.0";
+import "vnet/ip/ip_types.api";
 import "vnet/fib/fib_types.api";
 
 /** \brief Get the plugin version
diff --git a/src/plugins/abf/abf_api.c b/src/plugins/abf/abf_api.c
index e5c3bfa6f3f..a14c9008332 100644
--- a/src/plugins/abf/abf_api.c
+++ b/src/plugins/abf/abf_api.c
@@ -101,7 +101,7 @@ vl_api_abf_policy_add_del_t_handler (vl_api_abf_policy_add_del_t * mp)
   for (pi = 0; pi < mp->policy.n_paths; pi++)
     {
       path = &paths[pi];
-      rv = fib_path_api_parse (&mp->policy.paths[pi], path);
+      rv = fib_api_path_decode (&mp->policy.paths[pi], path);
 
       if (0 != rv)
 	{
@@ -158,9 +158,12 @@ typedef struct abf_dump_walk_ctx_t_
 static int
 abf_policy_send_details (u32 api, void *args)
 {
-  fib_route_path_encode_t *api_rpaths = NULL, *api_rpath;
+  fib_path_encode_ctx_t walk_ctx = {
+    .rpaths = NULL,
+  };
   vl_api_abf_policy_details_t *mp;
   abf_dump_walk_ctx_t *ctx;
+  fib_route_path_t *rpath;
   vl_api_fib_path_t *fp;
   size_t msg_size;
   abf_policy_t *ap;
@@ -181,17 +184,19 @@ abf_policy_send_details (u32 api, void *args)
   mp->policy.acl_index = htonl (ap->ap_acl);
   mp->policy.policy_id = htonl (ap->ap_id);
 
-  fib_path_list_walk_w_ext (ap->ap_pl, NULL, fib_path_encode, &api_rpaths);
+  fib_path_list_walk_w_ext (ap->ap_pl, NULL, fib_path_encode, &walk_ctx);
 
   fp = mp->policy.paths;
-  vec_foreach (api_rpath, api_rpaths)
+  vec_foreach (rpath, walk_ctx.rpaths)
   {
-    fib_api_path_encode (api_rpath, fp);
+    fib_api_path_encode (rpath, fp);
     fp++;
   }
 
   vl_api_send_msg (ctx->rp, (u8 *) mp);
 
+  vec_free (walk_ctx.rpaths);
+
   return (1);
 }
 
diff --git a/src/plugins/abf/abf_error.def b/src/plugins/abf/abf_error.def
index 71e798beb71..83349eae42a 100644
--- a/src/plugins/abf/abf_error.def
+++ b/src/plugins/abf/abf_error.def
@@ -17,3 +17,4 @@
 
 abf_error (NONE, "no match")
 abf_error (MATCHED, "matched")
+abf_error (MISSED, "missed")
diff --git a/src/plugins/abf/abf_itf_attach.c b/src/plugins/abf/abf_itf_attach.c
index 9569306ec2a..337eed8697d 100644
--- a/src/plugins/abf/abf_itf_attach.c
+++ b/src/plugins/abf/abf_itf_attach.c
@@ -505,12 +505,12 @@ abf_input_inline (vlib_main_t * vm,
 		  vlib_node_runtime_t * node,
 		  vlib_frame_t * frame, fib_protocol_t fproto)
 {
-  u32 n_left_from, *from, *to_next, next_index, matches;
+  u32 n_left_from, *from, *to_next, next_index, matches, misses;
 
   from = vlib_frame_vector_args (frame);
   n_left_from = frame->n_vectors;
   next_index = node->cached_next_index;
-  matches = 0;
+  matches = misses = 0;
 
   while (n_left_from > 0)
     {
@@ -530,6 +530,7 @@ abf_input_inline (vlib_main_t * vm,
 	  u32 match_acl_pos = ~0;
 	  u32 match_rule_index = ~0;
 	  u32 trace_bitmap = 0;
+	  u32 lc_index;
 	  u8 action;
 
 	  bi0 = from[0];
@@ -549,7 +550,7 @@ abf_input_inline (vlib_main_t * vm,
 	  /*
 	   * check if any of the policies attached to this interface matches.
 	   */
-	  u32 lc_index = abf_alctx_per_itf[fproto][sw_if_index0];
+	  lc_index = abf_alctx_per_itf[fproto][sw_if_index0];
 
 	  /*
 	     A non-inline version looks like this:
@@ -589,6 +590,7 @@ abf_input_inline (vlib_main_t * vm,
 	       *  move on down the feature arc
 	       */
 	      vnet_feature_next (&next0, b0);
+	      misses++;
 	    }
 
 	  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
@@ -614,6 +616,11 @@ abf_input_inline (vlib_main_t * vm,
 				abf_ip4_node.index :
 				abf_ip6_node.index),
 			       ABF_ERROR_MATCHED, matches);
+  vlib_node_increment_counter (vm,
+			       (fproto = FIB_PROTOCOL_IP6 ?
+				abf_ip4_node.index :
+				abf_ip6_node.index),
+			       ABF_ERROR_MISSED, misses);
 
   return frame->n_vectors;
 }
diff --git a/src/plugins/gtpu/gtpu.c b/src/plugins/gtpu/gtpu.c
index 7612055ed7f..dc5c689f5d3 100644
--- a/src/plugins/gtpu/gtpu.c
+++ b/src/plugins/gtpu/gtpu.c
@@ -520,8 +520,9 @@ int vnet_gtpu_add_del_tunnel
 		.frp_addr = zero_addr,
 		.frp_sw_if_index = 0xffffffff,
 		.frp_fib_index = ~0,
-		.frp_weight = 0,
+		.frp_weight = 1,
 		.frp_flags = FIB_ROUTE_PATH_LOCAL,
+		.frp_mitf_flags = MFIB_ITF_FLAG_FORWARD,
 	      };
 	      const mfib_prefix_t mpfx = {
 		.fp_proto = fp,
@@ -535,17 +536,14 @@ int vnet_gtpu_add_del_tunnel
 	       *  - the accepting interface is that from the API
 	       */
 	      mfib_table_entry_path_update (t->encap_fib_index,
-					    &mpfx,
-					    MFIB_SOURCE_GTPU,
-					    &path, MFIB_ITF_FLAG_FORWARD);
+					    &mpfx, MFIB_SOURCE_GTPU, &path);
 
 	      path.frp_sw_if_index = a->mcast_sw_if_index;
 	      path.frp_flags = FIB_ROUTE_PATH_FLAG_NONE;
+	      path.frp_mitf_flags = MFIB_ITF_FLAG_ACCEPT;
 	      mfei = mfib_table_entry_path_update (t->encap_fib_index,
 						   &mpfx,
-						   MFIB_SOURCE_GTPU,
-						   &path,
-						   MFIB_ITF_FLAG_ACCEPT);
+						   MFIB_SOURCE_GTPU, &path);
 
 	      /*
 	       * Create the mcast adjacency to send traffic to the group
diff --git a/src/plugins/igmp/igmp.c b/src/plugins/igmp/igmp.c
index 4e2fce1b916..1e9f647cd11 100644
--- a/src/plugins/igmp/igmp.c
+++ b/src/plugins/igmp/igmp.c
@@ -346,15 +346,6 @@ igmp_enable_disable (u32 sw_if_index, u8 enable, igmp_mode_t mode)
 	    format_vnet_sw_if_index_name, vnet_get_main (), sw_if_index);
 
   /* *INDENT-OFF* */
-  fib_route_path_t for_us_path =
-    {
-      .frp_proto = fib_proto_to_dpo (FIB_PROTOCOL_IP4),
-      .frp_addr = zero_addr,
-      .frp_sw_if_index = 0xffffffff,
-      .frp_fib_index = 0,
-      .frp_weight = 1,
-      .frp_flags = FIB_ROUTE_PATH_LOCAL,
-    };
   fib_route_path_t via_itf_path =
     {
       .frp_proto = fib_proto_to_dpo (FIB_PROTOCOL_IP4),
@@ -362,7 +353,18 @@ igmp_enable_disable (u32 sw_if_index, u8 enable, igmp_mode_t mode)
       .frp_sw_if_index = sw_if_index,
       .frp_fib_index = 0,
       .frp_weight = 1,
+    .frp_mitf_flags = MFIB_ITF_FLAG_ACCEPT,
     };
+  fib_route_path_t for_us_path = {
+    .frp_proto = fib_proto_to_dpo (FIB_PROTOCOL_IP4),
+    .frp_addr = zero_addr,
+    .frp_sw_if_index = 0xffffffff,
+    .frp_fib_index = 1,
+    .frp_weight = 0,
+    .frp_flags = FIB_ROUTE_PATH_LOCAL,
+    .frp_mitf_flags = MFIB_ITF_FLAG_FORWARD,
+  };
+
   /* *INDENT-ON* */
   /* find configuration, if it doesn't exist, create new */
   config = igmp_config_lookup (sw_if_index);
@@ -405,24 +407,19 @@ igmp_enable_disable (u32 sw_if_index, u8 enable, igmp_mode_t mode)
 	if (1 == im->n_configs_per_mfib_index[mfib_index])
 	  {
 	    /* first config in this FIB */
+	    mfib_table_lock (mfib_index, FIB_PROTOCOL_IP4, MFIB_SOURCE_IGMP);
 	    mfib_table_entry_path_update (mfib_index,
 					  &mpfx_general_query,
-					  MFIB_SOURCE_IGMP,
-					  &for_us_path,
-					  MFIB_ITF_FLAG_FORWARD);
+					  MFIB_SOURCE_IGMP, &for_us_path);
 	    mfib_table_entry_path_update (mfib_index,
 					  &mpfx_report,
-					  MFIB_SOURCE_IGMP,
-					  &for_us_path,
-					  MFIB_ITF_FLAG_FORWARD);
+					  MFIB_SOURCE_IGMP, &for_us_path);
 	  }
 	mfib_table_entry_path_update (mfib_index,
 				      &mpfx_general_query,
-				      MFIB_SOURCE_IGMP,
-				      &via_itf_path, MFIB_ITF_FLAG_ACCEPT);
+				      MFIB_SOURCE_IGMP, &via_itf_path);
 	mfib_table_entry_path_update (mfib_index, &mpfx_report,
-				      MFIB_SOURCE_IGMP, &via_itf_path,
-				      MFIB_ITF_FLAG_ACCEPT);
+				      MFIB_SOURCE_IGMP, &via_itf_path);
       }
     }
   else if (config && !enable)
@@ -438,6 +435,7 @@ igmp_enable_disable (u32 sw_if_index, u8 enable, igmp_mode_t mode)
 	  mfib_table_entry_path_remove (mfib_index,
 					&mpfx_report,
 					MFIB_SOURCE_IGMP, &for_us_path);
+	  mfib_table_unlock (mfib_index, FIB_PROTOCOL_IP4, MFIB_SOURCE_IGMP);
 	}
 
       mfib_table_entry_path_remove (mfib_index,
@@ -482,7 +480,6 @@ igmp_init (vlib_main_t * vm)
   igmp_main_t *im = &igmp_main;
 
   im->igmp_api_client_by_client_index = hash_create (0, sizeof (u32));
-
   im->logger = vlib_log_register_class ("igmp", 0);
 
   IGMP_DBG ("initialized");
diff --git a/src/plugins/igmp/igmp_proxy.c b/src/plugins/igmp/igmp_proxy.c
index 690b38a8001..c2f3e06fb1f 100644
--- a/src/plugins/igmp/igmp_proxy.c
+++ b/src/plugins/igmp/igmp_proxy.c
@@ -49,13 +49,13 @@ igmp_proxy_device_mfib_path_add_del (igmp_group_t * group, u8 add)
       .frp_sw_if_index = config->sw_if_index,
       .frp_fib_index = 0,
       .frp_weight = 1,
+      .frp_mitf_flags = MFIB_ITF_FLAG_FORWARD,
     };
   /* *INDENT-ON* */
 
   if (add)
     mfib_table_entry_path_update (mfib_index, &mpfx_group_addr,
-				  MFIB_SOURCE_IGMP, &via_itf_path,
-				  MFIB_ITF_FLAG_FORWARD);
+				  MFIB_SOURCE_IGMP, &via_itf_path);
   else
     mfib_table_entry_path_remove (mfib_index, &mpfx_group_addr,
 				  MFIB_SOURCE_IGMP, &via_itf_path);
diff --git a/src/plugins/l3xc/l3xc_api.c b/src/plugins/l3xc/l3xc_api.c
index 45c01fa4e7a..0427cca5f1c 100644
--- a/src/plugins/l3xc/l3xc_api.c
+++ b/src/plugins/l3xc/l3xc_api.c
@@ -108,7 +108,7 @@ vl_api_l3xc_update_t_handler (vl_api_l3xc_update_t * mp)
   for (pi = 0; pi < mp->l3xc.n_paths; pi++)
     {
       path = &paths[pi];
-      rv = fib_path_api_parse (&mp->l3xc.paths[pi], path);
+      rv = fib_api_path_decode (&mp->l3xc.paths[pi], path);
 
       if (0 != rv)
 	{
@@ -155,9 +155,12 @@ typedef struct l3xc_dump_walk_ctx_t_
 static int
 l3xc_send_details (u32 l3xci, void *args)
 {
-  fib_route_path_encode_t *api_rpaths = NULL, *api_rpath;
+  fib_path_encode_ctx_t path_ctx = {
+    .rpaths = NULL,
+  };
   vl_api_l3xc_details_t *mp;
   l3xc_dump_walk_ctx_t *ctx;
+  fib_route_path_t *rpath;
   vl_api_fib_path_t *fp;
   size_t msg_size;
   l3xc_t *l3xc;
@@ -177,13 +180,12 @@ l3xc_send_details (u32 l3xci, void *args)
   mp->l3xc.n_paths = n_paths;
   mp->l3xc.sw_if_index = htonl (l3xc->l3xc_sw_if_index);
 
-  fib_path_list_walk_w_ext (l3xc->l3xc_pl, NULL, fib_path_encode,
-			    &api_rpaths);
+  fib_path_list_walk_w_ext (l3xc->l3xc_pl, NULL, fib_path_encode, &path_ctx);
 
   fp = mp->l3xc.paths;
-  vec_foreach (api_rpath, api_rpaths)
+  vec_foreach (rpath, path_ctx.rpaths)
   {
-    fib_api_path_encode (api_rpath, fp);
+    fib_api_path_encode (rpath, fp);
     fp++;
   }
 
diff --git a/src/plugins/unittest/bier_test.c b/src/plugins/unittest/bier_test.c
index e6799eb4f42..8a609e5f20d 100644
--- a/src/plugins/unittest/bier_test.c
+++ b/src/plugins/unittest/bier_test.c
@@ -770,12 +770,12 @@ bier_test_mpls_imp (void)
         .frp_bier_imp = bii,
         .frp_weight = 0,
         .frp_flags = FIB_ROUTE_PATH_BIER_IMP,
+        .frp_mitf_flags = MFIB_ITF_FLAG_FORWARD,
     };
     mfib_table_entry_path_update(0, // default table
                                  &pfx_1_1_1_1_c_239_1_1_1 ,
                                  MFIB_SOURCE_API,
-                                 &path_via_bier_imp_1,
-                                 MFIB_ITF_FLAG_FORWARD);
+                                 &path_via_bier_imp_1);
     mfib_table_entry_delete(0,
                             &pfx_1_1_1_1_c_239_1_1_1 ,
                             MFIB_SOURCE_API);
diff --git a/src/plugins/unittest/mfib_test.c b/src/plugins/unittest/mfib_test.c
index 0707a6f57c8..2497a299c50 100644
--- a/src/plugins/unittest/mfib_test.c
+++ b/src/plugins/unittest/mfib_test.c
@@ -233,7 +233,7 @@ mfib_test_entry (fib_node_index_t fei,
 
     MFIB_TEST_REP((eflags == mfe->mfe_flags),
                   "%U has %U expect %U",
-                  format_mfib_prefix, &pfx,
+                  format_mfib_prefix, pfx,
                   format_mfib_entry_flags, mfe->mfe_flags,
                   format_mfib_entry_flags, eflags);
 
@@ -241,7 +241,7 @@ mfib_test_entry (fib_node_index_t fei,
     {
         MFIB_TEST_REP((DPO_DROP == mfe->mfe_rep.dpoi_type),
                       "%U links to %U",
-                      format_mfib_prefix, &pfx,
+                      format_mfib_prefix, pfx,
                       format_dpo_id, &mfe->mfe_rep, 0);
     }
     else
@@ -257,7 +257,7 @@ mfib_test_entry (fib_node_index_t fei,
 
         MFIB_TEST_REP((DPO_REPLICATE == tmp.dpoi_type),
                       "%U links to %U",
-                      format_mfib_prefix, &pfx,
+                      format_mfib_prefix, pfx,
                       format_dpo_type, tmp.dpoi_type);
 
         va_start(ap, n_buckets);
@@ -288,11 +288,11 @@ mfib_test_entry_itf (fib_node_index_t fei,
 
     MFIB_TEST_REP((NULL != mfi),
                   "%U has interface %d",
-                  format_mfib_prefix, &pfx, sw_if_index);
+                  format_mfib_prefix, pfx, sw_if_index);
 
     MFIB_TEST_REP((flags == mfi->mfi_flags),
                   "%U interface %d has flags %U expect %U",
-                  format_mfib_prefix, &pfx, sw_if_index,
+                  format_mfib_prefix, pfx, sw_if_index,
                   format_mfib_itf_flags, flags,
                   format_mfib_itf_flags, mfi->mfi_flags);
 
@@ -408,15 +408,15 @@ mfib_test_i (fib_protocol_t PROTO,
         .frp_addr = zero_addr,
         .frp_sw_if_index = tm->hw[0]->sw_if_index,
         .frp_fib_index = ~0,
-        .frp_weight = 0,
+        .frp_weight = 1,
         .frp_flags = 0,
+        .frp_mitf_flags = MFIB_ITF_FLAG_ACCEPT,
     };
 
     mfib_table_entry_path_update(fib_index,
                                  pfx_no_forward,
                                  MFIB_SOURCE_API,
-                                 &path_via_if0,
-                                 MFIB_ITF_FLAG_ACCEPT);
+                                 &path_via_if0);
 
     mfei_no_f = mfib_table_lookup_exact_match(fib_index, pfx_no_forward);
     MFIB_TEST(!mfib_test_entry(mfei_no_f,
@@ -424,41 +424,42 @@ mfib_test_i (fib_protocol_t PROTO,
                                0),
               "%U no replcaitions",
               format_mfib_prefix, pfx_no_forward);
-    MFIB_TEST_NS(!mfib_test_entry_itf(mfei_no_f, tm->hw[0]->sw_if_index,
-                                      MFIB_ITF_FLAG_ACCEPT));
+    MFIB_TEST(!mfib_test_entry_itf(mfei_no_f, tm->hw[0]->sw_if_index,
+                                   MFIB_ITF_FLAG_ACCEPT),
+              "%U interface not accepting",
+              format_mfib_prefix, pfx_no_forward);
 
     fib_route_path_t path_via_if1 = {
         .frp_proto = fib_proto_to_dpo(PROTO),
         .frp_addr = zero_addr,
         .frp_sw_if_index = tm->hw[1]->sw_if_index,
         .frp_fib_index = ~0,
-        .frp_weight = 0,
+        .frp_weight = 1,
         .frp_flags = 0,
+        .frp_mitf_flags = MFIB_ITF_FLAG_FORWARD,
     };
     fib_route_path_t path_via_if2 = {
         .frp_proto = fib_proto_to_dpo(PROTO),
         .frp_addr = zero_addr,
         .frp_sw_if_index = tm->hw[2]->sw_if_index,
         .frp_fib_index = ~0,
-        .frp_weight = 0,
+        .frp_weight = 1,
         .frp_flags = 0,
+        .frp_mitf_flags = MFIB_ITF_FLAG_FORWARD,
     };
     fib_route_path_t path_via_if3 = {
         .frp_proto = fib_proto_to_dpo(PROTO),
         .frp_addr = zero_addr,
         .frp_sw_if_index = tm->hw[3]->sw_if_index,
         .frp_fib_index = ~0,
-        .frp_weight = 0,
+        .frp_weight = 1,
         .frp_flags = 0,
+        .frp_mitf_flags = (MFIB_ITF_FLAG_FORWARD |
+                           MFIB_ITF_FLAG_NEGATE_SIGNAL),
     };
-    fib_route_path_t path_for_us = {
-        .frp_proto = fib_proto_to_dpo(PROTO),
-        .frp_addr = zero_addr,
-        .frp_sw_if_index = 0xffffffff,
-        .frp_fib_index = ~0,
-        .frp_weight = 0,
-        .frp_flags = FIB_ROUTE_PATH_LOCAL,
-    };
+    fib_route_path_t *two_paths = NULL;
+    vec_add1(two_paths, path_via_if2);
+    vec_add1(two_paths, path_via_if3);
 
     /*
      * An (S,G) with 1 accepting and 3 forwarding paths
@@ -466,24 +467,15 @@ mfib_test_i (fib_protocol_t PROTO,
     mfib_table_entry_path_update(fib_index,
                                  pfx_s_g,
                                  MFIB_SOURCE_API,
-                                 &path_via_if0,
-                                 MFIB_ITF_FLAG_ACCEPT);
-    mfib_table_entry_path_update(fib_index,
-                                 pfx_s_g,
-                                 MFIB_SOURCE_API,
-                                 &path_via_if1,
-                                 MFIB_ITF_FLAG_FORWARD);
-    mfib_table_entry_path_update(fib_index,
-                                 pfx_s_g,
-                                 MFIB_SOURCE_API,
-                                 &path_via_if2,
-                                 MFIB_ITF_FLAG_FORWARD);
+                                 &path_via_if0);
     mfib_table_entry_path_update(fib_index,
                                  pfx_s_g,
                                  MFIB_SOURCE_API,
-                                 &path_via_if3,
-                                 (MFIB_ITF_FLAG_FORWARD |
-                                  MFIB_ITF_FLAG_NEGATE_SIGNAL));
+                                 &path_via_if1);
+    mfib_table_entry_paths_update(fib_index,
+                                  pfx_s_g,
+                                  MFIB_SOURCE_API,
+                                  two_paths);
 
     mfei_s_g = mfib_table_lookup_exact_match(fib_index, pfx_s_g);
 
@@ -515,13 +507,11 @@ mfib_test_i (fib_protocol_t PROTO,
     mfei_g_1 = mfib_table_entry_path_update(fib_index,
                                             pfx_star_g_1,
                                             MFIB_SOURCE_API,
-                                            &path_via_if0,
-                                            MFIB_ITF_FLAG_ACCEPT);
+                                            &path_via_if0);
     mfib_table_entry_path_update(fib_index,
                                  pfx_star_g_1,
                                  MFIB_SOURCE_API,
-                                 &path_via_if1,
-                                 MFIB_ITF_FLAG_FORWARD);
+                                 &path_via_if1);
 
     /*
      * test we find the *,G and S,G via LPM and exact matches
@@ -583,16 +573,15 @@ mfib_test_i (fib_protocol_t PROTO,
      * A (*,G/m), which the same root G as the (*,G).
      * different paths. test our LPM.
      */
+    path_via_if2.frp_mitf_flags = MFIB_ITF_FLAG_ACCEPT;
     mfei_g_m = mfib_table_entry_path_update(fib_index,
                                             pfx_star_g_slash_m,
                                             MFIB_SOURCE_API,
-                                            &path_via_if2,
-                                            MFIB_ITF_FLAG_ACCEPT);
+                                            &path_via_if2);
     mfib_table_entry_path_update(fib_index,
                                  pfx_star_g_slash_m,
                                  MFIB_SOURCE_API,
-                                 &path_via_if3,
-                                 MFIB_ITF_FLAG_FORWARD);
+                                 &path_via_if3);
 
     /*
      * test we find the (*,G/m), (*,G) and (S,G) via LPM and exact matches
@@ -656,11 +645,20 @@ mfib_test_i (fib_protocol_t PROTO,
     /*
      * Add a for-us path
      */
+    fib_route_path_t path_for_us = {
+        .frp_proto = fib_proto_to_dpo(PROTO),
+        .frp_addr = zero_addr,
+        .frp_sw_if_index = 0xffffffff,
+        .frp_fib_index = ~0,
+        .frp_weight = 1,
+        .frp_flags = FIB_ROUTE_PATH_LOCAL,
+        .frp_mitf_flags = MFIB_ITF_FLAG_FORWARD,
+    };
+
     mfei = mfib_table_entry_path_update(fib_index,
                                         pfx_s_g,
                                         MFIB_SOURCE_API,
-                                        &path_for_us,
-                                        MFIB_ITF_FLAG_FORWARD);
+                                        &path_for_us);
 
     MFIB_TEST(!mfib_test_entry(mfei,
                                MFIB_ENTRY_FLAG_NONE,
@@ -693,11 +691,11 @@ mfib_test_i (fib_protocol_t PROTO,
      * update an existing forwarding path to be only accepting
      *   - expect it to be removed from the replication set.
      */
+    path_via_if3.frp_mitf_flags = MFIB_ITF_FLAG_ACCEPT;
     mfib_table_entry_path_update(fib_index,
                                  pfx_s_g,
                                  MFIB_SOURCE_API,
-                                 &path_via_if3,
-                                 MFIB_ITF_FLAG_ACCEPT);
+                                 &path_via_if3);
 
     MFIB_TEST(!mfib_test_entry(mfei,
                                MFIB_ENTRY_FLAG_NONE,
@@ -718,13 +716,13 @@ mfib_test_i (fib_protocol_t PROTO,
      * Make the path forwarding again
      *  - expect it to be added back to the replication set
      */
+    path_via_if3.frp_mitf_flags = (MFIB_ITF_FLAG_FORWARD |
+                              MFIB_ITF_FLAG_ACCEPT |
+                              MFIB_ITF_FLAG_NEGATE_SIGNAL);
     mfib_table_entry_path_update(fib_index,
                                  pfx_s_g,
                                  MFIB_SOURCE_API,
-                                 &path_via_if3,
-                                 (MFIB_ITF_FLAG_FORWARD |
-                                  MFIB_ITF_FLAG_ACCEPT |
-                                  MFIB_ITF_FLAG_NEGATE_SIGNAL));
+                                 &path_via_if3);
 
     mfei = mfib_table_lookup_exact_match(fib_index,
                                          pfx_s_g);
@@ -806,32 +804,37 @@ mfib_test_i (fib_protocol_t PROTO,
     MFIB_TEST_NS(!mfib_test_entry_no_itf(mfei, tm->hw[3]->sw_if_index));
 
     /*
-     * remove the accpeting only interface
+     * remove 
      */
-    mfib_table_entry_path_remove(fib_index,
-                                 pfx_s_g,
-                                 MFIB_SOURCE_API,
-                                 &path_via_if0);
-
-    MFIB_TEST(!mfib_test_entry(mfei,
-                               MFIB_ENTRY_FLAG_SIGNAL,
-                               1,
-                               DPO_ADJACENCY_MCAST, ai_2),
-              "%U replicate OK",
-              format_mfib_prefix, pfx_s_g);
-    MFIB_TEST_NS(!mfib_test_entry_itf(mfei, tm->hw[2]->sw_if_index,
-                                      MFIB_ITF_FLAG_FORWARD));
-    MFIB_TEST_NS(!mfib_test_entry_no_itf(mfei, tm->hw[0]->sw_if_index));
-    MFIB_TEST_NS(!mfib_test_entry_no_itf(mfei, tm->hw[1]->sw_if_index));
-    MFIB_TEST_NS(!mfib_test_entry_no_itf(mfei, tm->hw[3]->sw_if_index));
+    /* mfib_table_entry_path_remove(fib_index, */
+    /*                              pfx_s_g, */
+    /*                              MFIB_SOURCE_API, */
+    /*                              &path_via_if0); */
+
+    /* MFIB_TEST(!mfib_test_entry(mfei, */
+    /*                            MFIB_ENTRY_FLAG_SIGNAL, */
+    /*                            1, */
+    /*                            DPO_ADJACENCY_MCAST, ai_2), */
+    /*           "%U replicate OK", */
+    /*           format_mfib_prefix, pfx_s_g); */
+    /* MFIB_TEST_NS(!mfib_test_entry_itf(mfei, tm->hw[2]->sw_if_index, */
+    /*                                   MFIB_ITF_FLAG_FORWARD)); */
+    /* MFIB_TEST_NS(!mfib_test_entry_no_itf(mfei, tm->hw[0]->sw_if_index)); */
+    /* MFIB_TEST_NS(!mfib_test_entry_no_itf(mfei, tm->hw[1]->sw_if_index)); */
+    /* MFIB_TEST_NS(!mfib_test_entry_no_itf(mfei, tm->hw[3]->sw_if_index)); */
 
     /*
-     * remove the last path, the entry still has flags so it remains
+     * remove the last path and the accpeting only interface, 
+     * the entry still has flags so it remains
      */
-    mfib_table_entry_path_remove(fib_index,
-                                 pfx_s_g,
-                                 MFIB_SOURCE_API,
-                                 &path_via_if2);
+    vec_reset_length(two_paths);
+    vec_add1(two_paths, path_via_if0);
+    vec_add1(two_paths, path_via_if2);
+
+    mfib_table_entry_paths_remove(fib_index,
+                                  pfx_s_g,
+                                  MFIB_SOURCE_API,
+                                  two_paths);
 
     MFIB_TEST(!mfib_test_entry(mfei,
                                MFIB_ENTRY_FLAG_SIGNAL,
@@ -858,12 +861,12 @@ mfib_test_i (fib_protocol_t PROTO,
     /*
      * An entry with a NS interface
      */
+    path_via_if0.frp_mitf_flags = (MFIB_ITF_FLAG_ACCEPT |
+                              MFIB_ITF_FLAG_NEGATE_SIGNAL);
     mfei_g_2 = mfib_table_entry_path_update(fib_index,
                                             pfx_star_g_2,
                                             MFIB_SOURCE_API,
-                                            &path_via_if0,
-                                            (MFIB_ITF_FLAG_ACCEPT |
-                                             MFIB_ITF_FLAG_NEGATE_SIGNAL));
+                                            &path_via_if0);
     MFIB_TEST(!mfib_test_entry(mfei_g_2,
                                MFIB_ENTRY_FLAG_NONE,
                                0),
@@ -886,12 +889,12 @@ mfib_test_i (fib_protocol_t PROTO,
     /*
      * An entry with a NS interface
      */
+    path_via_if0.frp_mitf_flags = (MFIB_ITF_FLAG_ACCEPT |
+                              MFIB_ITF_FLAG_NEGATE_SIGNAL);
     mfei_g_3 = mfib_table_entry_path_update(fib_index,
                                             pfx_star_g_3,
                                             MFIB_SOURCE_API,
-                                            &path_via_if0,
-                                            (MFIB_ITF_FLAG_ACCEPT |
-                                             MFIB_ITF_NEGATE_SIGNAL));
+                                            &path_via_if0);
     MFIB_TEST(!mfib_test_entry(mfei_g_3,
                                MFIB_ENTRY_FLAG_NONE,
                                0),
@@ -1056,28 +1059,28 @@ mfib_test_i (fib_protocol_t PROTO,
         .frp_addr = *addr_nbr1,
         .frp_sw_if_index = tm->hw[0]->sw_if_index,
         .frp_fib_index = ~0,
-        .frp_weight = 0,
+        .frp_weight = 1,
         .frp_flags = 0,
+        .frp_mitf_flags = MFIB_ITF_FLAG_FORWARD,
     };
     fib_route_path_t path_via_nbr2 = {
         .frp_proto = fib_proto_to_dpo(PROTO),
         .frp_addr = *addr_nbr2,
         .frp_sw_if_index = tm->hw[0]->sw_if_index,
         .frp_fib_index = ~0,
-        .frp_weight = 0,
+        .frp_weight = 1,
         .frp_flags = 0,
+        .frp_mitf_flags = MFIB_ITF_FLAG_FORWARD,
     };
 
     mfei_g_1 = mfib_table_entry_path_update(fib_index,
                                             pfx_star_g_1,
                                             MFIB_SOURCE_API,
-                                            &path_via_nbr1,
-                                            (MFIB_ITF_FLAG_FORWARD));
+                                            &path_via_nbr1);
     mfei_g_1 = mfib_table_entry_path_update(fib_index,
                                             pfx_star_g_1,
                                             MFIB_SOURCE_API,
-                                            &path_via_nbr2,
-                                            (MFIB_ITF_FLAG_FORWARD));
+                                            &path_via_nbr2);
     MFIB_TEST(!mfib_test_entry(mfei_g_1,
                                MFIB_ENTRY_FLAG_NONE,
                                2,
@@ -1230,6 +1233,7 @@ mfib_test_i (fib_protocol_t PROTO,
         .frp_fib_index = 0,
         .frp_weight = 1,
         .frp_flags = FIB_ROUTE_PATH_FLAG_NONE,
+        .frp_mitf_flags = MFIB_ITF_FLAG_FORWARD,
     };
     dpo_id_t mldp_dpo = DPO_INVALID;
 
@@ -1240,8 +1244,7 @@ mfib_test_i (fib_protocol_t PROTO,
     mfei = mfib_table_entry_path_update(fib_index,
                                         pfx_s_g,
                                         MFIB_SOURCE_API,
-                                        &path_via_mldp,
-                                        MFIB_ITF_FLAG_FORWARD);
+                                        &path_via_mldp);
 
     MFIB_TEST(!mfib_test_entry(mfei,
                                MFIB_ENTRY_FLAG_NONE,
@@ -1256,8 +1259,7 @@ mfib_test_i (fib_protocol_t PROTO,
     mfei = mfib_table_entry_path_update(fib_index,
                                         pfx_s_g,
                                         MFIB_SOURCE_API,
-                                        &path_for_us,
-                                        MFIB_ITF_FLAG_FORWARD);
+                                        &path_for_us);
     MFIB_TEST(!mfib_test_entry(mfei,
                                MFIB_ENTRY_FLAG_NONE,
                                2,
@@ -1321,6 +1323,7 @@ mfib_test_i (fib_protocol_t PROTO,
     MFIB_TEST(n_itfs == pool_elts(mfib_itf_pool),
               " No more Interfaces %d!=%d",
               n_itfs, pool_elts(mfib_itf_pool));
+    vec_free(two_paths);
 
     return (res);
 }
@@ -1531,32 +1534,36 @@ mfib_test_rr_i (fib_protocol_t FPROTO,
         .frp_addr = zero_addr,
         .frp_sw_if_index = tm->hw[0]->sw_if_index,
         .frp_fib_index = ~0,
-        .frp_weight = 0,
+        .frp_weight = 1,
         .frp_flags = 0,
+        .frp_mitf_flags = MFIB_ITF_FLAG_ACCEPT,
     };
     fib_route_path_t path_via_if1 = {
         .frp_proto = DPROTO,
         .frp_addr = zero_addr,
         .frp_sw_if_index = tm->hw[1]->sw_if_index,
         .frp_fib_index = ~0,
-        .frp_weight = 0,
+        .frp_weight = 1,
         .frp_flags = 0,
+        .frp_mitf_flags = MFIB_ITF_FLAG_FORWARD,
     };
     fib_route_path_t path_via_if2 = {
         .frp_proto = DPROTO,
         .frp_addr = zero_addr,
         .frp_sw_if_index = tm->hw[2]->sw_if_index,
         .frp_fib_index = ~0,
-        .frp_weight = 0,
+        .frp_weight = 1,
         .frp_flags = 0,
+        .frp_mitf_flags = MFIB_ITF_FLAG_FORWARD,
     };
     fib_route_path_t path_for_us = {
         .frp_proto = DPROTO,
         .frp_addr = zero_addr,
         .frp_sw_if_index = 0xffffffff,
         .frp_fib_index = ~0,
-        .frp_weight = 0,
+        .frp_weight = 1,
         .frp_flags = FIB_ROUTE_PATH_LOCAL,
+        .frp_mitf_flags = MFIB_ITF_FLAG_FORWARD,
     };
 
     /*
@@ -1581,8 +1588,7 @@ mfib_test_rr_i (fib_protocol_t FPROTO,
     mfib_table_entry_path_update(fib_index,
                                  pfx_cover,
                                  MFIB_SOURCE_API,
-                                 &path_via_if1,
-                                 MFIB_ITF_FLAG_FORWARD);
+                                 &path_via_if1);
 
     mfei_cover = mfib_table_lookup_exact_match(fib_index, pfx_cover);
 
@@ -1609,8 +1615,7 @@ mfib_test_rr_i (fib_protocol_t FPROTO,
     mfib_table_entry_path_update(fib_index,
                                  pfx_cover,
                                  MFIB_SOURCE_API,
-                                 &path_via_if2,
-                                 MFIB_ITF_FLAG_FORWARD);
+                                 &path_via_if2);
 
     /*
      * expect the /32 and /28 to be via both boths
@@ -1667,8 +1672,7 @@ mfib_test_rr_i (fib_protocol_t FPROTO,
     mfib_table_entry_path_update(fib_index,
                                  pfx_cover,
                                  MFIB_SOURCE_API,
-                                 &path_via_if0,
-                                 MFIB_ITF_FLAG_ACCEPT);
+                                 &path_via_if0);
 
     /*
      * expect the /32 and /28 to be via both boths
@@ -1706,8 +1710,7 @@ mfib_test_rr_i (fib_protocol_t FPROTO,
     mfib_table_entry_path_update(fib_index,
                                  pfx_cover,
                                  MFIB_SOURCE_API,
-                                 &path_for_us,
-                                 MFIB_ITF_FLAG_FORWARD);
+                                 &path_for_us);
 
     /*
      * expect the /32 and /28 to be via all three paths
@@ -1775,11 +1778,10 @@ mfib_test_rr_i (fib_protocol_t FPROTO,
     /*
      * source the /32 with its own path
      */
-    mfib_table_entry_path_update(fib_index,
-                                 pfx_host1,
-                                 MFIB_SOURCE_API,
-                                 &path_via_if2,
-                                 MFIB_ITF_FLAG_FORWARD);
+    mfei_host1 = mfib_table_entry_path_update(fib_index,
+                                              pfx_host1,
+                                              MFIB_SOURCE_API,
+                                              &path_via_if2);
     MFIB_TEST(!mfib_test_entry(mfei_host1,
                                MFIB_ENTRY_FLAG_NONE,
                                1,
@@ -1809,17 +1811,16 @@ mfib_test_rr_i (fib_protocol_t FPROTO,
     /*
      * add the RR back then remove the path and RR
      */
-    mfib_table_entry_path_update(fib_index,
-                                 pfx_host1,
-                                 MFIB_SOURCE_API,
-                                 &path_via_if2,
-                                 MFIB_ITF_FLAG_FORWARD);
+    mfei_host1 = mfib_table_entry_path_update(fib_index,
+                                              pfx_host1,
+                                              MFIB_SOURCE_API,
+                                              &path_via_if2);
     MFIB_TEST(!mfib_test_entry(mfei_host1,
                                MFIB_ENTRY_FLAG_NONE,
                                1,
                                DPO_ADJACENCY_MCAST, ai_2),
               "%U replicate OK",
-              format_mfib_prefix, pfx_cover);
+              format_mfib_prefix, pfx_host1);
 
     mfib_table_entry_delete(fib_index, pfx_host1,
                             MFIB_SOURCE_API);