MPLS Mcast

 1 - interface-DPO
        Used in the Data-plane to change a packet's input interface
 2 - MPLS multicast FIB entry
        Same as a unicast entry but it links to a replicate not a load-balance DPO
 3 - Multicast MPLS tunnel
        Update MPLS tunnels to use a FIB path-list to describe the endpoint[s]. Use the path-list to generate the forwarding chain (DPOs) to link to .
 4 - Resolve a path via a local label (of an mLDP LSP)
        For IP multicast entries to use an LSP in the replication list, we need to decribe the 'resolve-via-label' where the label is that of a multicast LSP.
 5 - MPLS disposition path sets RPF-ID
        For a interface-less LSP (i.e. mLDP not RSVP-TE) at the tail of the LSP we still need to perform an RPF check. An MPLS disposition DPO performs the MPLS pop validation checks and sets the RPF-ID in the packet.
 6 - RPF check with per-entry RPF-ID
       An RPF-ID is used instead of a real interface SW if index in the case the IP traffic arrives from an LSP that does not have an associated interface.

Change-Id: Ib92e177be919147bafeb599729abf3d1abc2f4b3
Signed-off-by: Neale Ranns <nranns@cisco.com>

8 files changed, 417 insertions, 165 deletions

diff --git a/src/vnet/mfib/ip4_mfib.c b/src/vnet/mfib/ip4_mfib.c
index 164cafa118b..3ed7cba7b6f 100644
--- a/src/vnet/mfib/ip4_mfib.c
+++ b/src/vnet/mfib/ip4_mfib.c
@@ -72,6 +72,7 @@ ip4_create_mfib_with_table_id (u32 table_id)
         mfib_table_entry_update(mfib_table->mft_index,
                                 &prefix,
                                 MFIB_SOURCE_DEFAULT_ROUTE,
+                                MFIB_RPF_ID_NONE,
                                 MFIB_ENTRY_FLAG_DROP);
     }
 
diff --git a/src/vnet/mfib/ip6_mfib.c b/src/vnet/mfib/ip6_mfib.c
index 991b91c6498..116fee2222a 100644
--- a/src/vnet/mfib/ip6_mfib.c
+++ b/src/vnet/mfib/ip6_mfib.c
@@ -195,6 +195,7 @@ ip6_create_mfib_with_table_id (u32 table_id)
     mfib_table_entry_update(mfib_table->mft_index,
                             &all_zeros,
                             MFIB_SOURCE_DEFAULT_ROUTE,
+                            MFIB_RPF_ID_NONE,
                             MFIB_ENTRY_FLAG_DROP);
 
     /*
diff --git a/src/vnet/mfib/mfib_entry.c b/src/vnet/mfib/mfib_entry.c
index 1aa8e0864ca..847f25e7f1f 100644
--- a/src/vnet/mfib/mfib_entry.c
+++ b/src/vnet/mfib/mfib_entry.c
@@ -49,6 +49,15 @@
 #endif
 
 /**
+ * MFIB extensions to each path
+ */
+typedef struct mfib_path_ext_t_
+{
+    mfib_itf_flags_t mfpe_flags;
+    fib_node_index_t mfpe_path;
+} mfib_path_ext_t;
+
+/**
  * The source of an MFIB entry
  */
 typedef struct mfib_entry_src_t_
@@ -59,22 +68,39 @@ typedef struct mfib_entry_src_t_
     mfib_source_t mfes_src;
 
     /**
+     * Route flags
+     */
+    mfib_entry_flags_t mfes_flags;
+
+    /**
      * The path-list of forwarding interfaces
      */
     fib_node_index_t mfes_pl;
 
     /**
-     * Route flags
+     * RPF-ID
      */
-    mfib_entry_flags_t mfes_flags;
+    fib_rpf_id_t mfes_rpf_id;
+
+    /**
+     * Hash table of path extensions
+     */
+    mfib_path_ext_t *mfes_exts;
 
     /**
-     * The hash table of all interfaces
+     * The hash table of all interfaces.
+     *  This is forwarding time information derived from the paths
+     *  and their extensions.
      */
     mfib_itf_t *mfes_itfs;
 } mfib_entry_src_t;
 
 /**
+ * Pool of path extensions
+ */
+static mfib_path_ext_t *mfib_path_ext_pool;
+
+/**
  * String names for each source
  */
 static const char *mfib_source_names[] = MFIB_SOURCE_NAMES;
@@ -123,6 +149,24 @@ format_mfib_entry_dpo (u8 * s, va_list * args)
                    MFIB_ENTRY_FORMAT_BRIEF));
 }
 
+static inline mfib_path_ext_t *
+mfib_entry_path_ext_get (index_t mi)
+{
+    return (pool_elt_at_index(mfib_path_ext_pool, mi));
+}
+
+static u8 *
+format_mfib_entry_path_ext (u8 * s, va_list * args)
+{
+    mfib_path_ext_t *path_ext;
+    index_t mpi = va_arg(*args, index_t);
+
+    path_ext = mfib_entry_path_ext_get(mpi);
+    return (format(s, "path:%d flags:%U",
+                   path_ext->mfpe_path,
+                   format_mfib_itf_flags, path_ext->mfpe_flags));
+}
+
 u8 *
 format_mfib_entry (u8 * s, va_list * args)
 {
@@ -141,6 +185,8 @@ format_mfib_entry (u8 * s, va_list * args)
 
     if (level >= MFIB_ENTRY_FORMAT_DETAIL)
     {
+        fib_node_index_t path_index, mpi;
+
         s = format (s, "\n");
         s = format (s, " fib:%d", mfib_entry->mfe_fib_index);
         s = format (s, " index:%d", mfib_entry_get_index(mfib_entry));
@@ -153,6 +199,14 @@ format_mfib_entry (u8 * s, va_list * args)
             {
                 s = fib_path_list_format(msrc->mfes_pl, s);
             }
+            s = format (s, "    Extensions:\n",
+                        mfib_source_names[msrc->mfes_src]);
+            hash_foreach(path_index, mpi, msrc->mfes_exts,
+            ({
+                s = format(s, "     %U\n", format_mfib_entry_path_ext, mpi);
+            }));
+            s = format (s, "    Interface-Forwarding:\n",
+                        mfib_source_names[msrc->mfes_src]);
             hash_foreach(sw_if_index, mfi, msrc->mfes_itfs,
             ({
                 s = format(s, "    %U\n", format_mfib_itf, mfi);
@@ -165,7 +219,7 @@ format_mfib_entry (u8 * s, va_list * args)
     ({
         s = format(s, "\n  %U", format_mfib_itf, mfi);
     }));
-
+    s = format(s, "\n  RPF-ID:%d", mfib_entry->mfe_rpf_id);
     s = format(s, "\n  %U-chain\n  %U",
                format_fib_forw_chain_type,
                mfib_entry_get_default_chain_type(mfib_entry),
@@ -314,13 +368,6 @@ mfib_entry_src_remove (mfib_entry_t *mfib_entry,
     }
 }
 
-static int
-mfib_entry_src_n_itfs (const mfib_entry_src_t *msrc)
-{
-    return (hash_elts(msrc->mfes_itfs));
-}
-
-
 static void
 mfib_entry_last_lock_gone (fib_node_t *node)
 {
@@ -338,7 +385,6 @@ mfib_entry_last_lock_gone (fib_node_t *node)
         mfib_entry_src_flush(msrc);
     }
 
-    fib_path_list_unlock(mfib_entry->mfe_parent);
     vec_free(mfib_entry->mfe_srcs);
 
     fib_node_deinit(&mfib_entry->mfe_node);
@@ -417,10 +463,9 @@ mfib_entry_alloc (u32 fib_index,
     mfib_entry->mfe_flags = 0;
     mfib_entry->mfe_fib_index = fib_index;
     mfib_entry->mfe_prefix = *prefix;
-    mfib_entry->mfe_parent = FIB_NODE_INDEX_INVALID;
-    mfib_entry->mfe_sibling = FIB_NODE_INDEX_INVALID;
     mfib_entry->mfe_srcs = NULL;
     mfib_entry->mfe_itfs = NULL;
+    mfib_entry->mfe_rpf_id = MFIB_RPF_ID_NONE;
 
     dpo_reset(&mfib_entry->mfe_rep);
 
@@ -431,10 +476,57 @@ mfib_entry_alloc (u32 fib_index,
     return (mfib_entry);
 }
 
+static inline mfib_path_ext_t *
+mfib_entry_path_ext_find (mfib_path_ext_t *exts,
+                          fib_node_index_t path_index)
+{
+    uword *p;
+
+    p = hash_get(exts, path_index);
+
+    if (NULL != p)
+    {
+        return (mfib_entry_path_ext_get(p[0]));
+    }
+
+    return (NULL);
+}
+
+static mfib_path_ext_t*
+mfib_path_ext_add (mfib_entry_src_t *msrc,
+                   fib_node_index_t path_index,
+                   mfib_itf_flags_t mfi_flags)
+{
+    mfib_path_ext_t *path_ext;
+
+    pool_get(mfib_path_ext_pool, path_ext);
+
+    path_ext->mfpe_flags = mfi_flags;
+    path_ext->mfpe_path = path_index;
+
+    hash_set(msrc->mfes_exts, path_index,
+             path_ext - mfib_path_ext_pool);
+
+    return (path_ext);
+}
+
+static void
+mfib_path_ext_remove (mfib_entry_src_t *msrc,
+                      fib_node_index_t path_index)
+{
+    mfib_path_ext_t *path_ext;
+
+    path_ext = mfib_entry_path_ext_find(msrc->mfes_exts, path_index);
+
+    hash_unset(msrc->mfes_exts, path_index);
+    pool_put(mfib_path_ext_pool, path_ext);
+}
+
 typedef struct mfib_entry_collect_forwarding_ctx_t_
 {
     load_balance_path_t * next_hops;
     fib_forward_chain_type_t fct;
+    mfib_entry_src_t *msrc;
 } mfib_entry_collect_forwarding_ctx_t;
 
 static int
@@ -455,6 +547,20 @@ mfib_entry_src_collect_forwarding (fib_node_index_t pl_index,
         return (!0);
     }
 
+    /*
+     * If the path is not forwarding to use it
+     */
+    mfib_path_ext_t *path_ext;
+    
+    path_ext = mfib_entry_path_ext_find(ctx->msrc->mfes_exts,
+                                        path_index);
+
+    if (NULL != path_ext &&
+        !(path_ext->mfpe_flags & MFIB_ITF_FLAG_FORWARD))
+    {
+        return (!0);
+    }
+    
     switch (ctx->fct)
     {
     case FIB_FORW_CHAIN_TYPE_MCAST_IP4:
@@ -483,46 +589,61 @@ mfib_entry_src_collect_forwarding (fib_node_index_t pl_index,
 }
 
 static void
-mfib_entry_stack (mfib_entry_t *mfib_entry)
+mfib_entry_stack (mfib_entry_t *mfib_entry,
+                  mfib_entry_src_t *msrc)
 {
     dpo_proto_t dp;
 
     dp = fib_proto_to_dpo(mfib_entry_get_proto(mfib_entry));
 
-    if (FIB_NODE_INDEX_INVALID != mfib_entry->mfe_parent)
+    if (NULL != msrc &&
+        FIB_NODE_INDEX_INVALID != msrc->mfes_pl)
     {
         mfib_entry_collect_forwarding_ctx_t ctx = {
             .next_hops = NULL,
             .fct = mfib_entry_get_default_chain_type(mfib_entry),
+            .msrc = msrc,
         };
 
-        fib_path_list_walk(mfib_entry->mfe_parent,
+        fib_path_list_walk(msrc->mfes_pl,
                            mfib_entry_src_collect_forwarding,
                            &ctx);
 
         if (!(MFIB_ENTRY_FLAG_EXCLUSIVE & mfib_entry->mfe_flags))
         {
-            /*
-             * each path contirbutes a next-hop. form a replicate
-             * from those choices.
-             */
-            if (!dpo_id_is_valid(&mfib_entry->mfe_rep) ||
-                dpo_is_drop(&mfib_entry->mfe_rep))
+            if (NULL == ctx.next_hops)
             {
-                dpo_id_t tmp_dpo = DPO_INVALID;
-
-                dpo_set(&tmp_dpo,
-                        DPO_REPLICATE, dp,
-                        replicate_create(0, dp));
-
+                /*
+                 * no next-hops, stack directly on the drop
+                 */
                 dpo_stack(DPO_MFIB_ENTRY, dp,
                           &mfib_entry->mfe_rep,
-                          &tmp_dpo);
-
-                dpo_reset(&tmp_dpo);
+                          drop_dpo_get(dp));
+            }
+            else
+            {
+                /*
+                 * each path contirbutes a next-hop. form a replicate
+                 * from those choices.
+                 */
+                if (!dpo_id_is_valid(&mfib_entry->mfe_rep) ||
+                    dpo_is_drop(&mfib_entry->mfe_rep))
+                {
+                    dpo_id_t tmp_dpo = DPO_INVALID;
+
+                    dpo_set(&tmp_dpo,
+                            DPO_REPLICATE, dp,
+                            replicate_create(0, dp));
+
+                    dpo_stack(DPO_MFIB_ENTRY, dp,
+                              &mfib_entry->mfe_rep,
+                              &tmp_dpo);
+
+                    dpo_reset(&tmp_dpo);
+                }
+                replicate_multipath_update(&mfib_entry->mfe_rep,
+                                           ctx.next_hops);
             }
-            replicate_multipath_update(&mfib_entry->mfe_rep,
-                                       ctx.next_hops);
         }
         else
         {
@@ -548,11 +669,11 @@ mfib_entry_stack (mfib_entry_t *mfib_entry)
     }
 }
 
-static void
-mfib_entry_forwarding_path_add (mfib_entry_src_t *msrc,
-                                const fib_route_path_t *rpath)
+static fib_node_index_t
+mfib_entry_src_path_add (mfib_entry_src_t *msrc,
+                         const fib_route_path_t *rpath)
 {
-    fib_node_index_t old_pl_index;
+    fib_node_index_t path_index;
     fib_route_path_t *rpaths;
 
     ASSERT(!(MFIB_ENTRY_FLAG_EXCLUSIVE & msrc->mfes_flags));
@@ -563,32 +684,26 @@ mfib_entry_forwarding_path_add (mfib_entry_src_t *msrc,
     rpaths = NULL;
     vec_add1(rpaths, rpath[0]);
 
-    old_pl_index = msrc->mfes_pl;
-
     if (FIB_NODE_INDEX_INVALID == msrc->mfes_pl)
     {
-        msrc->mfes_pl =
-            fib_path_list_create(FIB_PATH_LIST_FLAG_NO_URPF,
-                                 rpaths);
-    }
-    else
-    {
-        msrc->mfes_pl =
-            fib_path_list_copy_and_path_add(msrc->mfes_pl,
-                                            FIB_PATH_LIST_FLAG_NO_URPF,
-                                            rpaths);
+        /* A non-shared path-list */
+        msrc->mfes_pl = fib_path_list_create(FIB_PATH_LIST_FLAG_NO_URPF,
+                                             NULL);
+        fib_path_list_lock(msrc->mfes_pl);
     }
-    fib_path_list_lock(msrc->mfes_pl);
-    fib_path_list_unlock(old_pl_index);
+
+    path_index = fib_path_list_path_add(msrc->mfes_pl, rpaths);
 
     vec_free(rpaths);
+
+    return (path_index);
 }
 
-static int
-mfib_entry_forwarding_path_remove (mfib_entry_src_t *msrc,
-                                   const fib_route_path_t *rpath)
+static fib_node_index_t
+mfib_entry_src_path_remove (mfib_entry_src_t *msrc,
+                            const fib_route_path_t *rpath)
 {
-    fib_node_index_t old_pl_index;
+    fib_node_index_t path_index;
     fib_route_path_t *rpaths;
 
     ASSERT(!(MFIB_ENTRY_FLAG_EXCLUSIVE & msrc->mfes_flags));
@@ -599,56 +714,31 @@ mfib_entry_forwarding_path_remove (mfib_entry_src_t *msrc,
     rpaths = NULL;
     vec_add1(rpaths, rpath[0]);
 
-    old_pl_index = msrc->mfes_pl;
-
-    msrc->mfes_pl =
-        fib_path_list_copy_and_path_remove(msrc->mfes_pl,
-                                           FIB_PATH_LIST_FLAG_NONE,
-                                           rpaths);
-
-    fib_path_list_lock(msrc->mfes_pl);
-    fib_path_list_unlock(old_pl_index);
+    path_index = fib_path_list_path_remove(msrc->mfes_pl, rpaths);
 
     vec_free(rpaths);
 
-    return (FIB_NODE_INDEX_INVALID != msrc->mfes_pl);
+    return (path_index);
 }
 
 static void
 mfib_entry_recalculate_forwarding (mfib_entry_t *mfib_entry)
 {
-    fib_node_index_t old_pl_index;
     mfib_entry_src_t *bsrc;
 
-    old_pl_index = mfib_entry->mfe_parent;
-
     /*
      * copy the forwarding data from the bast source
      */
     bsrc = mfib_entry_get_best_src(mfib_entry);
 
-    if (NULL == bsrc)
-    {
-        mfib_entry->mfe_parent = FIB_NODE_INDEX_INVALID;
-    }
-    else
+    if (NULL != bsrc)
     {
-        mfib_entry->mfe_parent = bsrc->mfes_pl;
         mfib_entry->mfe_flags = bsrc->mfes_flags;
         mfib_entry->mfe_itfs = bsrc->mfes_itfs;
+        mfib_entry->mfe_rpf_id = bsrc->mfes_rpf_id;
     }
 
-    /*
-     * re-stack the entry on the best forwarding info.
-     */
-    if (old_pl_index != mfib_entry->mfe_parent ||
-        FIB_NODE_INDEX_INVALID == old_pl_index)
-    {
-        mfib_entry_stack(mfib_entry);
-
-        fib_path_list_lock(mfib_entry->mfe_parent);
-        fib_path_list_unlock(old_pl_index);
-    }
+    mfib_entry_stack(mfib_entry, bsrc);
 }
 
 
@@ -656,6 +746,7 @@ fib_node_index_t
 mfib_entry_create (u32 fib_index,
                    mfib_source_t source,
                    const mfib_prefix_t *prefix,
+                   fib_rpf_id_t rpf_id,
                    mfib_entry_flags_t entry_flags)
 {
     fib_node_index_t mfib_entry_index;
@@ -666,6 +757,7 @@ mfib_entry_create (u32 fib_index,
                                   &mfib_entry_index);
     msrc = mfib_entry_src_find_or_create(mfib_entry, source);
     msrc->mfes_flags = entry_flags;
+    msrc->mfes_rpf_id = rpf_id;
 
     mfib_entry_recalculate_forwarding(mfib_entry);
 
@@ -682,13 +774,14 @@ static int
 mfib_entry_src_ok_for_delete (const mfib_entry_src_t *msrc)
 {
     return ((MFIB_ENTRY_FLAG_NONE == msrc->mfes_flags &&
-             0 == mfib_entry_src_n_itfs(msrc)));
+             0 == fib_path_list_get_n_paths(msrc->mfes_pl)));
 }
 
 int
 mfib_entry_update (fib_node_index_t mfib_entry_index,
                    mfib_source_t source,
                    mfib_entry_flags_t entry_flags,
+                   fib_rpf_id_t rpf_id,
                    index_t repi)
 {
     mfib_entry_t *mfib_entry;
@@ -697,6 +790,7 @@ mfib_entry_update (fib_node_index_t mfib_entry_index,
     mfib_entry = mfib_entry_get(mfib_entry_index);
     msrc = mfib_entry_src_find_or_create(mfib_entry, source);
     msrc->mfes_flags = entry_flags;
+    msrc->mfes_rpf_id = rpf_id;
 
     if (INDEX_INVALID != repi)
     {
@@ -768,55 +862,79 @@ mfib_entry_path_update (fib_node_index_t mfib_entry_index,
                         const fib_route_path_t *rpath,
                         mfib_itf_flags_t itf_flags)
 {
+    fib_node_index_t path_index;
+    mfib_path_ext_t *path_ext;
+    mfib_itf_flags_t old, new;
     mfib_entry_t *mfib_entry;
     mfib_entry_src_t *msrc;
-    mfib_itf_t *mfib_itf;
 
     mfib_entry = mfib_entry_get(mfib_entry_index);
     ASSERT(NULL != mfib_entry);
     msrc = mfib_entry_src_find_or_create(mfib_entry, source);
 
     /*
-     * search for the interface in the current set
+     * add the path to the path-list. If it's a duplicate we'll get
+     * back the original path.
+     */
+    path_index = mfib_entry_src_path_add(msrc, rpath);
+
+    /*
+     * find the path extension for that path
      */
-    mfib_itf = mfib_entry_itf_find(msrc->mfes_itfs,
-                                   rpath[0].frp_sw_if_index);
+    path_ext = mfib_entry_path_ext_find(msrc->mfes_exts, path_index);
 
-    if (NULL == mfib_itf)
+    if (NULL == path_ext)
     {
-        /*
-         * this is a path we do not yet have. If it is forwarding then we
-         * add it to the replication set
-         */
-        if (itf_flags & MFIB_ITF_FLAG_FORWARD)
-        {
-            mfib_entry_forwarding_path_add(msrc, rpath);
-        }
-        /*
-         * construct a new ITF for this entry's list
-         */
-        mfib_entry_itf_add(msrc,
-                           rpath[0].frp_sw_if_index,
-                           mfib_itf_create(rpath[0].frp_sw_if_index,
-                                           itf_flags));
+        old = MFIB_ITF_FLAG_NONE;
+        path_ext = mfib_path_ext_add(msrc, path_index, itf_flags);
     }
     else
     {
-        int was_forwarding = !!(mfib_itf->mfi_flags & MFIB_ITF_FLAG_FORWARD);
-        int is_forwarding  = !!(itf_flags & MFIB_ITF_FLAG_FORWARD);
+        old = path_ext->mfpe_flags;
+        path_ext->mfpe_flags = itf_flags;
+    }
 
-        if (!was_forwarding && is_forwarding)
-        {
-            mfib_entry_forwarding_path_add(msrc, rpath);
-        }
-        else if (was_forwarding && !is_forwarding)
+    /*
+     * Has the path changed its contribution to the input interface set.
+     * Which only paths with interfaces can do...
+     */
+    if (~0 != rpath[0].frp_sw_if_index)
+    {
+        mfib_itf_t *mfib_itf;
+
+        new = itf_flags;
+
+        if (old != new)
         {
-            mfib_entry_forwarding_path_remove(msrc, rpath);
+            if (MFIB_ITF_FLAG_NONE == new)
+            {
+                /*
+                 * no more interface flags on this path, remove
+                 * from the data-plane set
+                 */
+                mfib_entry_itf_remove(msrc, rpath[0].frp_sw_if_index);
+            }
+            else if (MFIB_ITF_FLAG_NONE == old)
+            {
+                /*
+                 * This interface is now contributing
+                 */
+                mfib_entry_itf_add(msrc,
+                                   rpath[0].frp_sw_if_index,
+                                   mfib_itf_create(rpath[0].frp_sw_if_index,
+                                                   itf_flags));
+            }
+            else
+            {
+                /*
+                 * change of flag contributions
+                 */
+                mfib_itf = mfib_entry_itf_find(msrc->mfes_itfs,
+                                               rpath[0].frp_sw_if_index);
+                /* Seen by packets inflight */
+                mfib_itf->mfi_flags = new;
+            }
         }
-        /*
-         * packets in flight see these updates.
-         */
-        mfib_itf->mfi_flags = itf_flags;
     }
 
     mfib_entry_recalculate_forwarding(mfib_entry);
@@ -833,9 +951,9 @@ mfib_entry_path_remove (fib_node_index_t mfib_entry_index,
                         mfib_source_t source,
                         const fib_route_path_t *rpath)
 {
+    fib_node_index_t path_index;
     mfib_entry_t *mfib_entry;
     mfib_entry_src_t *msrc;
-    mfib_itf_t *mfib_itf;
 
     mfib_entry = mfib_entry_get(mfib_entry_index);
     ASSERT(NULL != mfib_entry);
@@ -850,33 +968,23 @@ mfib_entry_path_remove (fib_node_index_t mfib_entry_index,
     }
 
     /*
-     * search for the interface in the current set
+     * remove the path from the path-list. If it's not there we'll get
+     * back invalid
      */
-    mfib_itf = mfib_entry_itf_find(msrc->mfes_itfs,
-                                   rpath[0].frp_sw_if_index);
+    path_index = mfib_entry_src_path_remove(msrc, rpath);
 
-    if (NULL == mfib_itf)
+    if (FIB_NODE_INDEX_INVALID != path_index)
     {
         /*
-         * removing a path that does not exist
+         * don't need the extension, nor the interface anymore
          */
-        return (mfib_entry_ok_for_delete(mfib_entry));
-    }
-
-    /*
-     * we have this path. If it is forwarding then we
-     * remove it to the replication set
-     */
-    if (mfib_itf->mfi_flags & MFIB_ITF_FLAG_FORWARD)
-    {
-        mfib_entry_forwarding_path_remove(msrc, rpath);
+        mfib_path_ext_remove(msrc, path_index);
+        if (~0 != rpath[0].frp_sw_if_index)
+        {
+            mfib_entry_itf_remove(msrc, rpath[0].frp_sw_if_index);
+        }
     }
 
-    /*
-     * remove the interface/path from this entry's list
-     */
-    mfib_entry_itf_remove(msrc, rpath[0].frp_sw_if_index);
-
     if (mfib_entry_src_ok_for_delete(msrc))
     {
         /*
@@ -1057,11 +1165,14 @@ mfib_entry_encode (fib_node_index_t mfib_entry_index,
                   fib_route_path_encode_t **api_rpaths)
 {
     mfib_entry_t *mfib_entry;
+    mfib_entry_src_t *bsrc;
 
     mfib_entry = mfib_entry_get(mfib_entry_index);
-    if (FIB_NODE_INDEX_INVALID != mfib_entry->mfe_parent)
+    bsrc = mfib_entry_get_best_src(mfib_entry);
+
+    if (FIB_NODE_INDEX_INVALID != bsrc->mfes_pl)
     {
-        fib_path_list_walk(mfib_entry->mfe_parent,
+        fib_path_list_walk(bsrc->mfes_pl,
                            fib_path_encode,
                            api_rpaths);
     }
diff --git a/src/vnet/mfib/mfib_entry.h b/src/vnet/mfib/mfib_entry.h
index dc8f49aad1a..4f62b18ec65 100644
--- a/src/vnet/mfib/mfib_entry.h
+++ b/src/vnet/mfib/mfib_entry.h
@@ -42,17 +42,6 @@ typedef struct mfib_entry_t_ {
      * The index of the FIB table this entry is in
      */
     u32 mfe_fib_index;
-    /**
-     * the path-list for which this entry is a child. This is also the path-list
-     * that is contributing forwarding for this entry.
-     */
-    fib_node_index_t mfe_parent;
-    /**
-     * index of this entry in the parent's child list.
-     * This is set when this entry is added as a child, but can also
-     * be changed by the parent as it manages its list.
-     */
-    u32 mfe_sibling;
 
     /**
      * A vector of sources contributing forwarding
@@ -65,7 +54,7 @@ typedef struct mfib_entry_t_ {
     CLIB_CACHE_LINE_ALIGN_MARK(cacheline1);
 
     /**
-     * The Replicate DPO used for forwarding.
+     * The DPO used for forwarding; replicate, drop, etc..
      */
     dpo_id_t mfe_rep;
 
@@ -75,6 +64,11 @@ typedef struct mfib_entry_t_ {
     mfib_entry_flags_t mfe_flags;
 
     /**
+     * RPF-ID used when the packets ingress not from an interface
+     */
+    fib_rpf_id_t mfe_rpf_id;
+
+    /**
      * A hash table of interfaces
      */
     mfib_itf_t *mfe_itfs;
@@ -90,11 +84,13 @@ extern u8 *format_mfib_entry(u8 * s, va_list * args);
 extern fib_node_index_t mfib_entry_create(u32 fib_index,
                                           mfib_source_t source,
                                           const mfib_prefix_t *prefix,
+                                          fib_rpf_id_t rpf_id,
                                           mfib_entry_flags_t entry_flags);
 
 extern int mfib_entry_update(fib_node_index_t fib_entry_index,
                              mfib_source_t source,
                              mfib_entry_flags_t entry_flags,
+                             fib_rpf_id_t rpf_id,
                              index_t rep_dpo);
 
 extern void mfib_entry_path_update(fib_node_index_t fib_entry_index,
diff --git a/src/vnet/mfib/mfib_forward.c b/src/vnet/mfib/mfib_forward.c
index 5fe0a57c03b..3d8f4f98cb0 100644
--- a/src/vnet/mfib/mfib_forward.c
+++ b/src/vnet/mfib/mfib_forward.c
@@ -380,13 +380,27 @@ mfib_forward_rpf (vlib_main_t * vm,
              * for the case of throughput traffic that is not replicated
              * to the host stack nor sets local flags
              */
-            if (PREDICT_TRUE(NULL != mfi0))
+
+            /*
+             * If the mfib entry has a configured RPF-ID check that
+             * in preference to an interface based RPF
+             */
+            if (MFIB_RPF_ID_NONE != mfe0->mfe_rpf_id)
             {
-                iflags0 = mfi0->mfi_flags;
+                iflags0 = (mfe0->mfe_rpf_id == vnet_buffer(b0)->ip.rpf_id ?
+                           MFIB_ITF_FLAG_ACCEPT :
+                           MFIB_ITF_FLAG_NONE);
             }
             else
             {
-                iflags0 = MFIB_ITF_FLAG_NONE;
+                if (PREDICT_TRUE(NULL != mfi0))
+                {
+                    iflags0 = mfi0->mfi_flags;
+                }
+                else
+                {
+                    iflags0 = MFIB_ITF_FLAG_NONE;
+                }
             }
             eflags0 = mfe0->mfe_flags;
 
@@ -436,17 +450,16 @@ mfib_forward_rpf (vlib_main_t * vm,
             {
                 mfib_forward_rpf_trace_t *t0;
 
-                t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
+                t0 = vlib_add_trace (vm, node, b0, sizeof (*t0));
                 t0->entry_index = mfei0;
+                t0->itf_flags = iflags0;
                 if (NULL == mfi0)
                 {
                     t0->sw_if_index = ~0;
-                    t0->itf_flags = MFIB_ITF_FLAG_NONE;
                 }
                 else
                 {
                     t0->sw_if_index = mfi0->mfi_sw_if_index;
-                    t0->itf_flags = mfi0->mfi_flags;
                 }
             }
             vlib_validate_buffer_enqueue_x1 (vm, node, next,
@@ -478,7 +491,7 @@ VLIB_REGISTER_NODE (ip4_mfib_forward_rpf_node, static) = {
 
     .n_next_nodes = MFIB_FORWARD_RPF_N_NEXT,
     .next_nodes = {
-        [MFIB_FORWARD_RPF_NEXT_DROP] = "error-drop",
+        [MFIB_FORWARD_RPF_NEXT_DROP] = "ip4-drop",
     },
 };
 
@@ -503,7 +516,7 @@ VLIB_REGISTER_NODE (ip6_mfib_forward_rpf_node, static) = {
 
     .n_next_nodes = MFIB_FORWARD_RPF_N_NEXT,
     .next_nodes = {
-        [MFIB_FORWARD_RPF_NEXT_DROP] = "error-drop",
+        [MFIB_FORWARD_RPF_NEXT_DROP] = "ip6-drop",
     },
 };
 
diff --git a/src/vnet/mfib/mfib_table.c b/src/vnet/mfib/mfib_table.c
index 3b4bd985893..7ffe8941e75 100644
--- a/src/vnet/mfib/mfib_table.c
+++ b/src/vnet/mfib/mfib_table.c
@@ -165,6 +165,7 @@ fib_node_index_t
 mfib_table_entry_update (u32 fib_index,
                          const mfib_prefix_t *prefix,
                          mfib_source_t source,
+                         fib_rpf_id_t rpf_id,
                          mfib_entry_flags_t entry_flags)
 {
     fib_node_index_t mfib_entry_index;
@@ -181,7 +182,8 @@ mfib_table_entry_update (u32 fib_index,
              * update to a non-existing entry with non-zero flags
              */
             mfib_entry_index = mfib_entry_create(fib_index, source,
-                                                 prefix, entry_flags);
+                                                 prefix, rpf_id,
+                                                 entry_flags);
 
             mfib_table_entry_insert(mfib_table, prefix, mfib_entry_index);
         }
@@ -198,6 +200,7 @@ mfib_table_entry_update (u32 fib_index,
         if (mfib_entry_update(mfib_entry_index,
                               source,
                               entry_flags,
+                              rpf_id,
                               INDEX_INVALID))
         {
             /*
@@ -230,6 +233,7 @@ mfib_table_entry_path_update (u32 fib_index,
         mfib_entry_index = mfib_entry_create(fib_index,
                                              source,
                                              prefix,
+                                             MFIB_RPF_ID_NONE,
                                              MFIB_ENTRY_FLAG_NONE);
 
         mfib_table_entry_insert(mfib_table, prefix, mfib_entry_index);
@@ -304,6 +308,7 @@ mfib_table_entry_special_add (u32 fib_index,
         mfib_entry_index = mfib_entry_create(fib_index,
                                              source,
                                              prefix,
+                                             MFIB_RPF_ID_NONE,
                                              MFIB_ENTRY_FLAG_NONE);
 
         mfib_table_entry_insert(mfib_table, prefix, mfib_entry_index);
@@ -311,6 +316,7 @@ mfib_table_entry_special_add (u32 fib_index,
 
     mfib_entry_update(mfib_entry_index, source,
                       (MFIB_ENTRY_FLAG_EXCLUSIVE | entry_flags),
+                      MFIB_RPF_ID_NONE,
                       rep_dpo);
 
     return (mfib_entry_index);
diff --git a/src/vnet/mfib/mfib_table.h b/src/vnet/mfib/mfib_table.h
index 95239f7cb17..83aa04ef797 100644
--- a/src/vnet/mfib/mfib_table.h
+++ b/src/vnet/mfib/mfib_table.h
@@ -122,6 +122,7 @@ extern fib_node_index_t mfib_table_lookup_exact_match(u32 fib_index,
 extern fib_node_index_t mfib_table_entry_update(u32 fib_index,
                                                 const mfib_prefix_t *prefix,
                                                 mfib_source_t source,
+                                                fib_rpf_id_t rpf_id,
                                                 mfib_entry_flags_t flags);
 
 /**
diff --git a/src/vnet/mfib/mfib_test.c b/src/vnet/mfib/mfib_test.c
index 36a303e844d..7c92ae99b1d 100644
--- a/src/vnet/mfib/mfib_test.c
+++ b/src/vnet/mfib/mfib_test.c
@@ -20,6 +20,8 @@
 #include <vnet/mfib/mfib_signal.h>
 #include <vnet/mfib/ip6_mfib.h>
 #include <vnet/fib/fib_path_list.h>
+#include <vnet/fib/fib_test.h>
+#include <vnet/fib/fib_table.h>
 
 #include <vnet/dpo/replicate_dpo.h>
 #include <vnet/adj/adj_mcast.h>
@@ -201,8 +203,8 @@ mfib_test_validate_rep_v (const replicate_t *rep,
         if (DPO_RECEIVE != dt)
         {
             MFIB_TEST_REP((ai == dpo->dpoi_index),
-                          "bucket %d stacks on %U",
-                          bucket,
+                          "bucket %d [exp:%d] stacks on %U",
+                          bucket, ai,
                           format_dpo_id, dpo, 0);
         }
     }
@@ -734,6 +736,7 @@ mfib_test_i (fib_protocol_t PROTO,
     mfib_table_entry_update(fib_index,
                             pfx_s_g,
                             MFIB_SOURCE_API,
+                            MFIB_RPF_ID_NONE,
                             MFIB_ENTRY_FLAG_SIGNAL);
     MFIB_TEST(mfib_test_entry(mfei,
                               MFIB_ENTRY_FLAG_SIGNAL,
@@ -824,6 +827,7 @@ mfib_test_i (fib_protocol_t PROTO,
     mfib_table_entry_update(fib_index,
                             pfx_s_g,
                             MFIB_SOURCE_API,
+                            MFIB_RPF_ID_NONE,
                             (MFIB_ENTRY_FLAG_SIGNAL |
                              MFIB_ENTRY_FLAG_CONNECTED));
     MFIB_TEST(mfib_test_entry(mfei,
@@ -965,6 +969,7 @@ mfib_test_i (fib_protocol_t PROTO,
     mfib_table_entry_update(fib_index,
                             pfx_s_g,
                             MFIB_SOURCE_API,
+                            MFIB_RPF_ID_NONE,
                             MFIB_ENTRY_FLAG_NONE);
     mfei = mfib_table_lookup_exact_match(fib_index,
                                          pfx_s_g);
@@ -1074,6 +1079,117 @@ mfib_test_i (fib_protocol_t PROTO,
     dpo_reset(&td);
 
     /*
+     * A Multicast LSP. This a mLDP head-end
+     */
+    fib_node_index_t ai_mpls_10_10_10_1, lfei;
+    ip46_address_t nh_10_10_10_1 = {
+	.ip4 = {
+	    .as_u32 = clib_host_to_net_u32(0x0a0a0a01),
+	},
+    };
+    ai_mpls_10_10_10_1 = adj_nbr_add_or_lock(FIB_PROTOCOL_IP4,
+                                             VNET_LINK_MPLS,
+                                             &nh_10_10_10_1,
+                                             tm->hw[0]->sw_if_index);
+
+    fib_prefix_t pfx_3500 = {
+	.fp_len = 21,
+	.fp_proto = FIB_PROTOCOL_MPLS,
+	.fp_label = 3500,
+	.fp_eos = MPLS_EOS,
+	.fp_payload_proto = DPO_PROTO_IP4,
+    };
+    fib_test_rep_bucket_t mc_0 = {
+        .type = FT_REP_LABEL_O_ADJ,
+	.label_o_adj = {
+	    .adj = ai_mpls_10_10_10_1,
+	    .label = 3300,
+	    .eos = MPLS_EOS,
+	},
+    };
+    mpls_label_t *l3300 = NULL;
+    vec_add1(l3300, 3300);
+
+    /*
+     * MPLS enable an interface so we get the MPLS table created
+     */
+    mpls_sw_interface_enable_disable(&mpls_main,
+                                     tm->hw[0]->sw_if_index,
+                                     1);
+
+    lfei = fib_table_entry_update_one_path(0, // default MPLS Table
+                                           &pfx_3500,
+                                           FIB_SOURCE_API,
+                                           FIB_ENTRY_FLAG_MULTICAST,
+                                           FIB_PROTOCOL_IP4,
+                                           &nh_10_10_10_1,
+                                           tm->hw[0]->sw_if_index,
+                                           ~0, // invalid fib index
+                                           1,
+                                           l3300,
+                                           FIB_ROUTE_PATH_FLAG_NONE);
+    MFIB_TEST(fib_test_validate_entry(lfei,
+                                      FIB_FORW_CHAIN_TYPE_MPLS_EOS,
+                                      1,
+                                      &mc_0),
+              "3500 via replicate over 10.10.10.1");
+
+    /*
+     * An (S,G) that resolves via the mLDP head-end
+     */
+    fib_route_path_t path_via_mldp = {
+        .frp_proto = FIB_PROTOCOL_MPLS,
+        .frp_local_label = pfx_3500.fp_label,
+        .frp_eos = MPLS_EOS,
+        .frp_sw_if_index = 0xffffffff,
+        .frp_fib_index = 0,
+        .frp_weight = 1,
+        .frp_flags = FIB_ROUTE_PATH_FLAG_NONE,
+    };
+    dpo_id_t mldp_dpo = DPO_INVALID;
+
+    fib_entry_contribute_forwarding(lfei,
+                                    FIB_FORW_CHAIN_TYPE_MPLS_EOS,
+                                    &mldp_dpo);
+
+    mfei = mfib_table_entry_path_update(fib_index,
+                                        pfx_s_g,
+                                        MFIB_SOURCE_API,
+                                        &path_via_mldp,
+                                        MFIB_ITF_FLAG_FORWARD);
+
+    MFIB_TEST(mfib_test_entry(mfei,
+                              MFIB_ENTRY_FLAG_NONE,
+                              1,
+                              DPO_REPLICATE, mldp_dpo.dpoi_index),
+              "%U over-mLDP replicate OK",
+              format_mfib_prefix, pfx_s_g);
+
+    /*
+     * add a for-us path. this tests two types of non-attached paths on one entry
+     */
+    mfei = mfib_table_entry_path_update(fib_index,
+                                        pfx_s_g,
+                                        MFIB_SOURCE_API,
+                                        &path_for_us,
+                                        MFIB_ITF_FLAG_FORWARD);
+    MFIB_TEST(mfib_test_entry(mfei,
+                              MFIB_ENTRY_FLAG_NONE,
+                              2,
+                              DPO_REPLICATE, mldp_dpo.dpoi_index,
+                              DPO_RECEIVE, 0),
+              "%U mLDP+for-us replicate OK",
+              format_mfib_prefix, pfx_s_g);
+
+    mfib_table_entry_delete(fib_index,
+                            pfx_s_g,
+                            MFIB_SOURCE_API);
+    fib_table_entry_delete(0,
+                           &pfx_3500,
+                           FIB_SOURCE_API);
+    dpo_reset(&mldp_dpo);
+
+    /*
      * Unlock the table - it's the last lock so should be gone thereafter
      */
     mfib_table_unlock(fib_index, PROTO);
@@ -1087,6 +1203,13 @@ mfib_test_i (fib_protocol_t PROTO,
     adj_unlock(ai_3);
 
     /*
+     * MPLS disable the interface
+     */
+    mpls_sw_interface_enable_disable(&mpls_main,
+                                     tm->hw[0]->sw_if_index,
+                                     0);
+
+    /*
      * test we've leaked no resources
      */
     MFIB_TEST(0 == adj_mcast_db_size(), "%d MCAST adjs", adj_mcast_db_size());