fib: Table Replace

Type: feature

from the API doc, a table replace is:

"
    The use-case is that, for some unspecified reason, the control plane
    has a very different set of entries it wants in the table than VPP
    currently has. The CP would thus like to 'replace' VPP's current table
    only by specifying what the new set of entries shall be, i.e. it is not
    going to delete anything that already eixts.
    the CP delcartes the start of this procedure with this begin_replace
    API Call, and when it has populated all the entries it wants, it calls
    the below end_replace API. From this point on it is of coursce free
    to add and delete entries as usual.
    The underlying mechanism by which VPP implements this replace is
    purposefully left unspecified.
"

In the FIB, the algorithm is implemented using mark and sweep.

Algorithm goes:
1) replace_begin: this marks all the entries in that table as 'stale'
2) download all the entries that should be in this table
 - this clears the stale flag on those entries
3) signal the table converged: ip_table_replace_end
 - this removes all entries that are still stale

this procedure can be used when an agent first connects to VPP,
as an alternative to dump and diff state reconciliation.

Change-Id: I168edec10cf7670866076b129ebfe6149ea8222e
Signed-off-by: Neale Ranns <nranns@cisco.com>

21 files changed, 666 insertions, 219 deletions

diff --git a/src/vnet/ethernet/arp.c b/src/vnet/ethernet/arp.c
index ca9a21cbbd1..6b1069ecb6d 100644
--- a/src/vnet/ethernet/arp.c
+++ b/src/vnet/ethernet/arp.c
@@ -2476,41 +2476,6 @@ proxy_arp_intfc_walk (proxy_arp_intf_walk_t cb, void *data)
   }
 }
 
-/*
- * Remove any proxy arp entries associated with the
- * specified fib.
- */
-int
-vnet_proxy_arp_fib_reset (u32 fib_id)
-{
-  ethernet_arp_main_t *am = &ethernet_arp_main;
-  ethernet_proxy_arp_t *pa;
-  u32 *entries_to_delete = 0;
-  u32 fib_index;
-  int i;
-
-  fib_index = fib_table_find (FIB_PROTOCOL_IP4, fib_id);
-  if (~0 == fib_index)
-    return VNET_API_ERROR_NO_SUCH_ENTRY;
-
-  vec_foreach (pa, am->proxy_arps)
-  {
-    if (pa->fib_index == fib_index)
-      {
-	vec_add1 (entries_to_delete, pa - am->proxy_arps);
-      }
-  }
-
-  for (i = 0; i < vec_len (entries_to_delete); i++)
-    {
-      vec_delete (am->proxy_arps, 1, entries_to_delete[i]);
-    }
-
-  vec_free (entries_to_delete);
-
-  return 0;
-}
-
 static clib_error_t *
 ip_arp_add_del_command_fn (vlib_main_t * vm,
 			   unformat_input_t * input, vlib_cli_command_t * cmd)
diff --git a/src/vnet/ethernet/arp.h b/src/vnet/ethernet/arp.h
index 56d9f29e09f..de92c4f419c 100644
--- a/src/vnet/ethernet/arp.h
+++ b/src/vnet/ethernet/arp.h
@@ -62,8 +62,6 @@ extern int vnet_arp_unset_ip4_over_ethernet (vnet_main_t * vnm,
 					     ethernet_arp_ip4_over_ethernet_address_t
 					     * a);
 
-extern int vnet_proxy_arp_fib_reset (u32 fib_id);
-
 void vnet_register_ip4_arp_resolution_event (vnet_main_t * vnm,
 					     void *address_arg,
 					     uword node_index,
diff --git a/src/vnet/fib/fib_entry.h b/src/vnet/fib/fib_entry.h
index 4ce28b1ae9d..f0e6e8d8aae 100644
--- a/src/vnet/fib/fib_entry.h
+++ b/src/vnet/fib/fib_entry.h
@@ -315,6 +315,10 @@ typedef enum fib_entry_src_attribute_t_ {
      */
     FIB_ENTRY_SRC_ATTRIBUTE_ACTIVE,
     /**
+     * the source is stale
+     */
+    FIB_ENTRY_SRC_ATTRIBUTE_STALE,
+    /**
      * the source is inherited from its cover
      */
     FIB_ENTRY_SRC_ATTRIBUTE_INHERITED,
@@ -329,6 +333,7 @@ typedef enum fib_entry_src_attribute_t_ {
     [FIB_ENTRY_SRC_ATTRIBUTE_ADDED]  = "added",	 \
     [FIB_ENTRY_SRC_ATTRIBUTE_CONTRIBUTING] = "contributing", \
     [FIB_ENTRY_SRC_ATTRIBUTE_ACTIVE] = "active", \
+    [FIB_ENTRY_SRC_ATTRIBUTE_STALE] = "stale",      \
     [FIB_ENTRY_SRC_ATTRIBUTE_INHERITED] = "inherited", \
 }
 
@@ -342,6 +347,7 @@ typedef enum fib_entry_src_flag_t_ {
     FIB_ENTRY_SRC_FLAG_ADDED  = (1 << FIB_ENTRY_SRC_ATTRIBUTE_ADDED),
     FIB_ENTRY_SRC_FLAG_CONTRIBUTING = (1 << FIB_ENTRY_SRC_ATTRIBUTE_CONTRIBUTING),
     FIB_ENTRY_SRC_FLAG_ACTIVE = (1 << FIB_ENTRY_SRC_ATTRIBUTE_ACTIVE),
+    FIB_ENTRY_SRC_FLAG_STALE = (1 << FIB_ENTRY_SRC_ATTRIBUTE_STALE),
     FIB_ENTRY_SRC_FLAG_INHERITED = (1 << FIB_ENTRY_SRC_ATTRIBUTE_INHERITED),
 } __attribute__ ((packed)) fib_entry_src_flag_t;
 
@@ -627,6 +633,8 @@ extern int fib_entry_is_sourced(fib_node_index_t fib_entry_index,
 extern fib_node_index_t fib_entry_get_path_list(fib_node_index_t fib_entry_index);
 extern int fib_entry_is_resolved(fib_node_index_t fib_entry_index);
 extern int fib_entry_is_host(fib_node_index_t fib_entry_index);
+extern int fib_entry_is_marked(fib_node_index_t fib_entry_index, fib_source_t source);
+extern void fib_entry_mark(fib_node_index_t fib_entry_index, fib_source_t source);
 extern void fib_entry_set_flow_hash_config(fib_node_index_t fib_entry_index,
                                            flow_hash_config_t hash_config);
 
diff --git a/src/vnet/fib/fib_entry_src.c b/src/vnet/fib/fib_entry_src.c
index 6ed13a328bd..176818abc0a 100644
--- a/src/vnet/fib/fib_entry_src.c
+++ b/src/vnet/fib/fib_entry_src.c
@@ -141,6 +141,44 @@ fib_entry_is_sourced (fib_node_index_t fib_entry_index,
     return (NULL != fib_entry_src_find(fib_entry, source));
 }
 
+int
+fib_entry_is_marked (fib_node_index_t fib_entry_index,
+                      fib_source_t source)
+{
+    fib_entry_t *fib_entry;
+    fib_entry_src_t *esrc;
+
+    fib_entry = fib_entry_get(fib_entry_index);
+
+    esrc = fib_entry_src_find(fib_entry, source);
+
+    if (NULL == esrc)
+    {
+        return (0);
+    }
+    else
+    {
+        return (!!(esrc->fes_flags & FIB_ENTRY_SRC_FLAG_STALE));
+    }
+}
+
+void
+fib_entry_mark (fib_node_index_t fib_entry_index,
+                fib_source_t source)
+{
+    fib_entry_t *fib_entry;
+    fib_entry_src_t *esrc;
+
+    fib_entry = fib_entry_get(fib_entry_index);
+
+    esrc = fib_entry_src_find(fib_entry, source);
+
+    if (NULL != esrc)
+    {
+        esrc->fes_flags |= FIB_ENTRY_SRC_FLAG_STALE;
+    }
+}
+
 static fib_entry_src_t *
 fib_entry_src_find_or_create (fib_entry_t *fib_entry,
 			      fib_source_t source,
diff --git a/src/vnet/fib/fib_entry_src.h b/src/vnet/fib/fib_entry_src.h
index a859b9c734a..8f13ae894bb 100644
--- a/src/vnet/fib/fib_entry_src.h
+++ b/src/vnet/fib/fib_entry_src.h
@@ -226,20 +226,24 @@ typedef struct fib_entry_src_vft_t_ {
     }								\
 }
 
-#define FIB_ENTRY_SRC_VFT_INVOKE(esrc, func, args)  \
-{                                                   \
-    const fib_entry_src_vft_t *_vft;                \
-    _vft = fib_entry_src_get_vft(esrc);             \
-    if (_vft->func)                                 \
-        _vft->func args;                            \
+#define FIB_ENTRY_SRC_VFT_INVOKE(esrc, func, args)             \
+{                                                              \
+    const fib_entry_src_vft_t *_vft;                           \
+    _vft = fib_entry_src_get_vft(esrc);                        \
+    if (_vft->func) {                                          \
+        (esrc)->fes_flags &= ~FIB_ENTRY_SRC_FLAG_STALE;        \
+        _vft->func args;                                       \
+    }                                                          \
 }
 
 #define FIB_ENTRY_SRC_VFT_INVOKE_AND_RETURN(esrc, func, args)  \
-{                                                   \
-    const fib_entry_src_vft_t *_vft;                \
-    _vft = fib_entry_src_get_vft(esrc);             \
-    if (_vft->func)                                 \
-        return (_vft->func args);                   \
+{                                                              \
+    const fib_entry_src_vft_t *_vft;                           \
+    _vft = fib_entry_src_get_vft(esrc);                        \
+    if (_vft->func) {                                          \
+        (esrc)->fes_flags &= ~FIB_ENTRY_SRC_FLAG_STALE;        \
+        return (_vft->func args);                              \
+    }                                                          \
 }
 
 #define FIB_ENTRY_SRC_VFT_EXISTS(esrc, func)        \
diff --git a/src/vnet/fib/fib_table.c b/src/vnet/fib/fib_table.c
index 3778fa9a944..d3cf5dc88b1 100644
--- a/src/vnet/fib/fib_table.c
+++ b/src/vnet/fib/fib_table.c
@@ -23,6 +23,8 @@
 #include <vnet/fib/ip6_fib.h>
 #include <vnet/fib/mpls_fib.h>
 
+const static char * fib_table_flags_strings[] = FIB_TABLE_ATTRIBUTES;
+
 fib_table_t *
 fib_table_get (fib_node_index_t index,
 	       fib_protocol_t proto)
@@ -1305,6 +1307,26 @@ format_fib_table_name (u8* s, va_list* ap)
     return (s);
 }
 
+u8*
+format_fib_table_flags (u8 *s, va_list *args)
+{
+    fib_table_flags_t flags = va_arg(*args, int);
+    fib_table_attribute_t attr;
+
+    if (!flags)
+    {
+        return format(s, "none");
+    }
+
+    FOR_EACH_FIB_TABLE_ATTRIBUTE(attr) {
+        if (1 << attr & flags) {
+            s = format(s, "%s", fib_table_flags_strings[attr]);
+        }
+    }
+
+    return (s);
+}
+
 /**
  * @brief Table flush context. Store the indicies of matching FIB entries
  * that need to be removed.
@@ -1335,7 +1357,6 @@ fib_table_flush_cb (fib_node_index_t fib_entry_index,
     return (FIB_TABLE_WALK_CONTINUE);
 }
 
-
 void
 fib_table_flush (u32 fib_index,
 		 fib_protocol_t proto,
@@ -1359,6 +1380,79 @@ fib_table_flush (u32 fib_index,
     vec_free(ctx.ftf_entries);
 }
 
+static fib_table_walk_rc_t
+fib_table_mark_cb (fib_node_index_t fib_entry_index,
+                   void *arg)
+{
+    fib_table_flush_ctx_t *ctx = arg;
+
+    if (fib_entry_is_sourced(fib_entry_index, ctx->ftf_source))
+    {
+        fib_entry_mark(fib_entry_index, ctx->ftf_source);
+    }
+    return (FIB_TABLE_WALK_CONTINUE);
+}
+
+void
+fib_table_mark (u32 fib_index,
+                fib_protocol_t proto,
+                fib_source_t source)
+{
+    fib_table_flush_ctx_t ctx = {
+        .ftf_source = source,
+    };
+    fib_table_t *fib_table;
+
+    fib_table = fib_table_get(fib_index, proto);
+
+    fib_table->ft_epoch++;
+    fib_table->ft_flags |= FIB_TABLE_FLAG_RESYNC;
+
+    fib_table_walk(fib_index, proto,
+                   fib_table_mark_cb,
+                   &ctx);
+}
+
+static fib_table_walk_rc_t
+fib_table_sweep_cb (fib_node_index_t fib_entry_index,
+                    void *arg)
+{
+    fib_table_flush_ctx_t *ctx = arg;
+
+    if (fib_entry_is_marked(fib_entry_index, ctx->ftf_source))
+    {
+        vec_add1(ctx->ftf_entries, fib_entry_index);
+    }
+    return (FIB_TABLE_WALK_CONTINUE);
+}
+
+void
+fib_table_sweep (u32 fib_index,
+                 fib_protocol_t proto,
+                 fib_source_t source)
+{
+    fib_table_flush_ctx_t ctx = {
+        .ftf_source = source,
+    };
+    fib_node_index_t *fib_entry_index;
+    fib_table_t *fib_table;
+
+    fib_table = fib_table_get(fib_index, proto);
+
+    fib_table->ft_flags &= ~FIB_TABLE_FLAG_RESYNC;
+
+    fib_table_walk(fib_index, proto,
+                   fib_table_sweep_cb,
+                   &ctx);
+
+    vec_foreach(fib_entry_index, ctx.ftf_entries)
+    {
+        fib_table_entry_delete_index(*fib_entry_index, source);
+    }
+
+    vec_free(ctx.ftf_entries);
+}
+
 u8 *
 format_fib_table_memory (u8 *s, va_list *args)
 {
diff --git a/src/vnet/fib/fib_table.h b/src/vnet/fib/fib_table.h
index f13dd77c8b4..74be63d326a 100644
--- a/src/vnet/fib/fib_table.h
+++ b/src/vnet/fib/fib_table.h
@@ -41,15 +41,20 @@ typedef enum fib_table_attribute_t_ {
      */
     FIB_TABLE_ATTRIBUTE_IP6_LL = FIB_TABLE_ATTRIBUTE_FIRST,
     /**
+     * the table is currently resync-ing
+     */
+    FIB_TABLE_ATTRIBUTE_RESYNC,
+    /**
      * Marker. add new entries before this one.
      */
-    FIB_TABLE_ATTRIBUTE_LAST = FIB_TABLE_ATTRIBUTE_IP6_LL,
+    FIB_TABLE_ATTRIBUTE_LAST = FIB_TABLE_ATTRIBUTE_RESYNC,
 } fib_table_attribute_t;
 
 #define FIB_TABLE_ATTRIBUTE_MAX (FIB_TABLE_ATTRIBUTE_LAST+1)
 
 #define FIB_TABLE_ATTRIBUTES {		         \
     [FIB_TABLE_ATTRIBUTE_IP6_LL]  = "ip6-ll",	 \
+    [FIB_TABLE_ATTRIBUTE_RESYNC]  = "resync",    \
 }
 
 #define FOR_EACH_FIB_TABLE_ATTRIBUTE(_item)      	\
@@ -60,8 +65,11 @@ typedef enum fib_table_attribute_t_ {
 typedef enum fib_table_flags_t_ {
     FIB_TABLE_FLAG_NONE   = 0,
     FIB_TABLE_FLAG_IP6_LL  = (1 << FIB_TABLE_ATTRIBUTE_IP6_LL),
+    FIB_TABLE_FLAG_RESYNC  = (1 << FIB_TABLE_ATTRIBUTE_RESYNC),
 } __attribute__ ((packed)) fib_table_flags_t;
 
+extern u8* format_fib_table_flags(u8 *s, va_list *args);
+
 /**
  * @brief 
  *   A protocol Independent FIB table
@@ -109,6 +117,11 @@ typedef struct fib_table_t_
     u32 ft_total_route_counts;
 
     /**
+     * Epoch - number of resyncs performed
+     */
+    u32 ft_epoch;
+
+    /**
      * Table description
      */
     u8* ft_desc;
@@ -624,6 +637,46 @@ extern void fib_table_flush(u32 fib_index,
 
 /**
  * @brief
+ *  Resync all entries from a table for the source
+ *  this is the mark part of the mark and sweep algorithm.
+ *  All entries in this FIB that are sourced by 'source' are marked
+ *  as stale.
+ *
+ * @param fib_index
+ *  The index of the FIB
+ *
+ * @paran proto
+ *  The protocol of the entries in the table
+ *
+ * @param source
+ *  the source to flush
+ */
+extern void fib_table_mark(u32 fib_index,
+                           fib_protocol_t proto,
+                           fib_source_t source);
+
+/**
+ * @brief
+ *  Signal that the table has converged, i.e. all updates are complete.
+ *  this is the sweep part of the mark and sweep algorithm.
+ *  All entries in this FIB that are sourced by 'source' and marked
+ *  as stale are flushed.
+ *
+ * @param fib_index
+ *  The index of the FIB
+ *
+ * @paran proto
+ *  The protocol of the entries in the table
+ *
+ * @param source
+ *  the source to flush
+ */
+extern void fib_table_sweep(u32 fib_index,
+                            fib_protocol_t proto,
+                            fib_source_t source);
+
+/**
+ * @brief
  *  Get the index of the FIB bound to the interface
  *
  * @paran proto
diff --git a/src/vnet/fib/ip4_fib.c b/src/vnet/fib/ip4_fib.c
index 0204b7a129a..02348af87d8 100644
--- a/src/vnet/fib/ip4_fib.c
+++ b/src/vnet/fib/ip4_fib.c
@@ -671,12 +671,14 @@ ip4_show_fib (vlib_main_t * vm,
             continue;
         }
 
-	s = format(s, "%U, fib_index:%d, flow hash:[%U] locks:[",
+	s = format(s, "%U, fib_index:%d, flow hash:[%U] epoch:%d flags:%U locks:[",
                    format_fib_table_name, fib->index,
                    FIB_PROTOCOL_IP4,
                    fib->index,
                    format_ip_flow_hash_config,
-                   fib_table->ft_flow_hash_config);
+                   fib_table->ft_flow_hash_config,
+                   fib_table->ft_epoch,
+                   format_fib_table_flags, fib_table->ft_flags);
 	FOR_EACH_FIB_SOURCE(source)
         {
             if (0 != fib_table->ft_locks[source])
diff --git a/src/vnet/fib/ip6_fib.c b/src/vnet/fib/ip6_fib.c
index cecfcbd1560..991fbc1151b 100644
--- a/src/vnet/fib/ip6_fib.c
+++ b/src/vnet/fib/ip6_fib.c
@@ -679,12 +679,15 @@ ip6_show_fib (vlib_main_t * vm,
         if (fib_table->ft_flags & FIB_TABLE_FLAG_IP6_LL)
             continue;
 
-	s = format(s, "%U, fib_index:%d, flow hash:[%U] locks:[",
+	s = format(s, "%U, fib_index:%d, flow hash:[%U] epoch:%d flags:%U locks:[",
                    format_fib_table_name, fib->index,
                    FIB_PROTOCOL_IP6,
                    fib->index,
                    format_ip_flow_hash_config,
-                   fib_table->ft_flow_hash_config);
+                   fib_table->ft_flow_hash_config,
+                   fib_table->ft_epoch,
+                   format_fib_table_flags, fib_table->ft_flags);
+
 	FOR_EACH_FIB_SOURCE(source)
         {
             if (0 != fib_table->ft_locks[source])
diff --git a/src/vnet/ip/ip.api b/src/vnet/ip/ip.api
index 69c7b88f2e6..b37530ef955 100644
--- a/src/vnet/ip/ip.api
+++ b/src/vnet/ip/ip.api
@@ -65,6 +65,59 @@ define ip_table_dump
   u32 context;
 };
 
+/** \brief IP table replace being
+
+    The use-case is that, for some unspecified reason, the control plane
+    has a very different set of entries it wants in the table than VPP
+    currently has. The CP would thus like to 'replace' VPP's current table
+    only by specifying what the new set of entries shall be, i.e. it is not
+    going to delete anything that already eixts.
+    the CP delcartes the start of this procedure with this begin_replace
+    API Call, and when it has populated all the entries it wants, it calls
+    the below end_replace API. From this point on it is of coursce free
+    to add and delete entries as usual.
+    The underlying mechanism by which VPP implements this replace is
+    purposefully left unspecified.
+
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param table - The table to resync
+*/
+autoreply define ip_table_replace_begin
+{
+  u32 client_index;
+  u32 context;
+  vl_api_ip_table_t table;
+};
+
+/** \brief IP table replace end
+
+    see replace start/
+
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param table - The table that has converged
+*/
+autoreply define ip_table_replace_end
+{
+  u32 client_index;
+  u32 context;
+  vl_api_ip_table_t table;
+};
+
+/** \brief IP table flush
+    Flush a table of all routes
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param table - The table to flush
+*/
+autoreply define ip_table_flush
+{
+  u32 client_index;
+  u32 context;
+  vl_api_ip_table_t table;
+};
+
 /** \brief IP FIB table response
     @param context - sender context
     @param table - description of the table
@@ -957,20 +1010,6 @@ define proxy_arp_intfc_details
   u32 sw_if_index;
 };
 
-/** \brief Reset fib table request
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param vrf_id - vrf/table id of the fib table to reset
-    @param is_ipv6 - an ipv6 fib to reset if non-zero, else ipv4
-*/
-autoreply define reset_fib
-{
-  u32 client_index;
-  u32 context;
-  u32 vrf_id;
-  u8 is_ipv6;
-};
-
 /** \brief Set max allowed ARP or ip6 neighbor entries request
     @param client_index - opaque cookie to identify the sender
     @param context - sender context, to match reply w/ request
diff --git a/src/vnet/ip/ip_api.c b/src/vnet/ip/ip_api.c
index 322a3ed115a..0f9e3709643 100644
--- a/src/vnet/ip/ip_api.c
+++ b/src/vnet/ip/ip_api.c
@@ -92,7 +92,9 @@ _(PROXY_ARP_ADD_DEL, proxy_arp_add_del)                                 \
 _(PROXY_ARP_DUMP, proxy_arp_dump)                                       \
 _(PROXY_ARP_INTFC_ENABLE_DISABLE, proxy_arp_intfc_enable_disable)       \
  _(PROXY_ARP_INTFC_DUMP, proxy_arp_intfc_dump)                          \
-_(RESET_FIB, reset_fib)							\
+_(IP_TABLE_REPLACE_BEGIN, ip_table_replace_begin)                       \
+_(IP_TABLE_REPLACE_END, ip_table_replace_end)                           \
+_(IP_TABLE_FLUSH, ip_table_flush)                                       \
 _(IP_ROUTE_ADD_DEL, ip_route_add_del)                                   \
 _(IP_TABLE_ADD_DEL, ip_table_add_del)                                   \
 _(IP_PUNT_POLICE, ip_punt_police)                                       \
@@ -313,6 +315,7 @@ send_ip_route_details (vpe_api_main_t * am,
   }
 
   vl_api_send_msg (reg, (u8 *) mp);
+  vec_free (rpaths);
 }
 
 typedef struct apt_ip6_fib_show_ctx_t_
@@ -399,14 +402,14 @@ typedef struct vl_api_ip_mfib_dump_ctx_t_
   fib_node_index_t *entries;
 } vl_api_ip_mfib_dump_ctx_t;
 
-static int
+static walk_rc_t
 mfib_route_dump_walk (fib_node_index_t fei, void *arg)
 {
   vl_api_ip_mfib_dump_ctx_t *ctx = arg;
 
   vec_add1 (ctx->entries, fei);
 
-  return (0);
+  return (WALK_CONTINUE);
 }
 
 static void
@@ -2572,137 +2575,90 @@ static void
   REPLY_MACRO (VL_API_IP_SCAN_NEIGHBOR_ENABLE_DISABLE_REPLY);
 }
 
-static int
-ip4_reset_fib_t_handler (vl_api_reset_fib_t * mp)
+static void
+vl_api_ip_table_replace_begin_t_handler (vl_api_ip_table_replace_begin_t * mp)
 {
-  vnet_main_t *vnm = vnet_get_main ();
-  vnet_interface_main_t *im = &vnm->interface_main;
-  ip4_main_t *im4 = &ip4_main;
-  static u32 *sw_if_indices_to_shut;
-  fib_table_t *fib_table;
-  ip4_fib_t *fib;
-  u32 sw_if_index;
-  int i;
-  int rv = VNET_API_ERROR_NO_SUCH_FIB;
-  u32 target_fib_id = ntohl (mp->vrf_id);
-
-  /* *INDENT-OFF* */
-  pool_foreach (fib_table, im4->fibs,
-  ({
-    vnet_sw_interface_t * si;
-
-    fib = pool_elt_at_index (im4->v4_fibs, fib_table->ft_index);
-
-    if (fib->table_id != target_fib_id)
-      continue;
-
-    /* remove any mpls encap/decap labels */
-    mpls_fib_reset_labels (fib->table_id);
-
-    /* remove any proxy arps in this fib */
-    vnet_proxy_arp_fib_reset (fib->table_id);
-
-    /* Set the flow hash for this fib to the default */
-    vnet_set_ip4_flow_hash (fib->table_id, IP_FLOW_HASH_DEFAULT);
-
-    vec_reset_length (sw_if_indices_to_shut);
-
-    /* Shut down interfaces in this FIB / clean out intfc routes */
-    pool_foreach (si, im->sw_interfaces,
-    ({
-      u32 sw_if_index = si->sw_if_index;
-
-      if (sw_if_index < vec_len (im4->fib_index_by_sw_if_index)
-          && (im4->fib_index_by_sw_if_index[si->sw_if_index] ==
-              fib->index))
-        vec_add1 (sw_if_indices_to_shut, si->sw_if_index);
-    }));
+  vl_api_ip_table_replace_begin_reply_t *rmp;
+  fib_protocol_t fproto;
+  u32 fib_index;
+  int rv = 0;
 
-    for (i = 0; i < vec_len (sw_if_indices_to_shut); i++) {
-      sw_if_index = sw_if_indices_to_shut[i];
+  fproto = (mp->table.is_ip6 ? FIB_PROTOCOL_IP6 : FIB_PROTOCOL_IP4);
+  fib_index = fib_table_find (fproto, ntohl (mp->table.table_id));
 
-      u32 flags = vnet_sw_interface_get_flags (vnm, sw_if_index);
-      flags &= ~(VNET_SW_INTERFACE_FLAG_ADMIN_UP);
-      vnet_sw_interface_set_flags (vnm, sw_if_index, flags);
+  if (INDEX_INVALID == fib_index)
+    rv = VNET_API_ERROR_NO_SUCH_FIB;
+  else
+    {
+      fib_table_mark (fib_index, fproto, FIB_SOURCE_API);
+      mfib_table_mark (mfib_table_find (fproto, ntohl (mp->table.table_id)),
+		       fproto, MFIB_SOURCE_API);
     }
-
-    fib_table_flush(fib->index, FIB_PROTOCOL_IP4, FIB_SOURCE_API);
-
-    rv = 0;
-    break;
-    })); /* pool_foreach (fib) */
-    /* *INDENT-ON* */
-
-  return rv;
+  REPLY_MACRO (VL_API_IP_TABLE_REPLACE_BEGIN_REPLY);
 }
 
-static int
-ip6_reset_fib_t_handler (vl_api_reset_fib_t * mp)
+static void
+vl_api_ip_table_replace_end_t_handler (vl_api_ip_table_replace_end_t * mp)
 {
-  vnet_main_t *vnm = vnet_get_main ();
-  vnet_interface_main_t *im = &vnm->interface_main;
-  ip6_main_t *im6 = &ip6_main;
-  static u32 *sw_if_indices_to_shut;
-  fib_table_t *fib_table;
-  ip6_fib_t *fib;
-  u32 sw_if_index;
-  int i;
-  int rv = VNET_API_ERROR_NO_SUCH_FIB;
-  u32 target_fib_id = ntohl (mp->vrf_id);
-
-  /* *INDENT-OFF* */
-  pool_foreach (fib_table, im6->fibs,
-  ({
-    vnet_sw_interface_t * si;
-
-    fib = pool_elt_at_index (im6->v6_fibs, fib_table->ft_index);
-
-    if (fib->table_id != target_fib_id)
-      continue;
-
-    vec_reset_length (sw_if_indices_to_shut);
-
-    /* Set the flow hash for this fib to the default */
-    vnet_set_ip6_flow_hash (fib->table_id, IP_FLOW_HASH_DEFAULT);
-
-    /* Shut down interfaces in this FIB / clean out intfc routes */
-    pool_foreach (si, im->sw_interfaces,
-    ({
-      if (im6->fib_index_by_sw_if_index[si->sw_if_index] ==
-          fib->index)
-        vec_add1 (sw_if_indices_to_shut, si->sw_if_index);
-    }));
+  vl_api_ip_table_replace_end_reply_t *rmp;
+  fib_protocol_t fproto;
+  u32 fib_index;
+  int rv = 0;
 
-    for (i = 0; i < vec_len (sw_if_indices_to_shut); i++) {
-      sw_if_index = sw_if_indices_to_shut[i];
+  fproto = (mp->table.is_ip6 ? FIB_PROTOCOL_IP6 : FIB_PROTOCOL_IP4);
+  fib_index = fib_table_find (fproto, ntohl (mp->table.table_id));
 
-      u32 flags = vnet_sw_interface_get_flags (vnm, sw_if_index);
-      flags &= ~(VNET_SW_INTERFACE_FLAG_ADMIN_UP);
-      vnet_sw_interface_set_flags (vnm, sw_if_index, flags);
+  if (INDEX_INVALID == fib_index)
+    rv = VNET_API_ERROR_NO_SUCH_FIB;
+  else
+    {
+      fib_table_sweep (fib_index, fproto, FIB_SOURCE_API);
+      mfib_table_sweep (mfib_table_find
+			(fproto, ntohl (mp->table.table_id)), fproto,
+			MFIB_SOURCE_API);
     }
-
-    fib_table_flush(fib->index, FIB_PROTOCOL_IP6, FIB_SOURCE_API);
-
-    rv = 0;
-    break;
-  })); /* pool_foreach (fib) */
-  /* *INDENT-ON* */
-
-  return rv;
+  REPLY_MACRO (VL_API_IP_TABLE_REPLACE_END_REPLY);
 }
 
 static void
-vl_api_reset_fib_t_handler (vl_api_reset_fib_t * mp)
+vl_api_ip_table_flush_t_handler (vl_api_ip_table_flush_t * mp)
 {
-  int rv;
-  vl_api_reset_fib_reply_t *rmp;
+  vl_api_ip_table_flush_reply_t *rmp;
+  fib_protocol_t fproto;
+  u32 fib_index;
+  int rv = 0;
 
-  if (mp->is_ipv6)
-    rv = ip6_reset_fib_t_handler (mp);
+  fproto = (mp->table.is_ip6 ? FIB_PROTOCOL_IP6 : FIB_PROTOCOL_IP4);
+  fib_index = fib_table_find (fproto, ntohl (mp->table.table_id));
+
+  if (INDEX_INVALID == fib_index)
+    rv = VNET_API_ERROR_NO_SUCH_FIB;
   else
-    rv = ip4_reset_fib_t_handler (mp);
+    {
+      vnet_main_t *vnm = vnet_get_main ();
+      vnet_interface_main_t *im = &vnm->interface_main;
+      vnet_sw_interface_t *si;
+
+      /* Shut down interfaces in this FIB / clean out intfc routes */
+      /* *INDENT-OFF* */
+      pool_foreach (si, im->sw_interfaces,
+      ({
+        if (fib_index == fib_table_get_index_for_sw_if_index (fproto,
+                                                              si->sw_if_index))
+          {
+            u32 flags = si->flags;
+            flags &= ~VNET_SW_INTERFACE_FLAG_ADMIN_UP;
+            vnet_sw_interface_set_flags (vnm, si->sw_if_index, flags);
+          }
+      }));
+      /* *INDENT-ON* */
+
+      fib_table_flush (fib_index, fproto, FIB_SOURCE_API);
+      mfib_table_flush (mfib_table_find (fproto, ntohl (mp->table.table_id)),
+			fproto, MFIB_SOURCE_API);
+    }
 
-  REPLY_MACRO (VL_API_RESET_FIB_REPLY);
+  REPLY_MACRO (VL_API_IP_TABLE_FLUSH_REPLY);
 }
 
 static void
diff --git a/src/vnet/mfib/ip4_mfib.c b/src/vnet/mfib/ip4_mfib.c
index 1d872f974fd..4da1be61113 100644
--- a/src/vnet/mfib/ip4_mfib.c
+++ b/src/vnet/mfib/ip4_mfib.c
@@ -506,9 +506,10 @@ ip4_show_mfib (vlib_main_t * vm,
             continue;
         }
 
-        vlib_cli_output (vm, "%U, fib_index %d",
+        vlib_cli_output (vm, "%U, fib_index:%d flags:%U",
                          format_mfib_table_name, mfib->index, FIB_PROTOCOL_IP4,
-                         mfib->index);
+                         mfib->index,
+                         format_mfib_table_flags, mfib_table->mft_flags);
 
         /* Show summary? */
         if (! verbose)
diff --git a/src/vnet/mfib/ip6_mfib.c b/src/vnet/mfib/ip6_mfib.c
index 5b15c8d1736..690f4ed9dfd 100644
--- a/src/vnet/mfib/ip6_mfib.c
+++ b/src/vnet/mfib/ip6_mfib.c
@@ -562,14 +562,14 @@ typedef struct ip6_mfib_show_ctx_t_ {
 } ip6_mfib_show_ctx_t;
 
 
-static int
+static walk_rc_t
 ip6_mfib_table_collect_entries (fib_node_index_t mfei, void *arg)
 {
     ip6_mfib_show_ctx_t *ctx = arg;
 
     vec_add1(ctx->entries, mfei);
 
-    return (0);
+    return (WALK_CONTINUE);
 }
 
 static void
@@ -609,7 +609,7 @@ typedef struct ip6_mfib_walk_ctx_t_
     void *i6w_ctx;
 } ip6_mfib_walk_ctx_t;
 
-static int
+static void
 ip6_mfib_walk_cb (clib_bihash_kv_40_8_t * kvp,
                  void *arg)
 {
@@ -617,9 +617,8 @@ ip6_mfib_walk_cb (clib_bihash_kv_40_8_t * kvp,
 
     if ((kvp->key[4] >> 32) == ctx->i6w_mfib_index)
     {
-        return (ctx->i6w_fn(kvp->value, ctx->i6w_ctx));
+        ctx->i6w_fn(kvp->value, ctx->i6w_ctx);
     }
-    return (FIB_TABLE_WALK_CONTINUE);
 }
 
 void
diff --git a/src/vnet/mfib/mfib_entry.c b/src/vnet/mfib/mfib_entry.c
index f169dc0886e..448e6c5e0a8 100644
--- a/src/vnet/mfib/mfib_entry.c
+++ b/src/vnet/mfib/mfib_entry.c
@@ -38,6 +38,7 @@ static mfib_path_ext_t *mfib_path_ext_pool;
  * String names for each source
  */
 static const char *mfib_source_names[] = MFIB_SOURCE_NAMES;
+static const char *mfib_src_attribute_names[] = MFIB_ENTRY_SRC_ATTRIBUTES;
 
 /*
  * Pool for all fib_entries
@@ -102,6 +103,26 @@ format_mfib_entry_path_ext (u8 * s, va_list * args)
 }
 
 u8 *
+format_mfib_entry_src_flags (u8 *s, va_list *args)
+{
+    mfib_entry_src_attribute_t sattr;
+    mfib_entry_src_flags_t flag = va_arg(*args, int);
+
+    if (!flag)
+    {
+        return format(s, "none");
+    }
+
+    FOR_EACH_MFIB_SRC_ATTRIBUTE(sattr) {
+        if ((1 << sattr) & flag) {
+            s = format (s, "%s,", mfib_src_attribute_names[sattr]);
+        }
+    }
+
+    return (s);
+}
+
+u8 *
 format_mfib_entry (u8 * s, va_list * args)
 {
     fib_node_index_t fei, mfi;
@@ -127,14 +148,15 @@ format_mfib_entry (u8 * s, va_list * args)
         s = format (s, " locks:%d\n", mfib_entry->mfe_node.fn_locks);
         vec_foreach(msrc, mfib_entry->mfe_srcs)
         {
-            s = format (s, "  src:%s locks:%d:",
+            s = format (s, "  src:%s flags:%U locks:%d:",
                         mfib_source_names[msrc->mfes_src],
+                        format_mfib_entry_src_flags, msrc->mfes_flags,
                         msrc->mfes_ref_count);
             if (msrc->mfes_cover != FIB_NODE_INDEX_INVALID)
             {
                 s = format (s, " cover:%d", msrc->mfes_cover);
             }
-            s = format (s, " %U\n", format_mfib_entry_flags, msrc->mfes_flags);
+            s = format (s, " %U\n", format_mfib_entry_flags, msrc->mfes_route_flags);
             if (FIB_NODE_INDEX_INVALID != msrc->mfes_pl)
             {
                 s = fib_path_list_format(msrc->mfes_pl, s);
@@ -201,7 +223,7 @@ mfib_entry_src_init (mfib_entry_t *mfib_entry,
 {
     mfib_entry_src_t esrc = {
         .mfes_pl = FIB_NODE_INDEX_INVALID,
-        .mfes_flags = MFIB_ENTRY_FLAG_NONE,
+        .mfes_route_flags = MFIB_ENTRY_FLAG_NONE,
         .mfes_src = source,
         .mfes_cover = FIB_NODE_INDEX_INVALID,
         .mfes_sibling = FIB_NODE_INDEX_INVALID,
@@ -215,8 +237,8 @@ mfib_entry_src_init (mfib_entry_t *mfib_entry,
 
 static mfib_entry_src_t *
 mfib_entry_src_find (const mfib_entry_t *mfib_entry,
-                    mfib_source_t source,
-                    u32 *index)
+                     mfib_source_t source,
+                     u32 *index)
 
 {
     mfib_entry_src_t *esrc;
@@ -269,8 +291,9 @@ mfib_entry_src_update (mfib_entry_t *mfib_entry,
 
     msrc = mfib_entry_src_find_or_create(mfib_entry, source);
 
-    msrc->mfes_flags = entry_flags;
+    msrc->mfes_route_flags = entry_flags;
     msrc->mfes_rpf_id = rpf_id;
+    msrc->mfes_flags &= ~MFIB_ENTRY_SRC_FLAG_STALE;
 
     return (msrc);
 }
@@ -286,6 +309,7 @@ mfib_entry_src_update_and_lock (mfib_entry_t *mfib_entry,
     msrc = mfib_entry_src_update(mfib_entry, source, rpf_id, entry_flags);
 
     msrc->mfes_ref_count++;
+    msrc->mfes_flags &= ~MFIB_ENTRY_SRC_FLAG_STALE;
 
     return (msrc);
 }
@@ -332,6 +356,44 @@ mfib_entry_is_sourced (fib_node_index_t mfib_entry_index,
 }
 
 int
+mfib_entry_is_marked (fib_node_index_t mfib_entry_index,
+                      mfib_source_t source)
+{
+    mfib_entry_t *mfib_entry;
+    mfib_entry_src_t *esrc;
+
+    mfib_entry = mfib_entry_get(mfib_entry_index);
+
+    esrc = mfib_entry_src_find(mfib_entry, source, NULL);
+
+    if (NULL == esrc)
+    {
+        return (0);
+    }
+    else
+    {
+        return (!!(esrc->mfes_flags & MFIB_ENTRY_SRC_FLAG_STALE));
+    }
+}
+
+void
+mfib_entry_mark (fib_node_index_t fib_entry_index,
+                 mfib_source_t source)
+{
+    mfib_entry_t *mfib_entry;
+    mfib_entry_src_t *esrc;
+
+    mfib_entry = mfib_entry_get(fib_entry_index);
+
+    esrc = mfib_entry_src_find(mfib_entry, source, NULL);
+
+    if (NULL != esrc)
+    {
+        esrc->mfes_flags |= MFIB_ENTRY_SRC_FLAG_STALE;
+    }
+}
+
+int
 mfib_entry_is_host (fib_node_index_t mfib_entry_index)
 {
     return (mfib_prefix_is_host(mfib_entry_get_prefix(mfib_entry_index)));
@@ -578,7 +640,7 @@ mfib_entry_stack (mfib_entry_t *mfib_entry,
          * updates to recalculate forwarding.
          */
         mfib_entry->mfe_pl = msrc->mfes_pl;
-        mfib_entry->mfe_flags = msrc->mfes_flags;
+        mfib_entry->mfe_flags = msrc->mfes_route_flags;
         mfib_entry->mfe_itfs = msrc->mfes_itfs;
         mfib_entry->mfe_rpf_id = msrc->mfes_rpf_id;
 
@@ -677,7 +739,9 @@ static fib_node_index_t*
 mfib_entry_src_paths_add (mfib_entry_src_t *msrc,
                           const fib_route_path_t *rpaths)
 {
-    ASSERT(!(MFIB_ENTRY_FLAG_EXCLUSIVE & msrc->mfes_flags));
+    ASSERT(!(MFIB_ENTRY_FLAG_EXCLUSIVE & msrc->mfes_route_flags));
+
+    msrc->mfes_flags &= ~MFIB_ENTRY_SRC_FLAG_STALE;
 
     if (FIB_NODE_INDEX_INVALID == msrc->mfes_pl)
     {
@@ -694,7 +758,9 @@ static fib_node_index_t*
 mfib_entry_src_paths_remove (mfib_entry_src_t *msrc,
                              const fib_route_path_t *rpaths)
 {
-    ASSERT(!(MFIB_ENTRY_FLAG_EXCLUSIVE & msrc->mfes_flags));
+    ASSERT(!(MFIB_ENTRY_FLAG_EXCLUSIVE & msrc->mfes_route_flags));
+
+    msrc->mfes_flags &= ~MFIB_ENTRY_SRC_FLAG_STALE;
 
     return (fib_path_list_paths_remove(msrc->mfes_pl, rpaths));
 }
@@ -792,11 +858,11 @@ static int
 mfib_entry_src_ok_for_delete (const mfib_entry_src_t *msrc)
 {
     return ((INDEX_INVALID == msrc->mfes_cover &&
-             MFIB_ENTRY_FLAG_NONE == msrc->mfes_flags &&
+             MFIB_ENTRY_FLAG_NONE == msrc->mfes_route_flags &&
              0 == fib_path_list_get_n_paths(msrc->mfes_pl)) &&
             (0 == hash_elts(msrc->mfes_itfs)));
 
-    /* return ((MFIB_ENTRY_FLAG_NONE == msrc->mfes_flags) && */
+    /* return ((MFIB_ENTRY_FLAG_NONE == msrc->mfes_route_flags) && */
     /*         (0 == fib_path_list_get_n_paths(msrc->mfes_pl)) && */
     /*         (0 == hash_elts(msrc->mfes_itfs))); */
 }
diff --git a/src/vnet/mfib/mfib_entry.h b/src/vnet/mfib/mfib_entry.h
index 4a1121bc9d5..7087fe87a4d 100644
--- a/src/vnet/mfib/mfib_entry.h
+++ b/src/vnet/mfib/mfib_entry.h
@@ -161,6 +161,8 @@ extern u32 mfib_entry_get_fib_index(fib_node_index_t fib_entry_index);
 extern int mfib_entry_is_sourced(fib_node_index_t fib_entry_index,
                                  mfib_source_t source);
 extern int mfib_entry_is_host(fib_node_index_t fib_entry_index);
+extern int mfib_entry_is_marked(fib_node_index_t fib_entry_index, mfib_source_t source);
+extern void mfib_entry_mark(fib_node_index_t fib_entry_index, mfib_source_t source);
 extern u32 mfib_entry_get_stats_index(fib_node_index_t fib_entry_index);
 extern void mfib_entry_cover_changed(fib_node_index_t fib_entry_index);
 extern void mfib_entry_cover_updated(fib_node_index_t fib_entry_index);
diff --git a/src/vnet/mfib/mfib_entry_src.h b/src/vnet/mfib/mfib_entry_src.h
index 86752e071ec..b85c010779c 100644
--- a/src/vnet/mfib/mfib_entry_src.h
+++ b/src/vnet/mfib/mfib_entry_src.h
@@ -28,6 +28,48 @@ typedef struct mfib_path_ext_t_
 } mfib_path_ext_t;
 
 /**
+ * Flags for the source data
+ */
+typedef enum mfib_entry_src_attribute_t_ {
+    /**
+     * Marker. Add new values after this one.
+     */
+    MFIB_ENTRY_SRC_ATTRIBUTE_FIRST,
+    /**
+     * the source has been added to the entry
+     */
+    MFIB_ENTRY_SRC_ATTRIBUTE_STALE = MFIB_ENTRY_SRC_ATTRIBUTE_FIRST,
+    /**
+     * Marker. add new entries before this one.
+     */
+    MFIB_ENTRY_SRC_ATTRIBUTE_LAST = MFIB_ENTRY_SRC_ATTRIBUTE_STALE,
+} mfib_entry_src_attribute_t;
+
+
+#define MFIB_ENTRY_SRC_ATTRIBUTES {                  \
+    [MFIB_ENTRY_SRC_ATTRIBUTE_STALE] = "stale",      \
+}
+
+#define FOR_EACH_MFIB_SRC_ATTRIBUTE(_item)                      \
+    for (_item = MFIB_ENTRY_SRC_ATTRIBUTE_FIRST;		\
+	 _item <= MFIB_ENTRY_SRC_ATTRIBUTE_LAST;		\
+	 _item++)
+
+typedef enum mfib_entry_src_flag_t_ {
+    MFIB_ENTRY_SRC_FLAG_NONE   = 0,
+    MFIB_ENTRY_SRC_FLAG_STALE = (1 << MFIB_ENTRY_SRC_ATTRIBUTE_STALE),
+} __attribute__ ((packed)) mfib_entry_src_flags_t;
+
+extern u8 * format_mfib_entry_src_flags(u8 *s, va_list *args);
+
+/*
+ * Keep the size of the flags field to 2 bytes, so it
+ * can be placed next to the 2 bytes reference count
+ */
+STATIC_ASSERT (sizeof(mfib_entry_src_flags_t) <= 2,
+	       "FIB entry flags field size too big");
+
+/**
  * The source of an MFIB entry
  */
 typedef struct mfib_entry_src_t_
@@ -40,7 +82,12 @@ typedef struct mfib_entry_src_t_
     /**
      * Route flags
      */
-    mfib_entry_flags_t mfes_flags;
+    mfib_entry_flags_t mfes_route_flags;
+
+    /**
+     * Source flags
+     */
+    mfib_entry_src_flags_t mfes_flags;
 
     /**
      * The reference count on the entry. this is a u32
diff --git a/src/vnet/mfib/mfib_entry_src_rr.c b/src/vnet/mfib/mfib_entry_src_rr.c
index 4512f74a9f5..a6a1e0d8aa5 100644
--- a/src/vnet/mfib/mfib_entry_src_rr.c
+++ b/src/vnet/mfib/mfib_entry_src_rr.c
@@ -62,7 +62,7 @@ mfib_entry_src_rr_activiate (mfib_entry_t *mfib_entry,
 
     msrc->mfes_pl = csrc->mfes_pl;
     fib_path_list_lock(msrc->mfes_pl);
-    msrc->mfes_flags = csrc->mfes_flags;
+    msrc->mfes_route_flags = csrc->mfes_route_flags;
     msrc->mfes_itfs = csrc->mfes_itfs;
     msrc->mfes_exts = csrc->mfes_exts;
     msrc->mfes_rpf_id = csrc->mfes_rpf_id;
@@ -91,7 +91,7 @@ mfib_entry_src_rr_cover_update (mfib_entry_t *mfib_entry,
 
     cover = mfib_entry_get(msrc->mfes_cover);
 
-    msrc->mfes_flags = cover->mfe_flags;
+    msrc->mfes_route_flags = cover->mfe_flags;
     msrc->mfes_itfs = cover->mfe_itfs;
     msrc->mfes_rpf_id = cover->mfe_rpf_id;
 
diff --git a/src/vnet/mfib/mfib_table.c b/src/vnet/mfib/mfib_table.c
index 504333a2474..a6a82774794 100644
--- a/src/vnet/mfib/mfib_table.c
+++ b/src/vnet/mfib/mfib_table.c
@@ -24,6 +24,8 @@
 #include <vnet/mfib/mfib_entry_cover.h>
 #include <vnet/mfib/mfib_signal.h>
 
+const static char * mfib_table_flags_strings[] = MFIB_TABLE_ATTRIBUTES;
+
 mfib_table_t *
 mfib_table_get (fib_node_index_t index,
                 fib_protocol_t proto)
@@ -643,7 +645,7 @@ typedef struct mfib_table_flush_ctx_t_
     mfib_source_t mftf_source;
 } mfib_table_flush_ctx_t;
 
-static int
+static walk_rc_t
 mfib_table_flush_cb (fib_node_index_t mfib_entry_index,
                      void *arg)
 {
@@ -653,7 +655,7 @@ mfib_table_flush_cb (fib_node_index_t mfib_entry_index,
     {
         vec_add1(ctx->mftf_entries, mfib_entry_index);
     }
-    return (1);
+    return (WALK_CONTINUE);
 }
 
 void
@@ -679,6 +681,79 @@ mfib_table_flush (u32 mfib_index,
     vec_free(ctx.mftf_entries);
 }
 
+static walk_rc_t
+mfib_table_mark_cb (fib_node_index_t fib_entry_index,
+                   void *arg)
+{
+    mfib_table_flush_ctx_t *ctx = arg;
+
+    if (mfib_entry_is_sourced(fib_entry_index, ctx->mftf_source))
+    {
+        mfib_entry_mark(fib_entry_index, ctx->mftf_source);
+    }
+    return (WALK_CONTINUE);
+}
+
+void
+mfib_table_mark (u32 fib_index,
+                 fib_protocol_t proto,
+                 mfib_source_t source)
+{
+    mfib_table_flush_ctx_t ctx = {
+        .mftf_source = source,
+    };
+    mfib_table_t *mfib_table;
+
+    mfib_table = mfib_table_get(fib_index, proto);
+
+    mfib_table->mft_epoch++;
+    mfib_table->mft_flags |= MFIB_TABLE_FLAG_RESYNC;
+
+    mfib_table_walk(fib_index, proto,
+                   mfib_table_mark_cb,
+                   &ctx);
+}
+
+static walk_rc_t
+mfib_table_sweep_cb (fib_node_index_t fib_entry_index,
+                    void *arg)
+{
+    mfib_table_flush_ctx_t *ctx = arg;
+
+    if (mfib_entry_is_marked(fib_entry_index, ctx->mftf_source))
+    {
+        vec_add1(ctx->mftf_entries, fib_entry_index);
+    }
+    return (WALK_CONTINUE);
+}
+
+void
+mfib_table_sweep (u32 fib_index,
+                  fib_protocol_t proto,
+                  mfib_source_t source)
+{
+    mfib_table_flush_ctx_t ctx = {
+        .mftf_source = source,
+    };
+    fib_node_index_t *fib_entry_index;
+    mfib_table_t *mfib_table;
+
+    mfib_table = mfib_table_get(fib_index, proto);
+
+    mfib_table->mft_flags &= ~MFIB_TABLE_FLAG_RESYNC;
+
+    mfib_table_walk(fib_index, proto,
+                    mfib_table_sweep_cb,
+                    &ctx);
+
+    vec_foreach(fib_entry_index, ctx.mftf_entries)
+    {
+        mfib_table_entry_delete_index(*fib_entry_index, source);
+    }
+
+    vec_free(ctx.mftf_entries);
+}
+
 static void
 mfib_table_destroy (mfib_table_t *mfib_table)
 {
@@ -770,6 +845,26 @@ mfib_table_walk (u32 fib_index,
 }
 
 u8*
+format_mfib_table_flags (u8 *s, va_list *args)
+{
+    mfib_table_flags_t flags = va_arg(*args, int);
+    mfib_table_attribute_t attr;
+
+    if (!flags)
+    {
+        return format(s, "none");
+    }
+
+    FOR_EACH_MFIB_TABLE_ATTRIBUTE(attr) {
+        if (1 << attr & flags) {
+            s = format(s, "%s", mfib_table_flags_strings[attr]);
+        }
+    }
+
+    return (s);
+}
+
+u8*
 format_mfib_table_name (u8* s, va_list *ap)
 {
     fib_node_index_t fib_index = va_arg(*ap, fib_node_index_t);
diff --git a/src/vnet/mfib/mfib_table.h b/src/vnet/mfib/mfib_table.h
index 47461375a10..d6b6dc3845d 100644
--- a/src/vnet/mfib/mfib_table.h
+++ b/src/vnet/mfib/mfib_table.h
@@ -29,6 +29,42 @@
 #define MFIB_TABLE_TOTAL_LOCKS MFIB_N_SOURCES
 
 /**
+ * Flags for the source data
+ */
+typedef enum mfib_table_attribute_t_ {
+    /**
+     * Marker. Add new values after this one.
+     */
+    MFIB_TABLE_ATTRIBUTE_FIRST,
+    /**
+     * the table is currently resync-ing
+     */
+    MFIB_TABLE_ATTRIBUTE_RESYNC = MFIB_TABLE_ATTRIBUTE_FIRST,
+    /**
+     * Marker. add new entries before this one.
+     */
+    MFIB_TABLE_ATTRIBUTE_LAST = MFIB_TABLE_ATTRIBUTE_RESYNC,
+} mfib_table_attribute_t;
+
+#define MFIB_TABLE_ATTRIBUTE_MAX (MFIB_TABLE_ATTRIBUTE_LAST+1)
+
+#define MFIB_TABLE_ATTRIBUTES {		         \
+    [MFIB_TABLE_ATTRIBUTE_RESYNC]  = "resync",    \
+}
+
+#define FOR_EACH_MFIB_TABLE_ATTRIBUTE(_item)      	\
+    for (_item = MFIB_TABLE_ATTRIBUTE_FIRST;		\
+	 _item < MFIB_TABLE_ATTRIBUTE_MAX;		\
+	 _item++)
+
+typedef enum mfib_table_flags_t_ {
+    MFIB_TABLE_FLAG_NONE   = 0,
+    MFIB_TABLE_FLAG_RESYNC  = (1 << MFIB_TABLE_ATTRIBUTE_RESYNC),
+} __attribute__ ((packed)) mfib_table_flags_t;
+
+extern u8* format_mfib_table_flags(u8 *s, va_list *args);
+
+/**
  * @brief
  *   A protocol Independent IP multicast FIB table
  */
@@ -56,6 +92,11 @@ typedef struct mfib_table_t_
     fib_protocol_t mft_proto;
 
     /**
+     * table falgs
+     */
+    mfib_table_flags_t mft_flags;
+
+    /**
      * number of locks on the table
      */
     u16 mft_locks[MFIB_TABLE_N_LOCKS];
@@ -66,6 +107,11 @@ typedef struct mfib_table_t_
     u32 mft_table_id;
 
     /**
+     * resync epoch
+     */
+    u32 mft_epoch;
+
+    /**
      * Index into FIB vector.
      */
     fib_node_index_t mft_index;
@@ -282,6 +328,46 @@ extern void mfib_table_flush(u32 fib_index,
 
 /**
  * @brief
+ *  Resync all entries from a table for the source
+ *  this is the mark part of the mark and sweep algorithm.
+ *  All entries in this FIB that are sourced by 'source' are marked
+ *  as stale.
+ *
+ * @param fib_index
+ *  The index of the FIB
+ *
+ * @paran proto
+ *  The protocol of the entries in the table
+ *
+ * @param source
+ *  the source to flush
+ */
+extern void mfib_table_mark(u32 fib_index,
+                            fib_protocol_t proto,
+                            mfib_source_t source);
+
+/**
+ * @brief
+ *  Signal that the table has converged, i.e. all updates are complete.
+ *  this is the sweep part of the mark and sweep algorithm.
+ *  All entries in this FIB that are sourced by 'source' and marked
+ *  as stale are flushed.
+ *
+ * @param fib_index
+ *  The index of the FIB
+ *
+ * @paran proto
+ *  The protocol of the entries in the table
+ *
+ * @param source
+ *  the source to flush
+ */
+extern void mfib_table_sweep(u32 fib_index,
+                             fib_protocol_t proto,
+                             mfib_source_t source);
+
+/**
+ * @brief
  *  Get the index of the FIB bound to the interface
  *
  * @paran proto
@@ -465,8 +551,8 @@ extern mfib_table_t *mfib_table_get(fib_node_index_t index,
 /**
  * @brief Call back function when walking entries in a FIB table
  */
-typedef int (*mfib_table_walk_fn_t)(fib_node_index_t fei,
-                                    void *ctx);
+typedef walk_rc_t (*mfib_table_walk_fn_t)(fib_node_index_t fei,
+                                          void *ctx);
 
 /**
  * @brief Walk all entries in a FIB table
diff --git a/src/vnet/mpls/mpls.c b/src/vnet/mpls/mpls.c
index 05361de505d..4076a8980a9 100644
--- a/src/vnet/mpls/mpls.c
+++ b/src/vnet/mpls/mpls.c
@@ -443,13 +443,6 @@ VLIB_CLI_COMMAND (mpls_table_command, static) = {
   .is_mp_safe = 1,
 };
 
-int
-mpls_fib_reset_labels (u32 fib_id)
-{
-  // FIXME
-  return 0;
-}
-
 static clib_error_t *
 mpls_init (vlib_main_t * vm)
 {
diff --git a/src/vnet/mpls/mpls.h b/src/vnet/mpls/mpls.h
index 918a3d83271..2cd41ea01c6 100644
--- a/src/vnet/mpls/mpls.h
+++ b/src/vnet/mpls/mpls.h
@@ -90,8 +90,6 @@ int mpls_sw_interface_enable_disable (mpls_main_t * mm,
 
 u8 mpls_sw_interface_is_enabled (u32 sw_if_index);
 
-int mpls_fib_reset_labels (u32 fib_id);
-
 int mpls_dest_cmp (void *a1, void *a2);
 
 int mpls_fib_index_cmp (void *a1, void *a2);