21 files changed, 666 insertions, 219 deletions

diff --git a/src/vnet/ethernet/arp.c b/src/vnet/ethernet/arp.c
index ca9a21cbbd1..6b1069ecb6d 100644
--- a/src/vnet/ethernet/arp.c
+++ b/src/vnet/ethernet/arp.c
@@ -2476,41 +2476,6 @@ proxy_arp_intfc_walk (proxy_arp_intf_walk_t cb, void *data)
   }
 }
 
-/*
- * Remove any proxy arp entries associated with the
- * specified fib.
- */
-int
-vnet_proxy_arp_fib_reset (u32 fib_id)
-{
-  ethernet_arp_main_t *am = &ethernet_arp_main;
-  ethernet_proxy_arp_t *pa;
-  u32 *entries_to_delete = 0;
-  u32 fib_index;
-  int i;
-
-  fib_index = fib_table_find (FIB_PROTOCOL_IP4, fib_id);
-  if (~0 == fib_index)
-    return VNET_API_ERROR_NO_SUCH_ENTRY;
-
-  vec_foreach (pa, am->proxy_arps)
-  {
-    if (pa->fib_index == fib_index)
-      {
-	vec_add1 (entries_to_delete, pa - am->proxy_arps);
-      }
-  }
-
-  for (i = 0; i < vec_len (entries_to_delete); i++)
-    {
-      vec_delete (am->proxy_arps, 1, entries_to_delete[i]);
-    }
-
-  vec_free (entries_to_delete);
-
-  return 0;
-}
-
 static clib_error_t *
 ip_arp_add_del_command_fn (vlib_main_t * vm,
 			   unformat_input_t * input, vlib_cli_command_t * cmd)
diff --git a/src/vnet/ethernet/arp.h b/src/vnet/ethernet/arp.h
index 56d9f29e09f..de92c4f419c 100644
--- a/src/vnet/ethernet/arp.h
+++ b/src/vnet/ethernet/arp.h
@@ -62,8 +62,6 @@ extern int vnet_arp_unset_ip4_over_ethernet (vnet_main_t * vnm,
 					     ethernet_arp_ip4_over_ethernet_address_t
 					     * a);
 
-extern int vnet_proxy_arp_fib_reset (u32 fib_id);
-
 void vnet_register_ip4_arp_resolution_event (vnet_main_t * vnm,
 					     void *address_arg,
 					     uword node_index,
diff --git a/src/vnet/fib/fib_entry.h b/src/vnet/fib/fib_entry.h
index 4ce28b1ae9d..f0e6e8d8aae 100644
--- a/src/vnet/fib/fib_entry.h
+++ b/src/vnet/fib/fib_entry.h
@@ -315,6 +315,10 @@ typedef enum fib_entry_src_attribute_t_ {
      */
     FIB_ENTRY_SRC_ATTRIBUTE_ACTIVE,
     /**
+     * the source is stale
+     */
+    FIB_ENTRY_SRC_ATTRIBUTE_STALE,
+    /**
      * the source is inherited from its cover
      */
     FIB_ENTRY_SRC_ATTRIBUTE_INHERITED,
@@ -329,6 +333,7 @@ typedef enum fib_entry_src_attribute_t_ {
     [FIB_ENTRY_SRC_ATTRIBUTE_ADDED]  = "added",	 \
     [FIB_ENTRY_SRC_ATTRIBUTE_CONTRIBUTING] = "contributing", \
     [FIB_ENTRY_SRC_ATTRIBUTE_ACTIVE] = "active", \
+    [FIB_ENTRY_SRC_ATTRIBUTE_STALE] = "stale",      \
     [FIB_ENTRY_SRC_ATTRIBUTE_INHERITED] = "inherited", \
 }
 
@@ -342,6 +347,7 @@ typedef enum fib_entry_src_flag_t_ {
     FIB_ENTRY_SRC_FLAG_ADDED  = (1 << FIB_ENTRY_SRC_ATTRIBUTE_ADDED),
     FIB_ENTRY_SRC_FLAG_CONTRIBUTING = (1 << FIB_ENTRY_SRC_ATTRIBUTE_CONTRIBUTING),
     FIB_ENTRY_SRC_FLAG_ACTIVE = (1 << FIB_ENTRY_SRC_ATTRIBUTE_ACTIVE),
+    FIB_ENTRY_SRC_FLAG_STALE = (1 << FIB_ENTRY_SRC_ATTRIBUTE_STALE),
     FIB_ENTRY_SRC_FLAG_INHERITED = (1 << FIB_ENTRY_SRC_ATTRIBUTE_INHERITED),
 } __attribute__ ((packed)) fib_entry_src_flag_t;
 
@@ -627,6 +633,8 @@ extern int fib_entry_is_sourced(fib_node_index_t fib_entry_index,
 extern fib_node_index_t fib_entry_get_path_list(fib_node_index_t fib_entry_index);
 extern int fib_entry_is_resolved(fib_node_index_t fib_entry_index);
 extern int fib_entry_is_host(fib_node_index_t fib_entry_index);
+extern int fib_entry_is_marked(fib_node_index_t fib_entry_index, fib_source_t source);
+extern void fib_entry_mark(fib_node_index_t fib_entry_index, fib_source_t source);
 extern void fib_entry_set_flow_hash_config(fib_node_index_t fib_entry_index,
                                            flow_hash_config_t hash_config);
 
diff --git a/src/vnet/fib/fib_entry_src.c b/src/vnet/fib/fib_entry_src.c
index 6ed13a328bd..176818abc0a 100644
--- a/src/vnet/fib/fib_entry_src.c
+++ b/src/vnet/fib/fib_entry_src.c
@@ -141,6 +141,44 @@ fib_entry_is_sourced (fib_node_index_t fib_entry_index,
     return (NULL != fib_entry_src_find(fib_entry, source));
 }
 
+int
+fib_entry_is_marked (fib_node_index_t fib_entry_index,
+                      fib_source_t source)
+{
+    fib_entry_t *fib_entry;
+    fib_entry_src_t *esrc;
+
+    fib_entry = fib_entry_get(fib_entry_index);
+
+    esrc = fib_entry_src_find(fib_entry, source);
+
+    if (NULL == esrc)
+    {
+        return (0);
+    }
+    else
+    {
+        return (!!(esrc->fes_flags & FIB_ENTRY_SRC_FLAG_STALE));
+    }
+}
+
+void
+fib_entry_mark (fib_node_index_t fib_entry_index,
+                fib_source_t source)
+{
+    fib_entry_t *fib_entry;
+    fib_entry_src_t *esrc;
+
+    fib_entry = fib_entry_get(fib_entry_index);
+
+    esrc = fib_entry_src_find(fib_entry, source);
+
+    if (NULL != esrc)
+    {
+        esrc->fes_flags |= FIB_ENTRY_SRC_FLAG_STALE;
+    }
+}
+
 static fib_entry_src_t *
 fib_entry_src_find_or_create (fib_entry_t *fib_entry,
 			      fib_source_t source,
diff --git a/src/vnet/fib/fib_entry_src.h b/src/vnet/fib/fib_entry_src.h
index a859b9c734a..8f13ae894bb 100644
--- a/src/vnet/fib/fib_entry_src.h
+++ b/src/vnet/fib/fib_entry_src.h
@@ -226,20 +226,24 @@ typedef struct fib_entry_src_vft_t_ {
     }								\
 }
 
-#define FIB_ENTRY_SRC_VFT_INVOKE(esrc, func, args)  \
-{                                                   \
-    const fib_entry_src_vft_t *_vft;                \
-    _vft = fib_entry_src_get_vft(esrc);             \
-    if (_vft->func)                                 \
-        _vft->func args;                            \
+#define FIB_ENTRY_SRC_VFT_INVOKE(esrc, func, args)             \
+{                                                              \
+    const fib_entry_src_vft_t *_vft;                           \
+    _vft = fib_entry_src_get_vft(esrc);                        \
+    if (_vft->func) {                                          \
+        (esrc)->fes_flags &= ~FIB_ENTRY_SRC_FLAG_STALE;        \
+        _vft->func args;                                       \
+    }                                                          \
 }
 
 #define FIB_ENTRY_SRC_VFT_INVOKE_AND_RETURN(esrc, func, args)  \
-{                                                   \
-    const fib_entry_src_vft_t *_vft;                \
-    _vft = fib_entry_src_get_vft(esrc);             \
-    if (_vft->func)                                 \
-        return (_vft->func args);                   \
+{                                                              \
+    const fib_entry_src_vft_t *_vft;                           \
+    _vft = fib_entry_src_get_vft(esrc);                        \
+    if (_vft->func) {                                          \
+        (esrc)->fes_flags &= ~FIB_ENTRY_SRC_FLAG_STALE;        \
+        return (_vft->func args);                              \
+    }                                                          \
 }
 
 #define FIB_ENTRY_SRC_VFT_EXISTS(esrc, func)        \
diff --git a/src/vnet/fib/fib_table.c b/src/vnet/fib/fib_table.c
index 3778fa9a944..d3cf5dc88b1 100644
--- a/src/vnet/fib/fib_table.c
+++ b/src/vnet/fib/fib_table.c
@@ -23,6 +23,8 @@
 #include <vnet/fib/ip6_fib.h>
 #include <vnet/fib/mpls_fib.h>
 
+const static char * fib_table_flags_strings[] = FIB_TABLE_ATTRIBUTES;
+
 fib_table_t *
 fib_table_get (fib_node_index_t index,
 	       fib_protocol_t proto)
@@ -1305,6 +1307,26 @@ format_fib_table_name (u8* s, va_list* ap)
     return (s);
 }
 
+u8*
+format_fib_table_flags (u8 *s, va_list *args)
+{
+    fib_table_flags_t flags = va_arg(*args, int);
+    fib_table_attribute_t attr;
+
+    if (!flags)
+    {
+        return format(s, "none");
+    }
+
+    FOR_EACH_FIB_TABLE_ATTRIBUTE(attr) {
+        if (1 << attr & flags) {
+            s = format(s, "%s", fib_table_flags_strings[attr]);
+        }
+    }
+
+    return (s);
+}
+
 /**
  * @brief Table flush context. Store the indicies of matching FIB entries
  * that need to be removed.
@@ -1335,7 +1357,6 @@ fib_table_flush_cb (fib_node_index_t fib_entry_index,
     return (FIB_TABLE_WALK_CONTINUE);
 }
 
-
 void
 fib_table_flush (u32 fib_index,
 		 fib_protocol_t proto,
@@ -1359,6 +1380,79 @@ fib_table_flush (u32 fib_index,
     vec_free(ctx.ftf_entries);
 }
 
+static fib_table_walk_rc_t
+fib_table_mark_cb (fib_node_index_t fib_entry_index,
+                   void *arg)
+{
+    fib_table_flush_ctx_t *ctx = arg;
+
+    if (fib_entry_is_sourced(fib_entry_index, ctx->ftf_source))
+    {
+        fib_entry_mark(fib_entry_index, ctx->ftf_source);
+    }
+    return (FIB_TABLE_WALK_CONTINUE);
+}
+
+void
+fib_table_mark (u32 fib_index,
+                fib_protocol_t proto,
+                fib_source_t source)
+{
+    fib_table_flush_ctx_t ctx = {
+        .ftf_source = source,
+    };
+    fib_table_t *fib_table;
+
+    fib_table = fib_table_get(fib_index, proto);
+
+    fib_table->ft_epoch++;
+    fib_table->ft_flags |= FIB_TABLE_FLAG_RESYNC;
+
+    fib_table_walk(fib_index, proto,
+                   fib_table_mark_cb,
+                   &ctx);
+}
+
+static fib_table_walk_rc_t
+fib_table_sweep_cb (fib_node_index_t fib_entry_index,
+                    void *arg)
+{
+    fib_table_flush_ctx_t *ctx = arg;
+
+    if (fib_entry_is_marked(fib_entry_index, ctx->ftf_source))
+    {
+        vec_add1(ctx->ftf_entries, fib_entry_index);
+    }
+    return (FIB_TABLE_WALK_CONTINUE);
+}
+
+void
+fib_table_sweep (u32 fib_index,
+                 fib_protocol_t proto,
+                 fib_source_t source)
+{
+    fib_table_flush_ctx_t ctx = {
+        .ftf_source = source,
+    };
+    fib_node_index_t *fib_entry_index;
+    fib_table_t *fib_table;
+
+    fib_table = fib_table_get(fib_index, proto);
+
+    fib_table->ft_flags &= ~FIB_TABLE_FLAG_RESYNC;
+
+    fib_table_walk(fib_index, proto,
+                   fib_table_sweep_cb,
+                   &ctx);
+
+    vec_foreach(fib_entry_index, ctx.ftf_entries)
+    {
+        fib_table_entry_delete_index(*fib_entry_index, source);
+    }
+
+    vec_free(ctx.ftf_entries);
+}
+
 u8 *
 format_fib_table_memory (u8 *s, va_list *args)
 {
diff --git a/src/vnet/fib/fib_table.h b/src/vnet/fib/fib_table.h
index f13dd77c8b4..74be63d326a 100644
--- a/src/vnet/fib/fib_table.h
+++ b/src/vnet/fib/fib_table.h
@@ -41,15 +41,20 @@ typedef enum fib_table_attribute_t_ {
      */
     FIB_TABLE_ATTRIBUTE_IP6_LL = FIB_TABLE_ATTRIBUTE_FIRST,
     /**
+     * the table is currently resync-ing
+     */
+    FIB_TABLE_ATTRIBUTE_RESYNC,
+    /**
      * Marker. add new entries before this one.
      */
-    FIB_TABLE_ATTRIBUTE_LAST = FIB_TABLE_ATTRIBUTE_IP6_LL,
+    FIB_TABLE_ATTRIBUTE_LAST = FIB_TABLE_ATTRIBUTE_RESYNC,
 } fib_table_attribute_t;
 
 #define FIB_TABLE_ATTRIBUTE_MAX (FIB_TABLE_ATTRIBUTE_LAST+1)
 
 #define FIB_TABLE_ATTRIBUTES {		         \
     [FIB_TABLE_ATTRIBUTE_IP6_LL]  = "ip6-ll",	 \
+    [FIB_TABLE_ATTRIBUTE_RESYNC]  = "resync",    \
 }
 
 #define FOR_EACH_FIB_TABLE_ATTRIBUTE(_item)      	\
@@ -60,8 +65,11 @@ typedef enum fib_table_attribute_t_ {
 typedef enum fib_table_flags_t_ {
     FIB_TABLE_FLAG_NONE   = 0,
     FIB_TABLE_FLAG_IP6_LL  = (1 << FIB_TABLE_ATTRIBUTE_IP6_LL),
+    FIB_TABLE_FLAG_RESYNC  = (1 << FIB_TABLE_ATTRIBUTE_RESYNC),
 } __attribute__ ((packed)) fib_table_flags_t;
 
+extern u8* format_fib_table_flags(u8 *s, va_list *args);
+
 /**
  * @brief 
  *   A protocol Independent FIB table
@@ -109,6 +117,11 @@ typedef struct fib_table_t_
     u32 ft_total_route_counts;
 
     /**
+     * Epoch - number of resyncs performed
+     */
+    u32 ft_epoch;
+
+    /**
      * Table description
      */
     u8* ft_desc;
@@ -624,6 +637,46 @@ extern void fib_table_flush(u32 fib_index,
 
 /**
  * @brief
+ *  Resync all entries from a table for the source
+ *  this is the mark part of the mark and sweep algorithm.
+ *  All entries in this FIB that are sourced by 'source' are marked
+ *  as stale.
+ *
+ * @param fib_index
+ *  The index of the FIB
+ *
+ * @paran proto
+ *  The protocol of the entries in the table
+ *
+ * @param source
+ *  the source to flush
+ */
+extern void fib_table_mark(u32 fib_index,
+                           fib_protocol_t proto,
+                           fib_source_t source);
+
+/**
+ * @brief
+ *  Signal that the table has converged, i.e. all updates are complete.
+ *  this is the sweep part of the mark and sweep algorithm.
+ *  All entries in this FIB that are sourced by 'source' and marked
+ *  as stale are flushed.
+ *
+ * @param fib_index
+ *  The index of the FIB
+ *
+ * @paran proto
+ *  The protocol of the entries in the table
+ *
+ * @param source
+ *  the source to flush
+ */
+extern void fib_table_sweep(u32 fib_index,
+                            fib_protocol_t proto,
+                            fib_source_t source);
+
+/**
+ * @brief
  *  Get the index of the FIB bound to the interface
  *
  * @paran proto
diff --git a/src/vnet/fib/ip4_fib.c b/src/vnet/fib/ip4_fib.c
index 0204b7a129a..02348af87d8 100644
--- a/src/vnet/fib/ip4_fib.c
+++ b/src/vnet/fib/ip4_fib.c
@@ -671,12 +671,14 @@ ip4_show_fib (vlib_main_t * vm,
             continue;
         }
 
-	s = format(s, "%U, fib_index:%d, flow hash:[%U] locks:[",
+	s = format(s, "%U, fib_index:%d, flow hash:[%U] epoch:%d flags:%U locks:[",
                    format_fib_table_name, fib->index,
                    FIB_PROTOCOL_IP4,
                    fib->index,
                    format_ip_flow_hash_config,
-                   fib_table->ft_flow_hash_config);
+                   fib_table->ft_flow_hash_config,
+                   fib_table->ft_epoch,
+                   format_fib_table_flags, fib_table->ft_flags);
 	FOR_EACH_FIB_SOURCE(source)
         {
             if (0 != fib_table->ft_locks[source])
diff --git a/src/vnet/fib/ip6_fib.c b/src/vnet/fib/ip6_fib.c
index cecfcbd1560..991fbc1151b 100644
--- a/src/vnet/fib/ip6_fib.c
+++ b/src/vnet/fib/ip6_fib.c
@@ -679,12 +679,15 @@ ip6_show_fib (vlib_main_t * vm,
         if (fib_table->ft_flags & FIB_TABLE_FLAG_IP6_LL)
             continue;
 
-	s = format(s, "%U, fib_index:%d, flow hash:[%U] locks:[",
+	s = format(s, "%U, fib_index:%d, flow hash:[%U] epoch:%d flags:%U locks:[",
                    format_fib_table_name, fib->index,
                    FIB_PROTOCOL_IP6,
                    fib->index,
                    format_ip_flow_hash_config,
-                   fib_table->ft_flow_hash_config);
+                   fib_table->ft_flow_hash_config,
+                   fib_table->ft_epoch,
+                   format_fib_table_flags, fib_table->ft_flags);
+
 	FOR_EACH_FIB_SOURCE(source)
         {
             if (0 != fib_table->ft_locks[source])
diff --git a/src/vnet/ip/ip.api b/src/vnet/ip/ip.api
index 69c7b88f2e6..b37530ef955 100644
--- a/src/vnet/ip/ip.api
+++ b/src/vnet/ip/ip.api
@@ -65,6 +65,59 @@ define ip_table_dump
   u32 context;
 };
 
+/** \brief IP table replace being
+
+    The use-case is that, for some unspecified reason, the control plane
+    has a very different set of entries it wants in the table than VPP
+    currently has. The CP would thus like to 'replace' VPP's current table
+    only by specifying what the new set of entries shall be, i.e. it is not
+    going to delete anything that already eixts.
+    the CP delcartes the start of this procedure with this begin_replace
+    API Call, and when it has populated all the entries it wants, it calls
+    the below end_replace API. From this point on it is of coursce free
+    to add and delete entries as usual.
+    The underlying mechanism by which VPP implements this replace is
+    purposefully left unspecified.
+
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param table - The table to resync
+*/
+autoreply define ip_table_replace_begin
+{
+  u32 client_index;
+  u32 context;
+  vl_api_ip_table_t table;
+};
+
+/** \brief IP table replace end
+
+    see replace start/
+
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param table - The table that has converged
+*/
+autoreply define ip_table_replace_end
+{
+  u32 client_index;
+  u32 context;
+  vl_api_ip_table_t table;
+};
+
+/** \brief IP table flush
+    Flush a table of all routes
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param table - The table to flush
+*/
+autoreply define ip_table_flush
+{
+  u32 client_index;
+  u32 context;
+  vl_api_ip_table_t table;
+};
+
 /** \brief IP FIB table response
     @param context - sender context
     @param table - description of the table
@@ -957,20 +1010,6 @@ define proxy_arp_intfc_details
   u32 sw_if_index;
 };
 
-/** \brief Reset fib table request
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param vrf_id - vrf/table id of the fib table to reset
-    @param is_ipv6 - an ipv6 fib to reset if non-zero, else ipv4
-*/
-autoreply define reset_fib
-{
-  u32 client_index;
-  u32 context;
-  u32 vrf_id;
-  u8 is_ipv6;
-};
-
 /** \brief Set max allowed ARP or ip6 neighbor entries request
     @param client_index - opaque cookie to identify the sender
     @param context - sender context, to match reply w/ request
diff --git a/src/vnet/ip/ip_api.c b/src/vnet/ip/ip_api.c
index 322a3ed115a..0f9e3709643 100644
--- a/src/vnet/ip/ip_api.c
+++ b/src/vnet/ip/ip_api.c
@@ -92,7 +92,9 @@ _(PROXY_ARP_ADD_DEL, proxy_arp_add_del)                                 \
 _(PROXY_ARP_DUMP, proxy_arp_dump)                                       \
 _(PROXY_ARP_INTFC_ENABLE_DISABLE, proxy_arp_intfc_enable_disable)       \
  _(PROXY_ARP_INTFC_DUMP, proxy_arp_intfc_dump)                          \
-_(RESET_FIB, reset_fib)							\
+_(IP_TABLE_REPLACE_BEGIN, ip_table_replace_begin)                       \
+_(IP_TABLE_REPLACE_END, ip_table_replace_end)                           \
+_(IP_TABLE_FLUSH, ip_table_flush)                                       \
 _(IP_ROUTE_ADD_DEL, ip_route_add_del)                                   \
 _(IP_TABLE_ADD_DEL, ip_table_add_del)                                   \
 _(IP_PUNT_POLICE, ip_punt_police)                                       \
@@ -313,6 +315,7 @@ send_ip_route_details (vpe_api_main_t * am,
   }
 
   vl_api_send_msg (reg, (u8 *) mp);
+  vec_free (rpaths);
 }
 
 typedef struct apt_ip6_fib_show_ctx_t_
@@ -399,14 +402,14 @@ typedef struct vl_api_ip_mfib_dump_ctx_t_
   fib_node_index_t *entries;
 } vl_api_ip_mfib_dump_ctx_t;
 
-static int
+static walk_rc_t
 mfib_route_dump_walk (fib_node_index_t fei, void *arg)
 {
   vl_api_ip_mfib_dump_ctx_t *ctx = arg;
 
   vec_add1 (ctx->entries, fei);
 
-  return (0);
+  return (WALK_CONTINUE);
 }
 
 static void
@@ -2572,137 +2575,90 @@ static void
   REPLY_MACRO (VL_API_IP_SCAN_NEIGHBOR_ENABLE_DISABLE_REPLY);
 }
 
-static int
-ip4_reset_fib_t_handler (vl_api_reset_fib_t * mp)
+static void
+vl_api_ip_table_replace_begin_t_handler (vl_api_ip_table_replace_begin_t * mp)
 {
-  vnet_main_t *vnm = vnet_get_main ();
-  vnet_interface_main_t *im = &vnm->interface_main;
-  ip4_main_t *im4 = &ip4_main;
-  static u32 *sw_if_indices_to_shut;
-  fib_table_t *fib_table;
-  ip4_fib_t *fib;
-  u32 sw_if_index;
-  int i;
-  int rv = VNET_API_ERROR_NO_SUCH_FIB;
-  u32 target_fib_id = ntohl (mp->vrf_id);
-
-  /* *INDENT-OFF* */
-  pool_foreach (fib_table, im4->fibs,
-  ({
-    vnet_sw_interface_t * si;
-
-    fib = pool_elt_at_index (im4->v4_fibs, fib_table->ft_index);
-
-    if (fib->table_id != target_fib_id)
-      continue;
-
-    /* remove any mpls encap/decap labels */
-    mpls_fib_reset_labels (fib->table_id);
-
-    /* remove any proxy arps in this fib */
-    vnet_proxy_arp_fib_reset (fib->table_id);
-
-    /* Set the flow hash for this fib to the default */
-    vnet_set_ip4_flow_hash (fib->table_id, IP_FLOW_HASH_DEFAULT);
-
-    vec_reset_length (sw_if_indices_to_shut);
-
-    /* Shut down interfaces in this FIB / clean out intfc routes */
-    pool_foreach (si, im->sw_interfaces,
-    ({
-      u32 sw_if_index = si->sw_if_index;
-
-      if (sw_if_index < vec_len (im4->fib_index_by_sw_if_index)
-          && (im4->fib_index_by_sw_if_index[si->sw_if_index] ==
-              fib->index))
-        vec_add1 (sw_if_indices_to_shut, si->sw_if_index);
-    }));
+  vl_api_ip_table_replace_begin_reply_t *rmp;
+  fib_protocol_t fproto;
+  u32 fib_index;
+  int rv = 0;
 
-    for (i = 0; i < vec_len (sw_if_indices_to_shut); i++) {
-      sw_if_index = sw_if_indices_to_shut[i];
+  fproto = (mp->table.is_ip6 ? FIB_PROTOCOL_IP6 : FIB_PROTOCOL_IP4);
+  fib_index = fib_table_find (fproto, ntohl (mp->table.table_id));
 
-      u32 flags = vnet_sw_interface_get_flags (vnm, sw_if_index);
-      flags &= ~(VNET_SW_INTERFACE_FLAG_ADMIN_UP);
-      vnet_sw_interface_set_flags (vnm, sw_if_index, flags);
+  if (INDEX_INVALID == fib_index)
+    rv = VNET_API_ERROR_NO_SUCH_FIB;
+  else
+    {
+      fib_table_mark (fib_index, fproto, FIB_SOURCE_API);
+      mfib_table_mark (mfib_table_find (fproto, ntohl (mp->table.table_id)),
+		       fproto, MFIB_SOURCE_API);
     }
-
-    fib_table_flush(fib->index, FIB_PROTOCOL_IP4, FIB_SOURCE_API);
-
-    rv = 0;
-    break;
-    })); /* pool_foreach (fib) */
-    /* *INDENT-ON* */
-
-  return rv;
+  REPLY_MACRO (VL_API_IP_TABLE_REPLACE_BEGIN_REPLY);
 }
 
-static int
-ip6_reset_fib_t_handler (vl_api_reset_fib_t * mp)
+static void
+vl_api_ip_table_replace_end_t_handler (vl_api_ip_table_replace_end_t * mp)
 {
-  vnet_main_t *vnm = vnet_get_main ();
-  vnet_interface_main_t *im = &vnm->interface_main;
-  ip6_main_t *im6 = &ip6_main;
-  static u32 *sw_if_indices_to_shut;
-  fib_table_t *fib_table;
-  ip6_fib_t *fib;
-  u32 sw_if_index;
-  int i;
-  int rv = VNET_API_ERROR_NO_SUCH_FIB;
-  u32 target_fib_id = ntohl (mp->vrf_id);
-
-  /* *INDENT-OFF* */
-  pool_foreach (fib_table, im6->fibs,
-  ({
-    vnet_sw_interface_t * si;
-
-    fib = pool_elt_at_index (im6->v6_fibs, fib_table->ft_index);
-
-    if (fib->table_id != target_fib_id)
-      continue;
-
-    vec_reset_length (sw_if_indices_to_shut);
-
-    /* Set the flow hash for this fib to the default */
-    vnet_set_ip6_flow_hash (fib->table_id, IP_FLOW_HASH_DEFAULT);
-
-    /* Shut down interfaces in this FIB / clean out intfc routes */
-    pool_foreach (si, im->sw_interfaces,
-    ({
-      if (im6->fib_index_by_sw_if_index[si->sw_if_index] ==
-          fib->index)
-        vec_add1 (sw_if_indices_to_shut, si->sw_if_index);
-    }));
+  vl_api_ip_table_replace_end_reply_t *rmp;
+  fib_protocol_t fproto;
+  u32 fib_index;
+  int rv = 0;
 
-    for (i = 0; i < vec_len (sw_if_indices_to_shut); i++) {
-      sw_if_index = sw_if_indices_to_shut[i];
+  fproto = (mp->table.is_ip6 ? FIB_PROTOCOL_IP6 : FIB_PROTOCOL_IP4);
+  fib_index = fib_table_find (fproto, ntohl (mp->table.table_id));
 
-      u32 flags = vnet_sw_interface_get_flags (vnm, sw_if_index);
-      flags &= ~(VNET_SW_INTERFACE_FLAG_ADMIN_UP);
-      vnet_sw_interface_set_flags (vnm, sw_if_index, flags);
+  if (INDEX_INVALID == fib_index)
+    rv = VNET_API_ERROR_NO_SUCH_FIB;
+  else
+    {
+      fib_table_sweep (fib_index, fproto, FIB_SOURCE_API);
+      mfib_table_sweep (mfib_table_find
+			(fproto, ntohl (mp->table.table_id)), fproto,
+			MFIB_SOURCE_API);
     }
-
-    fib_table_flush(fib->index, FIB_PROTOCOL_IP6, FIB_SOURCE_API);
-
-    rv = 0;
-    break;
-  })); /* pool_foreach (fib) */
-  /* *INDENT-ON* */
-
-  return rv;
+  REPLY_MACRO (VL_API_IP_TABLE_REPLACE_END_REPLY);
 }
 
 static void
-vl_api_reset_fib_t_handler (vl_api_reset_fib_t * mp)
+vl_api_ip_table_flush_t_handler (vl_api_ip_table_flush_t * mp)
 {
-  int rv;
-  vl_api_reset_fib_reply_t *rmp;
+  vl_api_ip_table_flush_reply_t *rmp;
+  fib_protocol_t fproto;
+  u32 fib_index;
+  int rv = 0;
 
-  if (mp->is_ipv6)
-    rv = ip6_reset_fib_t_handler (mp);
+  fproto = (mp->table.is_ip6 ? FIB_PROTOCOL_IP6 : FIB_PROTOCOL_IP4);
+  fib_index = fib_table_find (fproto, ntohl (mp->table.table_id));
+
+  if (INDEX_INVALID == fib_index)
+    rv = VNET_API_ERROR_NO_SUCH_FIB;
   else
-    rv = ip4_reset_fib_t_handler (mp);
+    {
+      vnet_main_t *vnm = vnet_get_main ();
+      vnet_interface_main_t *im = &vnm->interface_main;
+      vnet_sw_interface_t *si;
+
+      /* Shut down interfaces in this FIB / clean out intfc routes */
+      /* *INDENT-OFF* */
+      pool_foreach (si, im->sw_interfaces,
+      ({
+        if (fib_index == fib_table_get_index_for_sw_if_index (fproto,
+                                                              si->sw_if_index))
+          {
+            u32 flags = si->flags;
+            flags &= ~VNET_SW_INTERFACE_FLAG_ADMIN_UP;
+            vnet_sw_interface_set_flags (vnm, si->sw_if_index, flags);
+          }
+      }));
+      /* *INDENT-ON* */
+
+      fib_table_flush (fib_index, fproto, FIB_SOURCE_API);
+      mfib_table_flush (mfib_table_find (fproto, ntohl (mp->table.table_id)),
+			fproto, MFIB_SOURCE_API);
+    }
 
-  REPLY_MACRO (VL_API_RESET_FIB_REPLY);
+  REPLY_MACRO (VL_API_IP_TABLE_FLUSH_REPLY);
 }
 
 static void
diff --git a/src/vnet/mfib/ip4_mfib.c b/src/vnet/mfib/ip4_mfib.c
index 1d872f974fd..4da1be61113 100644
--- a/src/vnet/mfib/ip4_mfib.c
+++ b/src/vnet/mfib/ip4_mfib.c
@@ -506,9 +506,10 @@ ip4_show_mfib (vlib_main_t * vm,
             continue;
         }
 
-        vlib_cli_output (vm, "%U, fib_index %d",
+        vlib_cli_output (vm, "%U, fib_index:%d flags:%U",
                          format_mfib_table_name, mfib->index, FIB_PROTOCOL_IP4,
-                         mfib->index);
+                         mfib->index,
+                         format_mfib_table_flags, mfib_table->mft_flags);
 
         /* Show summary? */
         if (! verbose)
diff --git a/src/vnet/mfib/ip6_mfib.c b/src/vnet/mfib/ip6_mfib.c
index 5b15c8d1736..690f4ed9dfd 100644
--- a/src/vnet/mfib/ip6_mfib.c
+++ b/src/vnet/mfib/ip6_mfib.c
@@ -562,14 +562,14 @@ typedef struct ip6_mfib_show_ctx_t_ {
 } ip6_mfib_show_ctx_t;
 
 
-static int
+static walk_rc_t
 ip6_mfib_table_collect_entries (fib_node_index_t mfei, void *arg)
 {
     ip6_mfib_show_ctx_t *ctx = arg;
 
     vec_add1(ctx->entries, mfei);
 
-    return (0);
+    return (WALK_CONTINUE);
 }
 
 static void
@@ -609,7 +609,7 @@ typedef struct ip6_mfib_walk_ctx_t_
     void *i6w_ctx;
 } ip6_mfib_walk_ctx_t;
 
-static int
+static void
 ip6_mfib_walk_cb (clib_bihash_kv_40_8_t * kvp,
                  void *arg)
 {
@@ -617,9 +617,8 @@ ip6_mfib_walk_cb (clib_bihash_kv_40_8_t * kvp,
 
     if ((kvp->key[4] >> 32) == ctx->i6w_mfib_index)
     {
-        return (ctx->i6w_fn(kvp->value, ctx->i6w_ctx));
+        ctx->i6w_fn(kvp->value, ctx->i6w_ctx);
     }
-    return (FIB_TABLE_WALK_CONTINUE);
 }
 
 void
diff --git a/src/vnet/mfib/mfib_entry.c b/src/vnet/mfib/mfib_entry.c
index f169dc0886e..448e6c5e0a8 100644
--- a/src/vnet/mfib/mfib_entry.c
+++ b/src/vnet/mfib/mfib_entry.c
@@ -38,6 +38,7 @@ static mfib_path_ext_t *mfib_path_ext_pool;
  * String names for each source
  */
 static const char *mfib_source_names[] = MFIB_SOURCE_NAMES;
+static const char *mfib_src_attribute_names[] = MFIB_ENTRY_SRC_ATTRIBUTES;
 
 /*
  * Pool for all fib_entries
@@ -102,6 +103,26 @@ format_mfib_entry_path_ext (u8 * s, va_list * args)
 }
 
 u8 *
+format_mfib_entry_src_flags (u8 *s, va_list *args)
+{
+    mfib_entry_src_attribute_t sattr;
+    mfib_entry_src_flags_t flag = va_arg(*args, int);
+
+    if (!flag)
+    {
+        return format(s, "none");
+    }
+
+    FOR_EACH_MFIB_SRC_ATTRIBUTE(sattr) {
+        if ((1 << sattr) & flag) {
+            s = format (s, "%s,", mfib_src_attribute_names[sattr]);
+        }
+    }
+
+    return (s);
+}
+
+u8 *
 format_mfib_entry (u8 * s, va_list * args)
 {
     fib_node_index_t fei, mfi;
@@ -127,14 +148,15 @@ format_mfib_entry (u8 * s, va_list * args)
         s = format (s, " locks:%d\n", mfib_entry->mfe_node.fn_locks);
         vec_foreach(msrc, mfib_entry->mfe_srcs)
         {
-            s = format (s, "  src:%s locks:%d:",
+            s = format (s, "  src:%s flags:%U locks:%d:",
                         mfib_source_names[msrc->mfes_src],
+                        format_mfib_entry_src_flags, msrc->mfes_flags,
                         msrc->mfes_ref_count);
             if (msrc->mfes_cover != FIB_NODE_INDEX_INVALID)
             {
                 s = format (s, " cover:%d", msrc->mfes_cover);
             }
-            s = format (s, " %U\n", format_mfib_entry_flags, msrc->mfes_flags);
+            s = format (s, " %U\n", format_mfib_entry_flags, msrc->mfes_route_flags);
             if (FIB_NODE_INDEX_INVALID != msrc->mfes_pl)
             {
                 s = fib_path_list_format(msrc->mfes_pl, s);
@@ -201,7 +223,7 @@ mfib_entry_src_init (mfib_entry_t *mfib_entry,
 {
     mfib_entry_src_t esrc = {
         .mfes_pl = FIB_NODE_INDEX_INVALID,
-        .mfes_flags = MFIB_ENTRY_FLAG_NONE,
+        .mfes_route_flags = MFIB_ENTRY_FLAG_NONE,
         .mfes_src = source,
         .mfes_cover = FIB_NODE_INDEX_INVALID,
         .mfes_sibling = FIB_NODE_INDEX_INVALID,
@@ -215,8 +237,8 @@ mfib_entry_src_init (mfib_entry_t *mfib_entry,
 
 static mfib_entry_src_t *
 mfib_entry_src_find (const mfib_entry_t *mfib_entry,
-                    mfib_source_t source,
-                    u32 *index)
+                     mfib_source_t source,
+                     u32 *index)
 
 {
     mfib_entry_src_t *esrc;
@@ -269,8 +291,9 @@ mfib_entry_src_update (mfib_entry_t *mfib_entry,
 
     msrc = mfib_entry_src_find_or_create(mfib_entry, source);
 
-    msrc->mfes_flags = entry_flags;
+    msrc->mfes_route_flags = entry_flags;
     msrc->mfes_rpf_id = rpf_id;
+    msrc->mfes_flags &= ~MFIB_ENTRY_SRC_FLAG_STALE;
 
     return (msrc);
 }
@@ -286,6 +309,7 @@ mfib_entry_src_update_and_lock (mfib_entry_t *mfib_entry,
     msrc = mfib_entry_src_update(mfib_entry, source, rpf_id, entry_flags);
 
     msrc->mfes_ref_count++;
+    msrc->mfes_flags &= ~MFIB_ENTRY_SRC_FLAG_STALE;
 
     return (msrc);
 }
@@ -332,6 +356,44 @@ mfib_entry_is_sourced (fib_node_index_t mfib_entry_index,
 }
 
 int
+mfib_entry_is_marked (fib_node_index_t mfib_entry_index,
+                      mfib_source_t source)
+{
+    mfib_entry_t *mfib_entry;
+    mfib_entry_src_t *esrc;
+
+    mfib_entry = mfib_entry_get(mfib_entry_index);
+
+    esrc = mfib_entry_src_find(mfib_entry, source, NULL);
+
+    if (NULL == esrc)
+    {
+        return (0);
+    }
+    else
+    {
+        return (!!(esrc->mfes_flags & MFIB_ENTRY_SRC_FLAG_STALE));
+    }
+}
+
+void
+mfib_entry_mark (fib_node_index_t fib_entry_index,
+                 mfib_source_t source)
+{
+    mfib_entry_t *mfib_entry;
+    mfib_entry_src_t *esrc;
+
+    mfib_entry = mfib_entry_get(fib_entry_index);
+
+    esrc = mfib_entry_src_find(mfib_entry, source, NULL);
+
+    if (NULL != esrc)
+    {
+        esrc->mfes_flags |= MFIB_ENTRY_SRC_FLAG_STALE;
+    }
+}
+
+int
 mfib_entry_is_host (fib_node_index_t mfib_entry_index)
 {
     return (mfib_prefix_is_host(mfib_entry_get_prefix(mfib_entry_index)));
@@ -578,7 +640,7 @@ mfib_entry_stack (mfib_entry_t *mfib_entry,
          * updates to recalculate forwarding.
          */
         mfib_entry->mfe_pl = msrc->mfes_pl;
-        mfib_entry->mfe_flags = msrc->mfes_flags;
+        mfib_entry->mfe_flags = msrc->mfes_route_flags;
         mfib_entry->mfe_itfs = msrc->mfes_itfs;
         mfib_entry->mfe_rpf_id = msrc->mfes_rpf_id;
 
@@ -677,7 +739,9 @@ static fib_node_index_t*
 mfib_entry_src_paths_add (mfib_entry_src_t *msrc,
                           const fib_route_path_t *rpaths)
 {
-    ASSERT(!(MFIB_ENTRY_FLAG_EXCLUSIVE & msrc->mfes_flags));
+    ASSERT(!(MFIB_ENTRY_FLAG_EXCLUSIVE & msrc->mfes_route_flags));
+
+    msrc->mfes_flags &= ~MFIB_ENTRY_SRC_FLAG_STALE;
 
     if (FIB_NODE_INDEX_INVALID == msrc->mfes_pl)
     {
@@ -694,7 +758,9 @@ static fib_node_index_t*
 mfib_entry_src_paths_remove (mfib_entry_src_t *msrc,
                              const fib_route_path_t *rpaths)
 {
-    ASSERT(!(MFIB_ENTRY_FLAG_EXCLUSIVE & msrc->mfes_flags));
+    ASSERT(!(MFIB_ENTRY_FLAG_EXCLUSIVE & msrc->mfes_route_flags));
+
+    msrc->mfes_flags &= ~MFIB_ENTRY_SRC_FLAG_STALE;
 
     return (fib_path_list_paths_remove(msrc->mfes_pl, rpaths));
 }
@@ -792,11 +858,11 @@ static int
 mfib_entry_src_ok_for_delete (const mfib_entry_src_t *msrc)
 {
     return ((INDEX_INVALID == msrc->mfes_cover &&
-             MFIB_ENTRY_FLAG_NONE == msrc->mfes_flags &&
+             MFIB_ENTRY_FLAG_NONE == msrc->mfes_route_flags &&
              0 == fib_path_list_get_n_paths(msrc->mfes_pl)) &&
             (0 == hash_elts(msrc->mfes_itfs)));
 
-    /* return ((MFIB_ENTRY_FLAG_NONE == msrc->mfes_flags) && */
+    /* return ((MFIB_ENTRY_FLAG_NONE == msrc->mfes_route_flags) && */
     /*         (0 == fib_path_list_get_n_paths(msrc->mfes_pl)) && */
     /*         (0 == hash_elts(msrc->mfes_itfs))); */
 }
diff --git a/src/vnet/mfib/mfib_entry.h b/src/vnet/mfib/mfib_entry.h
index 4a1121bc9d5..7087fe87a4d 100644
--- a/src/vnet/mfib/mfib_entry.h
+++ b/src/vnet/mfib/mfib_entry.h
@@ -161,6 +161,8 @@ extern u32 mfib_entry_get_fib_index(fib_node_index_t fib_entry_index);
 extern int mfib_entry_is_sourced(fib_node_index_t fib_entry_index,
                                  mfib_source_t source);
 extern int mfib_entry_is_host(fib_node_index_t fib_entry_index);
+extern int mfib_entry_is_marked(fib_node_index_t fib_entry_index, mfib_source_t source);
+extern void mfib_entry_mark(fib_node_index_t fib_entry_index, mfib_source_t source);
 extern u32 mfib_entry_get_stats_index(fib_node_index_t fib_entry_index);
 extern void mfib_entry_cover_changed(fib_node_index_t fib_entry_index);
 extern void mfib_entry_cover_updated(fib_node_index_t fib_entry_index);
diff --git a/src/vnet/mfib/mfib_entry_src.h b/src/vnet/mfib/mfib_entry_src.h
index 86752e071ec..b85c010779c 100644
--- a/src/vnet/mfib/mfib_entry_src.h
+++ b/src/vnet/mfib/mfib_entry_src.h
@@ -28,6 +28,48 @@ typedef struct mfib_path_ext_t_
 } mfib_path_ext_t;
 
 /**
+ * Flags for the source data
+ */
+typedef enum mfib_entry_src_attribute_t_ {
+    /**
+     * Marker. Add new values after this one.
+     */
+    MFIB_ENTRY_SRC_ATTRIBUTE_FIRST,
+    /**
+     * the source has been added to the entry
+     */
+    MFIB_ENTRY_SRC_ATTRIBUTE_STALE = MFIB_ENTRY_SRC_ATTRIBUTE_FIRST,
+    /**
+     * Marker. add new entries before this one.
+     */
+    MFIB_ENTRY_SRC_ATTRIBUTE_LAST = MFIB_ENTRY_SRC_ATTRIBUTE_STALE,
+} mfib_entry_src_attribute_t;
+
+
+#define MFIB_ENTRY_SRC_ATTRIBUTES {                  \
+    [MFIB_ENTRY_SRC_ATTRIBUTE_STALE] = "stale",      \
+}
+
+#define FOR_EACH_MFIB_SRC_ATTRIBUTE(_item)                      \
+    for (_item = MFIB_ENTRY_SRC_ATTRIBUTE_FIRST;		\
+	 _item <= MFIB_ENTRY_SRC_ATTRIBUTE_LAST;		\
+	 _item++)
+
+typedef enum mfib_entry_src_flag_t_ {
+    MFIB_ENTRY_SRC_FLAG_NONE   = 0,
+    MFIB_ENTRY_SRC_FLAG_STALE = (1 << MFIB_ENTRY_SRC_ATTRIBUTE_STALE),
+} __attribute__ ((packed)) mfib_entry_src_flags_t;
+
+extern u8 * format_mfib_entry_src_flags(u8 *s, va_list *args);
+
+/*
+ * Keep the size of the flags field to 2 bytes, so it
+ * can be placed next to the 2 bytes reference count
+ */
+STATIC_ASSERT (sizeof(mfib_entry_src_flags_t) <= 2,
+	       "FIB entry flags field size too big");
+
+/**
  * The source of an MFIB entry
  */
 typedef struct mfib_entry_src_t_
@@ -40,7 +82,12 @@ typedef struct mfib_entry_src_t_
     /**
      * Route flags
      */
-    mfib_entry_flags_t mfes_flags;
+    mfib_entry_flags_t mfes_route_flags;
+
+    /**
+     * Source flags
+     */
+    mfib_entry_src_flags_t mfes_flags;
 
     /**
      * The reference count on the entry. this is a u32
diff --git a/src/vnet/mfib/mfib_entry_src_rr.c b/src/vnet/mfib/mfib_entry_src_rr.c
index 4512f74a9f5..a6a1e0d8aa5 100644
--- a/src/vnet/mfib/mfib_entry_src_rr.c
+++ b/src/vnet/mfib/mfib_entry_src_rr.c
@@ -62,7 +62,7 @@ mfib_entry_src_rr_activiate (mfib_entry_t *mfib_entry,
 
     msrc->mfes_pl = csrc->mfes_pl;
     fib_path_list_lock(msrc->mfes_pl);
-    msrc->mfes_flags = csrc->mfes_flags;
+    msrc->mfes_route_flags = csrc->mfes_route_flags;
     msrc->mfes_itfs = csrc->mfes_itfs;
     msrc->mfes_exts = csrc->mfes_exts;
     msrc->mfes_rpf_id = csrc->mfes_rpf_id;
@@ -91,7 +91,7 @@ mfib_entry_src_rr_cover_update (mfib_entry_t *mfib_entry,
 
     cover = mfib_entry_get(msrc->mfes_cover);
 
-    msrc->mfes_flags = cover->mfe_flags;
+    msrc->mfes_route_flags = cover->mfe_flags;
     msrc->mfes_itfs = cover->mfe_itfs;
     msrc->mfes_rpf_id = cover->mfe_rpf_id;
 
diff --git a/src/vnet/mfib/mfib_table.c b/src/vnet/mfib/mfib_table.c
index 504333a2474..a6a82774794 100644
--- a/src/vnet/mfib/mfib_table.c
+++ b/src/vnet/mfib/mfib_table.c
@@ -24,6 +24,8 @@
 #include <vnet/mfib/mfib_entry_cover.h>
 #include <vnet/mfib/mfib_signal.h>
 
+const static char * mfib_table_flags_strings[] = MFIB_TABLE_ATTRIBUTES;
+
 mfib_table_t *
 mfib_table_get (fib_node_index_t index,
                 fib_protocol_t proto)
@@ -643,7 +645,7 @@ typedef struct mfib_table_flush_ctx_t_
     mfib_source_t mftf_source;
 } mfib_table_flush_ctx_t;
 
-static int
+static walk_rc_t
 mfib_table_flush_cb (fib_node_index_t mfib_entry_index,
                      void *arg)
 {
@@ -653,7 +655,7 @@ mfib_table_flush_cb (fib_node_index_t mfib_entry_index,
     {
         vec_add1(ctx->mftf_entries, mfib_entry_index);
     }
-    return (1);
+    return (WALK_CONTINUE);
 }
 
 void
@@ -679,6 +681,79 @@ mfib_table_flush (u32 mfib_index,
     vec_free(ctx.mftf_entries);
 }
 
+static walk_rc_t
+mfib_table_mark_cb (fib_node_index_t fib_entry_index,
+                   void *arg)
+{
+    mfib_table_flush_ctx_t *ctx = arg;
+
+    if (mfib_entry_is_sourced(fib_entry_index, ctx->mftf_source))
+    {
+        mfib_entry_mark(fib_entry_index, ctx->mftf_source);
+    }
+    return (WALK_CONTINUE);
+}
+
+void
+mfib_table_mark (u32 fib_index,
+                 fib_protocol_t proto,
+                 mfib_source_t source)
+{
+    mfib_table_flush_ctx_t ctx = {
+        .mftf_source = source,
+    };
+    mfib_table_t *mfib_table;
+
+    mfib_table = mfib_table_get(fib_index, proto);
+
+    mfib_table->mft_epoch++;
+    mfib_table->mft_flags |= MFIB_TABLE_FLAG_RESYNC;
+
+    mfib_table_walk(fib_index, proto,
+                   mfib_table_mark_cb,
+                   &ctx);
+}
+
+static walk_rc_t
+mfib_table_sweep_cb (fib_node_index_t fib_entry_index,
+                    void *arg)
+{
+    mfib_table_flush_ctx_t *ctx = arg;
+
+    if (mfib_entry_is_marked(fib_entry_index, ctx->mftf_source))
+    {
+        vec_add1(ctx->mftf_entries, fib_entry_index);
+    }
+    return (WALK_CONTINUE);
+}
+
+void
+mfib_table_sweep (u32 fib_index,
+                  fib_protocol_t proto,
+                  mfib_source_t source)
+{
+    mfib_table_flush_ctx_t ctx = {
+        .mftf_source = source,
+    };
+    fib_node_index_t *fib_entry_index;
+    mfib_table_t *mfib_table;
+
+    mfib_table = mfib_table_get(fib_index, proto);
+
+    mfib_table->mft_flags &= ~MFIB_TABLE_FLAG_RESYNC;
+
+    mfib_table_walk(fib_index, proto,
+                    mfib_table_sweep_cb,
+                    &ctx);
+
+    vec_foreach(fib_entry_index, ctx.mftf_entries)
+    {
+        mfib_table_entry_delete_index(*fib_entry_index, source);
+    }
+
+    vec_free(ctx.mftf_entries);
+}
+
 static void
 mfib_table_destroy (mfib_table_t *mfib_table)
 {
@@ -770,6 +845,26 @@ mfib_table_walk (u32 fib_index,
 }
 
 u8*
+format_mfib_table_flags (u8 *s, va_list *args)
+{
+    mfib_table_flags_t flags = va_arg(*args, int);
+    mfib_table_attribute_t attr;
+
+    if (!flags)
+    {
+        return format(s, "none");
+    }
+
+    FOR_EACH_MFIB_TABLE_ATTRIBUTE(attr) {
+        if (1 << attr & flags) {
+            s = format(s, "%s", mfib_table_flags_strings[attr]);
+        }
+    }
+
+    return (s);
+}
+
+u8*
 format_mfib_table_name (u8* s, va_list *ap)
 {
     fib_node_index_t fib_index = va_arg(*ap, fib_node_index_t);
diff --git a/src/vnet/mfib/mfib_table.h b/src/vnet/mfib/mfib_table.h
index 47461375a10..d6b6dc3845d 100644
--- a/src/vnet/mfib/mfib_table.h
+++ b/src/vnet/mfib/mfib_table.h
@@ -29,6 +29,42 @@
 #define MFIB_TABLE_TOTAL_LOCKS MFIB_N_SOURCES
 
 /**
+ * Flags for the source data
+ */
+typedef enum mfib_table_attribute_t_ {
+    /**
+     * Marker. Add new values after this one.
+     */
+    MFIB_TABLE_ATTRIBUTE_FIRST,
+    /**
+     * the table is currently resync-ing
+     */
+    MFIB_TABLE_ATTRIBUTE_RESYNC = MFIB_TABLE_ATTRIBUTE_FIRST,
+    /**
+     * Marker. add new entries before this one.
+     */
+    MFIB_TABLE_ATTRIBUTE_LAST = MFIB_TABLE_ATTRIBUTE_RESYNC,
+} mfib_table_attribute_t;
+
+#define MFIB_TABLE_ATTRIBUTE_MAX (MFIB_TABLE_ATTRIBUTE_LAST+1)
+
+#define MFIB_TABLE_ATTRIBUTES {		         \
+    [MFIB_TABLE_ATTRIBUTE_RESYNC]  = "resync",    \
+}
+
+#define FOR_EACH_MFIB_TABLE_ATTRIBUTE(_item)      	\
+    for (_item = MFIB_TABLE_ATTRIBUTE_FIRST;		\
+	 _item < MFIB_TABLE_ATTRIBUTE_MAX;		\
+	 _item++)
+
+typedef enum mfib_table_flags_t_ {
+    MFIB_TABLE_FLAG_NONE   = 0,
+    MFIB_TABLE_FLAG_RESYNC  = (1 << MFIB_TABLE_ATTRIBUTE_RESYNC),
+} __attribute__ ((packed)) mfib_table_flags_t;
+
+extern u8* format_mfib_table_flags(u8 *s, va_list *args);
+
+/**
  * @brief
  *   A protocol Independent IP multicast FIB table
  */
@@ -56,6 +92,11 @@ typedef struct mfib_table_t_
     fib_protocol_t mft_proto;
 
     /**
+     * table falgs
+     */
+    mfib_table_flags_t mft_flags;
+
+    /**
      * number of locks on the table
      */
     u16 mft_locks[MFIB_TABLE_N_LOCKS];
@@ -66,6 +107,11 @@ typedef struct mfib_table_t_
     u32 mft_table_id;
 
     /**
+     * resync epoch
+     */
+    u32 mft_epoch;
+
+    /**
      * Index into FIB vector.
      */
     fib_node_index_t mft_index;
@@ -282,6 +328,46 @@ extern void mfib_table_flush(u32 fib_index,
 
 /**
  * @brief
+ *  Resync all entries from a table for the source
+ *  this is the mark part of the mark and sweep algorithm.
+ *  All entries in this FIB that are sourced by 'source' are marked
+ *  as stale.
+ *
+ * @param fib_index
+ *  The index of the FIB
+ *
+ * @paran proto
+ *  The protocol of the entries in the table
+ *
+ * @param source
+ *  the source to flush
+ */
+extern void mfib_table_mark(u32 fib_index,
+                            fib_protocol_t proto,
+                            mfib_source_t source);
+
+/**
+ * @brief
+ *  Signal that the table has converged, i.e. all updates are complete.
+ *  this is the sweep part of the mark and sweep algorithm.
+ *  All entries in this FIB that are sourced by 'source' and marked
+ *  as stale are flushed.
+ *
+ * @param fib_index
+ *  The index of the FIB
+ *
+ * @paran proto
+ *  The protocol of the entries in the table
+ *
+ * @param source
+ *  the source to flush
+ */
+extern void mfib_table_sweep(u32 fib_index,
+                             fib_protocol_t proto,
+                             mfib_source_t source);
+
+/**
+ * @brief
  *  Get the index of the FIB bound to the interface
  *
  * @paran proto
@@ -465,8 +551,8 @@ extern mfib_table_t *mfib_table_get(fib_node_index_t index,
 /**
  * @brief Call back function when walking entries in a FIB table
  */
-typedef int (*mfib_table_walk_fn_t)(fib_node_index_t fei,
-                                    void *ctx);
+typedef walk_rc_t (*mfib_table_walk_fn_t)(fib_node_index_t fei,
+                                          void *ctx);
 
 /**
  * @brief Walk all entries in a FIB table
diff --git a/src/vnet/mpls/mpls.c b/src/vnet/mpls/mpls.c
index 05361de505d..4076a8980a9 100644
--- a/src/vnet/mpls/mpls.c
+++ b/src/vnet/mpls/mpls.c
@@ -443,13 +443,6 @@ VLIB_CLI_COMMAND (mpls_table_command, static) = {
   .is_mp_safe = 1,
 };
 
-int
-mpls_fib_reset_labels (u32 fib_id)
-{
-  // FIXME
-  return 0;
-}
-
 static clib_error_t *
 mpls_init (vlib_main_t * vm)
 {
diff --git a/src/vnet/mpls/mpls.h b/src/vnet/mpls/mpls.h
index 918a3d83271..2cd41ea01c6 100644
--- a/src/vnet/mpls/mpls.h
+++ b/src/vnet/mpls/mpls.h
@@ -90,8 +90,6 @@ int mpls_sw_interface_enable_disable (mpls_main_t * mm,
 
 u8 mpls_sw_interface_is_enabled (u32 sw_if_index);
 
-int mpls_fib_reset_labels (u32 fib_id);
-
 int mpls_dest_cmp (void *a1, void *a2);
 
 int mpls_fib_index_cmp (void *a1, void *a2);