fib: Source Address Selection

Type: feature

Use the FIB to provide SAS (in so far as it is today)
 - Use the glean adjacency as the record of the connected prefixes
 = there's a glean per-{interface, protocol, connected-prefix}
 - Keep the glean up to date with whatever the recieve host prefix is
(since it can change)

Signed-off-by: Neale Ranns <neale.ranns@cisco.com>
Change-Id: I0f3dd1edb1f3fc965af1c7c586709028eb9cdeac

26 files changed, 697 insertions, 319 deletions

diff --git a/src/vnet/CMakeLists.txt b/src/vnet/CMakeLists.txt
index e387d25f913..c6c2b2e70e7 100644
--- a/src/vnet/CMakeLists.txt
+++ b/src/vnet/CMakeLists.txt
@@ -1195,6 +1195,7 @@ list(APPEND VNET_SOURCES
   fib/fib_path_list.c
   fib/fib_path.c
   fib/fib_path_ext.c
+  fib/fib_sas.c
   fib/fib_source.c
   fib/fib_urpf_list.c
   fib/fib_attached_export.c
@@ -1213,6 +1214,7 @@ list(APPEND VNET_HEADERS
   fib/fib_node_list.h
   fib/fib_entry.h
   fib/fib_entry_delegate.h
+  fib/fib_sas.h
   fib/fib_source.h
 )
 
diff --git a/src/vnet/adj/adj.c b/src/vnet/adj/adj.c
index db6d99a81d1..5c6ea9b71af 100644
--- a/src/vnet/adj/adj.c
+++ b/src/vnet/adj/adj.c
@@ -288,8 +288,7 @@ adj_last_lock_gone (ip_adjacency_t *adj)
 		       adj->rewrite_header.sw_if_index);
 	break;
     case IP_LOOKUP_NEXT_GLEAN:
-	adj_glean_remove(adj->ia_nh_proto,
-			 adj->rewrite_header.sw_if_index);
+	adj_glean_remove(adj);
 	break;
     case IP_LOOKUP_NEXT_MCAST_MIDCHAIN:
         adj_midchain_teardown(adj);
diff --git a/src/vnet/adj/adj.h b/src/vnet/adj/adj.h
index a53122711a8..b29b1b0c225 100644
--- a/src/vnet/adj/adj.h
+++ b/src/vnet/adj/adj.h
@@ -297,7 +297,7 @@ typedef struct ip_adjacency_t_
      */
     struct
     {
-      ip46_address_t receive_addr;
+      fib_prefix_t rx_pfx;
     } glean;
   } sub_type;
 
diff --git a/src/vnet/adj/adj_glean.c b/src/vnet/adj/adj_glean.c
index 65ccda1f5b8..c52e3d09693 100644
--- a/src/vnet/adj/adj_glean.c
+++ b/src/vnet/adj/adj_glean.c
@@ -19,10 +19,9 @@
 
 /*
  * The 'DB' of all glean adjs.
- * There is only one glean per-interface per-protocol, so this is a per-interface
- * vector
+ * There is one glean per-{interface, protocol, connected prefix}
  */
-static adj_index_t *adj_gleans[FIB_PROTOCOL_MAX];
+static uword **adj_gleans[FIB_PROTOCOL_IP_MAX];
 
 static inline u32
 adj_get_glean_node (fib_protocol_t proto)
@@ -39,6 +38,69 @@ adj_get_glean_node (fib_protocol_t proto)
     return (~0);
 }
 
+static adj_index_t
+adj_glean_db_lookup (fib_protocol_t proto,
+                     u32 sw_if_index,
+                     const ip46_address_t *nh_addr)
+{
+    uword *p;
+
+    if (vec_len(adj_gleans[proto]) <= sw_if_index)
+        return (ADJ_INDEX_INVALID);
+
+    p = hash_get_mem (adj_gleans[proto][sw_if_index], nh_addr);
+
+    if (p)
+        return (p[0]);
+
+    return (ADJ_INDEX_INVALID);
+}
+
+static void
+adj_glean_db_insert (fib_protocol_t proto,
+                     u32 sw_if_index,
+                     const ip46_address_t *nh_addr,
+                     adj_index_t ai)
+{
+    vlib_main_t *vm = vlib_get_main();
+
+    vlib_worker_thread_barrier_sync(vm);
+
+    vec_validate(adj_gleans[proto], sw_if_index);
+
+    if (NULL == adj_gleans[proto][sw_if_index])
+    {
+        adj_gleans[proto][sw_if_index] =
+            hash_create_mem (0, sizeof(ip46_address_t), sizeof(adj_index_t));
+    }
+
+    hash_set_mem_alloc (&adj_gleans[proto][sw_if_index],
+                        nh_addr, ai);
+
+    vlib_worker_thread_barrier_release(vm);
+}
+
+static void
+adj_glean_db_remove (fib_protocol_t proto,
+                     u32 sw_if_index,
+                     const ip46_address_t *nh_addr)
+{
+    vlib_main_t *vm = vlib_get_main();
+
+    vlib_worker_thread_barrier_sync(vm);
+
+    ASSERT(ADJ_INDEX_INVALID != adj_glean_db_lookup(proto, sw_if_index, nh_addr));
+    hash_unset_mem_free (&adj_gleans[proto][sw_if_index],
+                         nh_addr);
+
+    if (0 == hash_elts(adj_gleans[proto][sw_if_index]))
+    {
+        hash_free(adj_gleans[proto][sw_if_index]);
+        adj_gleans[proto][sw_if_index] = NULL;
+    }
+    vlib_worker_thread_barrier_release(vm);
+}
+
 /*
  * adj_glean_add_or_lock
  *
@@ -50,13 +112,14 @@ adj_index_t
 adj_glean_add_or_lock (fib_protocol_t proto,
                        vnet_link_t linkt,
 		       u32 sw_if_index,
-		       const ip46_address_t *nh_addr)
+		       const fib_prefix_t *conn)
 {
     ip_adjacency_t * adj;
+    adj_index_t ai;
 
-    vec_validate_init_empty(adj_gleans[proto], sw_if_index, ADJ_INDEX_INVALID);
+    ai = adj_glean_db_lookup(proto, sw_if_index, &conn->fp_addr);
 
-    if (ADJ_INDEX_INVALID == adj_gleans[proto][sw_if_index])
+    if (ADJ_INDEX_INVALID == ai)
     {
 	adj = adj_alloc(proto);
 
@@ -64,37 +127,33 @@ adj_glean_add_or_lock (fib_protocol_t proto,
 	adj->ia_nh_proto = proto;
         adj->ia_link = linkt;
         adj->ia_node_index = adj_get_glean_node(proto);
-	adj_gleans[proto][sw_if_index] = adj_get_index(adj);
-
-	if (NULL != nh_addr)
-	{
-	    adj->sub_type.glean.receive_addr = *nh_addr;
-	}
-        else
-        {
-            adj->sub_type.glean.receive_addr = zero_addr;
-        }
+        ai = adj_get_index(adj);
+        adj_lock(ai);
 
+	ASSERT(conn);
+        fib_prefix_normalize(conn, &adj->sub_type.glean.rx_pfx);
 	adj->rewrite_header.sw_if_index = sw_if_index;
 	adj->rewrite_header.data_bytes = 0;
         adj->rewrite_header.max_l3_packet_bytes =
 	  vnet_sw_interface_get_mtu(vnet_get_main(), sw_if_index,
                                     vnet_link_to_mtu(linkt));
-        adj_lock(adj_get_index(adj));
 
 	vnet_update_adjacency_for_sw_interface(vnet_get_main(),
                                                sw_if_index,
-                                               adj_get_index(adj));
+                                               ai);
+
+	adj_glean_db_insert(proto, sw_if_index,
+                            &adj->sub_type.glean.rx_pfx.fp_addr, ai);
     }
     else
     {
-	adj = adj_get(adj_gleans[proto][sw_if_index]);
-        adj_lock(adj_get_index(adj));
+	adj = adj_get(ai);
+        adj_lock(ai);
     }
 
     adj_delegate_adj_created(adj);
 
-    return (adj_get_index(adj));
+    return (ai);
 }
 
 /**
@@ -118,24 +177,143 @@ adj_glean_update_rewrite (adj_index_t adj_index)
                                   sizeof (adj->rewrite_data));
 }
 
+static adj_walk_rc_t
+adj_glean_update_rewrite_walk (adj_index_t ai,
+                               void *data)
+{
+    adj_glean_update_rewrite(ai);
+
+    return (ADJ_WALK_RC_CONTINUE);
+}
+
+void
+adj_glean_update_rewrite_itf (u32 sw_if_index)
+{
+    adj_glean_walk (sw_if_index, adj_glean_update_rewrite_walk, NULL);
+}
+
+void
+adj_glean_walk (u32 sw_if_index,
+                adj_walk_cb_t cb,
+                void *data)
+{
+    fib_protocol_t proto;
+
+    FOR_EACH_FIB_IP_PROTOCOL(proto)
+    {
+        adj_index_t ai, *aip, *ais = NULL;
+        ip46_address_t *conn;
+
+        if (vec_len(adj_gleans[proto]) <= sw_if_index ||
+            NULL == adj_gleans[proto][sw_if_index])
+            continue;
+
+        /*
+         * Walk first to collect the indices
+         * then walk the collection. This is safe
+         * to modifications of the hash table
+         */
+        hash_foreach_mem(conn, ai, adj_gleans[proto][sw_if_index],
+        ({
+            vec_add1(ais, ai);
+        }));
+
+        vec_foreach(aip, ais)
+        {
+            if (ADJ_WALK_RC_STOP == cb(*aip, data))
+                break;
+        }
+        vec_free(ais);
+    }
+}
+
 adj_index_t
 adj_glean_get (fib_protocol_t proto,
-               u32 sw_if_index)
+               u32 sw_if_index,
+               const ip46_address_t *nh)
 {
-    if (sw_if_index < vec_len(adj_gleans[proto]))
+    if (NULL != nh)
     {
-        return (adj_gleans[proto][sw_if_index]);
+        return adj_glean_db_lookup(proto, sw_if_index, nh);
+    }
+    else
+    {
+        ip46_address_t *conn;
+        adj_index_t ai;
+
+        if (vec_len(adj_gleans[proto]) <= sw_if_index ||
+            NULL == adj_gleans[proto][sw_if_index])
+            return (ADJ_INDEX_INVALID);
+
+        hash_foreach_mem(conn, ai, adj_gleans[proto][sw_if_index],
+        ({
+            return (ai);
+        }));
     }
     return (ADJ_INDEX_INVALID);
 }
 
+const ip46_address_t *
+adj_glean_get_src (fib_protocol_t proto,
+                   u32 sw_if_index,
+                   const ip46_address_t *nh)
+{
+    const ip_adjacency_t *adj;
+    ip46_address_t *conn;
+    adj_index_t ai;
+
+    if (vec_len(adj_gleans[proto]) <= sw_if_index ||
+        NULL == adj_gleans[proto][sw_if_index])
+        return (NULL);
+
+    fib_prefix_t pfx = {
+        .fp_len = fib_prefix_get_host_length(proto),
+        .fp_proto = proto,
+    };
+
+    if (nh)
+        pfx.fp_addr = *nh;
+
+    hash_foreach_mem(conn, ai, adj_gleans[proto][sw_if_index],
+    ({
+        adj = adj_get(ai);
+
+        if (adj->sub_type.glean.rx_pfx.fp_len > 0)
+        {
+            /* if no destination is specified use the just glean */
+            if (NULL == nh)
+                return (&adj->sub_type.glean.rx_pfx.fp_addr);
+
+            /* check the clean covers the desintation */
+            if (fib_prefix_is_cover(&adj->sub_type.glean.rx_pfx, &pfx))
+                return (&adj->sub_type.glean.rx_pfx.fp_addr);
+        }
+    }));
+
+    return (NULL);
+}
+
 void
-adj_glean_remove (fib_protocol_t proto,
-		  u32 sw_if_index)
+adj_glean_remove (ip_adjacency_t *adj)
 {
-    ASSERT(sw_if_index < vec_len(adj_gleans[proto]));
+    fib_prefix_t norm;
 
-    adj_gleans[proto][sw_if_index] = ADJ_INDEX_INVALID;
+    fib_prefix_normalize(&adj->sub_type.glean.rx_pfx,
+                         &norm);
+    adj_glean_db_remove(adj->ia_nh_proto,
+                        adj->rewrite_header.sw_if_index,
+                        &norm.fp_addr);
+}
+
+static adj_walk_rc_t
+adj_glean_start_backwalk (adj_index_t ai,
+                          void *data)
+{
+    fib_node_back_walk_ctx_t bw_ctx = *(fib_node_back_walk_ctx_t*) data;
+
+    fib_walk_sync(FIB_NODE_TYPE_ADJ, ai, &bw_ctx);
+
+    return (ADJ_WALK_RC_CONTINUE);
 }
 
 static clib_error_t *
@@ -146,25 +324,13 @@ adj_glean_interface_state_change (vnet_main_t * vnm,
     /*
      * for each glean on the interface trigger a walk back to the children
      */
-    fib_protocol_t proto;
-    ip_adjacency_t *adj;
-
-    FOR_EACH_FIB_IP_PROTOCOL(proto)
-    {
-	if (sw_if_index >= vec_len(adj_gleans[proto]) ||
-	    ADJ_INDEX_INVALID == adj_gleans[proto][sw_if_index])
-	    continue;
+    fib_node_back_walk_ctx_t bw_ctx = {
+        .fnbw_reason = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP ?
+                        FIB_NODE_BW_REASON_FLAG_INTERFACE_UP :
+                        FIB_NODE_BW_REASON_FLAG_INTERFACE_DOWN),
+    };
 
-	adj = adj_get(adj_gleans[proto][sw_if_index]);
-
-	fib_node_back_walk_ctx_t bw_ctx = {
-	    .fnbw_reason = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP ?
-			    FIB_NODE_BW_REASON_FLAG_INTERFACE_UP :
-			    FIB_NODE_BW_REASON_FLAG_INTERFACE_DOWN),
-	};
-
-	fib_walk_sync(FIB_NODE_TYPE_ADJ, adj_get_index(adj), &bw_ctx);
-    }
+    adj_glean_walk (sw_if_index, adj_glean_start_backwalk, &bw_ctx);
 
     return (NULL);
 }
@@ -217,12 +383,6 @@ adj_glean_interface_delete (vnet_main_t * vnm,
 			    u32 sw_if_index,
 			    u32 is_add)
 {
-    /*
-     * for each glean on the interface trigger a walk back to the children
-     */
-    fib_protocol_t proto;
-    ip_adjacency_t *adj;
-
     if (is_add)
     {
 	/*
@@ -241,20 +401,14 @@ adj_glean_interface_delete (vnet_main_t * vnm,
 	return (NULL);
     }
 
-    FOR_EACH_FIB_IP_PROTOCOL(proto)
-    {
-	if (sw_if_index >= vec_len(adj_gleans[proto]) ||
-	    ADJ_INDEX_INVALID == adj_gleans[proto][sw_if_index])
-	    continue;
-
-	adj = adj_get(adj_gleans[proto][sw_if_index]);
-
-	fib_node_back_walk_ctx_t bw_ctx = {
-	    .fnbw_reason =  FIB_NODE_BW_REASON_FLAG_INTERFACE_DELETE,
-	};
+    /*
+     * for each glean on the interface trigger a walk back to the children
+     */
+    fib_node_back_walk_ctx_t bw_ctx = {
+        .fnbw_reason =  FIB_NODE_BW_REASON_FLAG_INTERFACE_DELETE,
+    };
 
-	fib_walk_sync(FIB_NODE_TYPE_ADJ, adj_get_index(adj), &bw_ctx);
-    }
+    adj_glean_walk (sw_if_index, adj_glean_start_backwalk, &bw_ctx);
 
     return (NULL);
 }
@@ -268,14 +422,34 @@ format_adj_glean (u8* s, va_list *ap)
     CLIB_UNUSED(u32 indent) = va_arg(*ap, u32);
     ip_adjacency_t * adj = adj_get(index);
 
-    s = format(s, "%U-glean: %U",
+    s = format(s, "%U-glean: [src:%U] %U",
                format_fib_protocol, adj->ia_nh_proto,
-		format_vnet_rewrite,
-		&adj->rewrite_header, sizeof (adj->rewrite_data), 0);
+               format_fib_prefix, &adj->sub_type.glean.rx_pfx,
+               format_vnet_rewrite,
+               &adj->rewrite_header, sizeof (adj->rewrite_data), 0);
 
     return (s);
 }
 
+u32
+adj_glean_db_size (void)
+{
+    fib_protocol_t proto;
+    u32 sw_if_index = 0;
+    u64 count = 0;
+
+    FOR_EACH_FIB_IP_PROTOCOL(proto)
+    {
+	vec_foreach_index(sw_if_index, adj_gleans[proto])
+	{
+	    if (NULL != adj_gleans[proto][sw_if_index])
+	    {
+		count += hash_elts(adj_gleans[proto][sw_if_index]);
+	    }
+	}
+    }
+    return (count);
+}
 
 static void
 adj_dpo_lock (dpo_id_t *dpo)
diff --git a/src/vnet/adj/adj_glean.h b/src/vnet/adj/adj_glean.h
index 3ffbe36b51c..a06b9e81616 100644
--- a/src/vnet/adj/adj_glean.h
+++ b/src/vnet/adj/adj_glean.h
@@ -46,7 +46,7 @@
 extern adj_index_t adj_glean_add_or_lock(fib_protocol_t proto,
                                          vnet_link_t linkt,
 					 u32 sw_if_index,
-					 const ip46_address_t *nh_addr);
+					 const fib_prefix_t *conn);
 
 /**
  * @brief Get an existing glean
@@ -54,7 +54,8 @@ extern adj_index_t adj_glean_add_or_lock(fib_protocol_t proto,
  * @return INVALID if it does not exist
  */
 extern adj_index_t adj_glean_get(fib_protocol_t proto,
-                                 u32 sw_if_index);
+                                 u32 sw_if_index,
+                                 const ip46_address_t *nh_addr);
 
 /**
  * adj_glean_update_rewrite
@@ -66,6 +67,14 @@ extern adj_index_t adj_glean_get(fib_protocol_t proto,
  * glean behaviour on an adjacency liked to a connected prefix.
  */
 extern void adj_glean_update_rewrite(adj_index_t adj_index);
+extern void adj_glean_update_rewrite_itf(u32 sw_if_index);
+
+/**
+ * Return the source address from the glean
+ */
+const ip46_address_t *adj_glean_get_src(fib_protocol_t proto,
+                                        u32 sw_if_index,
+                                        const ip46_address_t *nh_addr);
 
 /**
  * @brief Format/display a glean adjacency.
@@ -73,9 +82,22 @@ extern void adj_glean_update_rewrite(adj_index_t adj_index);
 extern u8* format_adj_glean(u8* s, va_list *ap);
 
 /**
+ * Walk all the gleans on an interface
+ */
+extern void adj_glean_walk (u32 sw_if_index,
+                            adj_walk_cb_t,
+                            void *);
+
+/**
  * @brief
  *  Module initialisation
  */
 extern void adj_glean_module_init(void);
 
+/**
+ * @brief
+ *  Return the size of the adjacency database. for testing purposes
+ */
+extern u32 adj_glean_db_size(void);
+
 #endif
diff --git a/src/vnet/adj/adj_internal.h b/src/vnet/adj/adj_internal.h
index 11214932a3a..6639d32267f 100644
--- a/src/vnet/adj/adj_internal.h
+++ b/src/vnet/adj/adj_internal.h
@@ -120,8 +120,7 @@ extern void adj_nbr_remove(adj_index_t ai,
 			   vnet_link_t link_type,
 			   const ip46_address_t *nh_addr,
 			   u32 sw_if_index);
-extern void adj_glean_remove(fib_protocol_t proto,
-			     u32 sw_if_index);
+extern void adj_glean_remove(ip_adjacency_t *adj);
 extern void adj_mcast_remove(fib_protocol_t proto,
 			     u32 sw_if_index);
 extern void adj_midchain_teardown(ip_adjacency_t *adj);
diff --git a/src/vnet/fib/fib_entry_src.c b/src/vnet/fib/fib_entry_src.c
index ad8b23e4af1..7f4db6a071b 100644
--- a/src/vnet/fib/fib_entry_src.c
+++ b/src/vnet/fib/fib_entry_src.c
@@ -127,7 +127,7 @@ fib_entry_src_find_i (const fib_entry_t *fib_entry,
     return (NULL);
 }
 
-static fib_entry_src_t *
+fib_entry_src_t *
 fib_entry_src_find (const fib_entry_t *fib_entry,
 		    fib_source_t source)
 
@@ -1491,7 +1491,8 @@ fib_path_is_attached (const fib_route_path_t *rpath)
     {
 	return (!0);
     }
-    else if (rpath->frp_flags & FIB_ROUTE_PATH_ATTACHED)
+    else if (rpath->frp_flags & FIB_ROUTE_PATH_ATTACHED ||
+             rpath->frp_flags & FIB_ROUTE_PATH_GLEAN)
     {
         return (!0);
     }
diff --git a/src/vnet/fib/fib_entry_src.h b/src/vnet/fib/fib_entry_src.h
index edeaaf980e8..210507932c4 100644
--- a/src/vnet/fib/fib_entry_src.h
+++ b/src/vnet/fib/fib_entry_src.h
@@ -258,6 +258,8 @@ typedef struct fib_entry_src_vft_t_ {
 extern const fib_entry_src_vft_t*fib_entry_src_get_vft(
     const fib_entry_src_t *esrc);
 
+extern fib_entry_src_t * fib_entry_src_find (const fib_entry_t *fib_entry,
+                                             fib_source_t source);
 extern u8* fib_entry_src_format(fib_entry_t *entry,
 				fib_source_t source,
 				u8* s);
diff --git a/src/vnet/fib/fib_entry_src_interface.c b/src/vnet/fib/fib_entry_src_interface.c
index e1725773d93..402369d1dfc 100644
--- a/src/vnet/fib/fib_entry_src_interface.c
+++ b/src/vnet/fib/fib_entry_src_interface.c
@@ -48,45 +48,72 @@ static void
 fib_entry_src_interface_remove (fib_entry_src_t *src)
 {
     src->fes_pl = FIB_NODE_INDEX_INVALID;
+    ASSERT(src->u.interface.fesi_sibling == ~0);
 }
 
-static void
-fib_entry_src_interface_path_swap (fib_entry_src_t *src,
-				   const fib_entry_t *entry,
-				   fib_path_list_flags_t pl_flags,
-				   const fib_route_path_t *paths)
+static int
+fib_entry_src_interface_update_glean (fib_entry_t *cover,
+                                      const fib_entry_t *local)
 {
-    fib_node_index_t fib_entry_index;
-    ip_adjacency_t *adj;
+    fib_entry_src_t *src;
+    adj_index_t ai;
 
-    fib_entry_index = fib_entry_get_index(entry);
-    src->fes_pl = fib_path_list_create(pl_flags, paths);
+    src = fib_entry_src_find (cover, FIB_SOURCE_INTERFACE);
 
-    /*
-     * this is a hack to get the entry's prefix into the glean adjacency
-     * so that it is available for fast retrieval in the switch path.
-     */
-    if (!(FIB_ENTRY_FLAG_LOCAL & src->fes_entry_flags))
+    if (NULL == src)
     {
-        adj_index_t ai;
+        /*
+         * The cover is not an interface source, no work
+         */
+        return 0;
+    }
 
-        ai = fib_path_list_get_adj(src->fes_pl,
-                                   fib_entry_get_default_chain_type(
-                                       fib_entry_get(fib_entry_index)));
-        if (INDEX_INVALID != ai)
-        {
-            adj = adj_get(ai);
+    ai = fib_path_list_get_adj(src->fes_pl,
+                               fib_entry_get_default_chain_type(cover));
+
+    if (INDEX_INVALID != ai)
+    {
+        ip_adjacency_t *adj;
+
+        adj = adj_get(ai);
 
-            if (IP_LOOKUP_NEXT_GLEAN == adj->lookup_next_index)
+        if (IP_LOOKUP_NEXT_GLEAN == adj->lookup_next_index)
+        {
+            /*
+             * the connected prefix will link to a glean on a non-p2p
+             * interface.
+             * Ensure we are updating with a host in the connected's subnet
+             */
+            if (fib_prefix_is_cover(&adj->sub_type.glean.rx_pfx,
+                                    &local->fe_prefix))
             {
-                /*
-                 * the connected prefix will link to a glean on a non-p2p
-                 * u.interface.
-                 */
-                adj->sub_type.glean.receive_addr = entry->fe_prefix.fp_addr;
+                adj->sub_type.glean.rx_pfx.fp_addr = local->fe_prefix.fp_addr;
+                return (1);
             }
         }
     }
+
+    return (0);
+}
+
+static walk_rc_t
+fib_entry_src_interface_update_glean_walk (fib_entry_t *cover,
+                                           fib_node_index_t covered,
+                                           void *ctx)
+{
+    if (fib_entry_src_interface_update_glean(cover, fib_entry_get(covered)))
+        return (WALK_STOP);
+
+    return (WALK_CONTINUE);
+}
+
+static void
+fib_entry_src_interface_path_swap (fib_entry_src_t *src,
+				   const fib_entry_t *entry,
+				   fib_path_list_flags_t pl_flags,
+				   const fib_route_path_t *paths)
+{
+    src->fes_pl = fib_path_list_create(pl_flags, paths);
 }
 
 /*
@@ -116,6 +143,8 @@ fib_entry_src_interface_activate (fib_entry_src_t *src,
 
 	src->u.interface.fesi_sibling =
 	    fib_entry_cover_track(cover, fib_entry_get_index(fib_entry));
+
+        fib_entry_src_interface_update_glean(cover, fib_entry);
     }
 
     return (!0);
@@ -142,6 +171,11 @@ fib_entry_src_interface_deactivate (fib_entry_src_t *src,
 	fib_entry_cover_untrack(cover, src->u.interface.fesi_sibling);
 
 	src->u.interface.fesi_cover = FIB_NODE_INDEX_INVALID;
+	src->u.interface.fesi_sibling = ~0;
+
+        fib_entry_cover_walk(cover,
+                             fib_entry_src_interface_update_glean_walk,
+                             NULL);
     }
 }
 
diff --git a/src/vnet/fib/fib_path.c b/src/vnet/fib/fib_path.c
index 2cee8467647..2a4e6ab551f 100644
--- a/src/vnet/fib/fib_path.c
+++ b/src/vnet/fib/fib_path.c
@@ -246,6 +246,10 @@ typedef struct fib_path_t_ {
 	} attached_next_hop;
 	struct {
 	    /**
+	     * The Connected local address
+	     */
+	    fib_prefix_t fp_connected;
+	    /**
 	     * The interface
 	     */
 	    u32 fp_interface;
@@ -732,7 +736,7 @@ fib_path_attached_get_adj (fib_path_t *path,
 
         ai = adj_glean_add_or_lock(nh_proto, link,
                                    path->attached.fp_interface,
-                                   NULL);
+                                   &path->attached.fp_connected);
         dpo_set(dpo, DPO_ADJACENCY_GLEAN, vnet_link_to_dpo_proto(link), ai);
         adj_unlock(ai);
     }
@@ -1262,6 +1266,8 @@ fib_path_route_flags_to_cfg_flags (const fib_route_path_t *rpath)
 	cfg_flags |= FIB_PATH_CFG_FLAG_ICMP_UNREACH;
     if (rpath->frp_flags & FIB_ROUTE_PATH_ICMP_PROHIBIT)
 	cfg_flags |= FIB_PATH_CFG_FLAG_ICMP_PROHIBIT;
+    if (rpath->frp_flags & FIB_ROUTE_PATH_GLEAN)
+	cfg_flags |= FIB_PATH_CFG_FLAG_GLEAN;
 
     return (cfg_flags);
 }
@@ -1365,6 +1371,12 @@ fib_path_create (fib_node_index_t pl_index,
         path->fp_type = FIB_PATH_TYPE_SPECIAL;
         path->classify.fp_classify_table_id = rpath->frp_classify_table_id;
     }
+    else if (path->fp_cfg_flags & FIB_PATH_CFG_FLAG_GLEAN)
+    {
+        path->fp_type = FIB_PATH_TYPE_ATTACHED;
+        path->attached.fp_interface = rpath->frp_sw_if_index;
+        path->attached.fp_connected = rpath->frp_connected;
+    }
     else if (~0 != rpath->frp_sw_if_index)
     {
         if (ip46_address_is_zero(&rpath->frp_addr))
@@ -2105,7 +2117,7 @@ fib_path_resolve (fib_node_index_t path_index)
         break;
     }
     case FIB_PATH_TYPE_DVR:
-        dvr_dpo_add_or_lock(path->attached.fp_interface,
+        dvr_dpo_add_or_lock(path->dvr.fp_interface,
                             path->fp_nh_proto,
                             &path->fp_dpo);
         break;
diff --git a/src/vnet/fib/fib_path.h b/src/vnet/fib/fib_path.h
index 76f876d8f10..c0f76411390 100644
--- a/src/vnet/fib/fib_path.h
+++ b/src/vnet/fib/fib_path.h
@@ -100,9 +100,13 @@ typedef enum fib_path_cfg_attribute_t_ {
      */
     FIB_PATH_CFG_ATTRIBUTE_POP_PW_CW,
     /**
+     * The path is a glean
+     */
+    FIB_PATH_CFG_ATTRIBUTE_GLEAN,
+    /**
      * Marker. Add new types before this one, then update it.
      */
-    FIB_PATH_CFG_ATTRIBUTE_LAST = FIB_PATH_CFG_ATTRIBUTE_POP_PW_CW,
+    FIB_PATH_CFG_ATTRIBUTE_LAST = FIB_PATH_CFG_ATTRIBUTE_GLEAN,
 } __attribute__ ((packed)) fib_path_cfg_attribute_t;
 
 /**
@@ -123,7 +127,8 @@ typedef enum fib_path_cfg_attribute_t_ {
     [FIB_PATH_CFG_ATTRIBUTE_INTF_RX] = "interface-rx",	\
     [FIB_PATH_CFG_ATTRIBUTE_RPF_ID] = "rpf-id",         \
     [FIB_PATH_CFG_ATTRIBUTE_DEAG_SRC] = "deag-src",     \
-    [FIB_PATH_CFG_ATTRIBUTE_POP_PW_CW] = "pop-pw-cw",     \
+    [FIB_PATH_CFG_ATTRIBUTE_POP_PW_CW] = "pop-pw-cw",   \
+    [FIB_PATH_CFG_ATTRIBUTE_GLEAN] = "glean",           \
 }
 
 #define FOR_EACH_FIB_PATH_CFG_ATTRIBUTE(_item) \
@@ -149,6 +154,7 @@ typedef enum fib_path_cfg_flags_t_ {
     FIB_PATH_CFG_FLAG_RPF_ID = (1 << FIB_PATH_CFG_ATTRIBUTE_RPF_ID),
     FIB_PATH_CFG_FLAG_DEAG_SRC = (1 << FIB_PATH_CFG_ATTRIBUTE_DEAG_SRC),
     FIB_PATH_CFG_FLAG_POP_PW_CW = (1 << FIB_PATH_CFG_ATTRIBUTE_POP_PW_CW),
+    FIB_PATH_CFG_FLAG_GLEAN = (1 << FIB_PATH_CFG_ATTRIBUTE_GLEAN),
 } __attribute__ ((packed)) fib_path_cfg_flags_t;
 
 typedef enum fib_path_format_flags_t_
diff --git a/src/vnet/fib/fib_sas.c b/src/vnet/fib/fib_sas.c
new file mode 100644
index 00000000000..b607a0b5be8
--- /dev/null
+++ b/src/vnet/fib/fib_sas.c
@@ -0,0 +1,121 @@
+/*
+ * Copyright (c) 2020 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @brief FIB Source Address selection
+ *
+ * Use the FIB for source address selection on an interface
+ */
+
+#include <vnet/fib/fib_sas.h>
+#include <vnet/adj/adj_glean.h>
+#include <vnet/ip/ip6_link.h>
+
+
+bool
+fib_sas_get (u32 sw_if_index,
+             ip_address_family_t af,
+             const ip46_address_t *dst,
+             ip46_address_t *src)
+{
+    switch (af)
+    {
+    case AF_IP4:
+        if (dst)
+            return (fib_sas4_get(sw_if_index, &dst->ip4, &src->ip4));
+        else
+            return (fib_sas4_get(sw_if_index, NULL, &src->ip4));
+    case AF_IP6:
+        if (dst)
+            return (fib_sas6_get(sw_if_index, &dst->ip6, &src->ip6));
+        else
+            return (fib_sas6_get(sw_if_index, NULL, &src->ip6));
+    }
+    return (false);
+}
+
+bool
+fib_sas4_get (u32 sw_if_index,
+              const ip4_address_t *dst,
+              ip4_address_t *src)
+{
+    ip46_address_t d_tmp, *d_tmpp = NULL;
+    const ip46_address_t *s_tmp;
+    vnet_sw_interface_t *swif;
+
+    if (dst)
+    {
+        d_tmpp = &d_tmp;
+        d_tmp.ip4 = *dst;
+    }
+
+    /*
+     * If the interface is unnumbered then use the IP interface
+     */
+    swif = vnet_get_sw_interface (vnet_get_main(), sw_if_index);
+
+    if (swif->flags & VNET_SW_INTERFACE_FLAG_UNNUMBERED)
+        sw_if_index = swif->unnumbered_sw_if_index;
+
+    /*
+     * get the source address from the glean adjacency
+     */
+    s_tmp = adj_glean_get_src (FIB_PROTOCOL_IP4, sw_if_index, d_tmpp);
+
+    if (NULL != s_tmp)
+    {
+        src->as_u32 = s_tmp->ip4.as_u32;
+        return (true);
+    }
+
+    return (false);
+}
+
+bool
+fib_sas6_get (u32 sw_if_index,
+              const ip6_address_t *dst,
+              ip6_address_t *src)
+{
+    ip46_address_t d_tmp, *d_tmpp = NULL;
+    const ip46_address_t *s_tmp;
+
+    if (dst)
+    {
+        d_tmpp = &d_tmp;
+        d_tmp.ip6 = *dst;
+    }
+
+    /*
+     * if the dst is v6 and link local, use the source link local
+     */
+    if (ip6_address_is_link_local_unicast (dst))
+    {
+        ip6_address_copy (src, ip6_get_link_local_address (sw_if_index));
+        return (true);
+    }
+
+    /*
+     * get the source address from the glean adjacency
+     */
+    s_tmp = adj_glean_get_src (FIB_PROTOCOL_IP6, sw_if_index, d_tmpp);
+
+    if (NULL != s_tmp)
+    {
+        ip6_address_copy(src, &s_tmp->ip6);
+        return (true);
+    }
+
+    return (false);
+}
diff --git a/src/vnet/fib/fib_sas.h b/src/vnet/fib/fib_sas.h
new file mode 100644
index 00000000000..172a4d6b2dc
--- /dev/null
+++ b/src/vnet/fib/fib_sas.h
@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2020 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @brief FIB Source Address selection
+ *
+ * Use the FIB for source address selection on an interface
+ */
+
+#ifndef __FIB_SAS_H__
+#define __FIB_SAS_H__
+
+#include <vnet/fib/fib_types.h>
+#include <vnet/ip/ip_types.h>
+
+/**
+ * @brief Get a Source address to use in a packet being sent out
+ *        an interface
+ *
+ * @param sw_if_index The interface on which the packet is to be sent
+ * @param af The address family of the packet
+ * @param dst The destination of the packet (can be NULL in which case any
+ *            of the available address will be returned)
+ * @param src OUT the source address to use
+ *
+ * @return True if an address is available False (and src is unset) otherwise
+ */
+extern bool fib_sas_get (u32 sw_if_index,
+                         ip_address_family_t af,
+                         const ip46_address_t *dst,
+                         ip46_address_t *src);
+
+/**
+ * @brief Get an IPv4 Source address to use in a packet being sent out
+ *        an interface
+ *
+ * @param sw_if_index The interface on which the packet is to be sent
+ * @param dst The destination of the packet (can be NULL in which case any
+ *            of the available address will be returned)
+ * @param src OUT the source address to use
+ *
+ * @return True if an address is available False (and src is unset) otherwise
+ */
+extern bool fib_sas4_get (u32 sw_if_index,
+                          const ip4_address_t *dst,
+                          ip4_address_t *src);
+
+/**
+ * @brief Get an IPv6 Source address to use in a packet being sent out
+ *        an interface
+ *
+ * @param sw_if_index The interface on which the packet is to be sent
+ * @param dst The destination of the packet (can be NULL in which case any
+ *            of the available address will be returned)
+ * @param src OUT the source address to use
+ *
+ * @return True if an address is available False (and src is unset) otherwise
+ */
+extern bool fib_sas6_get (u32 sw_if_index,
+                          const ip6_address_t *dst,
+                          ip6_address_t *src);
+
+#endif
diff --git a/src/vnet/fib/fib_table.c b/src/vnet/fib/fib_table.c
index ec2acc59c52..e71e6c36bfb 100644
--- a/src/vnet/fib/fib_table.c
+++ b/src/vnet/fib/fib_table.c
@@ -510,7 +510,7 @@ fib_table_route_path_fixup (const fib_prefix_t *prefix,
         (~0 == path->frp_sw_if_index) &&
         (0 == ip46_address_cmp(&path->frp_addr, &prefix->fp_addr)))
     {
-        /* Prefix recurses via itse;f */
+        /* Prefix recurses via itself */
 	path->frp_flags |= FIB_ROUTE_PATH_DROP;
     }
     if (!(path->frp_flags & FIB_ROUTE_PATH_LOCAL) &&
@@ -522,6 +522,15 @@ fib_table_route_path_fixup (const fib_prefix_t *prefix,
 	path->frp_addr = prefix->fp_addr;
         path->frp_flags |= FIB_ROUTE_PATH_ATTACHED;
     }
+    else if ((*eflags & FIB_ENTRY_FLAG_CONNECTED) &&
+             !(*eflags & FIB_ENTRY_FLAG_LOCAL))
+    {
+        if (ip46_address_is_zero(&path->frp_addr))
+        {
+            path->frp_flags |= FIB_ROUTE_PATH_GLEAN;
+            fib_prefix_normalize(prefix, &path->frp_connected);
+        }
+    }
     if (*eflags & FIB_ENTRY_FLAG_DROP)
     {
 	path->frp_flags |= FIB_ROUTE_PATH_DROP;
diff --git a/src/vnet/fib/fib_types.c b/src/vnet/fib/fib_types.c
index b5576161d80..2fce6a85c74 100644
--- a/src/vnet/fib/fib_types.c
+++ b/src/vnet/fib/fib_types.c
@@ -260,6 +260,25 @@ fib_prefix_is_host (const fib_prefix_t *prefix)
     return (0);
 }
 
+void
+fib_prefix_normalize (const fib_prefix_t *p,
+                      fib_prefix_t *out)
+{
+    fib_prefix_copy (out, p);
+
+    switch (p->fp_proto)
+    {
+    case FIB_PROTOCOL_IP4:
+	ip4_address_normalize(&out->fp_addr.ip4, out->fp_len);
+        break;
+    case FIB_PROTOCOL_IP6:
+	ip6_address_normalize(&out->fp_addr.ip6, out->fp_len);
+        break;
+    case FIB_PROTOCOL_MPLS:
+	break;
+    }
+}
+
 u8 *
 format_fib_prefix (u8 * s, va_list * args)
 {
diff --git a/src/vnet/fib/fib_types.h b/src/vnet/fib/fib_types.h
index 832092c5679..b5a58e7b674 100644
--- a/src/vnet/fib/fib_types.h
+++ b/src/vnet/fib/fib_types.h
@@ -267,6 +267,13 @@ extern int fib_prefix_is_host(const fib_prefix_t *p);
 extern u8 fib_prefix_get_host_length (fib_protocol_t proto);
 
 /**
+ * normalise a prefix (i.e. mask the host bits according to the
+ * prefix length)
+ */
+extern void fib_prefix_normalize(const fib_prefix_t *p,
+                                 fib_prefix_t *out);
+
+/**
  * \brief Host prefix from ip
  */
 extern void fib_prefix_from_ip46_addr (const ip46_address_t *addr,
@@ -393,6 +400,10 @@ typedef enum fib_route_path_flags_t_
      * Pop a Psuedo Wire Control Word
      */
     FIB_ROUTE_PATH_POP_PW_CW = (1 << 18),
+    /**
+     * A path that resolves via a glean adjacency
+     */
+    FIB_ROUTE_PATH_GLEAN = (1 << 19),
 } fib_route_path_flags_t;
 
 /**
@@ -520,6 +531,11 @@ typedef struct fib_route_path_t_ {
                  * Present in an mfib path list
                  */
                 index_t frp_bier_imp;
+
+                /**
+                 * Glean prefix on a glean path
+                 */
+                fib_prefix_t frp_connected;
             };
 
             /**
diff --git a/src/vnet/ip-neighbor/ip4_neighbor.c b/src/vnet/ip-neighbor/ip4_neighbor.c
index 7c0cbdcb2a2..c268b96e00d 100644
--- a/src/vnet/ip-neighbor/ip4_neighbor.c
+++ b/src/vnet/ip-neighbor/ip4_neighbor.c
@@ -40,23 +40,23 @@
 #include <vnet/ip-neighbor/ip4_neighbor.h>
 #include <vnet/ethernet/ethernet.h>
 #include <vnet/util/throttle.h>
+#include <vnet/fib/fib_sas.h>
 
 /** ARP throttling */
 static throttle_t arp_throttle;
 
 void
-ip4_neighbor_probe_dst (const ip_adjacency_t * adj, const ip4_address_t * dst)
+ip4_neighbor_probe_dst (u32 sw_if_index, const ip4_address_t * dst)
 {
-  ip_interface_address_t *ia;
-  ip4_address_t *src;
+  ip4_address_t src;
+  adj_index_t ai;
 
-  src = ip4_interface_address_matching_destination
-    (&ip4_main,
-     &adj->sub_type.nbr.next_hop.ip4, adj->rewrite_header.sw_if_index, &ia);
-  if (!src)
-    return;
+  /* any glean will do, it's just for the rewrite */
+  ai = adj_glean_get (FIB_PROTOCOL_IP4, sw_if_index, NULL);
 
-  ip4_neighbor_probe (vlib_get_main (), vnet_get_main (), adj, src, dst);
+  if (ADJ_INDEX_INVALID != ai && fib_sas4_get (sw_if_index, dst, &src))
+    ip4_neighbor_probe (vlib_get_main (),
+			vnet_get_main (), adj_get (ai), &src, dst);
 }
 
 void
@@ -67,11 +67,12 @@ ip4_neighbor_advertise (vlib_main_t * vm,
   vnet_hw_interface_t *hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
   ip4_main_t *i4m = &ip4_main;
   u8 *rewrite, rewrite_len;
+  ip4_address_t tmp;
 
   if (NULL == addr)
     {
-      ip4_main_t *i4m = &ip4_main;
-      addr = ip4_interface_first_address (i4m, sw_if_index, 0);
+      fib_sas4_get (sw_if_index, NULL, &tmp);
+      addr = &tmp;
     }
 
   if (addr)
@@ -122,8 +123,6 @@ ip4_arp_inline (vlib_main_t * vm,
 		vlib_frame_t * frame, int is_glean)
 {
   vnet_main_t *vnm = vnet_get_main ();
-  ip4_main_t *im = &ip4_main;
-  ip_lookup_main_t *lm = &im->lookup_main;
   u32 *from, *to_next_drop;
   uword n_left_from, n_left_to_next_drop, next_index;
   u32 thread_index = vm->thread_index;
@@ -171,14 +170,14 @@ ip4_arp_inline (vlib_main_t * vm,
 	      /* resolve the packet's destination */
 	      ip4_header_t *ip0 = vlib_buffer_get_current (p0);
 	      resolve0 = ip0->dst_address;
-	      src0 = adj0->sub_type.glean.receive_addr.ip4;
+	      src0 = adj0->sub_type.glean.rx_pfx.fp_addr.ip4;
 	    }
 	  else
 	    {
 	      /* resolve the incomplete adj */
 	      resolve0 = adj0->sub_type.nbr.next_hop.ip4;
 	      /* Src IP address in ARP header. */
-	      if (ip4_src_address_for_packet (lm, sw_if_index0, &src0))
+	      if (!fib_sas4_get (sw_if_index0, &resolve0, &src0))
 		{
 		  /* No source address available */
 		  p0->error = node->errors[IP4_ARP_ERROR_NO_SOURCE_ADDRESS];
diff --git a/src/vnet/ip-neighbor/ip4_neighbor.h b/src/vnet/ip-neighbor/ip4_neighbor.h
index c52e2d446af..8805beadf23 100644
--- a/src/vnet/ip-neighbor/ip4_neighbor.h
+++ b/src/vnet/ip-neighbor/ip4_neighbor.h
@@ -19,7 +19,7 @@
 #include <vnet/ip/ip.h>
 #include <vnet/ethernet/arp_packet.h>
 
-extern void ip4_neighbor_probe_dst (const ip_adjacency_t * adj,
+extern void ip4_neighbor_probe_dst (u32 sw_if_index,
 				    const ip4_address_t * dst);
 extern void ip4_neighbor_advertise (vlib_main_t * vm,
 				    vnet_main_t * vnm,
diff --git a/src/vnet/ip-neighbor/ip6_neighbor.c b/src/vnet/ip-neighbor/ip6_neighbor.c
index ca67d85778d..478eca7fa27 100644
--- a/src/vnet/ip-neighbor/ip6_neighbor.c
+++ b/src/vnet/ip-neighbor/ip6_neighbor.c
@@ -17,23 +17,19 @@
 
 #include <vnet/ip-neighbor/ip6_neighbor.h>
 #include <vnet/util/throttle.h>
+#include <vnet/fib/fib_sas.h>
 
 /** ND throttling */
 static throttle_t nd_throttle;
 
 void
-ip6_neighbor_probe_dst (const ip_adjacency_t * adj, const ip6_address_t * dst)
+ip6_neighbor_probe_dst (u32 sw_if_index, const ip6_address_t * dst)
 {
-  ip_interface_address_t *ia;
-  ip6_address_t *src;
+  ip6_address_t src;
 
-  src = ip6_interface_address_matching_destination
-    (&ip6_main, dst, adj->rewrite_header.sw_if_index, &ia);
-
-  if (!src)
-    return;
-
-  ip6_neighbor_probe (vlib_get_main (), vnet_get_main (), adj, src, dst);
+  if (fib_sas6_get (sw_if_index, dst, &src))
+    ip6_neighbor_probe (vlib_get_main (), vnet_get_main (),
+			sw_if_index, &src, dst);
 }
 
 void
@@ -210,15 +206,15 @@ ip6_discover_neighbor_inline (vlib_main_t * vm,
 	   * Choose source address based on destination lookup
 	   * adjacency.
 	   */
-	  if (!ip6_src_address_for_packet (sw_if_index0,
-					   &ip0->dst_address, &src))
+	  if (!fib_sas6_get (sw_if_index0, &ip0->dst_address, &src))
 	    {
 	      /* There is no address on the interface */
 	      p0->error = node->errors[IP6_NBR_ERROR_NO_SOURCE_ADDRESS];
 	      continue;
 	    }
 
-	  b0 = ip6_neighbor_probe (vm, vnm, adj0, &src, &ip0->dst_address);
+	  b0 = ip6_neighbor_probe (vm, vnm, sw_if_index0,
+				   &src, &ip0->dst_address);
 
 	  if (PREDICT_TRUE (NULL != b0))
 	    {
diff --git a/src/vnet/ip-neighbor/ip6_neighbor.h b/src/vnet/ip-neighbor/ip6_neighbor.h
index 7f76efd2c86..681e634861c 100644
--- a/src/vnet/ip-neighbor/ip6_neighbor.h
+++ b/src/vnet/ip-neighbor/ip6_neighbor.h
@@ -34,17 +34,18 @@ extern void ip6_neighbor_advertise (vlib_main_t * vm,
 				    u32 sw_if_index,
 				    const ip6_address_t * addr);
 
-extern void ip6_neighbor_probe_dst (const ip_adjacency_t * adj,
+extern void ip6_neighbor_probe_dst (u32 sw_if_index,
 				    const ip6_address_t * dst);
 
 always_inline vlib_buffer_t *
 ip6_neighbor_probe (vlib_main_t * vm,
 		    vnet_main_t * vnm,
-		    const ip_adjacency_t * adj,
+		    u32 sw_if_index,
 		    const ip6_address_t * src, const ip6_address_t * dst)
 {
   icmp6_neighbor_solicitation_header_t *h0;
   vnet_hw_interface_t *hw_if0;
+  const ip_adjacency_t *adj;
   vlib_buffer_t *b0;
   int bogus_length;
   u32 bi0 = 0;
@@ -52,17 +53,17 @@ ip6_neighbor_probe (vlib_main_t * vm,
   h0 = vlib_packet_template_get_packet
     (vm, &ip6_neighbor_packet_template, &bi0);
   if (!h0)
-    return NULL;;
+    return NULL;
 
   /* if the interface has been disabled for ip6, later steps to retrieve
    * an adjacency will result in a segv.
    */
-  if (!ip6_link_is_enabled (adj->rewrite_header.sw_if_index))
+  if (!ip6_link_is_enabled (sw_if_index))
     return NULL;
 
   b0 = vlib_get_buffer (vm, bi0);
 
-  hw_if0 = vnet_get_sup_hw_interface (vnm, adj->rewrite_header.sw_if_index);
+  hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index);
 
   /*
    * Destination address is a solicited node multicast address.
@@ -87,11 +88,11 @@ ip6_neighbor_probe (vlib_main_t * vm,
   ASSERT (bogus_length == 0);
   VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
 
-  vnet_buffer (b0)->sw_if_index[VLIB_TX] = adj->rewrite_header.sw_if_index;
+  vnet_buffer (b0)->sw_if_index[VLIB_TX] = sw_if_index;
 
   /* Use the link's mcast adj to ship the packet */
   vnet_buffer (b0)->ip.adj_index[VLIB_TX] =
-    ip6_link_get_mcast_adj (adj->rewrite_header.sw_if_index);
+    ip6_link_get_mcast_adj (sw_if_index);
   adj = adj_get (vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
 
   b0->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
diff --git a/src/vnet/ip-neighbor/ip_neighbor.c b/src/vnet/ip-neighbor/ip_neighbor.c
index 5786775dc98..2dd8e748be6 100644
--- a/src/vnet/ip-neighbor/ip_neighbor.c
+++ b/src/vnet/ip-neighbor/ip_neighbor.c
@@ -1011,22 +1011,19 @@ ip_neighbor_register (ip_address_family_t af, const ip_neighbor_vft_t * vft)
 }
 
 void
-ip_neighbor_probe_dst (const ip_adjacency_t * adj, const ip46_address_t * dst)
+ip_neighbor_probe_dst (u32 sw_if_index,
+		       ip_address_family_t af, const ip46_address_t * dst)
 {
-  if (!vnet_sw_interface_is_admin_up (vnet_get_main (),
-				      adj->rewrite_header.sw_if_index))
+  if (!vnet_sw_interface_is_admin_up (vnet_get_main (), sw_if_index))
     return;
 
-  switch (adj->ia_nh_proto)
+  switch (af)
     {
-    case FIB_PROTOCOL_IP6:
-      ip6_neighbor_probe_dst (adj, &dst->ip6);
+    case AF_IP6:
+      ip6_neighbor_probe_dst (sw_if_index, &dst->ip6);
       break;
-    case FIB_PROTOCOL_IP4:
-      ip4_neighbor_probe_dst (adj, &dst->ip4);
-      break;
-    case FIB_PROTOCOL_MPLS:
-      ASSERT (0);
+    case AF_IP4:
+      ip4_neighbor_probe_dst (sw_if_index, &dst->ip4);
       break;
     }
 }
@@ -1034,7 +1031,9 @@ ip_neighbor_probe_dst (const ip_adjacency_t * adj, const ip46_address_t * dst)
 void
 ip_neighbor_probe (const ip_adjacency_t * adj)
 {
-  ip_neighbor_probe_dst (adj, &adj->sub_type.nbr.next_hop);
+  ip_neighbor_probe_dst (adj->rewrite_header.sw_if_index,
+			 ip_address_family_from_fib_proto (adj->ia_nh_proto),
+			 &adj->sub_type.nbr.next_hop);
 }
 
 void
@@ -1147,7 +1146,6 @@ ip_neighbor_ethernet_change_mac (ethernet_main_t * em,
 				 u32 sw_if_index, uword opaque)
 {
   ip_neighbor_t *ipn;
-  adj_index_t ai;
 
   IP_NEIGHBOR_DBG ("mac-change: %U",
 		   format_vnet_sw_if_index_name, vnet_get_main (),
@@ -1165,10 +1163,7 @@ ip_neighbor_ethernet_change_mac (ethernet_main_t * em,
   }));
   /* *INDENT-ON* */
 
-  ai = adj_glean_get (FIB_PROTOCOL_IP4, sw_if_index);
-
-  if (ADJ_INDEX_INVALID != ai)
-    adj_glean_update_rewrite (ai);
+  adj_glean_update_rewrite_itf (sw_if_index);
 }
 
 void
@@ -1543,14 +1538,8 @@ ip_neighbour_age_out (index_t ipni, f64 now, f64 * wait)
 	}
       else
 	{
-	  adj_index_t ai;
-
-	  ai = adj_glean_get (ip_address_family_to_fib_proto (af),
-			      ip_neighbor_get_sw_if_index (ipn));
-
-	  if (ADJ_INDEX_INVALID != ai)
-	    ip_neighbor_probe_dst (adj_get (ai),
-				   &ip_addr_46 (&ipn->ipn_key->ipnk_ip));
+	  ip_neighbor_probe_dst (ip_neighbor_get_sw_if_index (ipn),
+				 af, &ip_addr_46 (&ipn->ipn_key->ipnk_ip));
 
 	  ipn->ipn_n_probes++;
 	  *wait = 1;
diff --git a/src/vnet/ip-neighbor/ip_neighbor.h b/src/vnet/ip-neighbor/ip_neighbor.h
index 419c49491a3..064569b56ce 100644
--- a/src/vnet/ip-neighbor/ip_neighbor.h
+++ b/src/vnet/ip-neighbor/ip_neighbor.h
@@ -54,7 +54,8 @@ extern void ip_neighbor_learn (const ip_neighbor_learn_t * l);
 extern void ip_neighbor_update (vnet_main_t * vnm, adj_index_t ai);
 
 extern void ip_neighbor_probe (const ip_adjacency_t * adj);
-extern void ip_neighbor_probe_dst (const ip_adjacency_t * adj,
+extern void ip_neighbor_probe_dst (u32 sw_if_index,
+				   ip_address_family_t af,
 				   const ip46_address_t * ip);
 
 extern void ip_neighbor_mark (ip_address_family_t af);
diff --git a/src/vnet/ip/ip4.h b/src/vnet/ip/ip4.h
index 22de22f227a..3be2f7f1a7e 100644
--- a/src/vnet/ip/ip4.h
+++ b/src/vnet/ip/ip4.h
@@ -201,26 +201,6 @@ ip4_destination_matches_interface (ip4_main_t * im,
   return ip4_destination_matches_route (im, key, a, ia->address_length);
 }
 
-always_inline int
-ip4_src_address_for_packet (ip_lookup_main_t * lm,
-			    u32 sw_if_index, ip4_address_t * src)
-{
-  u32 if_add_index = lm->if_address_pool_index_by_sw_if_index[sw_if_index];
-  if (PREDICT_TRUE (if_add_index != ~0))
-    {
-      ip_interface_address_t *if_add =
-	pool_elt_at_index (lm->if_address_pool, if_add_index);
-      ip4_address_t *if_ip = ip_interface_address_get_address (lm, if_add);
-      *src = *if_ip;
-      return 0;
-    }
-  else
-    {
-      src->as_u32 = 0;
-    }
-  return (!0);
-}
-
 /* Find interface address which matches destination. */
 always_inline ip4_address_t *
 ip4_interface_address_matching_destination (ip4_main_t * im,
diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c
index 919718035a1..5903ef892ca 100644
--- a/src/vnet/ip/ip4_forward.c
+++ b/src/vnet/ip/ip4_forward.c
@@ -380,28 +380,28 @@ ip4_add_interface_prefix_routes (ip4_main_t *im,
   mhash_set (&lm->prefix_to_if_prefix_index, &key,
 	     if_prefix - lm->if_prefix_pool, 0 /* old value */);
 
+  pfx_special.fp_len = a->address_length;
+  pfx_special.fp_addr.ip4.as_u32 = address->as_u32;
+
+  /* set the glean route for the prefix */
+  fib_table_entry_update_one_path (fib_index, &pfx_special,
+                                   FIB_SOURCE_INTERFACE,
+                                   (FIB_ENTRY_FLAG_CONNECTED |
+                                    FIB_ENTRY_FLAG_ATTACHED),
+                                   DPO_PROTO_IP4,
+                                   /* No next-hop address */
+                                   NULL,
+                                   sw_if_index,
+                                   /* invalid FIB index */
+                                   ~0,
+                                   1,
+                                   /* no out-label stack */
+                                   NULL,
+                                   FIB_ROUTE_PATH_FLAG_NONE);
+
   /* length <= 30 - add glean, drop first address, maybe drop bcast address */
   if (a->address_length <= 30)
     {
-      pfx_special.fp_len = a->address_length;
-      pfx_special.fp_addr.ip4.as_u32 = address->as_u32;
-
-      /* set the glean route for the prefix */
-      fib_table_entry_update_one_path (fib_index, &pfx_special,
-				       FIB_SOURCE_INTERFACE,
-				       (FIB_ENTRY_FLAG_CONNECTED |
-					FIB_ENTRY_FLAG_ATTACHED),
-				       DPO_PROTO_IP4,
-				       /* No next-hop address */
-				       NULL,
-				       sw_if_index,
-                                       /* invalid FIB index */
-                                       ~0,
-                                       1,
-                                       /* no out-label stack */
-                                       NULL,
-                                       FIB_ROUTE_PATH_FLAG_NONE);
-
       /* set a drop route for the base address of the prefix */
       pfx_special.fp_len = 32;
       pfx_special.fp_addr.ip4.as_u32 =
@@ -528,90 +528,52 @@ ip4_del_interface_prefix_routes (ip4_main_t * im,
   if_prefix->ref_count -= 1;
 
   /*
-   * Routes need to be adjusted if:
-   * - deleting last intf addr in prefix
-   * - deleting intf addr used as default source address in glean adjacency
+   * Routes need to be adjusted if deleting last intf addr in prefix
    *
    * We're done now otherwise
    */
-  if ((if_prefix->ref_count > 0) &&
-      !pool_is_free_index (lm->if_address_pool, if_prefix->src_ia_index))
+  if (if_prefix->ref_count > 0)
     return;
 
   /* length <= 30, delete glean route, first address, last address */
   if (address_length <= 30)
     {
+      /* Less work to do in FIB if we remove the covered /32s first */
 
-      /* remove glean route for prefix */
-      pfx_special.fp_addr.ip4 = *address;
-      pfx_special.fp_len = address_length;
-      fib_table_entry_delete (fib_index, &pfx_special, FIB_SOURCE_INTERFACE);
-
-      /* if no more intf addresses in prefix, remove other special routes */
-      if (!if_prefix->ref_count)
-	{
-	  /* first address in prefix */
-	  pfx_special.fp_addr.ip4.as_u32 =
-	    address->as_u32 & im->fib_masks[address_length];
-	  pfx_special.fp_len = 32;
+      /* first address in prefix */
+      pfx_special.fp_addr.ip4.as_u32 =
+        address->as_u32 & im->fib_masks[address_length];
+      pfx_special.fp_len = 32;
 
-	  if (pfx_special.fp_addr.ip4.as_u32 != address->as_u32)
-	  fib_table_entry_special_remove (fib_index,
-					  &pfx_special,
-					  FIB_SOURCE_INTERFACE);
+      if (pfx_special.fp_addr.ip4.as_u32 != address->as_u32)
+        fib_table_entry_special_remove (fib_index,
+                                        &pfx_special,
+                                        FIB_SOURCE_INTERFACE);
 
-	  /* prefix broadcast address */
-	  pfx_special.fp_addr.ip4.as_u32 =
-	    address->as_u32 | ~im->fib_masks[address_length];
-	  pfx_special.fp_len = 32;
+      /* prefix broadcast address */
+      pfx_special.fp_addr.ip4.as_u32 =
+        address->as_u32 | ~im->fib_masks[address_length];
+      pfx_special.fp_len = 32;
 
-	  if (pfx_special.fp_addr.ip4.as_u32 != address->as_u32)
-	  fib_table_entry_special_remove (fib_index,
-					  &pfx_special,
-					  FIB_SOURCE_INTERFACE);
-	}
-      else
-	/* default source addr just got deleted, find another */
-	{
-	  ip_interface_address_t *new_src_ia = NULL;
-	  ip4_address_t *new_src_addr = NULL;
-
-	  new_src_addr =
-	    ip4_interface_address_matching_destination
-	      (im, address, sw_if_index, &new_src_ia);
-
-	  if_prefix->src_ia_index = new_src_ia - lm->if_address_pool;
-
-	  pfx_special.fp_len = address_length;
-	  pfx_special.fp_addr.ip4 = *new_src_addr;
-
-	  /* set new glean route for the prefix */
-	  fib_table_entry_update_one_path (fib_index, &pfx_special,
-					   FIB_SOURCE_INTERFACE,
-					   (FIB_ENTRY_FLAG_CONNECTED |
-					    FIB_ENTRY_FLAG_ATTACHED),
-					   DPO_PROTO_IP4,
-					   /* No next-hop address */
-					   NULL,
-					   sw_if_index,
-					   /* invalid FIB index */
-					   ~0,
-					   1,
-					   /* no out-label stack */
-					   NULL,
-					   FIB_ROUTE_PATH_FLAG_NONE);
-	  return;
-	}
+      if (pfx_special.fp_addr.ip4.as_u32 != address->as_u32)
+        fib_table_entry_special_remove (fib_index,
+                                        &pfx_special,
+                                        FIB_SOURCE_INTERFACE);
     }
-  /* length == 31, delete attached route for the other address */
   else if (address_length == 31)
     {
+      /* length == 31, delete attached route for the other address */
       pfx_special.fp_addr.ip4.as_u32 =
 	address->as_u32 ^ clib_host_to_net_u32(1);
 
       fib_table_entry_delete (fib_index, &pfx_special, FIB_SOURCE_INTERFACE);
     }
 
+  /* remove glean route for prefix */
+  pfx_special.fp_addr.ip4 = *address;
+  pfx_special.fp_len = address_length;
+  fib_table_entry_delete (fib_index, &pfx_special, FIB_SOURCE_INTERFACE);
+
   mhash_unset (&lm->prefix_to_if_prefix_index, &key, 0 /* old_value */);
   pool_put (lm->if_prefix_pool, if_prefix);
 }
@@ -623,16 +585,15 @@ ip4_del_interface_routes (u32 sw_if_index,
 			  ip4_address_t * address, u32 address_length)
 {
   fib_prefix_t pfx = {
-    .fp_len = address_length,
+    .fp_len = 32,
     .fp_proto = FIB_PROTOCOL_IP4,
     .fp_addr.ip4 = *address,
   };
 
+  fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
+
   ip4_del_interface_prefix_routes (im, sw_if_index, fib_index,
 				   address, address_length);
-
-  pfx.fp_len = 32;
-  fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 }
 
 #ifndef CLIB_MARCH_VARIANT
@@ -2540,9 +2501,8 @@ ip4_rewrite_inline_with_gso (vlib_main_t * vm,
 	       thread_index, adj_index0, 1,
 	       vlib_buffer_length_in_chain (vm, b[0]) + rw_len0);
 
-	  if (is_midchain && adj0->sub_type.midchain.fixup_func)
-	    adj0->sub_type.midchain.fixup_func
-	      (vm, adj0, b[0], adj0->sub_type.midchain.fixup_data);
+	  if (is_midchain)
+	    adj_midchain_fixup (vm, adj0, b[0]);
 
 	  if (is_mcast)
 	    /* copy bytes from the IP address into the MAC rewrite */
diff --git a/src/vnet/ip/ip6_link.c b/src/vnet/ip/ip6_link.c
index bd7ad73b695..082033a9de9 100644
--- a/src/vnet/ip/ip6_link.c
+++ b/src/vnet/ip/ip6_link.c
@@ -337,41 +337,6 @@ ip6_link_get_mcast_adj (u32 sw_if_index)
 }
 
 int
-ip6_src_address_for_packet (u32 sw_if_index,
-			    const ip6_address_t * dst, ip6_address_t * src)
-{
-  ip_lookup_main_t *lm;
-
-  lm = &ip6_main.lookup_main;
-
-  if (ip6_address_is_link_local_unicast (dst))
-    {
-      ip6_address_copy (src, ip6_get_link_local_address (sw_if_index));
-
-      return (!0);
-    }
-  else
-    {
-      u32 if_add_index =
-	lm->if_address_pool_index_by_sw_if_index[sw_if_index];
-      if (PREDICT_TRUE (if_add_index != ~0))
-	{
-	  ip_interface_address_t *if_add =
-	    pool_elt_at_index (lm->if_address_pool, if_add_index);
-	  ip6_address_t *if_ip =
-	    ip_interface_address_get_address (lm, if_add);
-	  *src = *if_ip;
-	  return (!0);
-	}
-    }
-
-  src->as_u64[0] = 0;
-  src->as_u64[1] = 0;
-
-  return (0);
-}
-
-int
 ip6_link_set_local_address (u32 sw_if_index, const ip6_address_t * address)
 {
   ip6_link_delegate_t *ild;
diff --git a/src/vnet/ip/ip6_link.h b/src/vnet/ip/ip6_link.h
index a9dfa5edcb0..d9f611950ea 100644
--- a/src/vnet/ip/ip6_link.h
+++ b/src/vnet/ip/ip6_link.h
@@ -30,10 +30,6 @@ extern int ip6_link_set_local_address (u32 sw_if_index,
 				       const ip6_address_t * address);
 extern adj_index_t ip6_link_get_mcast_adj (u32 sw_if_index);
 
-extern int
-ip6_src_address_for_packet (u32 sw_if_index,
-			    const ip6_address_t * dst, ip6_address_t * src);
-
 /**
  * Delegates for the interfaces
  *