ip: Replace Sematics for Interface IP addresses

Type: feature

 - replace functions for prefixes attached to interfaces
 - add ip_interface.[ch] to consoldate the functions

Signed-off-by: Neale Ranns <nranns@cisco.com>
Change-Id: I9c0c39c09dbf80ea1aadefee02c9bd16f094b6ad

13 files changed, 626 insertions, 295 deletions

diff --git a/src/vnet/CMakeLists.txt b/src/vnet/CMakeLists.txt
index bf1dba7f090..39774ba2682 100644
--- a/src/vnet/CMakeLists.txt
+++ b/src/vnet/CMakeLists.txt
@@ -458,6 +458,7 @@ list(APPEND VNET_SOURCES
   ip/ip_checksum.c
   ip/ip_frag.c
   ip/ip.c
+  ip/ip_interface.c
   ip/ip_init.c
   ip/ip_in_out_acl.c
   ip/ip_punt_drop.c
@@ -501,6 +502,7 @@ list(APPEND VNET_HEADERS
   ip/ip6_hop_by_hop_packet.h
   ip/ip6_packet.h
   ip/ip.h
+  ip/ip_interface.h
   ip/ip_packet.h
   ip/ip_source_and_port_range_check.h
   ip/ip_types.h
diff --git a/src/vnet/interface.api b/src/vnet/interface.api
index 1ecd3472f72..c262cd69fcf 100644
--- a/src/vnet/interface.api
+++ b/src/vnet/interface.api
@@ -228,6 +228,43 @@ autoreply define sw_interface_add_del_address
   vl_api_address_with_prefix_t prefix;
 };
 
+/** \brief IP interface address replace begin
+
+    The use-case is that, for some unspecified reason, the control plane
+    has a different set of interface addresses than VPP
+    currently has. The CP would thus like to 'replace' VPP's set
+    only by specifying what the new set shall be, i.e. it is not
+    going to delete anything that already eixts, rather, is wants any
+    unspecified interface addresses to be deleted implicitly.
+    The CP declares the start of this procedure with this replace_begin
+    API Call, and when it has populated all addresses it wants, it calls
+    the below replace_end API. From this point on it is of course free
+    to add and delete interface addresses as usual.
+    The underlying mechanism by which VPP implements this replace is
+    intentionally left unspecified.
+
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+autoreply define sw_interface_address_replace_begin
+{
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief IP interface address replace end
+
+    see ip_interface_address_replace_begin description.
+
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+autoreply define sw_interface_address_replace_end
+{
+  u32 client_index;
+  u32 context;
+};
+
 /** \brief Associate the specified interface with a fib table
     @param client_index - opaque cookie to identify the sender
     @param context - sender context, to match reply w/ request
diff --git a/src/vnet/interface_api.c b/src/vnet/interface_api.c
index 4ce0a9ab053..5b24a29efc9 100644
--- a/src/vnet/interface_api.c
+++ b/src/vnet/interface_api.c
@@ -79,8 +79,12 @@ _(CREATE_LOOPBACK_INSTANCE, create_loopback_instance)		\
 _(DELETE_LOOPBACK, delete_loopback)                             \
 _(INTERFACE_NAME_RENUMBER, interface_name_renumber)             \
 _(COLLECT_DETAILED_INTERFACE_STATS, collect_detailed_interface_stats) \
-_(SW_INTERFACE_SET_IP_DIRECTED_BROADCAST,                            \
-  sw_interface_set_ip_directed_broadcast)
+_(SW_INTERFACE_SET_IP_DIRECTED_BROADCAST,                       \
+  sw_interface_set_ip_directed_broadcast)                       \
+_(SW_INTERFACE_ADDRESS_REPLACE_BEGIN,                           \
+  sw_interface_address_replace_begin)                           \
+_(SW_INTERFACE_ADDRESS_REPLACE_END,                             \
+  sw_interface_address_replace_end)
 
 static void
 vl_api_sw_interface_set_flags_t_handler (vl_api_sw_interface_set_flags_t * mp)
@@ -1366,6 +1370,30 @@ static void
   REPLY_MACRO (VL_API_COLLECT_DETAILED_INTERFACE_STATS_REPLY);
 }
 
+static void
+  vl_api_sw_interface_address_replace_begin_t_handler
+  (vl_api_sw_interface_address_replace_begin_t * mp)
+{
+  vl_api_sw_interface_address_replace_begin_reply_t *rmp;
+  int rv = 0;
+
+  ip_interface_address_mark ();
+
+  REPLY_MACRO (VL_API_SW_INTERFACE_ADDRESS_REPLACE_BEGIN_REPLY);
+}
+
+static void
+  vl_api_sw_interface_address_replace_end_t_handler
+  (vl_api_sw_interface_address_replace_end_t * mp)
+{
+  vl_api_sw_interface_address_replace_end_reply_t *rmp;
+  int rv = 0;
+
+  ip_interface_address_sweep ();
+
+  REPLY_MACRO (VL_API_SW_INTERFACE_ADDRESS_REPLACE_END_REPLY);
+}
+
 /*
  * vpe_api_hookup
  * Add vpe's API message handlers to the table.
diff --git a/src/vnet/ip/ip.c b/src/vnet/ip/ip.c
index 8959b4c09cc..38e53fdef75 100644
--- a/src/vnet/ip/ip.c
+++ b/src/vnet/ip/ip.c
@@ -96,73 +96,6 @@ ip_set (ip46_address_t * dst, void *src, u8 is_ip4)
 		      sizeof (ip6_address_t));
 }
 
-u8
-ip_interface_has_address (u32 sw_if_index, ip46_address_t * ip, u8 is_ip4)
-{
-  ip_interface_address_t *ia = 0;
-
-  if (is_ip4)
-    {
-      ip_lookup_main_t *lm4 = &ip4_main.lookup_main;
-      ip4_address_t *ip4;
-      /* *INDENT-OFF* */
-      foreach_ip_interface_address (lm4, ia, sw_if_index, 1 /* unnumbered */ ,
-      ({
-        ip4 = ip_interface_address_get_address (lm4, ia);
-        if (ip4_address_compare (ip4, &ip->ip4) == 0)
-          return 1;
-      }));
-      /* *INDENT-ON* */
-    }
-  else
-    {
-      ip_lookup_main_t *lm6 = &ip6_main.lookup_main;
-      ip6_address_t *ip6;
-      /* *INDENT-OFF* */
-      foreach_ip_interface_address (lm6, ia, sw_if_index, 1 /* unnumbered */ ,
-      ({
-        ip6 = ip_interface_address_get_address (lm6, ia);
-        if (ip6_address_compare (ip6, &ip->ip6) == 0)
-          return 1;
-      }));
-      /* *INDENT-ON* */
-    }
-  return 0;
-}
-
-void *
-ip_interface_get_first_ip (u32 sw_if_index, u8 is_ip4)
-{
-  ip_lookup_main_t *lm4 = &ip4_main.lookup_main;
-  ip_lookup_main_t *lm6 = &ip6_main.lookup_main;
-  ip_interface_address_t *ia = 0;
-
-  if (is_ip4)
-    {
-      /* *INDENT-OFF* */
-      foreach_ip_interface_address (lm4, ia, sw_if_index, 1 /* unnumbered */ ,
-      ({
-        return ip_interface_address_get_address (lm4, ia);
-      }));
-      /* *INDENT-ON* */
-    }
-  else
-    {
-      /* *INDENT-OFF* */
-      foreach_ip_interface_address (lm6, ia, sw_if_index, 1 /* unnumbered */ ,
-      ({
-        ip6_address_t *rv;
-        rv = ip_interface_address_get_address (lm6, ia);
-        /* Trying to use a link-local ip6 src address is a fool's errand */
-        if (!ip6_address_is_link_local_unicast (rv))
-          return rv;
-      }));
-      /* *INDENT-ON* */
-    }
-
-  return 0;
-}
-
 u8 *
 format_ip_address_family (u8 * s, va_list * args)
 {
diff --git a/src/vnet/ip/ip.h b/src/vnet/ip/ip.h
index a6fcd410c0e..3fa97267b36 100644
--- a/src/vnet/ip/ip.h
+++ b/src/vnet/ip/ip.h
@@ -50,6 +50,7 @@
 #include <vnet/ip/format.h>
 #include <vnet/ip/ip_packet.h>
 #include <vnet/ip/lookup.h>
+#include <vnet/ip/ip_interface.h>
 
 #include <vnet/tcp/tcp_packet.h>
 #include <vnet/udp/udp_packet.h>
@@ -274,10 +275,8 @@ u8 ip_is_local_host (ip46_address_t * ip46_address, u8 is_ip4);
 u8 ip4_is_local_host (ip4_address_t * ip4_address);
 u8 ip6_is_local_host (ip6_address_t * ip6_address);
 u8 ip_is_local (u32 fib_index, ip46_address_t * ip46_address, u8 is_ip4);
-u8 ip_interface_has_address (u32 sw_if_index, ip46_address_t * ip, u8 is_ip4);
 void ip_copy (ip46_address_t * dst, ip46_address_t * src, u8 is_ip4);
 void ip_set (ip46_address_t * dst, void *src, u8 is_ip4);
-void *ip_interface_get_first_ip (u32 sw_if_index, u8 is_ip4);
 
 always_inline u32 vlib_buffer_get_ip4_fib_index (vlib_buffer_t * b);
 always_inline u32 vlib_buffer_get_ip6_fib_index (vlib_buffer_t * b);
diff --git a/src/vnet/ip/ip4.h b/src/vnet/ip/ip4.h
index 6e13cc8ea3e..b5bc2e2c33f 100644
--- a/src/vnet/ip/ip4.h
+++ b/src/vnet/ip/ip4.h
@@ -42,6 +42,7 @@
 
 #include <vnet/ip/ip4_packet.h>
 #include <vnet/ip/lookup.h>
+#include <vnet/ip/ip_interface.h>
 #include <vnet/buffer.h>
 #include <vnet/feature/feature.h>
 #include <vnet/ip/icmp46_packet.h>
diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c
index 952915f5317..acff66d994b 100644
--- a/src/vnet/ip/ip4_forward.c
+++ b/src/vnet/ip/ip4_forward.c
@@ -680,7 +680,7 @@ ip4_add_del_interface_address_internal (vlib_main_t * vm,
   ip4_main_t *im = &ip4_main;
   ip_lookup_main_t *lm = &im->lookup_main;
   clib_error_t *error = 0;
-  u32 if_address_index, elts_before;
+  u32 if_address_index;
   ip4_address_fib_t ip4_af, *addr_fib = 0;
 
   /* local0 interface doesn't support IP addressing  */
@@ -720,6 +720,7 @@ ip4_add_del_interface_address_internal (vlib_main_t * vm,
                    ip4_address_t * x =
                      ip_interface_address_get_address
                      (&im->lookup_main, ia);
+
                    if (ip4_destination_matches_route
                        (im, address, x, ia->address_length) ||
                        ip4_destination_matches_route (im,
@@ -733,11 +734,18 @@ ip4_add_del_interface_address_internal (vlib_main_t * vm,
 			   (x->as_u32 != address->as_u32))
 		         continue;
 
+                       if (ia->flags & IP_INTERFACE_ADDRESS_FLAG_STALE)
+                         /* if the address we're comparing against is stale
+                          * then the CP has not added this one back yet, maybe
+                          * it never will, so we have to assume it won't and
+                          * ignore it. if it does add it back, then it will fail
+                          * because this one is now present */
+                         continue;
+
 		       /* error if the length or intf was different */
-                       vnm->api_errno = VNET_API_ERROR_DUPLICATE_IF_ADDRESS;
+                       vnm->api_errno = VNET_API_ERROR_ADDRESS_IN_USE;
 
-                       return
-                         clib_error_create
+                       error = clib_error_create
                          ("failed to add %U on %U which conflicts with %U for interface %U",
                           format_ip4_address_and_length, address,
                           address_length,
@@ -747,6 +755,7 @@ ip4_add_del_interface_address_internal (vlib_main_t * vm,
                           ia->address_length,
                           format_vnet_sw_if_index_name, vnm,
                           sif->sw_if_index);
+                       goto done;
                      }
                  }));
             }
@@ -754,10 +763,70 @@ ip4_add_del_interface_address_internal (vlib_main_t * vm,
     }
   /* *INDENT-ON* */
 
-  elts_before = pool_elts (lm->if_address_pool);
+  if_address_index = ip_interface_address_find (lm, addr_fib, address_length);
+
+  if (is_del)
+    {
+      if (~0 == if_address_index)
+	{
+	  vnm->api_errno = VNET_API_ERROR_ADDRESS_NOT_FOUND_FOR_INTERFACE;
+	  error = clib_error_create ("%U not found for interface %U",
+				     lm->format_address_and_length,
+				     addr_fib, address_length,
+				     format_vnet_sw_if_index_name, vnm,
+				     sw_if_index);
+	  goto done;
+	}
+
+      ip_interface_address_del (lm, if_address_index, addr_fib);
+    }
+  else
+    {
+      if (~0 != if_address_index)
+	{
+	  ip_interface_address_t *ia;
+
+	  ia = pool_elt_at_index (lm->if_address_pool, if_address_index);
+
+	  if (ia->flags & IP_INTERFACE_ADDRESS_FLAG_STALE)
+	    {
+	      if (ia->sw_if_index == sw_if_index)
+		{
+		  /* re-adding an address during the replace action.
+		   * consdier this the update. clear the flag and
+		   * we're done */
+		  ia->flags &= ~IP_INTERFACE_ADDRESS_FLAG_STALE;
+		  goto done;
+		}
+	      else
+		{
+		  /* The prefix is moving from one interface to another.
+		   * delete the stale and add the new */
+		  ip4_add_del_interface_address_internal (vm,
+							  ia->sw_if_index,
+							  address,
+							  address_length, 1);
+		  ia = NULL;
+		  error = ip_interface_address_add (lm, sw_if_index,
+						    addr_fib, address_length,
+						    &if_address_index);
+		}
+	    }
+	  else
+	    {
+	      vnm->api_errno = VNET_API_ERROR_DUPLICATE_IF_ADDRESS;
+	      error = clib_error_create
+		("Prefix %U already found on interface %U",
+		 lm->format_address_and_length, addr_fib, address_length,
+		 format_vnet_sw_if_index_name, vnm, ia->sw_if_index);
+	    }
+	}
+      else
+	error = ip_interface_address_add (lm, sw_if_index,
+					  addr_fib, address_length,
+					  &if_address_index);
+    }
 
-  error = ip_interface_address_add_del
-    (lm, sw_if_index, addr_fib, address_length, is_del, &if_address_index);
   if (error)
     goto done;
 
@@ -778,14 +847,10 @@ ip4_add_del_interface_address_internal (vlib_main_t * vm,
 				  (lm->if_address_pool, if_address_index));
     }
 
-  /* If pool did not grow/shrink: add duplicate address. */
-  if (elts_before != pool_elts (lm->if_address_pool))
-    {
-      ip4_add_del_interface_address_callback_t *cb;
-      vec_foreach (cb, im->add_del_interface_address_callbacks)
-	cb->function (im, cb->function_opaque, sw_if_index,
-		      address, address_length, if_address_index, is_del);
-    }
+  ip4_add_del_interface_address_callback_t *cb;
+  vec_foreach (cb, im->add_del_interface_address_callbacks)
+    cb->function (im, cb->function_opaque, sw_if_index,
+		  address, address_length, if_address_index, is_del);
 
 done:
   vec_free (addr_fib);
diff --git a/src/vnet/ip/ip6.h b/src/vnet/ip/ip6.h
index ec8c772456c..5e16f990534 100644
--- a/src/vnet/ip/ip6.h
+++ b/src/vnet/ip/ip6.h
@@ -47,6 +47,7 @@
 #include <vnet/ip/ip46_address.h>
 #include <vnet/ip/ip6_hop_by_hop_packet.h>
 #include <vnet/ip/lookup.h>
+#include <vnet/ip/ip_interface.h>
 #include <stdbool.h>
 #include <vppinfra/bihash_24_8.h>
 #include <vppinfra/bihash_40_8.h>
diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c
index 91a93ee6cf0..0325627a1ef 100644
--- a/src/vnet/ip/ip6_forward.c
+++ b/src/vnet/ip/ip6_forward.c
@@ -295,7 +295,7 @@ ip6_add_del_interface_address (vlib_main_t * vm,
   vnet_main_t *vnm = vnet_get_main ();
   ip6_main_t *im = &ip6_main;
   ip_lookup_main_t *lm = &im->lookup_main;
-  clib_error_t *error;
+  clib_error_t *error = NULL;
   u32 if_address_index;
   ip6_address_fib_t ip6_af, *addr_fib = 0;
   const ip6_address_t *ll_addr;
@@ -371,6 +371,7 @@ ip6_add_del_interface_address (vlib_main_t * vm,
                    ip6_address_t * x =
                      ip_interface_address_get_address
                      (&im->lookup_main, ia);
+
                    if (ip6_destination_matches_route
                        (im, address, x, ia->address_length) ||
                        ip6_destination_matches_route (im,
@@ -384,10 +385,17 @@ ip6_add_del_interface_address (vlib_main_t * vm,
 			   !ip6_address_is_equal (x, address))
 		         continue;
 
+                       if (ia->flags & IP_INTERFACE_ADDRESS_FLAG_STALE)
+                         /* if the address we're comparing against is stale
+                          * then the CP has not added this one back yet, maybe
+                          * it never will, so we have to assume it won't and
+                          * ignore it. if it does add it back, then it will fail
+                          * because this one is now present */
+                         continue;
+
 		       /* error if the length or intf was different */
                        vnm->api_errno = VNET_API_ERROR_DUPLICATE_IF_ADDRESS;
-                       return
-                         clib_error_create
+                       error =  clib_error_create
                          ("failed to add %U which conflicts with %U for interface %U",
                           format_ip6_address_and_length, address,
                           address_length,
@@ -395,6 +403,7 @@ ip6_add_del_interface_address (vlib_main_t * vm,
                           ia->address_length,
                           format_vnet_sw_if_index_name, vnm,
                           sif->sw_if_index);
+                       goto done;
                      }
                  }));
             }
@@ -402,18 +411,71 @@ ip6_add_del_interface_address (vlib_main_t * vm,
     }
   /* *INDENT-ON* */
 
-  {
-    uword elts_before = pool_elts (lm->if_address_pool);
+  if_address_index = ip_interface_address_find (lm, addr_fib, address_length);
+
+  if (is_del)
+    {
+      if (~0 == if_address_index)
+	{
+	  vnm->api_errno = VNET_API_ERROR_ADDRESS_NOT_FOUND_FOR_INTERFACE;
+	  error = clib_error_create ("%U not found for interface %U",
+				     lm->format_address_and_length,
+				     addr_fib, address_length,
+				     format_vnet_sw_if_index_name, vnm,
+				     sw_if_index);
+	  goto done;
+	}
+
+      ip_interface_address_del (lm, if_address_index, addr_fib);
+    }
+  else
+    {
+      if (~0 != if_address_index)
+	{
+	  ip_interface_address_t *ia;
 
-    error = ip_interface_address_add_del
-      (lm, sw_if_index, addr_fib, address_length, is_del, &if_address_index);
-    if (error)
-      goto done;
+	  ia = pool_elt_at_index (lm->if_address_pool, if_address_index);
 
-    /* Pool did not grow: add duplicate address. */
-    if (elts_before == pool_elts (lm->if_address_pool))
-      goto done;
-  }
+	  if (ia->flags & IP_INTERFACE_ADDRESS_FLAG_STALE)
+	    {
+	      if (ia->sw_if_index == sw_if_index)
+		{
+		  /* re-adding an address during the replace action.
+		   * consdier this the update. clear the flag and
+		   * we're done */
+		  ia->flags &= ~IP_INTERFACE_ADDRESS_FLAG_STALE;
+		  goto done;
+		}
+	      else
+		{
+		  /* The prefix is moving from one interface to another.
+		   * delete the stale and add the new */
+		  ip6_add_del_interface_address (vm,
+						 ia->sw_if_index,
+						 address, address_length, 1);
+		  ia = NULL;
+		  error = ip_interface_address_add (lm, sw_if_index,
+						    addr_fib, address_length,
+						    &if_address_index);
+		}
+	    }
+	  else
+	    {
+	      vnm->api_errno = VNET_API_ERROR_DUPLICATE_IF_ADDRESS;
+	      error = clib_error_create
+		("Prefix %U already found on interface %U",
+		 lm->format_address_and_length, addr_fib, address_length,
+		 format_vnet_sw_if_index_name, vnm, ia->sw_if_index);
+	    }
+	}
+      else
+	error = ip_interface_address_add (lm, sw_if_index,
+					  addr_fib, address_length,
+					  &if_address_index);
+    }
+
+  if (error)
+    goto done;
 
   ip6_sw_interface_enable_disable (sw_if_index, !is_del);
   if (!is_del)
@@ -432,12 +494,12 @@ ip6_add_del_interface_address (vlib_main_t * vm,
 				  pool_elt_at_index (lm->if_address_pool,
 						     if_address_index));
     }
-  {
-    ip6_add_del_interface_address_callback_t *cb;
-    vec_foreach (cb, im->add_del_interface_address_callbacks)
-      cb->function (im, cb->function_opaque, sw_if_index,
-		    address, address_length, if_address_index, is_del);
-  }
+
+  ip6_add_del_interface_address_callback_t *cb;
+  vec_foreach (cb, im->add_del_interface_address_callbacks)
+    cb->function (im, cb->function_opaque, sw_if_index,
+		  address, address_length, if_address_index, is_del);
+
   if (is_del)
     ip6_link_disable (sw_if_index);
 
diff --git a/src/vnet/ip/ip_interface.c b/src/vnet/ip/ip_interface.c
new file mode 100644
index 00000000000..23c3df81638
--- /dev/null
+++ b/src/vnet/ip/ip_interface.c
@@ -0,0 +1,288 @@
+/*
+ * Copyright (c) 2020 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vnet/ip/ip.h>
+
+/**
+ * @file
+ * @brief IP prefix management on interfaces
+ */
+
+u32
+ip_interface_address_find (ip_lookup_main_t * lm,
+			   void *addr_fib, u32 address_length)
+{
+  uword *p = mhash_get (&lm->address_to_if_address_index, addr_fib);
+
+  if (p)
+    return (p[0]);
+
+  return (~0);
+}
+
+clib_error_t *
+ip_interface_address_add (ip_lookup_main_t * lm,
+			  u32 sw_if_index,
+			  void *addr_fib,
+			  u32 address_length, u32 * result_if_address_index)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+  ip_interface_address_t *a, *prev;
+  u32 pi;			/* previous index */
+  u32 ai;
+  u32 hi;			/* head index */
+
+  /* Verify given length. */
+  if ((address_length == 0) ||
+      (lm->is_ip6 && address_length > 128) ||
+      (!lm->is_ip6 && address_length > 32))
+    {
+      vnm->api_errno = VNET_API_ERROR_ADDRESS_LENGTH_MISMATCH;
+      return clib_error_create
+	("%U wrong length for interface %U",
+	 lm->format_address_and_length, addr_fib,
+	 address_length, format_vnet_sw_if_index_name, vnm, sw_if_index);
+    }
+
+  vec_validate_init_empty (lm->if_address_pool_index_by_sw_if_index,
+			   sw_if_index, ~0);
+
+  pool_get_zero (lm->if_address_pool, a);
+
+  ai = a - lm->if_address_pool;
+  hi = pi = lm->if_address_pool_index_by_sw_if_index[sw_if_index];
+
+  prev = 0;
+  while (pi != (u32) ~ 0)
+    {
+      prev = pool_elt_at_index (lm->if_address_pool, pi);
+      pi = prev->next_this_sw_interface;
+    }
+  pi = prev ? prev - lm->if_address_pool : (u32) ~ 0;
+
+  a->address_key = mhash_set (&lm->address_to_if_address_index,
+			      addr_fib, ai, /* old_value */ 0);
+  a->address_length = address_length;
+  a->sw_if_index = sw_if_index;
+  a->flags = 0;
+  a->prev_this_sw_interface = pi;
+  a->next_this_sw_interface = ~0;
+  if (prev)
+    prev->next_this_sw_interface = ai;
+
+  lm->if_address_pool_index_by_sw_if_index[sw_if_index] =
+    (hi != ~0) ? hi : ai;
+
+  *result_if_address_index = ai;
+
+  return (NULL);
+}
+
+void
+ip_interface_address_del (ip_lookup_main_t * lm,
+			  u32 address_index, void *addr_fib)
+{
+  ip_interface_address_t *a, *prev, *next;
+
+  a = pool_elt_at_index (lm->if_address_pool, address_index);
+
+  if (a->prev_this_sw_interface != ~0)
+    {
+      prev = pool_elt_at_index (lm->if_address_pool,
+				a->prev_this_sw_interface);
+      prev->next_this_sw_interface = a->next_this_sw_interface;
+    }
+  if (a->next_this_sw_interface != ~0)
+    {
+      next = pool_elt_at_index (lm->if_address_pool,
+				a->next_this_sw_interface);
+      next->prev_this_sw_interface = a->prev_this_sw_interface;
+
+      if (a->prev_this_sw_interface == ~0)
+	lm->if_address_pool_index_by_sw_if_index[a->sw_if_index] =
+	  a->next_this_sw_interface;
+    }
+
+  if ((a->next_this_sw_interface == ~0) && (a->prev_this_sw_interface == ~0))
+    lm->if_address_pool_index_by_sw_if_index[a->sw_if_index] = ~0;
+
+  mhash_unset (&lm->address_to_if_address_index, addr_fib,
+	       /* old_value */ 0);
+  pool_put (lm->if_address_pool, a);
+}
+
+u8
+ip_interface_has_address (u32 sw_if_index, ip46_address_t * ip, u8 is_ip4)
+{
+  ip_interface_address_t *ia = 0;
+
+  if (is_ip4)
+    {
+      ip_lookup_main_t *lm4 = &ip4_main.lookup_main;
+      ip4_address_t *ip4;
+      /* *INDENT-OFF* */
+      foreach_ip_interface_address (lm4, ia, sw_if_index, 1 /* unnumbered */ ,
+      ({
+        ip4 = ip_interface_address_get_address (lm4, ia);
+        if (ip4_address_compare (ip4, &ip->ip4) == 0)
+          return 1;
+      }));
+      /* *INDENT-ON* */
+    }
+  else
+    {
+      ip_lookup_main_t *lm6 = &ip6_main.lookup_main;
+      ip6_address_t *ip6;
+      /* *INDENT-OFF* */
+      foreach_ip_interface_address (lm6, ia, sw_if_index, 1 /* unnumbered */ ,
+      ({
+        ip6 = ip_interface_address_get_address (lm6, ia);
+        if (ip6_address_compare (ip6, &ip->ip6) == 0)
+          return 1;
+      }));
+      /* *INDENT-ON* */
+    }
+  return 0;
+}
+
+void *
+ip_interface_get_first_ip (u32 sw_if_index, u8 is_ip4)
+{
+  ip_lookup_main_t *lm4 = &ip4_main.lookup_main;
+  ip_lookup_main_t *lm6 = &ip6_main.lookup_main;
+  ip_interface_address_t *ia = 0;
+
+  if (is_ip4)
+    {
+      /* *INDENT-OFF* */
+      foreach_ip_interface_address (lm4, ia, sw_if_index, 1 /* unnumbered */ ,
+      ({
+        return ip_interface_address_get_address (lm4, ia);
+      }));
+      /* *INDENT-ON* */
+    }
+  else
+    {
+      /* *INDENT-OFF* */
+      foreach_ip_interface_address (lm6, ia, sw_if_index, 1 /* unnumbered */ ,
+      ({
+        ip6_address_t *rv;
+        rv = ip_interface_address_get_address (lm6, ia);
+        /* Trying to use a link-local ip6 src address is a fool's errand */
+        if (!ip6_address_is_link_local_unicast (rv))
+          return rv;
+      }));
+      /* *INDENT-ON* */
+    }
+
+  return 0;
+}
+
+static walk_rc_t
+ip_interface_address_mark_one_interface (vnet_main_t * vnm,
+					 vnet_sw_interface_t * si, void *ctx)
+{
+  ip_lookup_main_t *lm4 = &ip4_main.lookup_main;
+  ip_lookup_main_t *lm6 = &ip6_main.lookup_main;
+  ip_interface_address_t *ia = 0;
+
+  /* *INDENT-OFF* */
+  foreach_ip_interface_address (lm4, ia, si->sw_if_index, 1 /* unnumbered */ ,
+  ({
+    ia->flags |= IP_INTERFACE_ADDRESS_FLAG_STALE;
+  }));
+  foreach_ip_interface_address (lm6, ia, si->sw_if_index, 1 /* unnumbered */ ,
+  ({
+    ia->flags |= IP_INTERFACE_ADDRESS_FLAG_STALE;
+  }));
+  /* *INDENT-ON* */
+
+  return (WALK_CONTINUE);
+}
+
+void
+ip_interface_address_mark (void)
+{
+  vnet_sw_interface_walk (vnet_get_main (),
+			  ip_interface_address_mark_one_interface, NULL);
+}
+
+static walk_rc_t
+ip_interface_address_sweep_one_interface (vnet_main_t * vnm,
+					  vnet_sw_interface_t * si, void *ctx)
+{
+  vlib_main_t *vm = vlib_get_main ();
+  ip4_address_t *ip4_addrs = 0;
+  ip6_address_t *ip6_addrs = 0;
+  ip4_main_t *im4 = &ip4_main;
+  ip6_main_t *im6 = &ip6_main;
+  ip_interface_address_t *ia;
+  u32 *ip6_masks = 0;
+  u32 *ip4_masks = 0;
+  int i;
+
+  /* *INDENT-OFF* */
+  foreach_ip_interface_address (&im4->lookup_main, ia, si->sw_if_index, 1,
+  ({
+    if (ia->flags & IP_INTERFACE_ADDRESS_FLAG_STALE)
+      {
+        ip4_address_t * x = (ip4_address_t *)
+          ip_interface_address_get_address (&im4->lookup_main, ia);
+        vec_add1 (ip4_addrs, x[0]);
+        vec_add1 (ip4_masks, ia->address_length);
+      }
+  }));
+
+  foreach_ip_interface_address (&im6->lookup_main, ia, si->sw_if_index, 1,
+  ({
+    if (ia->flags & IP_INTERFACE_ADDRESS_FLAG_STALE)
+      {
+        ip6_address_t * x = (ip6_address_t *)
+          ip_interface_address_get_address (&im6->lookup_main, ia);
+        vec_add1 (ip6_addrs, x[0]);
+        vec_add1 (ip6_masks, ia->address_length);
+      }
+  }));
+  /* *INDENT-ON* */
+
+  for (i = 0; i < vec_len (ip4_addrs); i++)
+    ip4_add_del_interface_address (vm, si->sw_if_index, &ip4_addrs[i],
+				   ip4_masks[i], 1 /* is_del */ );
+  for (i = 0; i < vec_len (ip6_addrs); i++)
+    ip6_add_del_interface_address (vm, si->sw_if_index, &ip6_addrs[i],
+				   ip6_masks[i], 1 /* is_del */ );
+
+  vec_free (ip4_addrs);
+  vec_free (ip4_masks);
+  vec_free (ip6_addrs);
+  vec_free (ip6_masks);
+
+  return (WALK_CONTINUE);
+}
+
+void
+ip_interface_address_sweep (void)
+{
+  vnet_sw_interface_walk (vnet_get_main (),
+			  ip_interface_address_sweep_one_interface, NULL);
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vnet/ip/ip_interface.h b/src/vnet/ip/ip_interface.h
new file mode 100644
index 00000000000..f95b8deb07b
--- /dev/null
+++ b/src/vnet/ip/ip_interface.h
@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2020 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @file
+ * @brief IP prefix management on interfaces
+ */
+
+#ifndef included_ip_interface_h
+#define included_ip_interface_h
+
+#include <vnet/ip/lookup.h>
+
+clib_error_t *ip_interface_address_add (ip_lookup_main_t * lm,
+					u32 sw_if_index,
+					void *address,
+					u32 address_length,
+					u32 * result_index);
+void ip_interface_address_del (ip_lookup_main_t * lm,
+			       u32 addr_index, void *address);
+void *ip_interface_get_first_ip (u32 sw_if_index, u8 is_ip4);
+void ip_interface_address_mark (void);
+void ip_interface_address_sweep (void);
+u32 ip_interface_address_find (ip_lookup_main_t * lm,
+			       void *addr_fib, u32 address_length);
+u8 ip_interface_has_address (u32 sw_if_index, ip46_address_t * ip, u8 is_ip4);
+
+always_inline void *
+ip_interface_address_get_address (ip_lookup_main_t * lm,
+				  ip_interface_address_t * a)
+{
+  return mhash_key_to_mem (&lm->address_to_if_address_index, a->address_key);
+}
+
+always_inline ip_interface_prefix_t *
+ip_get_interface_prefix (ip_lookup_main_t * lm, ip_interface_prefix_key_t * k)
+{
+  uword *p = mhash_get (&lm->prefix_to_if_prefix_index, k);
+  return p ? pool_elt_at_index (lm->if_prefix_pool, p[0]) : 0;
+}
+
+/* *INDENT-OFF* */
+#define foreach_ip_interface_address(lm,a,sw_if_index,loop,body)        \
+do {                                                                    \
+    vnet_main_t *_vnm = vnet_get_main();                                \
+    u32 _sw_if_index = sw_if_index;                                     \
+    vnet_sw_interface_t *_swif;                                         \
+    _swif = vnet_get_sw_interface (_vnm, _sw_if_index);                 \
+                                                                        \
+    /*                                                                  \
+     * Loop => honor unnumbered interface addressing.                   \
+     */                                                                 \
+    if (_swif->flags & VNET_SW_INTERFACE_FLAG_UNNUMBERED)               \
+      {                                                                 \
+        if (loop)                                                       \
+          _sw_if_index = _swif->unnumbered_sw_if_index;                 \
+        else                                                            \
+          /* the interface is unnumbered, by the caller does not want   \
+           * unnumbered interfaces considered/honoured */               \
+          break;                                                        \
+      }                                                                 \
+    u32 _ia = ((vec_len((lm)->if_address_pool_index_by_sw_if_index)     \
+                > (_sw_if_index)) ?                                     \
+               vec_elt ((lm)->if_address_pool_index_by_sw_if_index,     \
+                        (_sw_if_index)) :                               \
+               (u32)~0);                                                \
+    ip_interface_address_t * _a;                                        \
+    while (_ia != ~0)                                                   \
+    {                                                                   \
+        _a = pool_elt_at_index ((lm)->if_address_pool, _ia);            \
+        _ia = _a->next_this_sw_interface;                               \
+        (a) = _a;                                                       \
+        body;                                                           \
+    }                                                                   \
+} while (0)
+/* *INDENT-ON* */
+
+#endif /* included_ip_interface_h */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c
index a0cb28b8906..5d4e137fb1c 100644
--- a/src/vnet/ip/lookup.c
+++ b/src/vnet/ip/lookup.c
@@ -57,132 +57,6 @@
  *
  */
 
-clib_error_t *
-ip_interface_address_add_del (ip_lookup_main_t * lm,
-			      u32 sw_if_index,
-			      void *addr_fib,
-			      u32 address_length,
-			      u32 is_del, u32 * result_if_address_index)
-{
-  vnet_main_t *vnm = vnet_get_main ();
-  ip_interface_address_t *a, *prev, *next;
-  uword *p = mhash_get (&lm->address_to_if_address_index, addr_fib);
-
-  vec_validate_init_empty (lm->if_address_pool_index_by_sw_if_index,
-			   sw_if_index, ~0);
-  a = p ? pool_elt_at_index (lm->if_address_pool, p[0]) : 0;
-
-  /* Verify given length. */
-  if ((a && (address_length != a->address_length)) ||
-      (address_length == 0) ||
-      (lm->is_ip6 && address_length > 128) ||
-      (!lm->is_ip6 && address_length > 32))
-    {
-      vnm->api_errno = VNET_API_ERROR_ADDRESS_LENGTH_MISMATCH;
-      return clib_error_create
-	("%U wrong length (expected %d) for interface %U",
-	 lm->format_address_and_length, addr_fib,
-	 address_length, a ? a->address_length : -1,
-	 format_vnet_sw_if_index_name, vnm, sw_if_index);
-    }
-
-  if (is_del)
-    {
-      if (!a)
-	{
-	  vnet_sw_interface_t *si = vnet_get_sw_interface (vnm, sw_if_index);
-	  vnm->api_errno = VNET_API_ERROR_ADDRESS_NOT_FOUND_FOR_INTERFACE;
-	  return clib_error_create ("%U not found for interface %U",
-				    lm->format_address_and_length,
-				    addr_fib, address_length,
-				    format_vnet_sw_interface_name, vnm, si);
-	}
-
-      if (a->prev_this_sw_interface != ~0)
-	{
-	  prev =
-	    pool_elt_at_index (lm->if_address_pool,
-			       a->prev_this_sw_interface);
-	  prev->next_this_sw_interface = a->next_this_sw_interface;
-	}
-      if (a->next_this_sw_interface != ~0)
-	{
-	  next =
-	    pool_elt_at_index (lm->if_address_pool,
-			       a->next_this_sw_interface);
-	  next->prev_this_sw_interface = a->prev_this_sw_interface;
-
-	  if (a->prev_this_sw_interface == ~0)
-	    lm->if_address_pool_index_by_sw_if_index[sw_if_index] =
-	      a->next_this_sw_interface;
-	}
-
-      if ((a->next_this_sw_interface == ~0)
-	  && (a->prev_this_sw_interface == ~0))
-	lm->if_address_pool_index_by_sw_if_index[sw_if_index] = ~0;
-
-      mhash_unset (&lm->address_to_if_address_index, addr_fib,
-		   /* old_value */ 0);
-      pool_put (lm->if_address_pool, a);
-
-      if (result_if_address_index)
-	*result_if_address_index = ~0;
-    }
-
-  else if (!a)
-    {
-      u32 pi;			/* previous index */
-      u32 ai;
-      u32 hi;			/* head index */
-
-      pool_get (lm->if_address_pool, a);
-      clib_memset (a, ~0, sizeof (a[0]));
-      ai = a - lm->if_address_pool;
-
-      hi = pi = lm->if_address_pool_index_by_sw_if_index[sw_if_index];
-      prev = 0;
-      while (pi != (u32) ~ 0)
-	{
-	  prev = pool_elt_at_index (lm->if_address_pool, pi);
-	  pi = prev->next_this_sw_interface;
-	}
-      pi = prev ? prev - lm->if_address_pool : (u32) ~ 0;
-
-      a->address_key = mhash_set (&lm->address_to_if_address_index,
-				  addr_fib, ai, /* old_value */ 0);
-      a->address_length = address_length;
-      a->sw_if_index = sw_if_index;
-      a->flags = 0;
-      a->prev_this_sw_interface = pi;
-      a->next_this_sw_interface = ~0;
-      if (prev)
-	prev->next_this_sw_interface = ai;
-
-      lm->if_address_pool_index_by_sw_if_index[sw_if_index] =
-	(hi != ~0) ? hi : ai;
-      if (result_if_address_index)
-	*result_if_address_index = ai;
-    }
-  else
-    {
-      if (sw_if_index != a->sw_if_index)
-	{
-	  if (result_if_address_index)
-	    *result_if_address_index = ~0;
-	  vnm->api_errno = VNET_API_ERROR_DUPLICATE_IF_ADDRESS;
-	  return clib_error_create
-	    ("Prefix %U already found on interface %U",
-	     lm->format_address_and_length, addr_fib, address_length,
-	     format_vnet_sw_if_index_name, vnm, a->sw_if_index);
-	}
-
-      if (result_if_address_index)
-	*result_if_address_index = a - lm->if_address_pool;
-    }
-
-  return /* no error */ 0;
-}
-
 static clib_error_t *
 ip_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
 {
diff --git a/src/vnet/ip/lookup.h b/src/vnet/ip/lookup.h
index 6340e57e095..49ed0bbd3ef 100644
--- a/src/vnet/ip/lookup.h
+++ b/src/vnet/ip/lookup.h
@@ -86,6 +86,11 @@ typedef u32 flow_hash_config_t;
 /* An all zeros address */
 extern const ip46_address_t zero_addr;
 
+typedef enum ip_interface_address_flags_t_
+{
+  IP_INTERFACE_ADDRESS_FLAG_STALE = (1 << 0),
+} __clib_packed ip_interface_address_flags_t;
+
 typedef struct
 {
   fib_prefix_t prefix;
@@ -116,8 +121,8 @@ typedef struct
   /* Address (prefix) length for this interface. */
   u16 address_length;
 
-  /* Will be used for something eventually.  Primary vs. secondary? */
-  u16 flags;
+  /* flags relating to this prefix */
+  ip_interface_address_flags_t flags;
 
   /* Next and previous pointers for doubly linked list of
      addresses per software interface. */
@@ -180,70 +185,8 @@ typedef struct ip_lookup_main_t
   u8 builtin_protocol_by_ip_protocol[256];
 } ip_lookup_main_t;
 
-clib_error_t *ip_interface_address_add_del (ip_lookup_main_t * lm,
-					    u32 sw_if_index,
-					    void *address,
-					    u32 address_length,
-					    u32 is_del, u32 * result_index);
-
 u8 *format_ip_flow_hash_config (u8 * s, va_list * args);
 
-always_inline ip_interface_address_t *
-ip_get_interface_address (ip_lookup_main_t * lm, void *addr_fib)
-{
-  uword *p = mhash_get (&lm->address_to_if_address_index, addr_fib);
-  return p ? pool_elt_at_index (lm->if_address_pool, p[0]) : 0;
-}
-
-always_inline void *
-ip_interface_address_get_address (ip_lookup_main_t * lm,
-				  ip_interface_address_t * a)
-{
-  return mhash_key_to_mem (&lm->address_to_if_address_index, a->address_key);
-}
-
-always_inline ip_interface_prefix_t *
-ip_get_interface_prefix (ip_lookup_main_t * lm, ip_interface_prefix_key_t * k)
-{
-  uword *p = mhash_get (&lm->prefix_to_if_prefix_index, k);
-  return p ? pool_elt_at_index (lm->if_prefix_pool, p[0]) : 0;
-}
-
-/* *INDENT-OFF* */
-#define foreach_ip_interface_address(lm,a,sw_if_index,loop,body)        \
-do {                                                                    \
-    vnet_main_t *_vnm = vnet_get_main();                                \
-    u32 _sw_if_index = sw_if_index;                                     \
-    vnet_sw_interface_t *_swif;                                         \
-    _swif = vnet_get_sw_interface (_vnm, _sw_if_index);                 \
-                                                                        \
-    /*                                                                  \
-     * Loop => honor unnumbered interface addressing.                   \
-     */                                                                 \
-    if (_swif->flags & VNET_SW_INTERFACE_FLAG_UNNUMBERED)               \
-      {                                                                 \
-        if (loop)                                                       \
-          _sw_if_index = _swif->unnumbered_sw_if_index;                 \
-        else                                                            \
-          /* the interface is unnumbered, by the caller does not want   \
-           * unnumbered interfaces considered/honoured */               \
-          break;                                                        \
-      }                                                                 \
-    u32 _ia = ((vec_len((lm)->if_address_pool_index_by_sw_if_index)     \
-                > (_sw_if_index)) ?                                     \
-               vec_elt ((lm)->if_address_pool_index_by_sw_if_index,     \
-                        (_sw_if_index)) :                               \
-               (u32)~0);                                                \
-    ip_interface_address_t * _a;                                        \
-    while (_ia != ~0)                                                   \
-    {                                                                   \
-        _a = pool_elt_at_index ((lm)->if_address_pool, _ia);            \
-        _ia = _a->next_this_sw_interface;                               \
-        (a) = _a;                                                       \
-        body;                                                           \
-    }                                                                   \
-} while (0)
-/* *INDENT-ON* */
 
 always_inline void
 ip_lookup_set_buffer_fib_index (u32 * fib_index_by_sw_if_index,