ip: Replace Sematics for Interface IP addresses

Type: feature

 - replace functions for prefixes attached to interfaces
 - add ip_interface.[ch] to consoldate the functions

Signed-off-by: Neale Ranns <nranns@cisco.com>
Change-Id: I9c0c39c09dbf80ea1aadefee02c9bd16f094b6ad

17 files changed, 1084 insertions, 370 deletions

diff --git a/src/plugins/gbp/test/test_gbp.py b/src/plugins/gbp/test/test_gbp.py
index 872ab4b5297..5ff71f42aa2 100644
--- a/src/plugins/gbp/test/test_gbp.py
+++ b/src/plugins/gbp/test/test_gbp.py
@@ -832,8 +832,10 @@ class TestGBP(VppTestCase):
         for epg in epgs:
             # IP config on the BVI interfaces
             if epg != epgs[1] and epg != epgs[4]:
-                VppIpInterfaceBind(self, epg.bvi, epg.rd.t4).add_vpp_config()
-                VppIpInterfaceBind(self, epg.bvi, epg.rd.t6).add_vpp_config()
+                b4 = VppIpInterfaceBind(self, epg.bvi,
+                                        epg.rd.t4).add_vpp_config()
+                b6 = VppIpInterfaceBind(self, epg.bvi,
+                                        epg.rd.t6).add_vpp_config()
                 epg.bvi.set_mac(self.router_mac)
 
                 # The BVIs are NAT inside interfaces
@@ -845,10 +847,12 @@ class TestGBP(VppTestCase):
                     is_add=1, flags=flags,
                     sw_if_index=epg.bvi.sw_if_index)
 
-            if_ip4 = VppIpInterfaceAddress(self, epg.bvi, epg.bvi_ip4, 32)
-            if_ip6 = VppIpInterfaceAddress(self, epg.bvi, epg.bvi_ip6, 128)
-            if_ip4.add_vpp_config()
-            if_ip6.add_vpp_config()
+            if_ip4 = VppIpInterfaceAddress(self, epg.bvi,
+                                           epg.bvi_ip4, 32,
+                                           bind=b4).add_vpp_config()
+            if_ip6 = VppIpInterfaceAddress(self, epg.bvi,
+                                           epg.bvi_ip6, 128,
+                                           bind=b6).add_vpp_config()
 
             # EPG uplink interfaces in the RD
             VppIpInterfaceBind(self, epg.uplink, epg.rd.t4).add_vpp_config()
@@ -2228,14 +2232,18 @@ class TestGBP(VppTestCase):
         for epg in epgs:
             # IP config on the BVI interfaces
             if epg != epgs[1]:
-                VppIpInterfaceBind(self, epg.bvi, epg.rd.t4).add_vpp_config()
-                VppIpInterfaceBind(self, epg.bvi, epg.rd.t6).add_vpp_config()
+                b4 = VppIpInterfaceBind(self, epg.bvi,
+                                        epg.rd.t4).add_vpp_config()
+                b6 = VppIpInterfaceBind(self, epg.bvi,
+                                        epg.rd.t6).add_vpp_config()
                 epg.bvi.set_mac(self.router_mac)
 
-            if_ip4 = VppIpInterfaceAddress(self, epg.bvi, epg.bvi_ip4, 32)
-            if_ip6 = VppIpInterfaceAddress(self, epg.bvi, epg.bvi_ip6, 128)
-            if_ip4.add_vpp_config()
-            if_ip6.add_vpp_config()
+            if_ip4 = VppIpInterfaceAddress(self, epg.bvi,
+                                           epg.bvi_ip4, 32,
+                                           bind=b4).add_vpp_config()
+            if_ip6 = VppIpInterfaceAddress(self, epg.bvi,
+                                           epg.bvi_ip6, 128,
+                                           bind=b6).add_vpp_config()
 
             # add the BD ARP termination entry for BVI IP
             epg.bd_arp_ip4 = VppBridgeDomainArpEntry(self, epg.bd.bd,
@@ -2455,8 +2463,8 @@ class TestGBP(VppTestCase):
         self.logger.info(self.vapi.cli("sh gbp bridge"))
 
         # ... and has a /32 applied
-        ip_addr = VppIpInterfaceAddress(self, gbd1.bvi, "10.0.0.128", 32)
-        ip_addr.add_vpp_config()
+        ip_addr = VppIpInterfaceAddress(self, gbd1.bvi,
+                                        "10.0.0.128", 32).add_vpp_config()
 
         #
         # The Endpoint-group
@@ -2536,8 +2544,8 @@ class TestGBP(VppTestCase):
         gbd1.add_vpp_config()
 
         # ... and has a /32 applied
-        ip_addr = VppIpInterfaceAddress(self, gbd1.bvi, "10.0.0.128", 32)
-        ip_addr.add_vpp_config()
+        ip_addr = VppIpInterfaceAddress(self, gbd1.bvi,
+                                        "10.0.0.128", 32).add_vpp_config()
 
         #
         # The Endpoint-group
@@ -2629,8 +2637,8 @@ class TestGBP(VppTestCase):
         self.logger.info(self.vapi.cli("sh gbp bridge"))
 
         # ... and has a /32 applied
-        ip_addr = VppIpInterfaceAddress(self, gbd1.bvi, "10.0.0.128", 32)
-        ip_addr.add_vpp_config()
+        ip_addr = VppIpInterfaceAddress(self, gbd1.bvi,
+                                        "10.0.0.128", 32).add_vpp_config()
 
         #
         # The Endpoint-group in which we are learning endpoints
@@ -2772,8 +2780,8 @@ class TestGBP(VppTestCase):
         #
         # Bind the BVI to the RD
         #
-        VppIpInterfaceBind(self, self.loop0, t4).add_vpp_config()
-        VppIpInterfaceBind(self, self.loop0, t6).add_vpp_config()
+        b4 = VppIpInterfaceBind(self, self.loop0, t4).add_vpp_config()
+        b6 = VppIpInterfaceBind(self, self.loop0, t6).add_vpp_config()
 
         #
         # Pg2 hosts the vxlan tunnel
@@ -2803,10 +2811,12 @@ class TestGBP(VppTestCase):
         self.logger.info(self.vapi.cli("sh gbp route"))
 
         # ... and has a /32 and /128 applied
-        ip4_addr = VppIpInterfaceAddress(self, gbd1.bvi, "10.0.0.128", 32)
-        ip4_addr.add_vpp_config()
-        ip6_addr = VppIpInterfaceAddress(self, gbd1.bvi, "2001:10::128", 128)
-        ip6_addr.add_vpp_config()
+        ip4_addr = VppIpInterfaceAddress(self, gbd1.bvi,
+                                         "10.0.0.128", 32,
+                                         bind=b4).add_vpp_config()
+        ip6_addr = VppIpInterfaceAddress(self, gbd1.bvi,
+                                         "2001:10::128", 128,
+                                         bind=b6).add_vpp_config()
 
         #
         # The Endpoint-group in which we are learning endpoints
@@ -3278,8 +3288,8 @@ class TestGBP(VppTestCase):
         #
         # Bind the BVI to the RD
         #
-        VppIpInterfaceBind(self, self.loop0, t4).add_vpp_config()
-        VppIpInterfaceBind(self, self.loop0, t6).add_vpp_config()
+        b_ip4 = VppIpInterfaceBind(self, self.loop0, t4).add_vpp_config()
+        b_ip6 = VppIpInterfaceBind(self, self.loop0, t6).add_vpp_config()
 
         #
         # Pg7 hosts a BD's UU-fwd
@@ -3301,14 +3311,16 @@ class TestGBP(VppTestCase):
         gbd2.add_vpp_config()
 
         # ... and has a /32 and /128 applied
-        ip4_addr = VppIpInterfaceAddress(self, gbd1.bvi, "10.0.0.128", 32)
-        ip4_addr.add_vpp_config()
-        ip6_addr = VppIpInterfaceAddress(self, gbd1.bvi, "2001:10::128", 128)
-        ip6_addr.add_vpp_config()
-        ip4_addr = VppIpInterfaceAddress(self, gbd2.bvi, "10.0.1.128", 32)
-        ip4_addr.add_vpp_config()
-        ip6_addr = VppIpInterfaceAddress(self, gbd2.bvi, "2001:11::128", 128)
-        ip6_addr.add_vpp_config()
+        ip4_addr = VppIpInterfaceAddress(self, gbd1.bvi,
+                                         "10.0.0.128", 32,
+                                         bind=b_ip4).add_vpp_config()
+        ip6_addr = VppIpInterfaceAddress(self, gbd1.bvi,
+                                         "2001:10::128", 128,
+                                         bind=b_ip6).add_vpp_config()
+        ip4_addr = VppIpInterfaceAddress(self, gbd2.bvi,
+                                         "10.0.1.128", 32).add_vpp_config()
+        ip6_addr = VppIpInterfaceAddress(self, gbd2.bvi,
+                                         "2001:11::128", 128).add_vpp_config()
 
         #
         # The Endpoint-groups in which we are learning endpoints
@@ -3867,8 +3879,8 @@ class TestGBP(VppTestCase):
         # add local l3out
         # the external bd
         self.loop4.set_mac(self.router_mac)
-        VppIpInterfaceBind(self, self.loop4, t4).add_vpp_config()
-        VppIpInterfaceBind(self, self.loop4, t6).add_vpp_config()
+        b_lo4_ip4 = VppIpInterfaceBind(self, self.loop4, t4).add_vpp_config()
+        b_lo4_ip6 = VppIpInterfaceBind(self, self.loop4, t6).add_vpp_config()
         ebd = VppBridgeDomain(self, 100)
         ebd.add_vpp_config()
         gebd = VppGbpBridgeDomain(self, ebd, rd1, self.loop4, None, None)
@@ -3885,12 +3897,12 @@ class TestGBP(VppTestCase):
             self,
             gebd.bvi,
             "10.1.0.128",
-            24).add_vpp_config()
+            24, bind=b_lo4_ip4).add_vpp_config()
         VppIpInterfaceAddress(
             self,
             gebd.bvi,
             "2001:10:1::128",
-            64).add_vpp_config()
+            64, bind=b_lo4_ip6).add_vpp_config()
         # ... which are L3-out subnets
         VppGbpSubnet(self, rd1, "10.1.0.0", 24,
                      VppEnum.vl_api_gbp_subnet_type_t.GBP_API_SUBNET_L3_OUT,
@@ -4197,12 +4209,12 @@ class TestGBP(VppTestCase):
         #
         # Bind the BVI to the RD
         #
-        VppIpInterfaceBind(self, self.loop0, t4).add_vpp_config()
-        VppIpInterfaceBind(self, self.loop0, t6).add_vpp_config()
-        VppIpInterfaceBind(self, self.loop1, t4).add_vpp_config()
-        VppIpInterfaceBind(self, self.loop1, t6).add_vpp_config()
-        VppIpInterfaceBind(self, self.loop2, t4).add_vpp_config()
-        VppIpInterfaceBind(self, self.loop2, t6).add_vpp_config()
+        b_lo0_ip4 = VppIpInterfaceBind(self, self.loop0, t4).add_vpp_config()
+        b_lo0_ip6 = VppIpInterfaceBind(self, self.loop0, t6).add_vpp_config()
+        b_lo1_ip4 = VppIpInterfaceBind(self, self.loop1, t4).add_vpp_config()
+        b_lo1_ip6 = VppIpInterfaceBind(self, self.loop1, t6).add_vpp_config()
+        b_lo2_ip4 = VppIpInterfaceBind(self, self.loop2, t4).add_vpp_config()
+        b_lo2_ip6 = VppIpInterfaceBind(self, self.loop2, t6).add_vpp_config()
 
         #
         # Pg7 hosts a BD's UU-fwd
@@ -4224,14 +4236,18 @@ class TestGBP(VppTestCase):
         gbd2.add_vpp_config()
 
         # ... and has a /32 and /128 applied
-        ip4_addr1 = VppIpInterfaceAddress(self, gbd1.bvi, "10.0.0.128", 32)
-        ip4_addr1.add_vpp_config()
-        ip6_addr1 = VppIpInterfaceAddress(self, gbd1.bvi, "2001:10::128", 128)
-        ip6_addr1.add_vpp_config()
-        ip4_addr2 = VppIpInterfaceAddress(self, gbd2.bvi, "10.0.1.128", 32)
-        ip4_addr2.add_vpp_config()
-        ip6_addr2 = VppIpInterfaceAddress(self, gbd2.bvi, "2001:11::128", 128)
-        ip6_addr2.add_vpp_config()
+        ip4_addr1 = VppIpInterfaceAddress(self, gbd1.bvi,
+                                          "10.0.0.128", 32,
+                                          bind=b_lo0_ip4).add_vpp_config()
+        ip6_addr1 = VppIpInterfaceAddress(self, gbd1.bvi,
+                                          "2001:10::128", 128,
+                                          bind=b_lo0_ip6).add_vpp_config()
+        ip4_addr2 = VppIpInterfaceAddress(self, gbd2.bvi,
+                                          "10.0.1.128", 32,
+                                          bind=b_lo1_ip4).add_vpp_config()
+        ip6_addr2 = VppIpInterfaceAddress(self, gbd2.bvi,
+                                          "2001:11::128", 128,
+                                          bind=b_lo1_ip6).add_vpp_config()
 
         #
         # The Endpoint-groups
@@ -4262,10 +4278,12 @@ class TestGBP(VppTestCase):
                                   bd_uu3, learn=False)
         gbd3.add_vpp_config()
 
-        ip4_addr3 = VppIpInterfaceAddress(self, gbd3.bvi, "12.0.0.128", 32)
-        ip4_addr3.add_vpp_config()
-        ip6_addr3 = VppIpInterfaceAddress(self, gbd3.bvi, "4001:10::128", 128)
-        ip6_addr3.add_vpp_config()
+        ip4_addr3 = VppIpInterfaceAddress(self, gbd3.bvi,
+                                          "12.0.0.128", 32,
+                                          bind=b_lo2_ip4).add_vpp_config()
+        ip6_addr3 = VppIpInterfaceAddress(self, gbd3.bvi,
+                                          "4001:10::128", 128,
+                                          bind=b_lo2_ip6).add_vpp_config()
 
         #
         # self.logger.info(self.vapi.cli("show gbp bridge"))
@@ -4624,8 +4642,8 @@ class TestGBP(VppTestCase):
         #
         # Bind the BVI to the RD
         #
-        VppIpInterfaceBind(self, self.loop0, t4).add_vpp_config()
-        VppIpInterfaceBind(self, self.loop0, t6).add_vpp_config()
+        b_ip4 = VppIpInterfaceBind(self, self.loop0, t4).add_vpp_config()
+        b_ip6 = VppIpInterfaceBind(self, self.loop0, t6).add_vpp_config()
 
         #
         # Pg7 hosts a BD's BUM
@@ -4661,10 +4679,10 @@ class TestGBP(VppTestCase):
         epg_220.add_vpp_config()
 
         # the BVIs have the subnets applied ...
-        ip4_addr = VppIpInterfaceAddress(self, gbd1.bvi, "10.0.0.128", 24)
-        ip4_addr.add_vpp_config()
-        ip6_addr = VppIpInterfaceAddress(self, gbd1.bvi, "2001:10::128", 64)
-        ip6_addr.add_vpp_config()
+        ip4_addr = VppIpInterfaceAddress(self, gbd1.bvi, "10.0.0.128",
+                                         24, bind=b_ip4).add_vpp_config()
+        ip6_addr = VppIpInterfaceAddress(self, gbd1.bvi, "2001:10::128",
+                                         64, bind=b_ip6).add_vpp_config()
 
         # ... which are L3-out subnets
         l3o_1 = VppGbpSubnet(
@@ -5410,8 +5428,8 @@ class TestGBP(VppTestCase):
         #
         # Bind the BVI to the RD
         #
-        VppIpInterfaceBind(self, self.loop0, t4).add_vpp_config()
-        VppIpInterfaceBind(self, self.loop0, t6).add_vpp_config()
+        bind_l0_ip4 = VppIpInterfaceBind(self, self.loop0, t4).add_vpp_config()
+        bind_l0_ip6 = VppIpInterfaceBind(self, self.loop0, t6).add_vpp_config()
 
         #
         # Pg7 hosts a BD's BUM
@@ -5439,8 +5457,9 @@ class TestGBP(VppTestCase):
         epg_220.add_vpp_config()
 
         # the BVIs have the subnet applied ...
-        ip4_addr = VppIpInterfaceAddress(self, gbd1.bvi, "10.0.0.128", 24)
-        ip4_addr.add_vpp_config()
+        ip4_addr = VppIpInterfaceAddress(self, gbd1.bvi,
+                                         "10.0.0.128", 24,
+                                         bind=bind_l0_ip4).add_vpp_config()
 
         # ... which is an Anonymous L3-out subnets
         l3o_1 = VppGbpSubnet(
diff --git a/src/vnet/CMakeLists.txt b/src/vnet/CMakeLists.txt
index bf1dba7f090..39774ba2682 100644
--- a/src/vnet/CMakeLists.txt
+++ b/src/vnet/CMakeLists.txt
@@ -458,6 +458,7 @@ list(APPEND VNET_SOURCES
   ip/ip_checksum.c
   ip/ip_frag.c
   ip/ip.c
+  ip/ip_interface.c
   ip/ip_init.c
   ip/ip_in_out_acl.c
   ip/ip_punt_drop.c
@@ -501,6 +502,7 @@ list(APPEND VNET_HEADERS
   ip/ip6_hop_by_hop_packet.h
   ip/ip6_packet.h
   ip/ip.h
+  ip/ip_interface.h
   ip/ip_packet.h
   ip/ip_source_and_port_range_check.h
   ip/ip_types.h
diff --git a/src/vnet/interface.api b/src/vnet/interface.api
index 1ecd3472f72..c262cd69fcf 100644
--- a/src/vnet/interface.api
+++ b/src/vnet/interface.api
@@ -228,6 +228,43 @@ autoreply define sw_interface_add_del_address
   vl_api_address_with_prefix_t prefix;
 };
 
+/** \brief IP interface address replace begin
+
+    The use-case is that, for some unspecified reason, the control plane
+    has a different set of interface addresses than VPP
+    currently has. The CP would thus like to 'replace' VPP's set
+    only by specifying what the new set shall be, i.e. it is not
+    going to delete anything that already eixts, rather, is wants any
+    unspecified interface addresses to be deleted implicitly.
+    The CP declares the start of this procedure with this replace_begin
+    API Call, and when it has populated all addresses it wants, it calls
+    the below replace_end API. From this point on it is of course free
+    to add and delete interface addresses as usual.
+    The underlying mechanism by which VPP implements this replace is
+    intentionally left unspecified.
+
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+autoreply define sw_interface_address_replace_begin
+{
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief IP interface address replace end
+
+    see ip_interface_address_replace_begin description.
+
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+autoreply define sw_interface_address_replace_end
+{
+  u32 client_index;
+  u32 context;
+};
+
 /** \brief Associate the specified interface with a fib table
     @param client_index - opaque cookie to identify the sender
     @param context - sender context, to match reply w/ request
diff --git a/src/vnet/interface_api.c b/src/vnet/interface_api.c
index 4ce0a9ab053..5b24a29efc9 100644
--- a/src/vnet/interface_api.c
+++ b/src/vnet/interface_api.c
@@ -79,8 +79,12 @@ _(CREATE_LOOPBACK_INSTANCE, create_loopback_instance)		\
 _(DELETE_LOOPBACK, delete_loopback)                             \
 _(INTERFACE_NAME_RENUMBER, interface_name_renumber)             \
 _(COLLECT_DETAILED_INTERFACE_STATS, collect_detailed_interface_stats) \
-_(SW_INTERFACE_SET_IP_DIRECTED_BROADCAST,                            \
-  sw_interface_set_ip_directed_broadcast)
+_(SW_INTERFACE_SET_IP_DIRECTED_BROADCAST,                       \
+  sw_interface_set_ip_directed_broadcast)                       \
+_(SW_INTERFACE_ADDRESS_REPLACE_BEGIN,                           \
+  sw_interface_address_replace_begin)                           \
+_(SW_INTERFACE_ADDRESS_REPLACE_END,                             \
+  sw_interface_address_replace_end)
 
 static void
 vl_api_sw_interface_set_flags_t_handler (vl_api_sw_interface_set_flags_t * mp)
@@ -1366,6 +1370,30 @@ static void
   REPLY_MACRO (VL_API_COLLECT_DETAILED_INTERFACE_STATS_REPLY);
 }
 
+static void
+  vl_api_sw_interface_address_replace_begin_t_handler
+  (vl_api_sw_interface_address_replace_begin_t * mp)
+{
+  vl_api_sw_interface_address_replace_begin_reply_t *rmp;
+  int rv = 0;
+
+  ip_interface_address_mark ();
+
+  REPLY_MACRO (VL_API_SW_INTERFACE_ADDRESS_REPLACE_BEGIN_REPLY);
+}
+
+static void
+  vl_api_sw_interface_address_replace_end_t_handler
+  (vl_api_sw_interface_address_replace_end_t * mp)
+{
+  vl_api_sw_interface_address_replace_end_reply_t *rmp;
+  int rv = 0;
+
+  ip_interface_address_sweep ();
+
+  REPLY_MACRO (VL_API_SW_INTERFACE_ADDRESS_REPLACE_END_REPLY);
+}
+
 /*
  * vpe_api_hookup
  * Add vpe's API message handlers to the table.
diff --git a/src/vnet/ip/ip.c b/src/vnet/ip/ip.c
index 8959b4c09cc..38e53fdef75 100644
--- a/src/vnet/ip/ip.c
+++ b/src/vnet/ip/ip.c
@@ -96,73 +96,6 @@ ip_set (ip46_address_t * dst, void *src, u8 is_ip4)
 		      sizeof (ip6_address_t));
 }
 
-u8
-ip_interface_has_address (u32 sw_if_index, ip46_address_t * ip, u8 is_ip4)
-{
-  ip_interface_address_t *ia = 0;
-
-  if (is_ip4)
-    {
-      ip_lookup_main_t *lm4 = &ip4_main.lookup_main;
-      ip4_address_t *ip4;
-      /* *INDENT-OFF* */
-      foreach_ip_interface_address (lm4, ia, sw_if_index, 1 /* unnumbered */ ,
-      ({
-        ip4 = ip_interface_address_get_address (lm4, ia);
-        if (ip4_address_compare (ip4, &ip->ip4) == 0)
-          return 1;
-      }));
-      /* *INDENT-ON* */
-    }
-  else
-    {
-      ip_lookup_main_t *lm6 = &ip6_main.lookup_main;
-      ip6_address_t *ip6;
-      /* *INDENT-OFF* */
-      foreach_ip_interface_address (lm6, ia, sw_if_index, 1 /* unnumbered */ ,
-      ({
-        ip6 = ip_interface_address_get_address (lm6, ia);
-        if (ip6_address_compare (ip6, &ip->ip6) == 0)
-          return 1;
-      }));
-      /* *INDENT-ON* */
-    }
-  return 0;
-}
-
-void *
-ip_interface_get_first_ip (u32 sw_if_index, u8 is_ip4)
-{
-  ip_lookup_main_t *lm4 = &ip4_main.lookup_main;
-  ip_lookup_main_t *lm6 = &ip6_main.lookup_main;
-  ip_interface_address_t *ia = 0;
-
-  if (is_ip4)
-    {
-      /* *INDENT-OFF* */
-      foreach_ip_interface_address (lm4, ia, sw_if_index, 1 /* unnumbered */ ,
-      ({
-        return ip_interface_address_get_address (lm4, ia);
-      }));
-      /* *INDENT-ON* */
-    }
-  else
-    {
-      /* *INDENT-OFF* */
-      foreach_ip_interface_address (lm6, ia, sw_if_index, 1 /* unnumbered */ ,
-      ({
-        ip6_address_t *rv;
-        rv = ip_interface_address_get_address (lm6, ia);
-        /* Trying to use a link-local ip6 src address is a fool's errand */
-        if (!ip6_address_is_link_local_unicast (rv))
-          return rv;
-      }));
-      /* *INDENT-ON* */
-    }
-
-  return 0;
-}
-
 u8 *
 format_ip_address_family (u8 * s, va_list * args)
 {
diff --git a/src/vnet/ip/ip.h b/src/vnet/ip/ip.h
index a6fcd410c0e..3fa97267b36 100644
--- a/src/vnet/ip/ip.h
+++ b/src/vnet/ip/ip.h
@@ -50,6 +50,7 @@
 #include <vnet/ip/format.h>
 #include <vnet/ip/ip_packet.h>
 #include <vnet/ip/lookup.h>
+#include <vnet/ip/ip_interface.h>
 
 #include <vnet/tcp/tcp_packet.h>
 #include <vnet/udp/udp_packet.h>
@@ -274,10 +275,8 @@ u8 ip_is_local_host (ip46_address_t * ip46_address, u8 is_ip4);
 u8 ip4_is_local_host (ip4_address_t * ip4_address);
 u8 ip6_is_local_host (ip6_address_t * ip6_address);
 u8 ip_is_local (u32 fib_index, ip46_address_t * ip46_address, u8 is_ip4);
-u8 ip_interface_has_address (u32 sw_if_index, ip46_address_t * ip, u8 is_ip4);
 void ip_copy (ip46_address_t * dst, ip46_address_t * src, u8 is_ip4);
 void ip_set (ip46_address_t * dst, void *src, u8 is_ip4);
-void *ip_interface_get_first_ip (u32 sw_if_index, u8 is_ip4);
 
 always_inline u32 vlib_buffer_get_ip4_fib_index (vlib_buffer_t * b);
 always_inline u32 vlib_buffer_get_ip6_fib_index (vlib_buffer_t * b);
diff --git a/src/vnet/ip/ip4.h b/src/vnet/ip/ip4.h
index 6e13cc8ea3e..b5bc2e2c33f 100644
--- a/src/vnet/ip/ip4.h
+++ b/src/vnet/ip/ip4.h
@@ -42,6 +42,7 @@
 
 #include <vnet/ip/ip4_packet.h>
 #include <vnet/ip/lookup.h>
+#include <vnet/ip/ip_interface.h>
 #include <vnet/buffer.h>
 #include <vnet/feature/feature.h>
 #include <vnet/ip/icmp46_packet.h>
diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c
index 952915f5317..acff66d994b 100644
--- a/src/vnet/ip/ip4_forward.c
+++ b/src/vnet/ip/ip4_forward.c
@@ -680,7 +680,7 @@ ip4_add_del_interface_address_internal (vlib_main_t * vm,
   ip4_main_t *im = &ip4_main;
   ip_lookup_main_t *lm = &im->lookup_main;
   clib_error_t *error = 0;
-  u32 if_address_index, elts_before;
+  u32 if_address_index;
   ip4_address_fib_t ip4_af, *addr_fib = 0;
 
   /* local0 interface doesn't support IP addressing  */
@@ -720,6 +720,7 @@ ip4_add_del_interface_address_internal (vlib_main_t * vm,
                    ip4_address_t * x =
                      ip_interface_address_get_address
                      (&im->lookup_main, ia);
+
                    if (ip4_destination_matches_route
                        (im, address, x, ia->address_length) ||
                        ip4_destination_matches_route (im,
@@ -733,11 +734,18 @@ ip4_add_del_interface_address_internal (vlib_main_t * vm,
 			   (x->as_u32 != address->as_u32))
 		         continue;
 
+                       if (ia->flags & IP_INTERFACE_ADDRESS_FLAG_STALE)
+                         /* if the address we're comparing against is stale
+                          * then the CP has not added this one back yet, maybe
+                          * it never will, so we have to assume it won't and
+                          * ignore it. if it does add it back, then it will fail
+                          * because this one is now present */
+                         continue;
+
 		       /* error if the length or intf was different */
-                       vnm->api_errno = VNET_API_ERROR_DUPLICATE_IF_ADDRESS;
+                       vnm->api_errno = VNET_API_ERROR_ADDRESS_IN_USE;
 
-                       return
-                         clib_error_create
+                       error = clib_error_create
                          ("failed to add %U on %U which conflicts with %U for interface %U",
                           format_ip4_address_and_length, address,
                           address_length,
@@ -747,6 +755,7 @@ ip4_add_del_interface_address_internal (vlib_main_t * vm,
                           ia->address_length,
                           format_vnet_sw_if_index_name, vnm,
                           sif->sw_if_index);
+                       goto done;
                      }
                  }));
             }
@@ -754,10 +763,70 @@ ip4_add_del_interface_address_internal (vlib_main_t * vm,
     }
   /* *INDENT-ON* */
 
-  elts_before = pool_elts (lm->if_address_pool);
+  if_address_index = ip_interface_address_find (lm, addr_fib, address_length);
+
+  if (is_del)
+    {
+      if (~0 == if_address_index)
+	{
+	  vnm->api_errno = VNET_API_ERROR_ADDRESS_NOT_FOUND_FOR_INTERFACE;
+	  error = clib_error_create ("%U not found for interface %U",
+				     lm->format_address_and_length,
+				     addr_fib, address_length,
+				     format_vnet_sw_if_index_name, vnm,
+				     sw_if_index);
+	  goto done;
+	}
+
+      ip_interface_address_del (lm, if_address_index, addr_fib);
+    }
+  else
+    {
+      if (~0 != if_address_index)
+	{
+	  ip_interface_address_t *ia;
+
+	  ia = pool_elt_at_index (lm->if_address_pool, if_address_index);
+
+	  if (ia->flags & IP_INTERFACE_ADDRESS_FLAG_STALE)
+	    {
+	      if (ia->sw_if_index == sw_if_index)
+		{
+		  /* re-adding an address during the replace action.
+		   * consdier this the update. clear the flag and
+		   * we're done */
+		  ia->flags &= ~IP_INTERFACE_ADDRESS_FLAG_STALE;
+		  goto done;
+		}
+	      else
+		{
+		  /* The prefix is moving from one interface to another.
+		   * delete the stale and add the new */
+		  ip4_add_del_interface_address_internal (vm,
+							  ia->sw_if_index,
+							  address,
+							  address_length, 1);
+		  ia = NULL;
+		  error = ip_interface_address_add (lm, sw_if_index,
+						    addr_fib, address_length,
+						    &if_address_index);
+		}
+	    }
+	  else
+	    {
+	      vnm->api_errno = VNET_API_ERROR_DUPLICATE_IF_ADDRESS;
+	      error = clib_error_create
+		("Prefix %U already found on interface %U",
+		 lm->format_address_and_length, addr_fib, address_length,
+		 format_vnet_sw_if_index_name, vnm, ia->sw_if_index);
+	    }
+	}
+      else
+	error = ip_interface_address_add (lm, sw_if_index,
+					  addr_fib, address_length,
+					  &if_address_index);
+    }
 
-  error = ip_interface_address_add_del
-    (lm, sw_if_index, addr_fib, address_length, is_del, &if_address_index);
   if (error)
     goto done;
 
@@ -778,14 +847,10 @@ ip4_add_del_interface_address_internal (vlib_main_t * vm,
 				  (lm->if_address_pool, if_address_index));
     }
 
-  /* If pool did not grow/shrink: add duplicate address. */
-  if (elts_before != pool_elts (lm->if_address_pool))
-    {
-      ip4_add_del_interface_address_callback_t *cb;
-      vec_foreach (cb, im->add_del_interface_address_callbacks)
-	cb->function (im, cb->function_opaque, sw_if_index,
-		      address, address_length, if_address_index, is_del);
-    }
+  ip4_add_del_interface_address_callback_t *cb;
+  vec_foreach (cb, im->add_del_interface_address_callbacks)
+    cb->function (im, cb->function_opaque, sw_if_index,
+		  address, address_length, if_address_index, is_del);
 
 done:
   vec_free (addr_fib);
diff --git a/src/vnet/ip/ip6.h b/src/vnet/ip/ip6.h
index ec8c772456c..5e16f990534 100644
--- a/src/vnet/ip/ip6.h
+++ b/src/vnet/ip/ip6.h
@@ -47,6 +47,7 @@
 #include <vnet/ip/ip46_address.h>
 #include <vnet/ip/ip6_hop_by_hop_packet.h>
 #include <vnet/ip/lookup.h>
+#include <vnet/ip/ip_interface.h>
 #include <stdbool.h>
 #include <vppinfra/bihash_24_8.h>
 #include <vppinfra/bihash_40_8.h>
diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c
index 91a93ee6cf0..0325627a1ef 100644
--- a/src/vnet/ip/ip6_forward.c
+++ b/src/vnet/ip/ip6_forward.c
@@ -295,7 +295,7 @@ ip6_add_del_interface_address (vlib_main_t * vm,
   vnet_main_t *vnm = vnet_get_main ();
   ip6_main_t *im = &ip6_main;
   ip_lookup_main_t *lm = &im->lookup_main;
-  clib_error_t *error;
+  clib_error_t *error = NULL;
   u32 if_address_index;
   ip6_address_fib_t ip6_af, *addr_fib = 0;
   const ip6_address_t *ll_addr;
@@ -371,6 +371,7 @@ ip6_add_del_interface_address (vlib_main_t * vm,
                    ip6_address_t * x =
                      ip_interface_address_get_address
                      (&im->lookup_main, ia);
+
                    if (ip6_destination_matches_route
                        (im, address, x, ia->address_length) ||
                        ip6_destination_matches_route (im,
@@ -384,10 +385,17 @@ ip6_add_del_interface_address (vlib_main_t * vm,
 			   !ip6_address_is_equal (x, address))
 		         continue;
 
+                       if (ia->flags & IP_INTERFACE_ADDRESS_FLAG_STALE)
+                         /* if the address we're comparing against is stale
+                          * then the CP has not added this one back yet, maybe
+                          * it never will, so we have to assume it won't and
+                          * ignore it. if it does add it back, then it will fail
+                          * because this one is now present */
+                         continue;
+
 		       /* error if the length or intf was different */
                        vnm->api_errno = VNET_API_ERROR_DUPLICATE_IF_ADDRESS;
-                       return
-                         clib_error_create
+                       error =  clib_error_create
                          ("failed to add %U which conflicts with %U for interface %U",
                           format_ip6_address_and_length, address,
                           address_length,
@@ -395,6 +403,7 @@ ip6_add_del_interface_address (vlib_main_t * vm,
                           ia->address_length,
                           format_vnet_sw_if_index_name, vnm,
                           sif->sw_if_index);
+                       goto done;
                      }
                  }));
             }
@@ -402,18 +411,71 @@ ip6_add_del_interface_address (vlib_main_t * vm,
     }
   /* *INDENT-ON* */
 
-  {
-    uword elts_before = pool_elts (lm->if_address_pool);
+  if_address_index = ip_interface_address_find (lm, addr_fib, address_length);
+
+  if (is_del)
+    {
+      if (~0 == if_address_index)
+	{
+	  vnm->api_errno = VNET_API_ERROR_ADDRESS_NOT_FOUND_FOR_INTERFACE;
+	  error = clib_error_create ("%U not found for interface %U",
+				     lm->format_address_and_length,
+				     addr_fib, address_length,
+				     format_vnet_sw_if_index_name, vnm,
+				     sw_if_index);
+	  goto done;
+	}
+
+      ip_interface_address_del (lm, if_address_index, addr_fib);
+    }
+  else
+    {
+      if (~0 != if_address_index)
+	{
+	  ip_interface_address_t *ia;
 
-    error = ip_interface_address_add_del
-      (lm, sw_if_index, addr_fib, address_length, is_del, &if_address_index);
-    if (error)
-      goto done;
+	  ia = pool_elt_at_index (lm->if_address_pool, if_address_index);
 
-    /* Pool did not grow: add duplicate address. */
-    if (elts_before == pool_elts (lm->if_address_pool))
-      goto done;
-  }
+	  if (ia->flags & IP_INTERFACE_ADDRESS_FLAG_STALE)
+	    {
+	      if (ia->sw_if_index == sw_if_index)
+		{
+		  /* re-adding an address during the replace action.
+		   * consdier this the update. clear the flag and
+		   * we're done */
+		  ia->flags &= ~IP_INTERFACE_ADDRESS_FLAG_STALE;
+		  goto done;
+		}
+	      else
+		{
+		  /* The prefix is moving from one interface to another.
+		   * delete the stale and add the new */
+		  ip6_add_del_interface_address (vm,
+						 ia->sw_if_index,
+						 address, address_length, 1);
+		  ia = NULL;
+		  error = ip_interface_address_add (lm, sw_if_index,
+						    addr_fib, address_length,
+						    &if_address_index);
+		}
+	    }
+	  else
+	    {
+	      vnm->api_errno = VNET_API_ERROR_DUPLICATE_IF_ADDRESS;
+	      error = clib_error_create
+		("Prefix %U already found on interface %U",
+		 lm->format_address_and_length, addr_fib, address_length,
+		 format_vnet_sw_if_index_name, vnm, ia->sw_if_index);
+	    }
+	}
+      else
+	error = ip_interface_address_add (lm, sw_if_index,
+					  addr_fib, address_length,
+					  &if_address_index);
+    }
+
+  if (error)
+    goto done;
 
   ip6_sw_interface_enable_disable (sw_if_index, !is_del);
   if (!is_del)
@@ -432,12 +494,12 @@ ip6_add_del_interface_address (vlib_main_t * vm,
 				  pool_elt_at_index (lm->if_address_pool,
 						     if_address_index));
     }
-  {
-    ip6_add_del_interface_address_callback_t *cb;
-    vec_foreach (cb, im->add_del_interface_address_callbacks)
-      cb->function (im, cb->function_opaque, sw_if_index,
-		    address, address_length, if_address_index, is_del);
-  }
+
+  ip6_add_del_interface_address_callback_t *cb;
+  vec_foreach (cb, im->add_del_interface_address_callbacks)
+    cb->function (im, cb->function_opaque, sw_if_index,
+		  address, address_length, if_address_index, is_del);
+
   if (is_del)
     ip6_link_disable (sw_if_index);
 
diff --git a/src/vnet/ip/ip_interface.c b/src/vnet/ip/ip_interface.c
new file mode 100644
index 00000000000..23c3df81638
--- /dev/null
+++ b/src/vnet/ip/ip_interface.c
@@ -0,0 +1,288 @@
+/*
+ * Copyright (c) 2020 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vnet/ip/ip.h>
+
+/**
+ * @file
+ * @brief IP prefix management on interfaces
+ */
+
+u32
+ip_interface_address_find (ip_lookup_main_t * lm,
+			   void *addr_fib, u32 address_length)
+{
+  uword *p = mhash_get (&lm->address_to_if_address_index, addr_fib);
+
+  if (p)
+    return (p[0]);
+
+  return (~0);
+}
+
+clib_error_t *
+ip_interface_address_add (ip_lookup_main_t * lm,
+			  u32 sw_if_index,
+			  void *addr_fib,
+			  u32 address_length, u32 * result_if_address_index)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+  ip_interface_address_t *a, *prev;
+  u32 pi;			/* previous index */
+  u32 ai;
+  u32 hi;			/* head index */
+
+  /* Verify given length. */
+  if ((address_length == 0) ||
+      (lm->is_ip6 && address_length > 128) ||
+      (!lm->is_ip6 && address_length > 32))
+    {
+      vnm->api_errno = VNET_API_ERROR_ADDRESS_LENGTH_MISMATCH;
+      return clib_error_create
+	("%U wrong length for interface %U",
+	 lm->format_address_and_length, addr_fib,
+	 address_length, format_vnet_sw_if_index_name, vnm, sw_if_index);
+    }
+
+  vec_validate_init_empty (lm->if_address_pool_index_by_sw_if_index,
+			   sw_if_index, ~0);
+
+  pool_get_zero (lm->if_address_pool, a);
+
+  ai = a - lm->if_address_pool;
+  hi = pi = lm->if_address_pool_index_by_sw_if_index[sw_if_index];
+
+  prev = 0;
+  while (pi != (u32) ~ 0)
+    {
+      prev = pool_elt_at_index (lm->if_address_pool, pi);
+      pi = prev->next_this_sw_interface;
+    }
+  pi = prev ? prev - lm->if_address_pool : (u32) ~ 0;
+
+  a->address_key = mhash_set (&lm->address_to_if_address_index,
+			      addr_fib, ai, /* old_value */ 0);
+  a->address_length = address_length;
+  a->sw_if_index = sw_if_index;
+  a->flags = 0;
+  a->prev_this_sw_interface = pi;
+  a->next_this_sw_interface = ~0;
+  if (prev)
+    prev->next_this_sw_interface = ai;
+
+  lm->if_address_pool_index_by_sw_if_index[sw_if_index] =
+    (hi != ~0) ? hi : ai;
+
+  *result_if_address_index = ai;
+
+  return (NULL);
+}
+
+void
+ip_interface_address_del (ip_lookup_main_t * lm,
+			  u32 address_index, void *addr_fib)
+{
+  ip_interface_address_t *a, *prev, *next;
+
+  a = pool_elt_at_index (lm->if_address_pool, address_index);
+
+  if (a->prev_this_sw_interface != ~0)
+    {
+      prev = pool_elt_at_index (lm->if_address_pool,
+				a->prev_this_sw_interface);
+      prev->next_this_sw_interface = a->next_this_sw_interface;
+    }
+  if (a->next_this_sw_interface != ~0)
+    {
+      next = pool_elt_at_index (lm->if_address_pool,
+				a->next_this_sw_interface);
+      next->prev_this_sw_interface = a->prev_this_sw_interface;
+
+      if (a->prev_this_sw_interface == ~0)
+	lm->if_address_pool_index_by_sw_if_index[a->sw_if_index] =
+	  a->next_this_sw_interface;
+    }
+
+  if ((a->next_this_sw_interface == ~0) && (a->prev_this_sw_interface == ~0))
+    lm->if_address_pool_index_by_sw_if_index[a->sw_if_index] = ~0;
+
+  mhash_unset (&lm->address_to_if_address_index, addr_fib,
+	       /* old_value */ 0);
+  pool_put (lm->if_address_pool, a);
+}
+
+u8
+ip_interface_has_address (u32 sw_if_index, ip46_address_t * ip, u8 is_ip4)
+{
+  ip_interface_address_t *ia = 0;
+
+  if (is_ip4)
+    {
+      ip_lookup_main_t *lm4 = &ip4_main.lookup_main;
+      ip4_address_t *ip4;
+      /* *INDENT-OFF* */
+      foreach_ip_interface_address (lm4, ia, sw_if_index, 1 /* unnumbered */ ,
+      ({
+        ip4 = ip_interface_address_get_address (lm4, ia);
+        if (ip4_address_compare (ip4, &ip->ip4) == 0)
+          return 1;
+      }));
+      /* *INDENT-ON* */
+    }
+  else
+    {
+      ip_lookup_main_t *lm6 = &ip6_main.lookup_main;
+      ip6_address_t *ip6;
+      /* *INDENT-OFF* */
+      foreach_ip_interface_address (lm6, ia, sw_if_index, 1 /* unnumbered */ ,
+      ({
+        ip6 = ip_interface_address_get_address (lm6, ia);
+        if (ip6_address_compare (ip6, &ip->ip6) == 0)
+          return 1;
+      }));
+      /* *INDENT-ON* */
+    }
+  return 0;
+}
+
+void *
+ip_interface_get_first_ip (u32 sw_if_index, u8 is_ip4)
+{
+  ip_lookup_main_t *lm4 = &ip4_main.lookup_main;
+  ip_lookup_main_t *lm6 = &ip6_main.lookup_main;
+  ip_interface_address_t *ia = 0;
+
+  if (is_ip4)
+    {
+      /* *INDENT-OFF* */
+      foreach_ip_interface_address (lm4, ia, sw_if_index, 1 /* unnumbered */ ,
+      ({
+        return ip_interface_address_get_address (lm4, ia);
+      }));
+      /* *INDENT-ON* */
+    }
+  else
+    {
+      /* *INDENT-OFF* */
+      foreach_ip_interface_address (lm6, ia, sw_if_index, 1 /* unnumbered */ ,
+      ({
+        ip6_address_t *rv;
+        rv = ip_interface_address_get_address (lm6, ia);
+        /* Trying to use a link-local ip6 src address is a fool's errand */
+        if (!ip6_address_is_link_local_unicast (rv))
+          return rv;
+      }));
+      /* *INDENT-ON* */
+    }
+
+  return 0;
+}
+
+static walk_rc_t
+ip_interface_address_mark_one_interface (vnet_main_t * vnm,
+					 vnet_sw_interface_t * si, void *ctx)
+{
+  ip_lookup_main_t *lm4 = &ip4_main.lookup_main;
+  ip_lookup_main_t *lm6 = &ip6_main.lookup_main;
+  ip_interface_address_t *ia = 0;
+
+  /* *INDENT-OFF* */
+  foreach_ip_interface_address (lm4, ia, si->sw_if_index, 1 /* unnumbered */ ,
+  ({
+    ia->flags |= IP_INTERFACE_ADDRESS_FLAG_STALE;
+  }));
+  foreach_ip_interface_address (lm6, ia, si->sw_if_index, 1 /* unnumbered */ ,
+  ({
+    ia->flags |= IP_INTERFACE_ADDRESS_FLAG_STALE;
+  }));
+  /* *INDENT-ON* */
+
+  return (WALK_CONTINUE);
+}
+
+void
+ip_interface_address_mark (void)
+{
+  vnet_sw_interface_walk (vnet_get_main (),
+			  ip_interface_address_mark_one_interface, NULL);
+}
+
+static walk_rc_t
+ip_interface_address_sweep_one_interface (vnet_main_t * vnm,
+					  vnet_sw_interface_t * si, void *ctx)
+{
+  vlib_main_t *vm = vlib_get_main ();
+  ip4_address_t *ip4_addrs = 0;
+  ip6_address_t *ip6_addrs = 0;
+  ip4_main_t *im4 = &ip4_main;
+  ip6_main_t *im6 = &ip6_main;
+  ip_interface_address_t *ia;
+  u32 *ip6_masks = 0;
+  u32 *ip4_masks = 0;
+  int i;
+
+  /* *INDENT-OFF* */
+  foreach_ip_interface_address (&im4->lookup_main, ia, si->sw_if_index, 1,
+  ({
+    if (ia->flags & IP_INTERFACE_ADDRESS_FLAG_STALE)
+      {
+        ip4_address_t * x = (ip4_address_t *)
+          ip_interface_address_get_address (&im4->lookup_main, ia);
+        vec_add1 (ip4_addrs, x[0]);
+        vec_add1 (ip4_masks, ia->address_length);
+      }
+  }));
+
+  foreach_ip_interface_address (&im6->lookup_main, ia, si->sw_if_index, 1,
+  ({
+    if (ia->flags & IP_INTERFACE_ADDRESS_FLAG_STALE)
+      {
+        ip6_address_t * x = (ip6_address_t *)
+          ip_interface_address_get_address (&im6->lookup_main, ia);
+        vec_add1 (ip6_addrs, x[0]);
+        vec_add1 (ip6_masks, ia->address_length);
+      }
+  }));
+  /* *INDENT-ON* */
+
+  for (i = 0; i < vec_len (ip4_addrs); i++)
+    ip4_add_del_interface_address (vm, si->sw_if_index, &ip4_addrs[i],
+				   ip4_masks[i], 1 /* is_del */ );
+  for (i = 0; i < vec_len (ip6_addrs); i++)
+    ip6_add_del_interface_address (vm, si->sw_if_index, &ip6_addrs[i],
+				   ip6_masks[i], 1 /* is_del */ );
+
+  vec_free (ip4_addrs);
+  vec_free (ip4_masks);
+  vec_free (ip6_addrs);
+  vec_free (ip6_masks);
+
+  return (WALK_CONTINUE);
+}
+
+void
+ip_interface_address_sweep (void)
+{
+  vnet_sw_interface_walk (vnet_get_main (),
+			  ip_interface_address_sweep_one_interface, NULL);
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vnet/ip/ip_interface.h b/src/vnet/ip/ip_interface.h
new file mode 100644
index 00000000000..f95b8deb07b
--- /dev/null
+++ b/src/vnet/ip/ip_interface.h
@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2020 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @file
+ * @brief IP prefix management on interfaces
+ */
+
+#ifndef included_ip_interface_h
+#define included_ip_interface_h
+
+#include <vnet/ip/lookup.h>
+
+clib_error_t *ip_interface_address_add (ip_lookup_main_t * lm,
+					u32 sw_if_index,
+					void *address,
+					u32 address_length,
+					u32 * result_index);
+void ip_interface_address_del (ip_lookup_main_t * lm,
+			       u32 addr_index, void *address);
+void *ip_interface_get_first_ip (u32 sw_if_index, u8 is_ip4);
+void ip_interface_address_mark (void);
+void ip_interface_address_sweep (void);
+u32 ip_interface_address_find (ip_lookup_main_t * lm,
+			       void *addr_fib, u32 address_length);
+u8 ip_interface_has_address (u32 sw_if_index, ip46_address_t * ip, u8 is_ip4);
+
+always_inline void *
+ip_interface_address_get_address (ip_lookup_main_t * lm,
+				  ip_interface_address_t * a)
+{
+  return mhash_key_to_mem (&lm->address_to_if_address_index, a->address_key);
+}
+
+always_inline ip_interface_prefix_t *
+ip_get_interface_prefix (ip_lookup_main_t * lm, ip_interface_prefix_key_t * k)
+{
+  uword *p = mhash_get (&lm->prefix_to_if_prefix_index, k);
+  return p ? pool_elt_at_index (lm->if_prefix_pool, p[0]) : 0;
+}
+
+/* *INDENT-OFF* */
+#define foreach_ip_interface_address(lm,a,sw_if_index,loop,body)        \
+do {                                                                    \
+    vnet_main_t *_vnm = vnet_get_main();                                \
+    u32 _sw_if_index = sw_if_index;                                     \
+    vnet_sw_interface_t *_swif;                                         \
+    _swif = vnet_get_sw_interface (_vnm, _sw_if_index);                 \
+                                                                        \
+    /*                                                                  \
+     * Loop => honor unnumbered interface addressing.                   \
+     */                                                                 \
+    if (_swif->flags & VNET_SW_INTERFACE_FLAG_UNNUMBERED)               \
+      {                                                                 \
+        if (loop)                                                       \
+          _sw_if_index = _swif->unnumbered_sw_if_index;                 \
+        else                                                            \
+          /* the interface is unnumbered, by the caller does not want   \
+           * unnumbered interfaces considered/honoured */               \
+          break;                                                        \
+      }                                                                 \
+    u32 _ia = ((vec_len((lm)->if_address_pool_index_by_sw_if_index)     \
+                > (_sw_if_index)) ?                                     \
+               vec_elt ((lm)->if_address_pool_index_by_sw_if_index,     \
+                        (_sw_if_index)) :                               \
+               (u32)~0);                                                \
+    ip_interface_address_t * _a;                                        \
+    while (_ia != ~0)                                                   \
+    {                                                                   \
+        _a = pool_elt_at_index ((lm)->if_address_pool, _ia);            \
+        _ia = _a->next_this_sw_interface;                               \
+        (a) = _a;                                                       \
+        body;                                                           \
+    }                                                                   \
+} while (0)
+/* *INDENT-ON* */
+
+#endif /* included_ip_interface_h */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c
index a0cb28b8906..5d4e137fb1c 100644
--- a/src/vnet/ip/lookup.c
+++ b/src/vnet/ip/lookup.c
@@ -57,132 +57,6 @@
  *
  */
 
-clib_error_t *
-ip_interface_address_add_del (ip_lookup_main_t * lm,
-			      u32 sw_if_index,
-			      void *addr_fib,
-			      u32 address_length,
-			      u32 is_del, u32 * result_if_address_index)
-{
-  vnet_main_t *vnm = vnet_get_main ();
-  ip_interface_address_t *a, *prev, *next;
-  uword *p = mhash_get (&lm->address_to_if_address_index, addr_fib);
-
-  vec_validate_init_empty (lm->if_address_pool_index_by_sw_if_index,
-			   sw_if_index, ~0);
-  a = p ? pool_elt_at_index (lm->if_address_pool, p[0]) : 0;
-
-  /* Verify given length. */
-  if ((a && (address_length != a->address_length)) ||
-      (address_length == 0) ||
-      (lm->is_ip6 && address_length > 128) ||
-      (!lm->is_ip6 && address_length > 32))
-    {
-      vnm->api_errno = VNET_API_ERROR_ADDRESS_LENGTH_MISMATCH;
-      return clib_error_create
-	("%U wrong length (expected %d) for interface %U",
-	 lm->format_address_and_length, addr_fib,
-	 address_length, a ? a->address_length : -1,
-	 format_vnet_sw_if_index_name, vnm, sw_if_index);
-    }
-
-  if (is_del)
-    {
-      if (!a)
-	{
-	  vnet_sw_interface_t *si = vnet_get_sw_interface (vnm, sw_if_index);
-	  vnm->api_errno = VNET_API_ERROR_ADDRESS_NOT_FOUND_FOR_INTERFACE;
-	  return clib_error_create ("%U not found for interface %U",
-				    lm->format_address_and_length,
-				    addr_fib, address_length,
-				    format_vnet_sw_interface_name, vnm, si);
-	}
-
-      if (a->prev_this_sw_interface != ~0)
-	{
-	  prev =
-	    pool_elt_at_index (lm->if_address_pool,
-			       a->prev_this_sw_interface);
-	  prev->next_this_sw_interface = a->next_this_sw_interface;
-	}
-      if (a->next_this_sw_interface != ~0)
-	{
-	  next =
-	    pool_elt_at_index (lm->if_address_pool,
-			       a->next_this_sw_interface);
-	  next->prev_this_sw_interface = a->prev_this_sw_interface;
-
-	  if (a->prev_this_sw_interface == ~0)
-	    lm->if_address_pool_index_by_sw_if_index[sw_if_index] =
-	      a->next_this_sw_interface;
-	}
-
-      if ((a->next_this_sw_interface == ~0)
-	  && (a->prev_this_sw_interface == ~0))
-	lm->if_address_pool_index_by_sw_if_index[sw_if_index] = ~0;
-
-      mhash_unset (&lm->address_to_if_address_index, addr_fib,
-		   /* old_value */ 0);
-      pool_put (lm->if_address_pool, a);
-
-      if (result_if_address_index)
-	*result_if_address_index = ~0;
-    }
-
-  else if (!a)
-    {
-      u32 pi;			/* previous index */
-      u32 ai;
-      u32 hi;			/* head index */
-
-      pool_get (lm->if_address_pool, a);
-      clib_memset (a, ~0, sizeof (a[0]));
-      ai = a - lm->if_address_pool;
-
-      hi = pi = lm->if_address_pool_index_by_sw_if_index[sw_if_index];
-      prev = 0;
-      while (pi != (u32) ~ 0)
-	{
-	  prev = pool_elt_at_index (lm->if_address_pool, pi);
-	  pi = prev->next_this_sw_interface;
-	}
-      pi = prev ? prev - lm->if_address_pool : (u32) ~ 0;
-
-      a->address_key = mhash_set (&lm->address_to_if_address_index,
-				  addr_fib, ai, /* old_value */ 0);
-      a->address_length = address_length;
-      a->sw_if_index = sw_if_index;
-      a->flags = 0;
-      a->prev_this_sw_interface = pi;
-      a->next_this_sw_interface = ~0;
-      if (prev)
-	prev->next_this_sw_interface = ai;
-
-      lm->if_address_pool_index_by_sw_if_index[sw_if_index] =
-	(hi != ~0) ? hi : ai;
-      if (result_if_address_index)
-	*result_if_address_index = ai;
-    }
-  else
-    {
-      if (sw_if_index != a->sw_if_index)
-	{
-	  if (result_if_address_index)
-	    *result_if_address_index = ~0;
-	  vnm->api_errno = VNET_API_ERROR_DUPLICATE_IF_ADDRESS;
-	  return clib_error_create
-	    ("Prefix %U already found on interface %U",
-	     lm->format_address_and_length, addr_fib, address_length,
-	     format_vnet_sw_if_index_name, vnm, a->sw_if_index);
-	}
-
-      if (result_if_address_index)
-	*result_if_address_index = a - lm->if_address_pool;
-    }
-
-  return /* no error */ 0;
-}
-
 static clib_error_t *
 ip_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
 {
diff --git a/src/vnet/ip/lookup.h b/src/vnet/ip/lookup.h
index 6340e57e095..49ed0bbd3ef 100644
--- a/src/vnet/ip/lookup.h
+++ b/src/vnet/ip/lookup.h
@@ -86,6 +86,11 @@ typedef u32 flow_hash_config_t;
 /* An all zeros address */
 extern const ip46_address_t zero_addr;
 
+typedef enum ip_interface_address_flags_t_
+{
+  IP_INTERFACE_ADDRESS_FLAG_STALE = (1 << 0),
+} __clib_packed ip_interface_address_flags_t;
+
 typedef struct
 {
   fib_prefix_t prefix;
@@ -116,8 +121,8 @@ typedef struct
   /* Address (prefix) length for this interface. */
   u16 address_length;
 
-  /* Will be used for something eventually.  Primary vs. secondary? */
-  u16 flags;
+  /* flags relating to this prefix */
+  ip_interface_address_flags_t flags;
 
   /* Next and previous pointers for doubly linked list of
      addresses per software interface. */
@@ -180,70 +185,8 @@ typedef struct ip_lookup_main_t
   u8 builtin_protocol_by_ip_protocol[256];
 } ip_lookup_main_t;
 
-clib_error_t *ip_interface_address_add_del (ip_lookup_main_t * lm,
-					    u32 sw_if_index,
-					    void *address,
-					    u32 address_length,
-					    u32 is_del, u32 * result_index);
-
 u8 *format_ip_flow_hash_config (u8 * s, va_list * args);
 
-always_inline ip_interface_address_t *
-ip_get_interface_address (ip_lookup_main_t * lm, void *addr_fib)
-{
-  uword *p = mhash_get (&lm->address_to_if_address_index, addr_fib);
-  return p ? pool_elt_at_index (lm->if_address_pool, p[0]) : 0;
-}
-
-always_inline void *
-ip_interface_address_get_address (ip_lookup_main_t * lm,
-				  ip_interface_address_t * a)
-{
-  return mhash_key_to_mem (&lm->address_to_if_address_index, a->address_key);
-}
-
-always_inline ip_interface_prefix_t *
-ip_get_interface_prefix (ip_lookup_main_t * lm, ip_interface_prefix_key_t * k)
-{
-  uword *p = mhash_get (&lm->prefix_to_if_prefix_index, k);
-  return p ? pool_elt_at_index (lm->if_prefix_pool, p[0]) : 0;
-}
-
-/* *INDENT-OFF* */
-#define foreach_ip_interface_address(lm,a,sw_if_index,loop,body)        \
-do {                                                                    \
-    vnet_main_t *_vnm = vnet_get_main();                                \
-    u32 _sw_if_index = sw_if_index;                                     \
-    vnet_sw_interface_t *_swif;                                         \
-    _swif = vnet_get_sw_interface (_vnm, _sw_if_index);                 \
-                                                                        \
-    /*                                                                  \
-     * Loop => honor unnumbered interface addressing.                   \
-     */                                                                 \
-    if (_swif->flags & VNET_SW_INTERFACE_FLAG_UNNUMBERED)               \
-      {                                                                 \
-        if (loop)                                                       \
-          _sw_if_index = _swif->unnumbered_sw_if_index;                 \
-        else                                                            \
-          /* the interface is unnumbered, by the caller does not want   \
-           * unnumbered interfaces considered/honoured */               \
-          break;                                                        \
-      }                                                                 \
-    u32 _ia = ((vec_len((lm)->if_address_pool_index_by_sw_if_index)     \
-                > (_sw_if_index)) ?                                     \
-               vec_elt ((lm)->if_address_pool_index_by_sw_if_index,     \
-                        (_sw_if_index)) :                               \
-               (u32)~0);                                                \
-    ip_interface_address_t * _a;                                        \
-    while (_ia != ~0)                                                   \
-    {                                                                   \
-        _a = pool_elt_at_index ((lm)->if_address_pool, _ia);            \
-        _ia = _a->next_this_sw_interface;                               \
-        (a) = _a;                                                       \
-        body;                                                           \
-    }                                                                   \
-} while (0)
-/* *INDENT-ON* */
 
 always_inline void
 ip_lookup_set_buffer_fib_index (u32 * fib_index_by_sw_if_index,
diff --git a/test/test_ip4.py b/test/test_ip4.py
index 18b350dcd2d..6f8047cace9 100644
--- a/test/test_ip4.py
+++ b/test/test_ip4.py
@@ -2155,5 +2155,167 @@ class TestIPCover(VppTestCase):
         # remove the default route
         r.remove_vpp_config()
 
+
+class TestIP4Replace(VppTestCase):
+    """ IPv4 Interface Address Replace """
+
+    @classmethod
+    def setUpClass(cls):
+        super(TestIP4Replace, cls).setUpClass()
+
+    @classmethod
+    def tearDownClass(cls):
+        super(TestIP4Replace, cls).tearDownClass()
+
+    def setUp(self):
+        super(TestIP4Replace, self).setUp()
+
+        self.create_pg_interfaces(range(4))
+
+        for i in self.pg_interfaces:
+            i.admin_up()
+
+    def tearDown(self):
+        super(TestIP4Replace, self).tearDown()
+        for i in self.pg_interfaces:
+            i.admin_down()
+
+    def get_n_pfxs(self, intf):
+        return len(self.vapi.ip_address_dump(intf.sw_if_index))
+
+    def test_replace(self):
+        """ IP interface address replace """
+
+        intf_pfxs = [[], [], [], []]
+
+        # add prefixes to each of the interfaces
+        for i in range(len(self.pg_interfaces)):
+            intf = self.pg_interfaces[i]
+
+            # 172.16.x.1/24
+            addr = "172.16.%d.1" % intf.sw_if_index
+            a = VppIpInterfaceAddress(self, intf, addr, 24).add_vpp_config()
+            intf_pfxs[i].append(a)
+
+            # 172.16.x.2/24 - a different address in the same subnet as above
+            addr = "172.16.%d.2" % intf.sw_if_index
+            a = VppIpInterfaceAddress(self, intf, addr, 24).add_vpp_config()
+            intf_pfxs[i].append(a)
+
+            # 172.15.x.2/24 - a different address and subnet
+            addr = "172.15.%d.2" % intf.sw_if_index
+            a = VppIpInterfaceAddress(self, intf, addr, 24).add_vpp_config()
+            intf_pfxs[i].append(a)
+
+        # a dump should n_address in it
+        for intf in self.pg_interfaces:
+            self.assertEqual(self.get_n_pfxs(intf), 3)
+
+        #
+        # remove all the address thru a replace
+        #
+        self.vapi.sw_interface_address_replace_begin()
+        self.vapi.sw_interface_address_replace_end()
+        for intf in self.pg_interfaces:
+            self.assertEqual(self.get_n_pfxs(intf), 0)
+
+        #
+        # add all the interface addresses back
+        #
+        for p in intf_pfxs:
+            for v in p:
+                v.add_vpp_config()
+        for intf in self.pg_interfaces:
+            self.assertEqual(self.get_n_pfxs(intf), 3)
+
+        #
+        # replace again, but this time update/re-add the address on the first
+        # two interfaces
+        #
+        self.vapi.sw_interface_address_replace_begin()
+
+        for p in intf_pfxs[:2]:
+            for v in p:
+                v.add_vpp_config()
+
+        self.vapi.sw_interface_address_replace_end()
+
+        # on the first two the address still exist,
+        # on the other two they do not
+        for intf in self.pg_interfaces[:2]:
+            self.assertEqual(self.get_n_pfxs(intf), 3)
+        for p in intf_pfxs[:2]:
+            for v in p:
+                self.assertTrue(v.query_vpp_config())
+        for intf in self.pg_interfaces[2:]:
+            self.assertEqual(self.get_n_pfxs(intf), 0)
+
+        #
+        # add all the interface addresses back on the last two
+        #
+        for p in intf_pfxs[2:]:
+            for v in p:
+                v.add_vpp_config()
+        for intf in self.pg_interfaces:
+            self.assertEqual(self.get_n_pfxs(intf), 3)
+
+        #
+        # replace again, this time add different prefixes on all the interfaces
+        #
+        self.vapi.sw_interface_address_replace_begin()
+
+        pfxs = []
+        for intf in self.pg_interfaces:
+            # 172.18.x.1/24
+            addr = "172.18.%d.1" % intf.sw_if_index
+            pfxs.append(VppIpInterfaceAddress(self, intf, addr,
+                                              24).add_vpp_config())
+
+        self.vapi.sw_interface_address_replace_end()
+
+        # only .18 should exist on each interface
+        for intf in self.pg_interfaces:
+            self.assertEqual(self.get_n_pfxs(intf), 1)
+        for pfx in pfxs:
+            self.assertTrue(pfx.query_vpp_config())
+
+        #
+        # remove everything
+        #
+        self.vapi.sw_interface_address_replace_begin()
+        self.vapi.sw_interface_address_replace_end()
+        for intf in self.pg_interfaces:
+            self.assertEqual(self.get_n_pfxs(intf), 0)
+
+        #
+        # add prefixes to each interface. post-begin add the prefix from
+        # interface X onto interface Y. this would normally be an error
+        # since it would generate a 'duplicate address' warning. but in
+        # this case, since what is newly downloaded is sane, it's ok
+        #
+        for intf in self.pg_interfaces:
+            # 172.18.x.1/24
+            addr = "172.18.%d.1" % intf.sw_if_index
+            VppIpInterfaceAddress(self, intf, addr, 24).add_vpp_config()
+
+        self.vapi.sw_interface_address_replace_begin()
+
+        pfxs = []
+        for intf in self.pg_interfaces:
+            # 172.18.x.1/24
+            addr = "172.18.%d.1" % (intf.sw_if_index + 1)
+            pfxs.append(VppIpInterfaceAddress(self, intf,
+                                              addr, 24).add_vpp_config())
+
+        self.vapi.sw_interface_address_replace_end()
+
+        self.logger.info(self.vapi.cli("sh int addr"))
+
+        for intf in self.pg_interfaces:
+            self.assertEqual(self.get_n_pfxs(intf), 1)
+        for pfx in pfxs:
+            self.assertTrue(pfx.query_vpp_config())
+
+
 if __name__ == '__main__':
     unittest.main(testRunner=VppTestRunner)
diff --git a/test/test_ip6.py b/test/test_ip6.py
index 63d3e41f915..990b53ec035 100644
--- a/test/test_ip6.py
+++ b/test/test_ip6.py
@@ -2560,5 +2560,166 @@ class TestIPReplace(VppTestCase):
             self.assertEqual(len(t.mdump()), 5)
 
 
+class TestIP6Replace(VppTestCase):
+    """ IPv4 Interface Address Replace """
+
+    @classmethod
+    def setUpClass(cls):
+        super(TestIP6Replace, cls).setUpClass()
+
+    @classmethod
+    def tearDownClass(cls):
+        super(TestIP6Replace, cls).tearDownClass()
+
+    def setUp(self):
+        super(TestIP6Replace, self).setUp()
+
+        self.create_pg_interfaces(range(4))
+
+        for i in self.pg_interfaces:
+            i.admin_up()
+
+    def tearDown(self):
+        super(TestIP6Replace, self).tearDown()
+        for i in self.pg_interfaces:
+            i.admin_down()
+
+    def get_n_pfxs(self, intf):
+        return len(self.vapi.ip_address_dump(intf.sw_if_index, True))
+
+    def test_replace(self):
+        """ IP interface address replace """
+
+        intf_pfxs = [[], [], [], []]
+
+        # add prefixes to each of the interfaces
+        for i in range(len(self.pg_interfaces)):
+            intf = self.pg_interfaces[i]
+
+            # 2001:16:x::1/64
+            addr = "2001:16:%d::1" % intf.sw_if_index
+            a = VppIpInterfaceAddress(self, intf, addr, 64).add_vpp_config()
+            intf_pfxs[i].append(a)
+
+            # 2001:16:x::2/64 - a different address in the same subnet as above
+            addr = "2001:16:%d::2" % intf.sw_if_index
+            a = VppIpInterfaceAddress(self, intf, addr, 64).add_vpp_config()
+            intf_pfxs[i].append(a)
+
+            # 2001:15:x::2/64 - a different address and subnet
+            addr = "2001:15:%d::2" % intf.sw_if_index
+            a = VppIpInterfaceAddress(self, intf, addr, 64).add_vpp_config()
+            intf_pfxs[i].append(a)
+
+        # a dump should n_address in it
+        for intf in self.pg_interfaces:
+            self.assertEqual(self.get_n_pfxs(intf), 3)
+
+        #
+        # remove all the address thru a replace
+        #
+        self.vapi.sw_interface_address_replace_begin()
+        self.vapi.sw_interface_address_replace_end()
+        for intf in self.pg_interfaces:
+            self.assertEqual(self.get_n_pfxs(intf), 0)
+
+        #
+        # add all the interface addresses back
+        #
+        for p in intf_pfxs:
+            for v in p:
+                v.add_vpp_config()
+        for intf in self.pg_interfaces:
+            self.assertEqual(self.get_n_pfxs(intf), 3)
+
+        #
+        # replace again, but this time update/re-add the address on the first
+        # two interfaces
+        #
+        self.vapi.sw_interface_address_replace_begin()
+
+        for p in intf_pfxs[:2]:
+            for v in p:
+                v.add_vpp_config()
+
+        self.vapi.sw_interface_address_replace_end()
+
+        # on the first two the address still exist,
+        # on the other two they do not
+        for intf in self.pg_interfaces[:2]:
+            self.assertEqual(self.get_n_pfxs(intf), 3)
+        for p in intf_pfxs[:2]:
+            for v in p:
+                self.assertTrue(v.query_vpp_config())
+        for intf in self.pg_interfaces[2:]:
+            self.assertEqual(self.get_n_pfxs(intf), 0)
+
+        #
+        # add all the interface addresses back on the last two
+        #
+        for p in intf_pfxs[2:]:
+            for v in p:
+                v.add_vpp_config()
+        for intf in self.pg_interfaces:
+            self.assertEqual(self.get_n_pfxs(intf), 3)
+
+        #
+        # replace again, this time add different prefixes on all the interfaces
+        #
+        self.vapi.sw_interface_address_replace_begin()
+
+        pfxs = []
+        for intf in self.pg_interfaces:
+            # 2001:18:x::1/64
+            addr = "2001:18:%d::1" % intf.sw_if_index
+            pfxs.append(VppIpInterfaceAddress(self, intf, addr,
+                                              64).add_vpp_config())
+
+        self.vapi.sw_interface_address_replace_end()
+
+        # only .18 should exist on each interface
+        for intf in self.pg_interfaces:
+            self.assertEqual(self.get_n_pfxs(intf), 1)
+        for pfx in pfxs:
+            self.assertTrue(pfx.query_vpp_config())
+
+        #
+        # remove everything
+        #
+        self.vapi.sw_interface_address_replace_begin()
+        self.vapi.sw_interface_address_replace_end()
+        for intf in self.pg_interfaces:
+            self.assertEqual(self.get_n_pfxs(intf), 0)
+
+        #
+        # add prefixes to each interface. post-begin add the prefix from
+        # interface X onto interface Y. this would normally be an error
+        # since it would generate a 'duplicate address' warning. but in
+        # this case, since what is newly downloaded is sane, it's ok
+        #
+        for intf in self.pg_interfaces:
+            # 2001:18:x::1/64
+            addr = "2001:18:%d::1" % intf.sw_if_index
+            VppIpInterfaceAddress(self, intf, addr, 64).add_vpp_config()
+
+        self.vapi.sw_interface_address_replace_begin()
+
+        pfxs = []
+        for intf in self.pg_interfaces:
+            # 2001:18:x::1/64
+            addr = "2001:18:%d::1" % (intf.sw_if_index + 1)
+            pfxs.append(VppIpInterfaceAddress(self, intf,
+                                              addr, 64).add_vpp_config())
+
+        self.vapi.sw_interface_address_replace_end()
+
+        self.logger.info(self.vapi.cli("sh int addr"))
+
+        for intf in self.pg_interfaces:
+            self.assertEqual(self.get_n_pfxs(intf), 1)
+        for pfx in pfxs:
+            self.assertTrue(pfx.query_vpp_config())
+
+
 if __name__ == '__main__':
     unittest.main(testRunner=VppTestRunner)
diff --git a/test/vpp_ip_route.py b/test/vpp_ip_route.py
index 67183d943c8..d871f7a0b11 100644
--- a/test/vpp_ip_route.py
+++ b/test/vpp_ip_route.py
@@ -8,7 +8,7 @@ from vpp_object import VppObject
 from socket import inet_pton, inet_ntop, AF_INET, AF_INET6
 from vpp_ip import DpoProto, INVALID_INDEX, VppIpAddressUnion, \
     VppIpMPrefix
-from ipaddress import ip_address, IPv4Network, IPv6Network
+from ipaddress import ip_network, ip_address, IPv4Network, IPv6Network
 
 # from vnet/vnet/mpls/mpls_types.h
 MPLS_IETF_MAX_LABEL = 0xfffff
@@ -93,7 +93,7 @@ def address_proto(ip_addr):
         return FibPathProto.FIB_PATH_NH_PROTO_IP6
 
 
-def find_route(test, addr, len, table_id=0):
+def find_route(test, addr, len, table_id=0, sw_if_index=None):
     prefix = mk_network(addr, len)
 
     if 4 == prefix.version:
@@ -104,7 +104,16 @@ def find_route(test, addr, len, table_id=0):
     for e in routes:
         if table_id == e.route.table_id \
            and str(e.route.prefix) == str(prefix):
-            return True
+            if not sw_if_index:
+                return True
+            else:
+                # should be only one path if the user is looking
+                # for the interface the route is reachable through
+                if e.route.n_paths != 1:
+                    return False
+                else:
+                    return (e.route.paths[0].sw_if_index == sw_if_index)
+
     return False
 
 
@@ -234,12 +243,16 @@ class VppIpTable(VppObject):
 
 class VppIpInterfaceAddress(VppObject):
 
-    def __init__(self, test, intf, addr, len):
+    def __init__(self, test, intf, addr, len, bind=None):
         self._test = test
         self.intf = intf
         self.addr = addr
         self.len = len
         self.prefix = "%s/%d" % (addr, len)
+        self.host_len = ip_network(self.prefix, strict=False).max_prefixlen
+        self.table_id = 0
+        if bind:
+            self.table_id = bind.table.table_id
 
     def add_vpp_config(self):
         self._test.vapi.sw_interface_add_del_address(
@@ -254,13 +267,40 @@ class VppIpInterfaceAddress(VppObject):
             is_add=0)
 
     def query_vpp_config(self):
-        return fib_interface_ip_prefix(self._test,
-                                       self.addr,
-                                       self.len,
-                                       self.intf.sw_if_index)
+        # search for the IP address mapping and the two expected
+        # FIB entries
+        v = ip_address(self.addr).version
+
+        if ((v == 4 and self.len < 31) or (v == 6 and self.len < 127)):
+            return (fib_interface_ip_prefix(self._test,
+                                            self.addr,
+                                            self.len,
+                                            self.intf.sw_if_index) &
+                    find_route(self._test,
+                               self.addr,
+                               self.len,
+                               table_id=self.table_id,
+                               sw_if_index=self.intf.sw_if_index) &
+                    find_route(self._test,
+                               self.addr,
+                               self.host_len,
+                               table_id=self.table_id,
+                               sw_if_index=self.intf.sw_if_index))
+        else:
+            return (fib_interface_ip_prefix(self._test,
+                                            self.addr,
+                                            self.len,
+                                            self.intf.sw_if_index) &
+                    find_route(self._test,
+                               self.addr,
+                               self.host_len,
+                               table_id=self.table_id,
+                               sw_if_index=self.intf.sw_if_index))
 
     def object_id(self):
-        return "interface-ip-%s-%s" % (self.intf, self.prefix)
+        return "interface-ip-%s-%d-%s" % (self.intf,
+                                          self.table_id,
+                                          self.prefix)
 
 
 class VppIpInterfaceBind(VppObject):
@@ -276,6 +316,7 @@ class VppIpInterfaceBind(VppObject):
         else:
             self.intf.set_table_ip4(self.table.table_id)
         self._test.registry.register(self, self._test.logger)
+        return self
 
     def remove_vpp_config(self):
         if 0 == self.table.table_id: