dhcp: fix crash on unicast renewal send

Type: fix

- when the addresses were learnt a copy of the client was sent to the
main thread, this meant the unicast adjacecny was saved on the copy
not on the original.
- Add logging.
- Improve the proxy-node that hands the clint packets so the DHCP
packets are traced.
 - allow a renewal to configure new address data

Change-Id: I6ab0afcccbc4a1cdefdd1b8beeda8fc7ba20ec1f
Signed-off-by: Neale Ranns <nranns@cisco.com>

12 files changed, 646 insertions, 421 deletions

diff --git a/src/plugins/dhcp/CMakeLists.txt b/src/plugins/dhcp/CMakeLists.txt
index b2dd630d461..9db38e391f6 100644
--- a/src/plugins/dhcp/CMakeLists.txt
+++ b/src/plugins/dhcp/CMakeLists.txt
@@ -17,6 +17,7 @@ add_vpp_plugin(dhcp
   dhcp_api.c
   dhcp_client_detect.c
   dhcp_proxy.c
+  dhcp4_packet.c
   dhcp4_proxy_node.c
   dhcp6_client_common_dp.c
   dhcp6_ia_na_client_dp.c
diff --git a/src/plugins/dhcp/client.c b/src/plugins/dhcp/client.c
index 800da3e504a..f38e3fdfa8d 100644
--- a/src/plugins/dhcp/client.c
+++ b/src/plugins/dhcp/client.c
@@ -19,10 +19,17 @@
 #include <vnet/fib/fib_table.h>
 #include <vnet/qos/qos_types.h>
 
+vlib_log_class_t dhcp_logger;
+
 dhcp_client_main_t dhcp_client_main;
-static u8 *format_dhcp_client_state (u8 * s, va_list * va);
 static vlib_node_registration_t dhcp_client_process_node;
 
+#define DHCP_DBG(...)                           \
+    vlib_log_debug (dhcp_logger, __VA_ARGS__);
+
+#define DHCP_INFO(...)                          \
+    vlib_log_notice (dhcp_logger, __VA_ARGS__);
+
 #define foreach_dhcp_sent_packet_stat           \
 _(DISCOVER, "DHCP discover packets sent")       \
 _(OFFER, "DHCP offer packets sent")             \
@@ -52,15 +59,124 @@ static char *dhcp_client_process_stat_strings[] = {
     "DHCP unknown packets sent",
 };
 
+static u8 *
+format_dhcp_client_state (u8 * s, va_list * va)
+{
+  dhcp_client_state_t state = va_arg (*va, dhcp_client_state_t);
+  char *str = "BOGUS!";
+
+  switch (state)
+    {
+#define _(a)                                    \
+    case a:                                     \
+      str = #a;                                 \
+        break;
+      foreach_dhcp_client_state;
+#undef _
+    default:
+      break;
+    }
+
+  s = format (s, "%s", str);
+  return s;
+}
+
+static u8 *
+format_dhcp_client (u8 * s, va_list * va)
+{
+  dhcp_client_main_t *dcm = va_arg (*va, dhcp_client_main_t *);
+  dhcp_client_t *c = va_arg (*va, dhcp_client_t *);
+  int verbose = va_arg (*va, int);
+  ip4_address_t *addr;
+
+  s = format (s, "[%d] %U state %U installed %d", c - dcm->clients,
+	      format_vnet_sw_if_index_name, dcm->vnet_main, c->sw_if_index,
+	      format_dhcp_client_state, c->state, c->addresses_installed);
+
+  if (0 != c->dscp)
+    s = format (s, " dscp %d", c->dscp);
+
+  if (c->installed.leased_address.as_u32)
+    {
+      s = format (s, " addr %U/%d gw %U server %U",
+		  format_ip4_address, &c->installed.leased_address,
+		  c->installed.subnet_mask_width,
+		  format_ip4_address, &c->installed.router_address,
+		  format_ip4_address, &c->installed.dhcp_server);
+
+      vec_foreach (addr, c->domain_server_address)
+	s = format (s, " dns %U", format_ip4_address, addr);
+    }
+  else
+    {
+      s = format (s, " no address");
+    }
+
+  if (verbose)
+    {
+      s =
+	format (s,
+		"\n lease: lifetime:%d renewal-interval:%d expires:%.2f (now:%.2f)",
+		c->lease_lifetime, c->lease_renewal_interval,
+		c->lease_expires, vlib_time_now (dcm->vlib_main));
+      s =
+	format (s, "\n retry-count:%d, next-xmt:%.2f", c->retry_count,
+		c->next_transmit);
+      s =
+	format (s, "\n adjacencies:[unicast:%d broadcast:%d]", c->ai_ucast,
+		c->ai_bcast);
+    }
+  return s;
+}
+
 static void
 dhcp_client_acquire_address (dhcp_client_main_t * dcm, dhcp_client_t * c)
 {
   /*
    * Install any/all info gleaned from dhcp, right here
    */
-  ip4_add_del_interface_address (dcm->vlib_main, c->sw_if_index,
-				 (void *) &c->leased_address,
-				 c->subnet_mask_width, 0 /*is_del */ );
+  if (!c->addresses_installed)
+    {
+      ip4_add_del_interface_address (dcm->vlib_main, c->sw_if_index,
+				     (void *) &c->learned.leased_address,
+				     c->learned.subnet_mask_width,
+				     0 /*is_del */ );
+      if (c->learned.router_address.as_u32)
+	{
+	  fib_prefix_t all_0s = {
+	    .fp_len = 0,
+	    .fp_proto = FIB_PROTOCOL_IP4,
+	  };
+	  ip46_address_t nh = {
+	    .ip4 = c->learned.router_address,
+	  };
+
+          /* *INDENT-OFF* */
+          fib_table_entry_path_add (
+              fib_table_get_index_for_sw_if_index (
+                  FIB_PROTOCOL_IP4,
+                  c->sw_if_index),
+              &all_0s,
+              FIB_SOURCE_DHCP,
+              FIB_ENTRY_FLAG_NONE,
+              DPO_PROTO_IP4,
+              &nh, c->sw_if_index,
+              ~0, 1, NULL,	// no label stack
+              FIB_ROUTE_PATH_FLAG_NONE);
+          /* *INDENT-ON* */
+	}
+      if (c->learned.dhcp_server.as_u32)
+	{
+	  ip46_address_t dst = {
+	    .ip4 = c->learned.dhcp_server,
+	  };
+	  c->ai_ucast = adj_nbr_add_or_lock (FIB_PROTOCOL_IP4,
+					     VNET_LINK_IP4, &dst,
+					     c->sw_if_index);
+	}
+    }
+  clib_memcpy (&c->installed, &c->learned, sizeof (c->installed));
+  c->addresses_installed = 1;
 }
 
 static void
@@ -70,10 +186,35 @@ dhcp_client_release_address (dhcp_client_main_t * dcm, dhcp_client_t * c)
    * Remove any/all info gleaned from dhcp, right here. Caller(s)
    * have not wiped out the info yet.
    */
+  if (c->addresses_installed)
+    {
+      ip4_add_del_interface_address (dcm->vlib_main, c->sw_if_index,
+				     (void *) &c->installed.leased_address,
+				     c->installed.subnet_mask_width,
+				     1 /*is_del */ );
 
-  ip4_add_del_interface_address (dcm->vlib_main, c->sw_if_index,
-				 (void *) &c->leased_address,
-				 c->subnet_mask_width, 1 /*is_del */ );
+      /* Remove the default route */
+      if (c->installed.router_address.as_u32)
+	{
+	  fib_prefix_t all_0s = {
+	    .fp_len = 0,
+	    .fp_proto = FIB_PROTOCOL_IP4,
+	  };
+	  ip46_address_t nh = {
+	    .ip4 = c->installed.router_address,
+	  };
+
+	  fib_table_entry_path_remove (fib_table_get_index_for_sw_if_index
+				       (FIB_PROTOCOL_IP4, c->sw_if_index),
+				       &all_0s, FIB_SOURCE_DHCP,
+				       DPO_PROTO_IP4, &nh, c->sw_if_index, ~0,
+				       1, FIB_ROUTE_PATH_FLAG_NONE);
+	}
+      adj_unlock (c->ai_ucast);
+      c->ai_ucast = ADJ_INDEX_INVALID;
+    }
+  clib_memset (&c->installed, 0, sizeof (c->installed));
+  c->addresses_installed = 0;
 }
 
 static void
@@ -86,9 +227,12 @@ dhcp_client_proc_callback (uword * client_index)
 }
 
 static void
-dhcp_client_addr_callback (dhcp_client_t * c)
+dhcp_client_addr_callback (u32 * cindex)
 {
   dhcp_client_main_t *dcm = &dhcp_client_main;
+  dhcp_client_t *c;
+
+  c = pool_elt_at_index (dcm->clients, *cindex);
 
   /* disable the feature */
   vnet_feature_enable_disable ("ip4-unicast",
@@ -96,48 +240,11 @@ dhcp_client_addr_callback (dhcp_client_t * c)
 			       c->sw_if_index, 0 /* disable */ , 0, 0);
   c->client_detect_feature_enabled = 0;
 
-  /* if renewing the lease, the address and route have already been added */
-  if (c->state == DHCP_BOUND)
-    return;
-
-  /* add the address to the interface */
-  dhcp_client_acquire_address (dcm, c);
-
-  /*
-   * Configure default IP route:
-   */
-  if (c->router_address.as_u32)
-    {
-      fib_prefix_t all_0s = {
-	.fp_len = 0,
-	.fp_addr.ip4.as_u32 = 0x0,
-	.fp_proto = FIB_PROTOCOL_IP4,
-      };
-      ip46_address_t nh = {
-	.ip4 = c->router_address,
-      };
-
-      /* *INDENT-OFF* */
-      fib_table_entry_path_add (
-	fib_table_get_index_for_sw_if_index (
-	  FIB_PROTOCOL_IP4,
-	  c->sw_if_index),
-	  &all_0s,
-	  FIB_SOURCE_DHCP,
-	  FIB_ENTRY_FLAG_NONE,
-          DPO_PROTO_IP4,
-          &nh, c->sw_if_index,
-          ~0, 1, NULL,	// no label stack
-          FIB_ROUTE_PATH_FLAG_NONE);
-      /* *INDENT-ON* */
-    }
-  if (c->dhcp_server.as_u32)
+  /* add the address to the interface if they've changed since the last time */
+  if (0 != clib_memcmp (&c->installed, &c->learned, sizeof (c->learned)))
     {
-      ip46_address_t dst = {
-	.ip4 = c->dhcp_server,
-      };
-      c->ai_ucast = adj_nbr_add_or_lock (FIB_PROTOCOL_IP4,
-					 VNET_LINK_IP4, &dst, c->sw_if_index);
+      dhcp_client_release_address (dcm, c);
+      dhcp_client_acquire_address (dcm, c);
     }
 
   /*
@@ -145,6 +252,28 @@ dhcp_client_addr_callback (dhcp_client_t * c)
    */
   if (c->event_callback)
     c->event_callback (c->client_index, c);
+
+  DHCP_INFO ("update: %U", format_dhcp_client, dcm, c);
+}
+
+static void
+dhcp_client_reset (dhcp_client_main_t * dcm, dhcp_client_t * c)
+{
+  if (c->client_detect_feature_enabled == 1)
+    {
+      vnet_feature_enable_disable ("ip4-unicast",
+				   "ip4-dhcp-client-detect",
+				   c->sw_if_index, 0, 0, 0);
+      c->client_detect_feature_enabled = 0;
+    }
+
+  dhcp_client_release_address (dcm, c);
+  clib_memset (&c->learned, 0, sizeof (c->installed));
+  c->state = DHCP_DISCOVER;
+  c->next_transmit = vlib_time_now (dcm->vlib_main);
+  c->retry_count = 0;
+  c->lease_renewal_interval = 0;
+  vec_free (c->domain_server_address);
 }
 
 /*
@@ -194,9 +323,9 @@ dhcp_client_for_us (u32 bi, vlib_buffer_t * b,
 
   /* parse through the packet, learn what we can */
   if (dhcp->your_ip_address.as_u32)
-    c->leased_address.as_u32 = dhcp->your_ip_address.as_u32;
+    c->learned.leased_address.as_u32 = dhcp->your_ip_address.as_u32;
 
-  c->dhcp_server.as_u32 = dhcp->server_ip_address.as_u32;
+  c->learned.dhcp_server.as_u32 = dhcp->server_ip_address.as_u32;
 
   o = (dhcp_option_t *) dhcp->options;
 
@@ -230,19 +359,19 @@ dhcp_client_for_us (u32 bi, vlib_buffer_t * b,
 	  break;
 
 	case 54:		/* dhcp server address */
-	  c->dhcp_server.as_u32 = o->data_as_u32[0];
+	  c->learned.dhcp_server.as_u32 = o->data_as_u32[0];
 	  break;
 
 	case 1:		/* subnet mask */
 	  {
 	    u32 subnet_mask = clib_host_to_net_u32 (o->data_as_u32[0]);
-	    c->subnet_mask_width = count_set_bits (subnet_mask);
+	    c->learned.subnet_mask_width = count_set_bits (subnet_mask);
 	  }
 	  break;
 	case 3:		/* router address */
 	  {
 	    u32 router_address = o->data_as_u32[0];
-	    c->router_address.as_u32 = router_address;
+	    c->learned.router_address.as_u32 = router_address;
 	  }
 	  break;
 	case 6:		/* domain server address */
@@ -297,29 +426,9 @@ dhcp_client_for_us (u32 bi, vlib_buffer_t * b,
 	{
 	  vlib_node_increment_counter (vm, dhcp_client_process_node.index,
 				       DHCP_STAT_NAK, 1);
-	  /* Probably never happens in bound state, but anyhow... */
-	  if (c->state == DHCP_BOUND)
-	    {
-	      ip4_add_del_interface_address (dcm->vlib_main, c->sw_if_index,
-					     (void *) &c->leased_address,
-					     c->subnet_mask_width,
-					     1 /*is_del */ );
-	      vnet_feature_enable_disable ("ip4-unicast",
-					   "ip4-dhcp-client-detect",
-					   c->sw_if_index, 1 /* enable */ ,
-					   0, 0);
-	      c->client_detect_feature_enabled = 1;
-	    }
-	  /* Wipe out any memory of the address we had... */
-	  c->state = DHCP_DISCOVER;
-	  c->next_transmit = now;
-	  c->retry_count = 0;
-	  c->leased_address.as_u32 = 0;
-	  c->subnet_mask_width = 0;
-	  c->router_address.as_u32 = 0;
-	  c->lease_renewal_interval = 0;
-	  c->dhcp_server.as_u32 = 0;
-	  vec_free (c->domain_server_address);
+	  /* Probably never happens in bound state, but anyhow...
+	     Wipe out any memory of the address we had... */
+	  dhcp_client_reset (dcm, c);
 	  break;
 	}
 
@@ -335,8 +444,13 @@ dhcp_client_for_us (u32 bi, vlib_buffer_t * b,
 	  break;
 	}
       /* OK, we own the address (etc), add to the routing table(s) */
-      vl_api_rpc_call_main_thread (dhcp_client_addr_callback,
-				   (u8 *) c, sizeof (*c));
+      {
+	/* Send the index over to the main thread, where it can retrieve
+	 * the original client */
+	u32 cindex = c - dcm->clients;
+	vl_api_force_rpc_call_main_thread (dhcp_client_addr_callback,
+					   (u8 *) & cindex, sizeof (u32));
+      }
 
       c->state = DHCP_BOUND;
       c->retry_count = 0;
@@ -351,8 +465,7 @@ dhcp_client_for_us (u32 bi, vlib_buffer_t * b,
       break;
     }
 
-  /* drop the pkt, return 1 */
-  vlib_buffer_free (vm, &bi, 1);
+  /* return 1 so the call disposes of this packet */
   return 1;
 }
 
@@ -377,6 +490,10 @@ send_dhcp_pkt (dhcp_client_main_t * dcm, dhcp_client_t * c,
   u16 udp_length, ip_length;
   u32 counter_index, node_index;
 
+  DHCP_INFO ("send: type:%U bcast:%d %U",
+	     format_dhcp_packet_type, type,
+	     is_broadcast, format_dhcp_client, dcm, c);
+
   /* Interface(s) down? */
   if ((hw->flags & VNET_HW_INTERFACE_FLAG_LINK_UP) == 0)
     return;
@@ -400,6 +517,9 @@ send_dhcp_pkt (dhcp_client_main_t * dcm, dhcp_client_t * c,
 
   vnet_buffer (b)->sw_if_index[VLIB_RX] = c->sw_if_index;
 
+  if (ADJ_INDEX_INVALID == c->ai_ucast)
+    is_broadcast = 1;
+
   if (is_broadcast)
     {
       node_index = ip4_rewrite_node.index;
@@ -456,8 +576,8 @@ send_dhcp_pkt (dhcp_client_main_t * dcm, dhcp_client_t * c,
   else
     {
       /* Renewing an active lease, plain old ip4 src/dst */
-      ip->src_address.as_u32 = c->leased_address.as_u32;
-      ip->dst_address.as_u32 = c->dhcp_server.as_u32;
+      ip->src_address.as_u32 = c->learned.leased_address.as_u32;
+      ip->dst_address.as_u32 = c->learned.dhcp_server.as_u32;
     }
 
   udp->src_port = clib_host_to_net_u16 (UDP_DST_PORT_dhcp_to_client);
@@ -473,7 +593,7 @@ send_dhcp_pkt (dhcp_client_main_t * dcm, dhcp_client_t * c,
 
   /* Lease renewal, set up client_ip_address */
   if (is_broadcast == 0)
-    dhcp->client_ip_address.as_u32 = c->leased_address.as_u32;
+    dhcp->client_ip_address.as_u32 = c->learned.leased_address.as_u32;
 
   dhcp->opcode = 1;		/* request, all we send */
   dhcp->hardware_type = 1;	/* ethernet */
@@ -508,20 +628,20 @@ send_dhcp_pkt (dhcp_client_main_t * dcm, dhcp_client_t * c,
    * If server ip address is available with non-zero value,
    * option 54 (DHCP Server Identifier) is sent.
    */
-  if (c->dhcp_server.as_u32)
+  if (c->learned.dhcp_server.as_u32)
     {
       o->option = 54;
       o->length = 4;
-      clib_memcpy (o->data, &c->dhcp_server.as_u32, 4);
+      clib_memcpy (o->data, &c->learned.dhcp_server.as_u32, 4);
       o = (dhcp_option_t *) (((uword) o) + (o->length + 2));
     }
 
   /* send option 50, requested IP address */
-  if (c->leased_address.as_u32)
+  if (c->learned.leased_address.as_u32)
     {
       o->option = 50;
       o->length = 4;
-      clib_memcpy (o->data, &c->leased_address.as_u32, 4);
+      clib_memcpy (o->data, &c->learned.leased_address.as_u32, 4);
       o = (dhcp_option_t *) (((uword) o) + (o->length + 2));
     }
 
@@ -594,7 +714,17 @@ dhcp_discover_state (dhcp_client_main_t * dcm, dhcp_client_t * c, f64 now)
    * State machine "DISCOVER" state. Send a dhcp discover packet,
    * eventually back off the retry rate.
    */
+  DHCP_INFO ("enter discover: %U", format_dhcp_client, dcm, c);
 
+  /*
+   * In order to accept any OFFER, whether broadcasted or unicasted, we
+   * need to configure the dhcp-client-detect feature as an input feature
+   * so the DHCP OFFER is sent to the ip4-local node. Without this a
+   * broadcasted OFFER hits the 255.255.255.255/32 address and a unicast
+   * hits 0.0.0.0/0 both of which default to drop and the latter may forward
+   * of box - not what we want. Nor to we want to change these route for
+   * all interfaces in this table
+   */
   if (c->client_detect_feature_enabled == 0)
     {
       vnet_feature_enable_disable ("ip4-unicast",
@@ -620,6 +750,8 @@ dhcp_request_state (dhcp_client_main_t * dcm, dhcp_client_t * c, f64 now)
    * State machine "REQUEST" state. Send a dhcp request packet,
    * eventually drop back to the discover state.
    */
+  DHCP_INFO ("enter request: %U", format_dhcp_client, dcm, c);
+
   send_dhcp_pkt (dcm, c, DHCP_PACKET_REQUEST, 1 /* is_broadcast */ );
 
   c->retry_count++;
@@ -638,13 +770,6 @@ static int
 dhcp_bound_state (dhcp_client_main_t * dcm, dhcp_client_t * c, f64 now)
 {
   /*
-   * State machine "BOUND" state. Send a dhcp request packet to renew
-   * the lease.
-   * Eventually, when the lease expires, forget the dhcp data
-   * and go back to the stone age.
-   */
-
-  /*
    * We disable the client detect feature when we bind a
    * DHCP address. Turn it back on again on first renew attempt.
    * Otherwise, if the DHCP server replies we'll never see it.
@@ -657,6 +782,23 @@ dhcp_bound_state (dhcp_client_main_t * dcm, dhcp_client_t * c, f64 now)
       c->client_detect_feature_enabled = 1;
     }
 
+  /*
+   * State machine "BOUND" state. Send a dhcp request packet to renew
+   * the lease.
+   * Eventually, when the lease expires, forget the dhcp data
+   * and go back to the stone age.
+   */
+  if (now > c->lease_expires)
+    {
+      DHCP_INFO ("lease expired: %U", format_dhcp_client, dcm, c);
+
+      /* reset all data for the client. do not send any more messages
+       * since the objects to do so have been lost */
+      dhcp_client_reset (dcm, c);
+      return 1;
+    }
+
+  DHCP_INFO ("enter bound: %U", format_dhcp_client, dcm, c);
   send_dhcp_pkt (dcm, c, DHCP_PACKET_REQUEST, 0 /* is_broadcast */ );
 
   c->retry_count++;
@@ -665,39 +807,6 @@ dhcp_bound_state (dhcp_client_main_t * dcm, dhcp_client_t * c, f64 now)
   else
     c->next_transmit = now + 1.0;
 
-  if (now > c->lease_expires)
-    {
-      /* Remove the default route */
-      if (c->router_address.as_u32)
-	{
-	  fib_prefix_t all_0s = {
-	    .fp_len = 0,
-	    .fp_addr.ip4.as_u32 = 0x0,
-	    .fp_proto = FIB_PROTOCOL_IP4,
-	  };
-	  ip46_address_t nh = {
-	    .ip4 = c->router_address,
-	  };
-
-	  fib_table_entry_path_remove (fib_table_get_index_for_sw_if_index
-				       (FIB_PROTOCOL_IP4, c->sw_if_index),
-				       &all_0s, FIB_SOURCE_DHCP,
-				       DPO_PROTO_IP4, &nh, c->sw_if_index, ~0,
-				       1, FIB_ROUTE_PATH_FLAG_NONE);
-	}
-      /* Remove the interface address */
-      dhcp_client_release_address (dcm, c);
-      c->state = DHCP_DISCOVER;
-      c->next_transmit = now;
-      c->retry_count = 0;
-      /* Wipe out any memory of the address we had... */
-      c->leased_address.as_u32 = 0;
-      c->subnet_mask_width = 0;
-      c->router_address.as_u32 = 0;
-      c->lease_renewal_interval = 0;
-      c->dhcp_server.as_u32 = 0;
-      return 1;
-    }
   return 0;
 }
 
@@ -776,6 +885,7 @@ dhcp_client_process (vlib_main_t * vm,
 
 	case ~0:
           /* *INDENT-OFF* */
+          DHCP_INFO ("timeout");
 	  pool_foreach (c, dcm->clients,
           ({
             timeout = dhcp_client_sm (now, timeout,
@@ -805,75 +915,6 @@ VLIB_REGISTER_NODE (dhcp_client_process_node,static) = {
 };
 /* *INDENT-ON* */
 
-static u8 *
-format_dhcp_client_state (u8 * s, va_list * va)
-{
-  dhcp_client_state_t state = va_arg (*va, dhcp_client_state_t);
-  char *str = "BOGUS!";
-
-  switch (state)
-    {
-#define _(a)                                    \
-    case a:                                     \
-      str = #a;                                 \
-        break;
-      foreach_dhcp_client_state;
-#undef _
-    default:
-      break;
-    }
-
-  s = format (s, "%s", str);
-  return s;
-}
-
-static u8 *
-format_dhcp_client (u8 * s, va_list * va)
-{
-  dhcp_client_main_t *dcm = va_arg (*va, dhcp_client_main_t *);
-  dhcp_client_t *c = va_arg (*va, dhcp_client_t *);
-  int verbose = va_arg (*va, int);
-  ip4_address_t *addr;
-
-  s = format (s, "[%d] %U state %U ", c - dcm->clients,
-	      format_vnet_sw_if_index_name, dcm->vnet_main, c->sw_if_index,
-	      format_dhcp_client_state, c->state);
-
-  if (0 != c->dscp)
-    s = format (s, "dscp %d ", c->dscp);
-
-  if (c->leased_address.as_u32)
-    {
-      s = format (s, "addr %U/%d gw %U",
-		  format_ip4_address, &c->leased_address,
-		  c->subnet_mask_width, format_ip4_address,
-		  &c->router_address);
-
-      vec_foreach (addr, c->domain_server_address)
-	s = format (s, " dns %U", format_ip4_address, addr);
-    }
-  else
-    {
-      s = format (s, "no address\n");
-    }
-
-  if (verbose)
-    {
-      s =
-	format (s,
-		"\n lease: lifetime:%d renewal-interval:%d expires:%.2f (now:%.2f)",
-		c->lease_lifetime, c->lease_renewal_interval,
-		c->lease_expires, vlib_time_now (dcm->vlib_main));
-      s =
-	format (s, "\n retry-count:%d, next-xmt:%.2f", c->retry_count,
-		c->next_transmit);
-      s =
-	format (s, "\n adjacencies:[unicast:%d broadcast:%d]", c->ai_ucast,
-		c->ai_bcast);
-    }
-  return s;
-}
-
 static clib_error_t *
 show_dhcp_client_command_fn (vlib_main_t * vm,
 			     unformat_input_t * input,
@@ -934,11 +975,6 @@ dhcp_client_add_del (dhcp_client_add_del_args_t * a)
   vlib_main_t *vm = dcm->vlib_main;
   dhcp_client_t *c;
   uword *p;
-  fib_prefix_t all_0s = {
-    .fp_len = 0,
-    .fp_addr.ip4.as_u32 = 0x0,
-    .fp_proto = FIB_PROTOCOL_IP4,
-  };
 
   p = hash_get (dcm->client_by_sw_if_index, a->sw_if_index);
 
@@ -973,42 +1009,17 @@ dhcp_client_add_del (dhcp_client_add_del_args_t * a)
 
       hash_set (dcm->client_by_sw_if_index, a->sw_if_index, c - dcm->clients);
 
-      /*
-       * In order to accept any OFFER, whether broadcasted or unicasted, we
-       * need to configure the dhcp-client-detect feature as an input feature
-       * so the DHCP OFFER is sent to the ip4-local node. Without this a
-       * broadcasted OFFER hits the 255.255.255.255/32 address and a unicast
-       * hits 0.0.0.0/0 both of which default to drop and the latter may forward
-       * of box - not what we want. Nor to we want to change these route for
-       * all interfaces in this table
-       */
-      vnet_feature_enable_disable ("ip4-unicast",
-				   "ip4-dhcp-client-detect",
-				   c->sw_if_index, 1 /* enable */ , 0, 0);
-      c->client_detect_feature_enabled = 1;
-
       vlib_process_signal_event (vm, dhcp_client_process_node.index,
 				 EVENT_DHCP_CLIENT_WAKEUP, c - dcm->clients);
+
+      DHCP_INFO ("create: %U", format_dhcp_client, dcm, c);
     }
   else
     {
       c = pool_elt_at_index (dcm->clients, p[0]);
 
-      if (c->router_address.as_u32)
-	{
-	  ip46_address_t nh = {
-	    .ip4 = c->router_address,
-	  };
+      dhcp_client_reset (dcm, c);
 
-	  fib_table_entry_path_remove (fib_table_get_index_for_sw_if_index
-				       (FIB_PROTOCOL_IP4, c->sw_if_index),
-				       &all_0s, FIB_SOURCE_DHCP,
-				       DPO_PROTO_IP4, &nh, c->sw_if_index, ~0,
-				       1, FIB_ROUTE_PATH_FLAG_NONE);
-	}
-      dhcp_client_release_address (dcm, c);
-
-      adj_unlock (c->ai_ucast);
       adj_unlock (c->ai_bcast);
 
       vec_free (c->domain_server_address);
@@ -1241,6 +1252,9 @@ dhcp_client_init (vlib_main_t * vm)
   dcm->vlib_main = vm;
   dcm->vnet_main = vnet_get_main ();
   dcm->seed = (u32) clib_cpu_time_now ();
+
+  dhcp_logger = vlib_log_register_class ("dhcp", "client");
+
   return 0;
 }
 
diff --git a/src/plugins/dhcp/client.h b/src/plugins/dhcp/client.h
index 68176abadf6..e9e3bcf67c4 100644
--- a/src/plugins/dhcp/client.h
+++ b/src/plugins/dhcp/client.h
@@ -42,6 +42,24 @@ struct dhcp_client_t_;
 typedef void (*dhcp_event_cb_t) (u32 client_index,
 				 const struct dhcp_client_t_ * client);
 
+/**
+ * The set of addresses/mask that contribute forwarding info
+ * and are installed.
+ */
+typedef struct dhcp_client_fwd_addresses_t_
+{
+  /** the address assigned to this client and it's mask */
+  ip4_address_t leased_address;
+  u32 subnet_mask_width;
+
+  /** the address of the DHCP server handing out the address.
+      this is used to send any unicast messages */
+  ip4_address_t dhcp_server;
+
+  /** The address of this client's default gateway - may not be present */
+  ip4_address_t router_address;
+} dhcp_client_fwd_addresses_t;
+
 typedef struct dhcp_client_t_
 {
   dhcp_client_state_t state;
@@ -59,11 +77,16 @@ typedef struct dhcp_client_t_
   /* DHCP transaction ID, a random number */
   u32 transaction_id;
 
-  /* leased address, other learned info DHCP */
-  ip4_address_t leased_address;	/* from your_ip_address field */
-  ip4_address_t dhcp_server;
-  u32 subnet_mask_width;	/* option 1 */
-  ip4_address_t router_address;	/* option 3 */
+  /**
+   * leased address, other learned info DHCP
+   *  the learned set is updated by new messages recieved in the DP
+   *  the installed set is what's actually been added
+   */
+  dhcp_client_fwd_addresses_t learned;
+  dhcp_client_fwd_addresses_t installed;
+  /* have local Addresses and default route been installed */
+  u8 addresses_installed;
+
   ip4_address_t *domain_server_address;	/* option 6 */
   u32 lease_renewal_interval;	/* option 51 */
   u32 lease_lifetime;		/* option 59 */
diff --git a/src/plugins/dhcp/dhcp.api b/src/plugins/dhcp/dhcp.api
index a91874be82c..9d8ca4c5b8d 100644
--- a/src/plugins/dhcp/dhcp.api
+++ b/src/plugins/dhcp/dhcp.api
@@ -105,7 +105,7 @@ define dhcp_plugin_control_ping_reply
     @param is_add - add the config if non-zero, else delete
     @param insert_circuit_id - option82 suboption 1 fib number
     @param dhcp_server[] - server address
-    @param dhcp_src_address[] - <fix this, need details>
+    @param dhcp_src_address[] - sc address for packets sent to the server
 */
 autoreply define dhcp_proxy_config
 {
diff --git a/src/plugins/dhcp/dhcp4_packet.c b/src/plugins/dhcp/dhcp4_packet.c
new file mode 100644
index 00000000000..7592120ffea
--- /dev/null
+++ b/src/plugins/dhcp/dhcp4_packet.c
@@ -0,0 +1,122 @@
+/*
+ * dhcp4_packet.c: dhcp packet format functions
+ *
+ * Copyright (c) 2013 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <dhcp/dhcp4_packet.h>
+#include <vnet/ip/format.h>
+
+u8 *
+format_dhcp_packet_type (u8 * s, va_list * args)
+{
+  dhcp_packet_type_t pt = va_arg (*args, dhcp_packet_type_t);
+
+  switch (pt)
+    {
+    case DHCP_PACKET_DISCOVER:
+      s = format (s, "discover");
+      break;
+    case DHCP_PACKET_OFFER:
+      s = format (s, "offer");
+      break;
+    case DHCP_PACKET_REQUEST:
+      s = format (s, "request");
+      break;
+    case DHCP_PACKET_ACK:
+      s = format (s, "ack");
+      break;
+    case DHCP_PACKET_NAK:
+      s = format (s, "nack");
+      break;
+    }
+  return (s);
+}
+
+u8 *
+format_dhcp_header (u8 * s, va_list * args)
+{
+  dhcp_header_t *d = va_arg (*args, dhcp_header_t *);
+  u32 max_bytes = va_arg (*args, u32);
+  dhcp_option_t *o;
+  u32 tmp;
+
+  s = format (s, "opcode:%s", (d->opcode == 1 ? "request" : "reply"));
+  s = format (s, " hw[type:%d addr-len:%d addr:%U]",
+	      d->hardware_type, d->hardware_address_length,
+	      format_hex_bytes, d->client_hardware_address,
+	      d->hardware_address_length);
+  s = format (s, " hops%d", d->hops);
+  s = format (s, " transaction-ID:0x%x", d->transaction_identifier);
+  s = format (s, " seconds:%d", d->seconds);
+  s = format (s, " flags:0x%x", d->flags);
+  s = format (s, " client:%U", format_ip4_address, &d->client_ip_address);
+  s = format (s, " your:%U", format_ip4_address, &d->your_ip_address);
+  s = format (s, " server:%U", format_ip4_address, &d->server_ip_address);
+  s = format (s, " gateway:%U", format_ip4_address, &d->gateway_ip_address);
+  s = format (s, " cookie:%U", format_ip4_address, &d->magic_cookie);
+
+  o = (dhcp_option_t *) d->options;
+
+  while (o->option != 0xFF /* end of options */  &&
+	 (u8 *) o < (u8 *) d + max_bytes)
+    {
+      switch (o->option)
+	{
+	case 53:		/* dhcp message type */
+	  tmp = o->data[0];
+	  s =
+	    format (s, ", option-53: type:%U", format_dhcp_packet_type, tmp);
+	  break;
+	case 54:		/* dhcp server address */
+	  s = format (s, ", option-54: server:%U",
+		      format_ip4_address, &o->data_as_u32[0]);
+	  break;
+	case 58:		/* lease renew time in seconds */
+	  s = format (s, ", option-58: renewal:%d",
+		      clib_host_to_net_u32 (o->data_as_u32[0]));
+	  break;
+	case 1:		/* subnet mask */
+	  s = format (s, ", option-1: subnet-mask:%d",
+		      clib_host_to_net_u32 (o->data_as_u32[0]));
+	  break;
+	case 3:		/* router address */
+	  s = format (s, ", option-3: router:%U",
+		      format_ip4_address, &o->data_as_u32[0]);
+	  break;
+	case 6:		/* domain server address */
+	  s = format (s, ", option-6: domian-server:%U",
+		      format_hex_bytes, o->data, o->length);
+	  break;
+	case 12:		/* hostname */
+	  s = format (s, ", option-12: hostname:%U",
+		      format_hex_bytes, o->data, o->length);
+	  break;
+	default:
+	  tmp = o->option;
+	  s = format (s, " option-%d: skipped", tmp);
+	  break;
+	}
+      o = (dhcp_option_t *) (((u8 *) o) + (o->length + 2));
+    }
+  return (s);
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/dhcp/dhcp4_packet.h b/src/plugins/dhcp/dhcp4_packet.h
index 3076dd9529d..abd60127d6c 100644
--- a/src/plugins/dhcp/dhcp4_packet.h
+++ b/src/plugins/dhcp/dhcp4_packet.h
@@ -51,6 +51,8 @@ typedef struct
   dhcp_option_t options[0];
 } dhcp_header_t;
 
+extern u8 *format_dhcp_header (u8 * s, va_list * args);
+
 typedef enum
 {
   DHCP_PACKET_DISCOVER = 1,
@@ -60,6 +62,8 @@ typedef enum
   DHCP_PACKET_NAK,
 } dhcp_packet_type_t;
 
+extern u8 *format_dhcp_packet_type (u8 * s, va_list * args);
+
 typedef enum dhcp_packet_option_t_
 {
   DHCP_PACKET_OPTION_MSG_TYPE = 53,
diff --git a/src/plugins/dhcp/dhcp4_proxy_error.def b/src/plugins/dhcp/dhcp4_proxy_error.def
index adf04808fa3..83f11830ab6 100644
--- a/src/plugins/dhcp/dhcp4_proxy_error.def
+++ b/src/plugins/dhcp/dhcp4_proxy_error.def
@@ -29,4 +29,5 @@ dhcp_proxy_error (OPTION_82_VSS_NOT_PROCESSED, "DHCP VSS not processed by DHCP s
 dhcp_proxy_error (BAD_YIADDR, "DHCP packets with bad your_ip_address fields")
 dhcp_proxy_error (BAD_SVR_FIB_OR_ADDRESS, "DHCP packets not from DHCP server or server FIB.")
 dhcp_proxy_error (PKT_TOO_BIG, "DHCP packets which are too big.")
+dhcp_proxy_error (FOR_US, "DHCP packets for local client.")
 
diff --git a/src/plugins/dhcp/dhcp4_proxy_node.c b/src/plugins/dhcp/dhcp4_proxy_node.c
index 10963c7ff47..53c95c09f12 100644
--- a/src/plugins/dhcp/dhcp4_proxy_node.c
+++ b/src/plugins/dhcp/dhcp4_proxy_node.c
@@ -48,7 +48,11 @@ typedef struct
   u32 error;
   u32 sw_if_index;
   u32 original_sw_if_index;
-} dhcp_proxy_trace_t;
+
+  /* enough space for the DHCP header plus some options */
+  u8 packet_data[2 * sizeof (dhcp_header_t)];
+}
+dhcp_proxy_trace_t;
 
 #define VPP_DHCP_OPTION82_SUB1_SIZE   6
 #define VPP_DHCP_OPTION82_SUB5_SIZE   6
@@ -79,6 +83,8 @@ format_dhcp_proxy_trace (u8 * s, va_list * args)
 
   s = format (s, "  original_sw_if_index: %d, sw_if_index: %d\n",
 	      t->original_sw_if_index, t->sw_if_index);
+  s = format (s, "  %U",
+	      format_dhcp_header, t->packet_data, sizeof (t->packet_data));
 
   return s;
 }
@@ -393,6 +399,10 @@ dhcp_proxy_to_server_input (vlib_main_t * vm,
 		      if (next0 == DHCP_PROXY_TO_SERVER_INPUT_NEXT_LOOKUP)
 			tr->trace_ip4_address.as_u32 =
 			  server->dhcp_server.ip4.as_u32;
+
+		      clib_memcpy_fast (tr->packet_data, h0,
+					sizeof (tr->packet_data));
+
 		    }
 
 		  if (PREDICT_FALSE (0 == n_left_to_next))
@@ -416,6 +426,8 @@ dhcp_proxy_to_server_input (vlib_main_t * vm,
 	      if (next0 == DHCP_PROXY_TO_SERVER_INPUT_NEXT_LOOKUP)
 		tr->trace_ip4_address.as_u32 =
 		  proxy->dhcp_servers[0].dhcp_server.ip4.as_u32;
+	      clib_memcpy_fast (tr->packet_data, h0,
+				sizeof (tr->packet_data));
 	    }
 
 	do_enqueue:
@@ -471,70 +483,86 @@ VLIB_REGISTER_NODE (dhcp_proxy_to_server_node, static) = {
 };
 /* *INDENT-ON* */
 
+typedef enum
+{
+  DHCP4_PROXY_NEXT_DROP,
+  DHCP4_PROXY_NEXT_TX,
+  DHCP4_PROXY_N_NEXT,
+} dhcp4_next_t;
+
 static uword
 dhcp_proxy_to_client_input (vlib_main_t * vm,
 			    vlib_node_runtime_t * node,
 			    vlib_frame_t * from_frame)
 {
-  u32 n_left_from, *from;
+  u32 n_left_from, *from, *to_next, n_left_to_next;
   ethernet_main_t *em = vnet_get_ethernet_main ();
   dhcp_proxy_main_t *dpm = &dhcp_proxy_main;
   vnet_main_t *vnm = vnet_get_main ();
   ip4_main_t *im = &ip4_main;
+  u32 next_index;
 
   from = vlib_frame_vector_args (from_frame);
   n_left_from = from_frame->n_vectors;
+  next_index = node->cached_next_index;
 
   while (n_left_from > 0)
     {
-      u32 bi0;
-      vlib_buffer_t *b0;
-      udp_header_t *u0;
-      dhcp_header_t *h0;
-      ip4_header_t *ip0 = 0;
-      ip4_address_t *ia0 = 0;
-      u32 old0, new0;
-      ip_csum_t sum0;
-      ethernet_interface_t *ei0;
-      ethernet_header_t *mac0;
-      vnet_hw_interface_t *hi0;
-      vlib_frame_t *f0;
-      u32 *to_next0;
-      u32 sw_if_index = ~0;
-      vnet_sw_interface_t *si0;
-      u32 error0 = (u32) ~ 0;
-      vnet_sw_interface_t *swif;
-      u32 fib_index;
-      dhcp_proxy_t *proxy;
-      dhcp_server_t *server;
-      u32 original_sw_if_index = (u32) ~ 0;
-      ip4_address_t relay_addr = {
-	.as_u32 = 0,
-      };
-
-      bi0 = from[0];
-      from += 1;
-      n_left_from -= 1;
-
-      b0 = vlib_get_buffer (vm, bi0);
-      h0 = vlib_buffer_get_current (b0);
-
-      /*
-       * udp_local hands us the DHCP header, need udp hdr,
-       * ip hdr to relay to client
-       */
-      vlib_buffer_advance (b0, -(sizeof (*u0)));
-      u0 = vlib_buffer_get_current (b0);
-
-      vlib_buffer_advance (b0, -(sizeof (*ip0)));
-      ip0 = vlib_buffer_get_current (b0);
-
-      /* Consumed by dhcp client code? */
-      if (dhcp_client_for_us (bi0, b0, ip0, u0, h0))
-	continue;
-
-      if (1 /* dpm->insert_option_82 */ )
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+      while (n_left_from > 0 && n_left_to_next > 0)
 	{
+	  u32 bi0;
+	  vlib_buffer_t *b0;
+	  udp_header_t *u0;
+	  dhcp_header_t *h0;
+	  ip4_header_t *ip0 = 0;
+	  ip4_address_t *ia0 = 0;
+	  u32 old0, new0;
+	  ip_csum_t sum0;
+	  ethernet_interface_t *ei0;
+	  ethernet_header_t *mac0;
+	  vnet_hw_interface_t *hi0;
+	  u32 sw_if_index = ~0;
+	  vnet_sw_interface_t *si0;
+	  u32 error0 = (u32) ~ 0;
+	  vnet_sw_interface_t *swif;
+	  u32 fib_index;
+	  dhcp_proxy_t *proxy;
+	  dhcp_server_t *server;
+	  u32 original_sw_if_index = (u32) ~ 0;
+	  dhcp4_next_t next0 = DHCP4_PROXY_NEXT_TX;
+	  ip4_address_t relay_addr = {
+	    .as_u32 = 0,
+	  };
+
+	  bi0 = to_next[0] = from[0];
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  h0 = vlib_buffer_get_current (b0);
+
+	  /*
+	   * udp_local hands us the DHCP header, need udp hdr,
+	   * ip hdr to relay to client
+	   */
+	  vlib_buffer_advance (b0, -(sizeof (*u0)));
+	  u0 = vlib_buffer_get_current (b0);
+
+	  vlib_buffer_advance (b0, -(sizeof (*ip0)));
+	  ip0 = vlib_buffer_get_current (b0);
+
+	  /* Consumed by dhcp client code? */
+	  if (dhcp_client_for_us (bi0, b0, ip0, u0, h0))
+	    {
+	      error0 = DHCP_PROXY_ERROR_FOR_US;
+	      goto drop_packet;
+	    }
+
+	  // if (1 /* dpm->insert_option_82 */ )
 	  /* linearize needed to "unclone" and scan options */
 	  int rv = vlib_buffer_chain_linearize (vm, b0);
 	  if ((b0->flags & VLIB_BUFFER_NEXT_PRESENT) != 0 || !rv)
@@ -594,134 +622,135 @@ dhcp_proxy_to_client_input (vlib_main_t * vm,
 		}
 	      o = (dhcp_option_t *) (o->data + o->length);
 	    }
-	}
 
-      if (sw_if_index == (u32) ~ 0)
-	{
-	  error0 = DHCP_PROXY_ERROR_NO_OPTION_82;
-
-	drop_packet:
-	  vlib_node_increment_counter (vm, dhcp_proxy_to_client_node.index,
-				       error0, 1);
-	  f0 = vlib_get_frame_to_node (vm, dpm->error_drop_node_index);
-	  to_next0 = vlib_frame_vector_args (f0);
-	  to_next0[0] = bi0;
-	  f0->n_vectors = 1;
-	  vlib_put_frame_to_node (vm, dpm->error_drop_node_index, f0);
-	  goto do_trace;
-	}
+	  if (sw_if_index == (u32) ~ 0)
+	    {
+	      error0 = DHCP_PROXY_ERROR_NO_OPTION_82;
+
+	    drop_packet:
+	      vlib_node_increment_counter (vm,
+					   dhcp_proxy_to_client_node.index,
+					   error0, 1);
+	      b0->error = node->errors[error0];
+	      next0 = DHCP4_PROXY_NEXT_DROP;
+	      goto do_trace;
+	    }
 
-      if (relay_addr.as_u32 == 0)
-	{
-	  error0 = DHCP_PROXY_ERROR_BAD_OPTION_82_ADDR;
-	  goto drop_packet;
-	}
+	  if (relay_addr.as_u32 == 0)
+	    {
+	      error0 = DHCP_PROXY_ERROR_BAD_OPTION_82_ADDR;
+	      goto drop_packet;
+	    }
 
-      if (sw_if_index >= vec_len (im->fib_index_by_sw_if_index))
-	{
-	  error0 = DHCP_PROXY_ERROR_BAD_OPTION_82_ITF;
-	  goto drop_packet;
-	}
+	  if (sw_if_index >= vec_len (im->fib_index_by_sw_if_index))
+	    {
+	      error0 = DHCP_PROXY_ERROR_BAD_OPTION_82_ITF;
+	      goto drop_packet;
+	    }
 
-      fib_index = im->fib_index_by_sw_if_index[sw_if_index];
-      proxy = dhcp_get_proxy (dpm, fib_index, FIB_PROTOCOL_IP4);
+	  fib_index = im->fib_index_by_sw_if_index[sw_if_index];
+	  proxy = dhcp_get_proxy (dpm, fib_index, FIB_PROTOCOL_IP4);
 
-      if (PREDICT_FALSE (NULL == proxy))
-	{
-	  error0 = DHCP_PROXY_ERROR_NO_SERVER;
-	  goto drop_packet;
-	}
+	  if (PREDICT_FALSE (NULL == proxy))
+	    {
+	      error0 = DHCP_PROXY_ERROR_NO_SERVER;
+	      goto drop_packet;
+	    }
 
-      vec_foreach (server, proxy->dhcp_servers)
-      {
-	if (ip0->src_address.as_u32 == server->dhcp_server.ip4.as_u32)
+	  vec_foreach (server, proxy->dhcp_servers)
 	  {
-	    goto server_found;
+	    if (ip0->src_address.as_u32 == server->dhcp_server.ip4.as_u32)
+	      {
+		goto server_found;
+	      }
 	  }
-      }
 
-      error0 = DHCP_PROXY_ERROR_BAD_SVR_FIB_OR_ADDRESS;
-      goto drop_packet;
+	  error0 = DHCP_PROXY_ERROR_BAD_SVR_FIB_OR_ADDRESS;
+	  goto drop_packet;
 
-    server_found:
-      vnet_buffer (b0)->sw_if_index[VLIB_TX] = sw_if_index;
+	server_found:
+	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = sw_if_index;
 
-      swif = vnet_get_sw_interface (vnm, sw_if_index);
-      original_sw_if_index = sw_if_index;
-      if (swif->flags & VNET_SW_INTERFACE_FLAG_UNNUMBERED)
-	sw_if_index = swif->unnumbered_sw_if_index;
+	  swif = vnet_get_sw_interface (vnm, sw_if_index);
+	  original_sw_if_index = sw_if_index;
+	  if (swif->flags & VNET_SW_INTERFACE_FLAG_UNNUMBERED)
+	    sw_if_index = swif->unnumbered_sw_if_index;
 
-      ia0 = ip4_interface_first_address (&ip4_main, sw_if_index, 0);
-      if (ia0 == 0)
-	{
-	  error0 = DHCP_PROXY_ERROR_NO_INTERFACE_ADDRESS;
-	  goto drop_packet;
-	}
+	  ia0 = ip4_interface_first_address (&ip4_main, sw_if_index, 0);
+	  if (ia0 == 0)
+	    {
+	      error0 = DHCP_PROXY_ERROR_NO_INTERFACE_ADDRESS;
+	      goto drop_packet;
+	    }
 
-      if (relay_addr.as_u32 != ia0->as_u32)
-	{
-	  error0 = DHCP_PROXY_ERROR_BAD_YIADDR;
-	  goto drop_packet;
-	}
+	  if (relay_addr.as_u32 != ia0->as_u32)
+	    {
+	      error0 = DHCP_PROXY_ERROR_BAD_YIADDR;
+	      goto drop_packet;
+	    }
 
-      u0->checksum = 0;
-      u0->dst_port = clib_net_to_host_u16 (UDP_DST_PORT_dhcp_to_client);
-      sum0 = ip0->checksum;
-      old0 = ip0->dst_address.as_u32;
-      new0 = 0xFFFFFFFF;
-      ip0->dst_address.as_u32 = new0;
-      sum0 = ip_csum_update (sum0, old0, new0, ip4_header_t /* structure */ ,
-			     dst_address /* offset of changed member */ );
-      ip0->checksum = ip_csum_fold (sum0);
-
-      sum0 = ip0->checksum;
-      old0 = ip0->src_address.as_u32;
-      new0 = ia0->as_u32;
-      ip0->src_address.as_u32 = new0;
-      sum0 = ip_csum_update (sum0, old0, new0, ip4_header_t /* structure */ ,
-			     src_address /* offset of changed member */ );
-      ip0->checksum = ip_csum_fold (sum0);
-
-      vlib_buffer_advance (b0, -(sizeof (ethernet_header_t)));
-      si0 = vnet_get_sw_interface (vnm, original_sw_if_index);
-      if (si0->type == VNET_SW_INTERFACE_TYPE_SUB)
-	vlib_buffer_advance (b0, -4 /* space for VLAN tag */ );
-
-      mac0 = vlib_buffer_get_current (b0);
-
-      hi0 = vnet_get_sup_hw_interface (vnm, original_sw_if_index);
-      ei0 = pool_elt_at_index (em->interfaces, hi0->hw_instance);
-      clib_memcpy (mac0->src_address, ei0->address, sizeof (ei0->address));
-      clib_memset (mac0->dst_address, 0xff, sizeof (mac0->dst_address));
-      mac0->type = (si0->type == VNET_SW_INTERFACE_TYPE_SUB) ?
-	clib_net_to_host_u16 (0x8100) : clib_net_to_host_u16 (0x0800);
-
-      if (si0->type == VNET_SW_INTERFACE_TYPE_SUB)
-	{
-	  u32 *vlan_tag = (u32 *) (mac0 + 1);
-	  u32 tmp;
-	  tmp = (si0->sub.id << 16) | 0x0800;
-	  *vlan_tag = clib_host_to_net_u32 (tmp);
-	}
+	  u0->checksum = 0;
+	  u0->dst_port = clib_net_to_host_u16 (UDP_DST_PORT_dhcp_to_client);
+	  sum0 = ip0->checksum;
+	  old0 = ip0->dst_address.as_u32;
+	  new0 = 0xFFFFFFFF;
+	  ip0->dst_address.as_u32 = new0;
+	  sum0 =
+	    ip_csum_update (sum0, old0, new0, ip4_header_t /* structure */ ,
+			    dst_address /* offset of changed member */ );
+	  ip0->checksum = ip_csum_fold (sum0);
+
+	  sum0 = ip0->checksum;
+	  old0 = ip0->src_address.as_u32;
+	  new0 = ia0->as_u32;
+	  ip0->src_address.as_u32 = new0;
+	  sum0 =
+	    ip_csum_update (sum0, old0, new0, ip4_header_t /* structure */ ,
+			    src_address /* offset of changed member */ );
+	  ip0->checksum = ip_csum_fold (sum0);
 
-      /* $$$ This needs to be rewritten, for sure */
-      f0 = vlib_get_frame_to_node (vm, hi0->output_node_index);
-      to_next0 = vlib_frame_vector_args (f0);
-      to_next0[0] = bi0;
-      f0->n_vectors = 1;
-      vlib_put_frame_to_node (vm, hi0->output_node_index, f0);
+	  vlib_buffer_advance (b0, -(sizeof (ethernet_header_t)));
+	  si0 = vnet_get_sw_interface (vnm, original_sw_if_index);
+	  if (si0->type == VNET_SW_INTERFACE_TYPE_SUB)
+	    vlib_buffer_advance (b0, -4 /* space for VLAN tag */ );
 
-    do_trace:
-      if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
-	{
-	  dhcp_proxy_trace_t *tr = vlib_add_trace (vm, node,
-						   b0, sizeof (*tr));
-	  tr->which = 1;	/* to client */
-	  tr->trace_ip4_address.as_u32 = ia0 ? ia0->as_u32 : 0;
-	  tr->error = error0;
-	  tr->original_sw_if_index = original_sw_if_index;
-	  tr->sw_if_index = sw_if_index;
+	  mac0 = vlib_buffer_get_current (b0);
+
+	  hi0 = vnet_get_sup_hw_interface (vnm, original_sw_if_index);
+	  ei0 = pool_elt_at_index (em->interfaces, hi0->hw_instance);
+	  clib_memcpy (mac0->src_address, ei0->address,
+		       sizeof (ei0->address));
+	  clib_memset (mac0->dst_address, 0xff, sizeof (mac0->dst_address));
+	  mac0->type = (si0->type == VNET_SW_INTERFACE_TYPE_SUB) ?
+	    clib_net_to_host_u16 (0x8100) : clib_net_to_host_u16 (0x0800);
+
+	  if (si0->type == VNET_SW_INTERFACE_TYPE_SUB)
+	    {
+	      u32 *vlan_tag = (u32 *) (mac0 + 1);
+	      u32 tmp;
+	      tmp = (si0->sub.id << 16) | 0x0800;
+	      *vlan_tag = clib_host_to_net_u32 (tmp);
+	    }
+
+	do_trace:
+	  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+	    {
+	      dhcp_proxy_trace_t *tr = vlib_add_trace (vm, node,
+						       b0, sizeof (*tr));
+	      tr->which = 1;	/* to client */
+	      tr->trace_ip4_address.as_u32 = ia0 ? ia0->as_u32 : 0;
+	      tr->error = error0;
+	      tr->original_sw_if_index = original_sw_if_index;
+	      tr->sw_if_index = sw_if_index;
+	      clib_memcpy_fast (tr->packet_data, h0,
+				sizeof (tr->packet_data));
+	    }
+
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, next0);
 	}
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
     }
 
   return from_frame->n_vectors;
@@ -741,6 +770,11 @@ VLIB_REGISTER_NODE (dhcp_proxy_to_client_node, static) = {
 #if 0
   .unformat_buffer = unformat_dhcp_proxy_header,
 #endif
+  .n_next_nodes = DHCP4_PROXY_N_NEXT,
+  .next_nodes = {
+    [DHCP4_PROXY_NEXT_DROP] = "error-drop",
+    [DHCP4_PROXY_NEXT_TX] = "interface-output",
+  },
 };
 /* *INDENT-ON* */
 
diff --git a/src/plugins/dhcp/dhcp_api.c b/src/plugins/dhcp/dhcp_api.c
index 61efaba09b0..744d8386d44 100644
--- a/src/plugins/dhcp/dhcp_api.c
+++ b/src/plugins/dhcp/dhcp_api.c
@@ -269,10 +269,12 @@ dhcp_client_lease_encode (vl_api_dhcp_lease_t * lease,
   clib_memcpy (&lease->hostname, client->hostname, len);
   lease->hostname[len] = 0;
 
-  lease->mask_width = client->subnet_mask_width;
-  clib_memcpy (&lease->host_address.un, (u8 *) & client->leased_address,
+  lease->mask_width = client->installed.subnet_mask_width;
+  clib_memcpy (&lease->host_address.un,
+	       (u8 *) & client->installed.leased_address,
 	       sizeof (ip4_address_t));
-  clib_memcpy (&lease->router_address.un, (u8 *) & client->router_address,
+  clib_memcpy (&lease->router_address.un,
+	       (u8 *) & client->installed.router_address,
 	       sizeof (ip4_address_t));
 
   lease->count = vec_len (client->domain_server_address);
diff --git a/src/plugins/dhcp/test/test_dhcp.py b/src/plugins/dhcp/test/test_dhcp.py
index 9b55510c868..e79787c5f23 100644
--- a/src/plugins/dhcp/test/test_dhcp.py
+++ b/src/plugins/dhcp/test/test_dhcp.py
@@ -1465,6 +1465,25 @@ class TestDHCP(VppTestCase):
         self.assertTrue(find_route(self, self.pg3.local_ip4, 32))
 
         #
+        # read the DHCP client details from a dump
+        #
+        clients = self.vapi.dhcp_client_dump()
+
+        self.assertEqual(clients[0].client.sw_if_index,
+                         self.pg3.sw_if_index)
+        self.assertEqual(clients[0].lease.sw_if_index,
+                         self.pg3.sw_if_index)
+        self.assertEqual(clients[0].client.hostname, hostname)
+        self.assertEqual(clients[0].lease.hostname, hostname)
+        # 0 = DISCOVER, 1 = REQUEST, 2 = BOUND
+        self.assertEqual(clients[0].lease.state, 2)
+        self.assertEqual(clients[0].lease.mask_width, 24)
+        self.assertEqual(str(clients[0].lease.router_address),
+                         self.pg3.remote_ip4)
+        self.assertEqual(str(clients[0].lease.host_address),
+                         self.pg3.local_ip4)
+
+        #
         # wait for the unicasted renewal
         #  the first attempt will be an ARP packet, since we have not yet
         #  responded to VPP's request
@@ -1492,6 +1511,25 @@ class TestDHCP(VppTestCase):
                                       l2_bc=False,
                                       broadcast=False)
 
+        # send an ACK with different data from the original offer *
+        self.pg3.generate_remote_hosts(4)
+        p_ack = (Ether(dst=self.pg3.local_mac, src=self.pg3.remote_mac) /
+                 IP(src=self.pg3.remote_ip4, dst=self.pg3.local_ip4) /
+                 UDP(sport=DHCP4_SERVER_PORT, dport=DHCP4_CLIENT_PORT) /
+                 BOOTP(op=1, yiaddr=self.pg3.remote_hosts[3].ip4,
+                       chaddr=mac_pton(self.pg3.local_mac)) /
+                 DHCP(options=[('message-type', 'ack'),
+                               ('subnet_mask', "255.255.255.0"),
+                               ('router', self.pg3.remote_hosts[1].ip4),
+                               ('server_id', self.pg3.remote_hosts[2].ip4),
+                               ('lease_time', 43200),
+                               ('renewal_time', 2),
+                               'end']))
+
+        self.pg3.add_stream(p_ack)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
         #
         # read the DHCP client details from a dump
         #
@@ -1501,23 +1539,15 @@ class TestDHCP(VppTestCase):
                          self.pg3.sw_if_index)
         self.assertEqual(clients[0].lease.sw_if_index,
                          self.pg3.sw_if_index)
-        self.assertEqual(clients[0].client.hostname.rstrip('\0'),
-                         hostname)
-        self.assertEqual(clients[0].lease.hostname.rstrip('\0'),
-                         hostname)
+        self.assertEqual(clients[0].client.hostname, hostname)
+        self.assertEqual(clients[0].lease.hostname, hostname)
         # 0 = DISCOVER, 1 = REQUEST, 2 = BOUND
         self.assertEqual(clients[0].lease.state, 2)
         self.assertEqual(clients[0].lease.mask_width, 24)
         self.assertEqual(str(clients[0].lease.router_address),
-                         self.pg3.remote_ip4)
+                         self.pg3.remote_hosts[1].ip4)
         self.assertEqual(str(clients[0].lease.host_address),
-                         self.pg3.local_ip4)
-
-        # remove the left over ARP entry
-        self.vapi.ip_neighbor_add_del(self.pg3.sw_if_index,
-                                      self.pg3.remote_mac,
-                                      self.pg3.remote_ip4,
-                                      is_add=0)
+                         self.pg3.remote_hosts[3].ip4)
 
         #
         # remove the DHCP config
@@ -1532,6 +1562,8 @@ class TestDHCP(VppTestCase):
 
         #
         # Start the procedure again. Use requested lease time option.
+        # this time wait for the lease to expire and the client to
+        # self-destruct
         #
         hostname += "-2"
         self.pg3.admin_down()
@@ -1601,12 +1633,6 @@ class TestDHCP(VppTestCase):
         self.assertTrue(find_route(self, self.pg3.local_ip4, 32))
         self.assertTrue(find_route(self, self.pg3.local_ip4, 24))
 
-        # remove the left over ARP entry
-        self.vapi.ip_neighbor_add_del(self.pg3.sw_if_index,
-                                      self.pg3.remote_mac,
-                                      self.pg3.remote_ip4,
-                                      is_add=0)
-
         #
         # the route should be gone after the lease expires
         #
diff --git a/src/vlibmemory/api.h b/src/vlibmemory/api.h
index dc1e75e484d..6cd645bb1d5 100644
--- a/src/vlibmemory/api.h
+++ b/src/vlibmemory/api.h
@@ -26,6 +26,7 @@
 #include <vlibmemory/socket_client.h>
 
 void vl_api_rpc_call_main_thread (void *fp, u8 * data, u32 data_length);
+void vl_api_force_rpc_call_main_thread (void *fp, u8 * data, u32 data_length);
 u16 vl_client_get_first_plugin_msg_id (const char *plugin_name);
 void vl_api_send_pending_rpc_requests (vlib_main_t * vm);
 u8 *vl_api_serialize_message_table (api_main_t * am, u8 * vector);
diff --git a/src/vnet/ip/ip4_punt_drop.c b/src/vnet/ip/ip4_punt_drop.c
index fc5a68fc66a..aee84b4df52 100644
--- a/src/vnet/ip/ip4_punt_drop.c
+++ b/src/vnet/ip/ip4_punt_drop.c
@@ -187,10 +187,7 @@ VLIB_REGISTER_NODE (ip4_not_enabled_node) =
   .name = "ip4-not-enabled",
   .vector_size = sizeof (u32),
   .format_trace = format_ip4_forward_next_trace,
-  .n_next_nodes = 1,
-  .next_nodes = {
-    [0] = "error-drop",
-  },
+  .sibling_of = "ip4-drop",
 };
 
 VLIB_REGISTER_NODE (ip4_punt_node) =