DHCP Multiple Servers (VPP-602, VPP-605)

Multiple DHCP (4 and/or 6) servers can be added and removed through multiple calls to the 'set dhcp server' API.
All 4/6/ discover/solicit messages will then be replicated to all servers in the list. The expectation is that the servers/system is configured in such a way that this is viable.
If VSS information is providied for the clinet VRF which also has multiple servers configured, then the same VSS information is sent to each server. Likewise the source address of packets sent to from VPP to each server is the same.

Change-Id: I3287cb084c84b3f612b78bc69cfcb5b9c1f8934d
Signed-off-by: Neale Ranns <nranns@cisco.com>

10 files changed, 440 insertions, 176 deletions

diff --git a/src/vnet/dhcp/client.c b/src/vnet/dhcp/client.c
index d34c5a645eb..29749a33737 100644
--- a/src/vnet/dhcp/client.c
+++ b/src/vnet/dhcp/client.c
@@ -366,7 +366,7 @@ send_dhcp_pkt (dhcp_client_main_t * dcm, dhcp_client_t * c,
   o = (dhcp_option_t * )dhcp->options;
 
   /* Send option 53, the DHCP message type */
-  o->option = 53;
+  o->option = DHCP_PACKET_OPTION_MSG_TYPE;
   o->length = 1;
   o->data[0] = type;
   o = (dhcp_option_t *) (((uword) o) + (o->length + 2));
diff --git a/src/vnet/dhcp/dhcp.api b/src/vnet/dhcp/dhcp.api
index 8daadd8c77a..2db85a797e9 100644
--- a/src/vnet/dhcp/dhcp.api
+++ b/src/vnet/dhcp/dhcp.api
@@ -137,19 +137,25 @@ define dhcp_proxy_dump
   u8  is_ip6;
 };
 
+typeonly manual_print manual_endian define dhcp_server
+{
+  u32 server_vrf_id;
+  u8 dhcp_server[16];
+};
+
 /** \brief Tell client about a DHCP completion event
     @param client_index - opaque cookie to identify the sender
 */
-define dhcp_proxy_details
+manual_endian manual_print define dhcp_proxy_details
 {
   u32 context;
   u32 rx_vrf_id;
-  u32 server_vrf_id;
   u32 vss_oui;
   u32 vss_fib_id;
   u8 is_ipv6;
-  u8 dhcp_server[16];
   u8 dhcp_src_address[16];
+  u8 count;
+  vl_api_dhcp_server_t servers[count];
 };
 
 /*
diff --git a/src/vnet/dhcp/dhcp4_packet.h b/src/vnet/dhcp/dhcp4_packet.h
index 28c4b156e5c..07829f4823c 100644
--- a/src/vnet/dhcp/dhcp4_packet.h
+++ b/src/vnet/dhcp/dhcp4_packet.h
@@ -55,6 +55,11 @@ typedef enum {
   DHCP_PACKET_ACK=5,
 } dhcp_packet_type_t;
 
+typedef enum dhcp_packet_option_t_
+{
+    DHCP_PACKET_OPTION_MSG_TYPE = 53,
+} dhcp_packet_option_t;
+
 /* charming antique: 99.130.83.99 is the dhcp magic cookie */
 #define DHCP_MAGIC (clib_host_to_net_u32(0x63825363))
 
diff --git a/src/vnet/dhcp/dhcp4_proxy_node.c b/src/vnet/dhcp/dhcp4_proxy_node.c
index 88a99249684..1c84881a520 100644
--- a/src/vnet/dhcp/dhcp4_proxy_node.c
+++ b/src/vnet/dhcp/dhcp4_proxy_node.c
@@ -135,18 +135,17 @@ dhcp_proxy_to_server_input (vlib_main_t * vm,
           u32 original_sw_if_index = 0;
           u8  *end = NULL;
           u32 fib_index;
-          dhcp_server_t * server;
+          dhcp_proxy_t *proxy;
+          dhcp_server_t *server;
           u32 rx_sw_if_index;
           dhcp_option_t *o;
           u32 len = 0;
           vlib_buffer_free_list_t *fl;
+          u8 is_discover = 0;
 
 	  bi0 = from[0];
-	  to_next[0] = bi0;
 	  from += 1;
-	  to_next += 1;
 	  n_left_from -= 1;
-	  n_left_to_next -= 1;
 
 	  b0 = vlib_get_buffer (vm, bi0);
 
@@ -172,16 +171,17 @@ dhcp_proxy_to_server_input (vlib_main_t * vm,
           rx_sw_if_index = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 
           fib_index = im->fib_index_by_sw_if_index [rx_sw_if_index];
-          server = dhcp_get_server(dpm, fib_index, FIB_PROTOCOL_IP4);
-          
-          if (PREDICT_FALSE (NULL == server))
+          proxy = dhcp_get_proxy(dpm, fib_index, FIB_PROTOCOL_IP4);
+
+          if (PREDICT_FALSE (NULL == proxy))
             {
               error0 = DHCP_PROXY_ERROR_NO_SERVER;
               next0 = DHCP_PROXY_TO_SERVER_INPUT_NEXT_DROP;
               pkts_no_server++;
               goto do_trace;
             }
-          
+
+          server = &proxy->dhcp_servers[0];
           vlib_buffer_advance (b0, -(sizeof(*ip0)));
           ip0 = vlib_buffer_get_current (b0);
 
@@ -198,7 +198,7 @@ dhcp_proxy_to_server_input (vlib_main_t * vm,
 
           sum0 = ip0->checksum;
           old0 = ip0->src_address.as_u32;
-          new0 = server->dhcp_src_address.ip4.as_u32;
+          new0 = proxy->dhcp_src_address.ip4.as_u32;
           ip0->src_address.as_u32 = new0;
           sum0 = ip_csum_update (sum0, old0, new0, 
                                 ip4_header_t /* structure */, 
@@ -209,7 +209,7 @@ dhcp_proxy_to_server_input (vlib_main_t * vm,
           vnet_buffer(b0)->sw_if_index[VLIB_TX] =
             server->server_fib_index;
 
-          h0->gateway_ip_address.as_u32 = server->dhcp_src_address.ip4.as_u32;
+          h0->gateway_ip_address.as_u32 = proxy->dhcp_src_address.ip4.as_u32;
           pkts_to_server++;
 
           o = (dhcp_option_t *) h0->options;
@@ -220,7 +220,16 @@ dhcp_proxy_to_server_input (vlib_main_t * vm,
           end = b0->data + b0->current_data + b0->current_length;
           /* TLVs are not performance-friendly... */
           while  (o->option != 0xFF /* end of options */ && (u8 *)o < end) 
+            {
+              if (DHCP_PACKET_OPTION_MSG_TYPE == o->option)
+                {
+                  if (DHCP_PACKET_DISCOVER == o->data[0])
+                    {
+                      is_discover = 1;
+                    }
+                }
               o = (dhcp_option_t *) (((uword) o) + (o->length + 2));
+          }
 
           fl = vlib_buffer_get_free_list (vm, b0->free_list_index);
           // start write at (option*)o, some packets have padding
@@ -340,6 +349,65 @@ dhcp_proxy_to_server_input (vlib_main_t * vm,
           
           next0 = DHCP_PROXY_TO_SERVER_INPUT_NEXT_LOOKUP;
 
+          /*
+           * If we have multiple servers configured and this is the
+           * client's discover message, then send copies to each of
+           * those servers
+           */
+          if (is_discover && vec_len(proxy->dhcp_servers) > 1)
+          {
+              u32 ii;
+
+              for (ii = 1; ii < vec_len(proxy->dhcp_servers); ii++)
+              {
+                  vlib_buffer_t *c0;
+                  u32 ci0;
+              
+                  c0 = vlib_buffer_copy(vm, b0);
+                  ci0 = vlib_get_buffer_index(vm, c0);
+                  server = &proxy->dhcp_servers[ii];
+
+                  ip0 = vlib_buffer_get_current (c0);
+
+                  sum0 = ip0->checksum;
+                  old0 = ip0->dst_address.as_u32;
+                  new0 = server->dhcp_server.ip4.as_u32;
+                  ip0->dst_address.as_u32 = server->dhcp_server.ip4.as_u32;
+                  sum0 = ip_csum_update (sum0, old0, new0, 
+                                         ip4_header_t /* structure */, 
+                                         dst_address /* changed member */);
+                  ip0->checksum = ip_csum_fold (sum0);
+
+                  to_next[0] = ci0;
+                  to_next += 1;
+                  n_left_to_next -= 1;
+
+                  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+                                                   to_next, n_left_to_next,
+                                                   ci0, next0);
+
+                  if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) 
+                  {
+                      dhcp_proxy_trace_t *tr;
+
+                      tr = vlib_add_trace (vm, node, c0, sizeof (*tr));
+                      tr->which = 0; /* to server */
+                      tr->error = error0;
+                      tr->original_sw_if_index = original_sw_if_index;
+                      tr->sw_if_index = sw_if_index;
+                      if (next0 == DHCP_PROXY_TO_SERVER_INPUT_NEXT_LOOKUP)
+                          tr->trace_ip4_address.as_u32 = server->dhcp_server.ip4.as_u32;
+                  }
+
+                  if (PREDICT_FALSE(0 == n_left_to_next))
+                  {
+                      vlib_put_next_frame (vm, node, next_index,
+                                           n_left_to_next);
+                      vlib_get_next_frame (vm, node, next_index,
+                                           to_next, n_left_to_next);
+                  }
+              }
+          }
         do_trace:
           if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) 
             {
@@ -350,10 +418,15 @@ dhcp_proxy_to_server_input (vlib_main_t * vm,
                tr->original_sw_if_index = original_sw_if_index;
                tr->sw_if_index = sw_if_index;
                if (next0 == DHCP_PROXY_TO_SERVER_INPUT_NEXT_LOOKUP)
-                 tr->trace_ip4_address.as_u32 = server->dhcp_server.ip4.as_u32;
+                 tr->trace_ip4_address.as_u32 =
+                     proxy->dhcp_servers[0].dhcp_server.ip4.as_u32;
             }
 
         do_enqueue:
+	  to_next[0] = bi0;
+	  to_next += 1;
+	  n_left_to_next -= 1;
+
 	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 					   to_next, n_left_to_next,
 					   bi0, next0);
@@ -437,7 +510,8 @@ dhcp_proxy_to_client_input (vlib_main_t * vm,
       u32 error0 = (u32)~0;
       vnet_sw_interface_t *swif;
       u32 fib_index;
-      dhcp_server_t * server;
+      dhcp_proxy_t *proxy;
+      dhcp_server_t *server;
       u32 original_sw_if_index = (u32) ~0;
       ip4_address_t relay_addr = {
           .as_u32 = 0,
@@ -547,20 +621,26 @@ dhcp_proxy_to_client_input (vlib_main_t * vm,
         }
 
       fib_index = im->fib_index_by_sw_if_index [sw_if_index];
-      server = dhcp_get_server(dpm, fib_index, FIB_PROTOCOL_IP4);
+      proxy = dhcp_get_proxy(dpm, fib_index, FIB_PROTOCOL_IP4);
 
-      if (PREDICT_FALSE (NULL == server))
+      if (PREDICT_FALSE (NULL == proxy))
         {
           error0 = DHCP_PROXY_ERROR_NO_SERVER;
           goto drop_packet;
         }
       
-      if (ip0->src_address.as_u32 != server->dhcp_server.ip4.as_u32)
-        {             
-          error0 = DHCP_PROXY_ERROR_BAD_SVR_FIB_OR_ADDRESS;
-          goto drop_packet;
+      vec_foreach(server, proxy->dhcp_servers)
+        {
+          if (ip0->src_address.as_u32 == server->dhcp_server.ip4.as_u32)
+            {
+              goto server_found;
+            }
         }
 
+      error0 = DHCP_PROXY_ERROR_BAD_SVR_FIB_OR_ADDRESS;
+      goto drop_packet;
+
+    server_found:
       vnet_buffer (b0)->sw_if_index[VLIB_TX] = sw_if_index;
 
       swif = vnet_get_sw_interface (vnm, sw_if_index);
@@ -709,9 +789,8 @@ dhcp4_proxy_set_server (ip46_address_t *addr,
 
   if (is_del)
     {
-      rc = dhcp_proxy_server_del (FIB_PROTOCOL_IP4, rx_fib_index);
-
-      if (0 == rc)
+      if (dhcp_proxy_server_del (FIB_PROTOCOL_IP4, rx_fib_index,
+                                 addr, server_table_id))
       {
           fib_table_entry_special_remove(rx_fib_index,
                                          &all_1s,
@@ -809,29 +888,35 @@ VLIB_CLI_COMMAND (dhcp_proxy_set_command, static) = {
 static u8 *
 format_dhcp4_proxy_server (u8 * s, va_list * args)
 {
-  dhcp_server_t * server = va_arg (*args, dhcp_server_t *);
+  dhcp_proxy_t *proxy = va_arg (*args, dhcp_proxy_t *);
   ip4_fib_t * rx_fib, * server_fib;
+  dhcp_server_t *server;
 
-  if (server == 0)
+  if (proxy == 0)
     {
-      s = format (s, "%=16s%=16s%=14s%=14s", "Server", "Src Address", 
-                  "Server FIB", "RX FIB");
+        s = format (s, "%=14s%=16s%s", "RX FIB", "Src Address", 
+                    "Servers FIB,Address");
       return s;
     }
 
-  server_fib = ip4_fib_get(server->server_fib_index);
-  rx_fib = ip4_fib_get(server->rx_fib_index);
+  rx_fib = ip4_fib_get(proxy->rx_fib_index);
+
+  s = format (s, "%=14u%=16U",
+              rx_fib->table_id,
+              format_ip46_address, &proxy->dhcp_src_address, IP46_TYPE_ANY);
 
-  s = format (s, "%=16U%=16U%=14u%=14u",
-              format_ip46_address, &server->dhcp_server, IP46_TYPE_ANY,
-              format_ip46_address, &server->dhcp_src_address, IP46_TYPE_ANY,
-              server_fib->table_id,
-              rx_fib->table_id);
+  vec_foreach(server, proxy->dhcp_servers)
+  {
+      server_fib = ip4_fib_get(server->server_fib_index);
+      s = format (s, "%u,%U  ",
+                  server_fib->table_id,
+                  format_ip46_address, &server->dhcp_server, IP46_TYPE_ANY);
+  }
   return s;
 }
 
 static int
-dhcp4_proxy_show_walk (dhcp_server_t *server,
+dhcp4_proxy_show_walk (dhcp_proxy_t *server,
                        void *ctx)
 {
     vlib_main_t * vm = ctx;
diff --git a/src/vnet/dhcp/dhcp6_proxy_node.c b/src/vnet/dhcp/dhcp6_proxy_node.c
index 58674209ca7..524cb095357 100644
--- a/src/vnet/dhcp/dhcp6_proxy_node.c
+++ b/src/vnet/dhcp/dhcp6_proxy_node.c
@@ -140,7 +140,8 @@ dhcpv6_proxy_to_server_input (vlib_main_t * vm,
   ip6_main_t * im = &ip6_main;
   ip6_address_t * src;
   int bogus_length;
-  dhcp_server_t * server;
+  dhcp_proxy_t *proxy;
+  dhcp_server_t *server;
   u32  rx_fib_idx = 0, server_fib_idx = 0;
 
   next_index = node->cached_next_index;
@@ -176,13 +177,11 @@ dhcpv6_proxy_to_server_input (vlib_main_t * vm,
           u8 client_src_mac[6];
           vlib_buffer_free_list_t *fl;
           dhcp_vss_t *vss;
+          u8 is_solicit = 0;
 
 	  bi0 = from[0];
-	  to_next[0] = bi0;
 	  from += 1;
-	  to_next += 1;
 	  n_left_from -= 1;
-	  n_left_to_next -= 1;
 
 	  b0 = vlib_get_buffer (vm, bi0);
 
@@ -227,9 +226,9 @@ dhcpv6_proxy_to_server_input (vlib_main_t * vm,
           /* Send to DHCPV6 server via the configured FIB */
           rx_sw_if_index = sw_if_index =  vnet_buffer(b0)->sw_if_index[VLIB_RX];
           rx_fib_idx = im->mfib_index_by_sw_if_index [rx_sw_if_index];
-          server = dhcp_get_server(dpm, rx_fib_idx, FIB_PROTOCOL_IP6);
+          proxy = dhcp_get_proxy(dpm, rx_fib_idx, FIB_PROTOCOL_IP6);
 
-          if (PREDICT_FALSE (NULL == server))
+          if (PREDICT_FALSE (NULL == proxy))
           {
               error0 = DHCPV6_PROXY_ERROR_NO_SERVER;
               next0 = DHCPV6_PROXY_TO_SERVER_INPUT_NEXT_DROP;
@@ -237,6 +236,7 @@ dhcpv6_proxy_to_server_input (vlib_main_t * vm,
               goto do_trace;
           }
 
+          server = &proxy->dhcp_servers[0];
           server_fib_idx = server->server_fib_index;
           vnet_buffer(b0)->sw_if_index[VLIB_TX] = server_fib_idx;
 
@@ -371,18 +371,19 @@ dhcpv6_proxy_to_server_input (vlib_main_t * vm,
           ip1->payload_length =  u1->length;
           ip1->protocol = PROTO_UDP;
           ip1->hop_limit = HOP_COUNT_LIMIT;
-	      src = (server->dhcp_server.ip6.as_u64[0] ||
-                     server->dhcp_server.ip6.as_u64[1]) ?
-	        &server->dhcp_server.ip6 : &all_dhcpv6_server_address;
+          src = ((server->dhcp_server.ip6.as_u64[0] ||
+                  server->dhcp_server.ip6.as_u64[1]) ?
+                 &server->dhcp_server.ip6 :
+                 &all_dhcpv6_server_address);
           copy_ip6_address(&ip1->dst_address, src);
 
 
           ia0 = ip6_interface_first_global_or_site_address
               (&ip6_main, vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 
-	      src = (server->dhcp_src_address.ip6.as_u64[0] ||
-                     server->dhcp_src_address.ip6.as_u64[1]) ?
-	        &server->dhcp_src_address.ip6 : ia0;
+	      src = (proxy->dhcp_src_address.ip6.as_u64[0] ||
+                     proxy->dhcp_src_address.ip6.as_u64[1]) ?
+	        &proxy->dhcp_src_address.ip6 : ia0;
           if (ia0 == 0)
             {
               error0 = DHCPV6_PROXY_ERROR_NO_SRC_ADDRESS;
@@ -400,6 +401,66 @@ dhcpv6_proxy_to_server_input (vlib_main_t * vm,
 
           next0 = DHCPV6_PROXY_TO_SERVER_INPUT_NEXT_LOOKUP;
 
+          is_solicit = (DHCPV6_MSG_SOLICIT == h0->u.msg_type);
+
+          /*
+           * If we have multiple servers configured and this is the
+           * client's discover message, then send copies to each of
+           * those servers
+           */
+          if (is_solicit && vec_len(proxy->dhcp_servers) > 1)
+          {
+              u32 ii;
+
+              for (ii = 1; ii < vec_len(proxy->dhcp_servers); ii++)
+              {
+                  vlib_buffer_t *c0;
+                  u32 ci0;
+              
+                  c0 = vlib_buffer_copy(vm, b0);
+                  ci0 = vlib_get_buffer_index(vm, c0);
+                  server = &proxy->dhcp_servers[ii];
+
+                  ip0 = vlib_buffer_get_current (c0);
+
+                  src = ((server->dhcp_server.ip6.as_u64[0] ||
+                          server->dhcp_server.ip6.as_u64[1]) ?
+                         &server->dhcp_server.ip6 :
+                         &all_dhcpv6_server_address);
+                  copy_ip6_address(&ip1->dst_address, src);
+
+                  to_next[0] = ci0;
+                  to_next += 1;
+                  n_left_to_next -= 1;
+
+                  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+                                                   to_next, n_left_to_next,
+                                                   ci0, next0);
+
+                  if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) 
+                  {
+                      dhcpv6_proxy_trace_t *tr;
+
+                      tr = vlib_add_trace (vm, node, c0, sizeof (*tr));
+                      tr->which = 0; /* to server */
+                      tr->error = error0;
+                      tr->original_sw_if_index = rx_sw_if_index;
+                      tr->sw_if_index = sw_if_index;
+                      if (next0 == DHCPV6_PROXY_TO_SERVER_INPUT_NEXT_LOOKUP)
+                          copy_ip6_address((ip6_address_t *)&tr->packet_data[0],
+                                           &server->dhcp_server.ip6);
+                  }
+
+                  if (PREDICT_FALSE(0 == n_left_to_next))
+                  {
+                      vlib_put_next_frame (vm, node, next_index,
+                                           n_left_to_next);
+                      vlib_get_next_frame (vm, node, next_index,
+                                           to_next, n_left_to_next);
+                  }
+              }
+          }
+
         do_trace:
           if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
             {
@@ -414,6 +475,10 @@ dhcpv6_proxy_to_server_input (vlib_main_t * vm,
             }
 
         do_enqueue:
+	  to_next[0] = bi0;
+	  to_next += 1;
+	  n_left_to_next -= 1;
+
 	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 					   to_next, n_left_to_next,
 					   bi0, next0);
@@ -475,7 +540,8 @@ dhcpv6_proxy_to_client_input (vlib_main_t * vm,
   u32 n_left_from, * from;
   ethernet_main_t *em = ethernet_get_main (vm);
   dhcp_proxy_main_t * dm = &dhcp_proxy_main;
-  dhcp_server_t * server;
+  dhcp_proxy_t *proxy;
+  dhcp_server_t *server;
   vnet_main_t * vnm = vnet_get_main();
   int bogus_length;
 
@@ -588,9 +654,9 @@ dhcpv6_proxy_to_client_input (vlib_main_t * vm,
       vlib_buffer_advance (b0, sizeof(*r0));
 
       client_fib_idx = im->mfib_index_by_sw_if_index[sw_if_index];
-      server = dhcp_get_server(dm, client_fib_idx, FIB_PROTOCOL_IP6);
+      proxy = dhcp_get_proxy(dm, client_fib_idx, FIB_PROTOCOL_IP6);
 
-      if (NULL == server)
+      if (NULL == proxy)
       {
     	  error0 = DHCPV6_PROXY_ERROR_NO_SERVER;
           goto drop_packet;
@@ -599,15 +665,21 @@ dhcpv6_proxy_to_client_input (vlib_main_t * vm,
       server_fib_idx = im->fib_index_by_sw_if_index
           [vnet_buffer(b0)->sw_if_index[VLIB_RX]];
 
-      if (server_fib_idx != server->server_fib_index ||
-          ip0->src_address.as_u64[0] != server->dhcp_server.ip6.as_u64[0] ||
-          ip0->src_address.as_u64[1] != server->dhcp_server.ip6.as_u64[1])
-        {
-          //drop packet if not from server with configured address or FIB
-          error0 = DHCPV6_PROXY_ERROR_BAD_SVR_FIB_OR_ADDRESS;
-          goto drop_packet;
-        }
+      vec_foreach(server, proxy->dhcp_servers)
+      {
+          if (server_fib_idx == server->server_fib_index &&
+              ip0->src_address.as_u64[0] == server->dhcp_server.ip6.as_u64[0] &&
+              ip0->src_address.as_u64[1] == server->dhcp_server.ip6.as_u64[1])
+          {
+              goto server_found;
+          }
+      }
+
+      //drop packet if not from server with configured address or FIB
+      error0 = DHCPV6_PROXY_ERROR_BAD_SVR_FIB_OR_ADDRESS;
+      goto drop_packet;
 
+    server_found:
       vnet_buffer (b0)->sw_if_index[VLIB_TX] = original_sw_if_index
           = sw_if_index;
 
@@ -773,9 +845,8 @@ dhcp6_proxy_set_server (ip46_address_t *addr,
 
   if (is_del)
     {
-      rc = dhcp_proxy_server_del (FIB_PROTOCOL_IP6, rx_fib_index);
-
-      if (0 == rc)
+      if (dhcp_proxy_server_del (FIB_PROTOCOL_IP6, rx_fib_index,
+                                 addr, server_table_id))
       {
           mfib_table_entry_delete(rx_fib_index,
                                   &all_dhcp_servers,
@@ -893,43 +964,50 @@ VLIB_CLI_COMMAND (dhcpv6_proxy_set_command, static) = {
 static u8 *
 format_dhcp6_proxy_server (u8 * s, va_list * args)
 {
-  dhcp_server_t * server = va_arg (*args, dhcp_server_t *);
+  dhcp_proxy_t * proxy = va_arg (*args, dhcp_proxy_t *);
   ip6_fib_t *server_fib;
+  dhcp_server_t *server;
   ip6_mfib_t *rx_fib;
 
-  if (NULL == server)
+  if (proxy == 0)
     {
-      s = format (s, "%=40s%=40s%=14s%=14s", "Server Address", "Source Address",
-                  "Server FIB", "RX FIB");
+        s = format (s, "%=14s%=16s%s", "RX FIB", "Src Address", 
+                    "Servers FIB,Address");
       return s;
     }
 
-  server_fib = ip6_fib_get(server->server_fib_index);
-  rx_fib = ip6_mfib_get(server->rx_fib_index);
+  rx_fib = ip6_mfib_get(proxy->rx_fib_index);
+
+  s = format (s, "%=14u%=16U",
+              rx_fib->table_id,
+              format_ip46_address, &proxy->dhcp_src_address, IP46_TYPE_ANY);
 
+  vec_foreach(server, proxy->dhcp_servers)
+  {
+      server_fib = ip6_fib_get(server->server_fib_index);
+      s = format (s, "%u,%U  ",
+                  server_fib->table_id,
+                  format_ip46_address, &server->dhcp_server, IP46_TYPE_ANY);
+  }
 
-  s = format (s, "%=40U%=40U%=14u%=14u",
-              format_ip46_address, &server->dhcp_server, IP46_TYPE_ANY,
-              format_ip46_address, &server->dhcp_src_address, IP46_TYPE_ANY,
-              server_fib->table_id, rx_fib->table_id);
   return s;
 }
 
 static int
-dhcp6_proxy_show_walk (dhcp_server_t *server,
+dhcp6_proxy_show_walk (dhcp_proxy_t *proxy,
                        void *ctx)
 {
     vlib_main_t * vm = ctx;
 
-    vlib_cli_output (vm, "%U", format_dhcp6_proxy_server, server);
+    vlib_cli_output (vm, "%U", format_dhcp6_proxy_server, proxy);
 
     return (1);
 }
 
 static clib_error_t *
 dhcpv6_proxy_show_command_fn (vlib_main_t * vm,
-                            unformat_input_t * input,
-                            vlib_cli_command_t * cmd)
+                              unformat_input_t * input,
+                              vlib_cli_command_t * cmd)
 {
   vlib_cli_output (vm, "%U", format_dhcp6_proxy_server, NULL /* header line */);
 
diff --git a/src/vnet/dhcp/dhcp_api.c b/src/vnet/dhcp/dhcp_api.c
index ce34f6a47b3..e9c757e85b6 100644
--- a/src/vnet/dhcp/dhcp_api.c
+++ b/src/vnet/dhcp/dhcp_api.c
@@ -24,6 +24,7 @@
 #include <vnet/api_errno.h>
 #include <vnet/dhcp/dhcp_proxy.h>
 #include <vnet/dhcp/client.h>
+#include <vnet/fib/fib_table.h>
 
 #include <vnet/vnet_msg_enum.h>
 
@@ -113,46 +114,73 @@ vl_api_dhcp_proxy_dump_t_handler (vl_api_dhcp_proxy_dump_t * mp)
   if (q == 0)
     return;
 
-  dhcp_proxy_dump ((mp->is_ip6 == 0 ?
+  dhcp_proxy_dump ((mp->is_ip6 == 1 ?
 		    FIB_PROTOCOL_IP6 : FIB_PROTOCOL_IP4), q, mp->context);
 }
 
 void
 dhcp_send_details (fib_protocol_t proto,
-		   void *opaque,
-		   u32 context,
-		   const ip46_address_t * server,
-		   const ip46_address_t * src,
-		   u32 server_fib_id,
-		   u32 rx_fib_id, u32 vss_fib_id, u32 vss_oui)
+		   void *opaque, u32 context, dhcp_proxy_t * proxy)
 {
   vl_api_dhcp_proxy_details_t *mp;
   unix_shared_memory_queue_t *q = opaque;
-
-  mp = vl_msg_api_alloc (sizeof (*mp));
+  vl_api_dhcp_server_t *v_server;
+  dhcp_server_t *server;
+  fib_table_t *s_fib;
+  dhcp_vss_t *vss;
+  u32 count;
+  size_t n;
+
+  count = vec_len (proxy->dhcp_servers);
+  n = sizeof (*mp) + (count * sizeof (vl_api_dhcp_server_t));
+  mp = vl_msg_api_alloc (n);
   if (!mp)
     return;
-  memset (mp, 0, sizeof (*mp));
+  memset (mp, 0, n);
   mp->_vl_msg_id = ntohs (VL_API_DHCP_PROXY_DETAILS);
   mp->context = context;
-
-  mp->rx_vrf_id = htonl (rx_fib_id);
-  mp->server_vrf_id = htonl (server_fib_id);
-  mp->vss_oui = htonl (vss_oui);
-  mp->vss_fib_id = htonl (vss_fib_id);
+  mp->count = count;
 
   mp->is_ipv6 = (proto == FIB_PROTOCOL_IP6);
+  mp->rx_vrf_id =
+    htonl (dhcp_proxy_rx_table_get_table_id (proto, proxy->rx_fib_index));
+
+  vss = dhcp_get_vss_info (&dhcp_proxy_main, proxy->rx_fib_index, proto);
+
+  if (NULL != vss)
+    {
+      mp->vss_oui = htonl (vss->oui);
+      mp->vss_fib_id = htonl (vss->fib_id);
+    }
+
+  vec_foreach_index (count, proxy->dhcp_servers)
+  {
+    server = &proxy->dhcp_servers[count];
+    v_server = &mp->servers[count];
+
+    s_fib = fib_table_get (server->server_fib_index, proto);
+
+    v_server->server_vrf_id = htonl (s_fib->ft_table_id);
+
+    if (mp->is_ipv6)
+      {
+	memcpy (v_server->dhcp_server, &server->dhcp_server.ip6, 16);
+      }
+    else
+      {
+	/* put the address in the first bytes */
+	memcpy (v_server->dhcp_server, &server->dhcp_server.ip4, 4);
+      }
+  }
 
   if (mp->is_ipv6)
     {
-      memcpy (mp->dhcp_server, server, 16);
-      memcpy (mp->dhcp_src_address, src, 16);
+      memcpy (mp->dhcp_src_address, &proxy->dhcp_src_address.ip6, 16);
     }
   else
     {
       /* put the address in the first bytes */
-      memcpy (mp->dhcp_server, &server->ip4, 4);
-      memcpy (mp->dhcp_src_address, &src->ip4, 4);
+      memcpy (mp->dhcp_src_address, &proxy->dhcp_src_address.ip4, 4);
     }
   vl_msg_api_send_shmem (q, (u8 *) & mp);
 }
diff --git a/src/vnet/dhcp/dhcp_proxy.c b/src/vnet/dhcp/dhcp_proxy.c
index 8e31c3dbecc..ba7f354e9fa 100644
--- a/src/vnet/dhcp/dhcp_proxy.c
+++ b/src/vnet/dhcp/dhcp_proxy.c
@@ -44,7 +44,7 @@ dhcp_proxy_rx_table_unlock (fib_protocol_t proto,
         mfib_table_unlock(fib_index, proto);
 }
 
-static u32
+ u32
 dhcp_proxy_rx_table_get_table_id (fib_protocol_t proto,
                                   u32 fib_index)
 {
@@ -72,7 +72,7 @@ dhcp_proxy_walk (fib_protocol_t proto,
                  void *ctx)
 {
   dhcp_proxy_main_t * dpm = &dhcp_proxy_main;
-  dhcp_server_t * server;
+  dhcp_proxy_t * server;
   u32 server_index, i;
 
   vec_foreach_index (i, dpm->dhcp_server_index_by_rx_fib_index[proto])
@@ -124,31 +124,68 @@ dhcp_vss_walk (fib_protocol_t proto,
     }
 }
 
+static u32
+dhcp_proxy_server_find (dhcp_proxy_t *proxy,
+                        fib_protocol_t proto,
+                        ip46_address_t *addr,
+                        u32 server_table_id)
+{
+    dhcp_server_t *server;
+    u32 ii, fib_index;
+
+    vec_foreach_index(ii, proxy->dhcp_servers)
+    {
+        server = &proxy->dhcp_servers[ii];
+        fib_index = fib_table_find(proto, server_table_id);
+
+        if (ip46_address_is_equal(&server->dhcp_server,
+                                  addr) &&
+            (server->server_fib_index == fib_index))
+        {
+            return (ii);
+        }
+    }
+    return (~0);
+}
+
 int
 dhcp_proxy_server_del (fib_protocol_t proto,
-                       u32 rx_fib_index)
+                       u32 rx_fib_index,
+                       ip46_address_t *addr,
+                       u32 server_table_id)
 {
   dhcp_proxy_main_t * dpm = &dhcp_proxy_main;
-  dhcp_server_t * server = 0;
-  int rc = 0;
+  dhcp_proxy_t *proxy = 0;
 
-  server = dhcp_get_server(dpm, rx_fib_index, proto);
+  proxy = dhcp_get_proxy(dpm, rx_fib_index, proto);
 
-  if (NULL == server)
-  {
-      rc = VNET_API_ERROR_NO_SUCH_ENTRY;
-  }
-  else
+  if (NULL != proxy)
   {
-      /* Use the default server again.  */
-      dpm->dhcp_server_index_by_rx_fib_index[proto][rx_fib_index] = ~0;
+      dhcp_server_t *server;
+      u32 index;
 
-      fib_table_unlock (server->server_fib_index, proto);
+      index = dhcp_proxy_server_find(proxy, proto, addr, server_table_id);
 
-      pool_put (dpm->dhcp_servers[proto], server);
+      if (~0 != index)
+      {
+          server = &proxy->dhcp_servers[index];
+          fib_table_unlock (server->server_fib_index, proto);
+
+          vec_del1(proxy->dhcp_servers, index);
+
+          if (0 == vec_len(proxy->dhcp_servers))
+          {
+              /* no servers left, delete the proxy config */
+              dpm->dhcp_server_index_by_rx_fib_index[proto][rx_fib_index] = ~0;
+              vec_free(proxy->dhcp_servers);
+              pool_put (dpm->dhcp_servers[proto], proxy);
+              return (1);
+          }
+      }
   }
 
-  return (rc);
+  /* the proxy still exists */
+  return (0);
 }
 
 int
@@ -159,48 +196,42 @@ dhcp_proxy_server_add (fib_protocol_t proto,
                        u32 server_table_id)
 {
   dhcp_proxy_main_t * dpm = &dhcp_proxy_main;
-  dhcp_server_t * server = 0;
+  dhcp_proxy_t * proxy = 0;
   int new = 0;
 
-  server = dhcp_get_server(dpm, rx_fib_index, proto);
+  proxy = dhcp_get_proxy(dpm, rx_fib_index, proto);
 
-  if (NULL == server)
+  if (NULL == proxy)
   {
       vec_validate_init_empty(dpm->dhcp_server_index_by_rx_fib_index[proto],
                               rx_fib_index,
                               ~0);
 
-      pool_get (dpm->dhcp_servers[proto], server);
-      memset (server, 0, sizeof (*server));
+      pool_get (dpm->dhcp_servers[proto], proxy);
+      memset (proxy, 0, sizeof (*proxy));
       new = 1;
 
       dpm->dhcp_server_index_by_rx_fib_index[proto][rx_fib_index] =
-          server - dpm->dhcp_servers[proto];
+          proxy - dpm->dhcp_servers[proto];
 
-      server->rx_fib_index = rx_fib_index;
-      server->server_fib_index = 
-          fib_table_find_or_create_and_lock(proto, server_table_id);
+      proxy->dhcp_src_address = *src_address;
+      proxy->rx_fib_index = rx_fib_index;
   }
   else
   {
-      /* modify, may need to swap server FIBs */
-      u32 tmp_index;
-
-      tmp_index = fib_table_find(proto, server_table_id);
-
-      if (tmp_index != server->server_fib_index)
+      if (~0 != dhcp_proxy_server_find(proxy, proto, addr, server_table_id))
       {
-          tmp_index = server->server_fib_index;
-
-          /* certainly swapping if the fib doesn't exist */
-          server->server_fib_index = 
-              fib_table_find_or_create_and_lock(proto, server_table_id);
-          fib_table_unlock (tmp_index, proto);
+          return (new);
       }
   }
 
-  server->dhcp_server = *addr;
-  server->dhcp_src_address = *src_address;
+  dhcp_server_t server = {
+      .dhcp_server = *addr,
+      .server_fib_index = fib_table_find_or_create_and_lock(proto,
+                                                            server_table_id),
+  };
+
+  vec_add1(proxy->dhcp_servers, server);
 
   return (new);
 }
@@ -213,31 +244,15 @@ typedef struct dhcp4_proxy_dump_walk_ctx_t_
 } dhcp_proxy_dump_walk_cxt_t;
 
 static int
-dhcp_proxy_dump_walk (dhcp_server_t *server,
+dhcp_proxy_dump_walk (dhcp_proxy_t *proxy,
                       void *arg)
 {
   dhcp_proxy_dump_walk_cxt_t *ctx = arg;
-  fib_table_t *s_fib;
-  u32 rx_table_id;
-  dhcp_vss_t *v;
-
-  v = dhcp_get_vss_info(&dhcp_proxy_main,
-                        server->rx_fib_index,
-                        ctx->proto);
-
-  s_fib = fib_table_get(server->server_fib_index, ctx->proto);
-  rx_table_id = dhcp_proxy_rx_table_get_table_id(server->rx_fib_index,
-                                                 ctx->proto);
 
   dhcp_send_details(ctx->proto,
                     ctx->opaque,
                     ctx->context,
-                    &server->dhcp_server,
-                    &server->dhcp_src_address,
-                    s_fib->ft_table_id,
-                    rx_table_id,
-                    (v ? v->fib_id : 0),
-                    (v ? v->oui : 0));
+                    proxy);
 
   return (1);
 }
diff --git a/src/vnet/dhcp/dhcp_proxy.h b/src/vnet/dhcp/dhcp_proxy.h
index 708e92f3c32..ef2bc0a1926 100644
--- a/src/vnet/dhcp/dhcp_proxy.h
+++ b/src/vnet/dhcp/dhcp_proxy.h
@@ -58,9 +58,10 @@ typedef struct dhcp_vss_t_ {
 } dhcp_vss_t;
 
 /**
- * @brief A DHCP proxy server represenation
+ * @brief A representation of a single DHCP Server within a given VRF config
  */
-typedef struct dhcp_server_t_ {
+typedef struct dhcp_server_t_
+{
     /**
      * @brief The address of the DHCP server to which to relay the client's
      *        messages
@@ -68,22 +69,47 @@ typedef struct dhcp_server_t_ {
     ip46_address_t dhcp_server;
 
     /**
-     * @brief The source address to use in relayed messaes
-     */
-    ip46_address_t dhcp_src_address;
-
-    /**
      * @brief The FIB index (not the external Table-ID) in which the server
      *        is reachable.
      */
     u32 server_fib_index;
+} dhcp_server_t;
+
+/**
+ * @brief A DHCP proxy represenation fpr per-client VRF config
+ */
+typedef struct dhcp_proxy_t_ {
+    /**
+     * @brief The set of DHCP servers to which messages are relayed.
+     *  If multiple servers are configured then discover/solict messages
+     * are relayed to each. A cookie is maintained for the relay, and only
+     * one message is replayed to the client, based on the presence of the
+     * cookie.
+     * The expectation is there are only 1 or 2 servers, hence no fancy DB.
+     */
+    dhcp_server_t *dhcp_servers;
+
+    /**
+     * @brief Hash table of pending requets key'd on the clients MAC address
+     */
+    uword *dhcp_pending;
+
+    /**
+     * @brief A lock for the pending request DB.
+     */
+    int lock;
+
+    /**
+     * @brief The source address to use in relayed messaes
+     */
+    ip46_address_t dhcp_src_address;
 
     /**
      * @brief The FIB index (not the external Table-ID) in which the client
      *        is resides.
      */
     u32 rx_fib_index;
-} dhcp_server_t;
+} dhcp_proxy_t;
 
 #define DHCP_N_PROTOS (FIB_PROTOCOL_IP6 + 1)
 
@@ -92,7 +118,7 @@ typedef struct dhcp_server_t_ {
  */
 typedef struct {
   /* Pool of DHCP servers */
-  dhcp_server_t *dhcp_servers[DHCP_N_PROTOS];
+  dhcp_proxy_t *dhcp_servers[DHCP_N_PROTOS];
 
   /* Pool of selected DHCP server. Zero is the default server */
   u32 * dhcp_server_index_by_rx_fib_index[DHCP_N_PROTOS];
@@ -114,12 +140,7 @@ extern dhcp_proxy_main_t dhcp_proxy_main;
 void dhcp_send_details (fib_protocol_t proto,
                         void *opaque,
                         u32 context,
-                        const ip46_address_t *server,
-                        const ip46_address_t *src,
-                        u32 server_fib_id,
-                        u32 rx_fib_id,
-                        u32 vss_fib_id,
-                        u32 vss_oui);
+                        dhcp_proxy_t *proxy);
 
 /**
  * @brief Show (on CLI) a VSS config during a show walk
@@ -157,16 +178,22 @@ int dhcp_proxy_server_add(fib_protocol_t proto,
 
 /**
  * @brief Delete a DHCP proxy config
- * @return 0 is deleted, otherwise an error code
+ * @return 1 if the proxy is deleted, 0 otherwise
  */
 int dhcp_proxy_server_del(fib_protocol_t proto,
-                          u32 rx_fib_index);
+                          u32 rx_fib_index,
+                          ip46_address_t *addr,
+                          u32 server_table_id);
+
+u32
+dhcp_proxy_rx_table_get_table_id (fib_protocol_t proto,
+                                  u32 fib_index);
 
 /**
  * @brief Callback function invoked for each DHCP proxy entry
  *  return 0 to break the walk, non-zero otherwise.
  */
-typedef int (*dhcp_proxy_walk_fn_t)(dhcp_server_t *server,
+typedef int (*dhcp_proxy_walk_fn_t)(dhcp_proxy_t *server,
                                     void *ctx);
 
 /**
@@ -192,6 +219,18 @@ void dhcp_vss_walk(fib_protocol_t proto,
                    void *ctx);
 
 /**
+ * @brief Lock a proxy object to prevent simultaneous access of its
+ *  pending store
+ */
+void dhcp_proxy_lock (dhcp_proxy_t *server);
+
+/**
+ * @brief Lock a proxy object to prevent simultaneous access of its
+ *  pending store
+ */
+void dhcp_proxy_unlock (dhcp_proxy_t *server);
+
+/**
  * @brief Get the VSS data for the FIB index
  */
 static inline dhcp_vss_t *
@@ -215,12 +254,12 @@ dhcp_get_vss_info (dhcp_proxy_main_t *dm,
 /**
  * @brief Get the DHCP proxy server data for the FIB index
  */
-static inline dhcp_server_t *
-dhcp_get_server (dhcp_proxy_main_t *dm,
-                 u32 rx_fib_index,
-                 fib_protocol_t proto)
+static inline dhcp_proxy_t *
+dhcp_get_proxy (dhcp_proxy_main_t *dm,
+                u32 rx_fib_index,
+                fib_protocol_t proto)
 {
-  dhcp_server_t *s = NULL;
+  dhcp_proxy_t *s = NULL;
 
   if (vec_len(dm->dhcp_server_index_by_rx_fib_index[proto]) > rx_fib_index &&
       dm->dhcp_server_index_by_rx_fib_index[proto][rx_fib_index] != ~0)
diff --git a/src/vnet/ip/ip6_packet.h b/src/vnet/ip/ip6_packet.h
index 6eabeef1f3d..9bf19edb5db 100644
--- a/src/vnet/ip/ip6_packet.h
+++ b/src/vnet/ip/ip6_packet.h
@@ -79,6 +79,8 @@ typedef CLIB_PACKED (union {
 #define ip46_address_reset(ip46)	((ip46)->as_u64[0] = (ip46)->as_u64[1] = 0)
 #define ip46_address_cmp(ip46_1, ip46_2) (memcmp(ip46_1, ip46_2, sizeof(*ip46_1)))
 #define ip46_address_is_zero(ip46)	(((ip46)->as_u64[0] == 0) && ((ip46)->as_u64[1] == 0))
+#define ip46_address_is_equal(a1, a2)	(((a1)->as_u64[0] == (a2)->as_u64[0]) \
+                                         && ((a1)->as_u64[1] == (a2)->as_u64[1]))
 
 always_inline void
 ip46_from_addr_buf (u32 is_ipv6, u8 * buf, ip46_address_t * ip)
diff --git a/src/vnet/pg/input.c b/src/vnet/pg/input.c
index 4a65b024f6c..2649798b109 100644
--- a/src/vnet/pg/input.c
+++ b/src/vnet/pg/input.c
@@ -1373,6 +1373,7 @@ typedef struct
   u32 stream_index;
 
   u32 packet_length;
+  u32 sw_if_index;
 
   /* Use pre data for packet data. */
   vlib_buffer_t buffer;
@@ -1399,6 +1400,7 @@ format_pg_input_trace (u8 * s, va_list * va)
     s = format (s, "stream %d", t->stream_index);
 
   s = format (s, ", %d bytes", t->packet_length);
+  s = format (s, ", %d sw_if_index", t->sw_if_index);
 
   s = format (s, "\n%U%U",
 	      format_white_space, indent, format_vlib_buffer, &t->buffer);
@@ -1458,6 +1460,9 @@ pg_input_trace (pg_main_t * pg,
       t0->packet_length = vlib_buffer_length_in_chain (vm, b0);
       t1->packet_length = vlib_buffer_length_in_chain (vm, b1);
 
+      t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
+      t1->sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_RX];
+
       clib_memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data));
       clib_memcpy (&t1->buffer, b1, sizeof (b1[0]) - sizeof (b1->pre_data));
 
@@ -1484,6 +1489,7 @@ pg_input_trace (pg_main_t * pg,
 
       t0->stream_index = stream_index;
       t0->packet_length = vlib_buffer_length_in_chain (vm, b0);
+      t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
       clib_memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data));
       clib_memcpy (t0->buffer.pre_data, b0->data,
 		   sizeof (t0->buffer.pre_data));