DVR: run L3 output features

- rename l2_bridged to is_dvr. Including on the ip.api
    this was new in the 18.01 release so no compatability issues.
- steal the free space in vnet_buffer_opaque_t for use with flags.
- run the ipX-output feature arc from the DVR DPO

Change-Id: I040e5976d1dbe076fcdda3a40a7804f56337ce3f
Signed-off-by: Neale Ranns <nranns@cisco.com>

6 files changed, 661 insertions, 436 deletions

diff --git a/src/vnet/dpo/dpo.c b/src/vnet/dpo/dpo.c
index 1ee0727135d..51adbc3d55a 100644
--- a/src/vnet/dpo/dpo.c
+++ b/src/vnet/dpo/dpo.c
@@ -40,7 +40,7 @@
 #include <vnet/dpo/interface_rx_dpo.h>
 #include <vnet/dpo/interface_tx_dpo.h>
 #include <vnet/dpo/mpls_disposition.h>
-#include <vnet/dpo/l2_bridge_dpo.h>
+#include <vnet/dpo/dvr_dpo.h>
 #include <vnet/dpo/l3_proxy_dpo.h>
 
 /**
@@ -537,7 +537,7 @@ dpo_module_init (vlib_main_t * vm)
     interface_rx_dpo_module_init();
     interface_tx_dpo_module_init();
     mpls_disp_dpo_module_init();
-    l2_bridge_dpo_module_init();
+    dvr_dpo_module_init();
     l3_proxy_dpo_module_init();
 
     return (NULL);
diff --git a/src/vnet/dpo/dpo.h b/src/vnet/dpo/dpo.h
index 27c129ed444..3290a5d239e 100644
--- a/src/vnet/dpo/dpo.h
+++ b/src/vnet/dpo/dpo.h
@@ -116,7 +116,7 @@ typedef enum dpo_type_t_ {
     DPO_MFIB_ENTRY,
     DPO_INTERFACE_RX,
     DPO_INTERFACE_TX,
-    DPO_L2_BRIDGE,
+    DPO_DVR,
     DPO_L3_PROXY,
     DPO_BIER_TABLE,
     DPO_BIER_FMASK,
@@ -147,10 +147,10 @@ typedef enum dpo_type_t_ {
     [DPO_CLASSIFY] = "dpo-classify",	\
     [DPO_MPLS_LABEL] = "dpo-mpls-label", \
     [DPO_MPLS_DISPOSITION] = "dpo-mpls-diposition", \
-    [DPO_MFIB_ENTRY] = "dpo-mfib_entry", \
+    [DPO_MFIB_ENTRY] = "dpo-mfib-entry", \
     [DPO_INTERFACE_RX] = "dpo-interface-rx",	\
     [DPO_INTERFACE_TX] = "dpo-interface-tx",	\
-    [DPO_L2_BRIDGE] = "dpo-l2-bridge",	\
+    [DPO_DVR] = "dpo-dvr",	\
     [DPO_L3_PROXY] = "dpo-l3-proxy",	\
     [DPO_BIER_TABLE] = "bier-table",	\
     [DPO_BIER_FMASK] = "bier-fmask",	\
diff --git a/src/vnet/dpo/dvr_dpo.c b/src/vnet/dpo/dvr_dpo.c
new file mode 100644
index 00000000000..1aa16546c82
--- /dev/null
+++ b/src/vnet/dpo/dvr_dpo.c
@@ -0,0 +1,591 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vnet/dpo/dvr_dpo.h>
+#include <vnet/fib/fib_node.h>
+#include <vnet/ip/ip.h>
+#include <vnet/ethernet/ethernet.h>
+
+/**
+ * The 'DB' of DVR DPOs.
+ * There is one per-interface per-L3 proto, so this is a per-interface vector
+ */
+static index_t *dvr_dpo_db[DPO_PROTO_NUM];
+
+static dvr_dpo_t *
+dvr_dpo_alloc (void)
+{
+    dvr_dpo_t *dd;
+
+    pool_get(dvr_dpo_pool, dd);
+
+    return (dd);
+}
+
+static inline dvr_dpo_t *
+dvr_dpo_get_from_dpo (const dpo_id_t *dpo)
+{
+    ASSERT(DPO_DVR == dpo->dpoi_type);
+
+    return (dvr_dpo_get(dpo->dpoi_index));
+}
+
+static inline index_t
+dvr_dpo_get_index (dvr_dpo_t *dd)
+{
+    return (dd - dvr_dpo_pool);
+}
+
+static void
+dvr_dpo_lock (dpo_id_t *dpo)
+{
+    dvr_dpo_t *dd;
+
+    dd = dvr_dpo_get_from_dpo(dpo);
+    dd->dd_locks++;
+}
+
+static void
+dvr_dpo_unlock (dpo_id_t *dpo)
+{
+    dvr_dpo_t *dd;
+
+    dd = dvr_dpo_get_from_dpo(dpo);
+    dd->dd_locks--;
+
+    if (0 == dd->dd_locks)
+    {
+        if (DPO_PROTO_IP4 == dd->dd_proto)
+        {
+            vnet_feature_enable_disable ("ip4-output", "ip4-dvr-reinject",
+                                         dd->dd_sw_if_index, 0, 0, 0);
+        }
+        else
+        {
+            vnet_feature_enable_disable ("ip6-output", "ip6-dvr-reinject",
+                                         dd->dd_sw_if_index, 0, 0, 0);
+        }
+
+        dvr_dpo_db[dd->dd_proto][dd->dd_sw_if_index] = INDEX_INVALID;
+        pool_put(dvr_dpo_pool, dd);
+    }
+}
+
+void
+dvr_dpo_add_or_lock (u32 sw_if_index,
+                     dpo_proto_t dproto,
+                     dpo_id_t *dpo)
+{
+    dvr_dpo_t *dd;
+
+    vec_validate_init_empty(dvr_dpo_db[dproto],
+                            sw_if_index,
+                            INDEX_INVALID);
+
+    if (INDEX_INVALID == dvr_dpo_db[dproto][sw_if_index])
+    {
+        dd = dvr_dpo_alloc();
+
+        dd->dd_sw_if_index = sw_if_index;
+        dd->dd_proto = dproto;
+
+        dvr_dpo_db[dproto][sw_if_index] = dvr_dpo_get_index(dd);
+
+        /*
+         * enable the reinject into L2 path feature on the interface
+         */
+        if (DPO_PROTO_IP4 == dproto)
+            vnet_feature_enable_disable ("ip4-output", "ip4-dvr-reinject",
+                                         dd->dd_sw_if_index, 1, 0, 0);
+        else if (DPO_PROTO_IP6 == dproto)
+            vnet_feature_enable_disable ("ip6-output", "ip6-dvr-reinject",
+                                         dd->dd_sw_if_index, 1, 0, 0);
+        else
+            ASSERT(0);
+    }
+    else
+    {
+        dd = dvr_dpo_get(dvr_dpo_db[dproto][sw_if_index]);
+    }
+
+    dpo_set(dpo, DPO_DVR, dproto, dvr_dpo_get_index(dd));
+}
+
+
+static clib_error_t *
+dvr_dpo_interface_state_change (vnet_main_t * vnm,
+                                      u32 sw_if_index,
+                                      u32 flags)
+{
+    /*
+     */
+    return (NULL);
+}
+
+VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION(
+    dvr_dpo_interface_state_change);
+
+/**
+ * @brief Registered callback for HW interface state changes
+ */
+static clib_error_t *
+dvr_dpo_hw_interface_state_change (vnet_main_t * vnm,
+                                         u32 hw_if_index,
+                                         u32 flags)
+{
+    return (NULL);
+}
+
+VNET_HW_INTERFACE_LINK_UP_DOWN_FUNCTION(
+    dvr_dpo_hw_interface_state_change);
+
+static clib_error_t *
+dvr_dpo_interface_delete (vnet_main_t * vnm,
+                                u32 sw_if_index,
+                                u32 is_add)
+{
+    return (NULL);
+}
+
+VNET_SW_INTERFACE_ADD_DEL_FUNCTION(
+    dvr_dpo_interface_delete);
+
+u8*
+format_dvr_dpo (u8* s, va_list *ap)
+{
+    index_t index = va_arg(*ap, index_t);
+    CLIB_UNUSED(u32 indent) = va_arg(*ap, u32);
+    vnet_main_t * vnm = vnet_get_main();
+    dvr_dpo_t *dd = dvr_dpo_get(index);
+
+    return (format(s, "dvr-%U-dpo",
+                   format_vnet_sw_interface_name,
+                   vnm,
+                   vnet_get_sw_interface(vnm, dd->dd_sw_if_index)));
+}
+
+static void
+dvr_dpo_mem_show (void)
+{
+    fib_show_memory_usage("DVR",
+                          pool_elts(dvr_dpo_pool),
+                          pool_len(dvr_dpo_pool),
+                          sizeof(dvr_dpo_t));
+}
+
+
+const static dpo_vft_t dvr_dpo_vft = {
+    .dv_lock = dvr_dpo_lock,
+    .dv_unlock = dvr_dpo_unlock,
+    .dv_format = format_dvr_dpo,
+    .dv_mem_show = dvr_dpo_mem_show,
+};
+
+/**
+ * @brief The per-protocol VLIB graph nodes that are assigned to a glean
+ *        object.
+ *
+ * this means that these graph nodes are ones from which a glean is the
+ * parent object in the DPO-graph.
+ */
+const static char* const dvr_dpo_ip4_nodes[] =
+{
+    "ip4-dvr-dpo",
+    NULL,
+};
+const static char* const dvr_dpo_ip6_nodes[] =
+{
+    "ip6-dvr-dpo",
+    NULL,
+};
+
+const static char* const * const dvr_dpo_nodes[DPO_PROTO_NUM] =
+{
+    [DPO_PROTO_IP4]  = dvr_dpo_ip4_nodes,
+    [DPO_PROTO_IP6]  = dvr_dpo_ip6_nodes,
+};
+
+void
+dvr_dpo_module_init (void)
+{
+    dpo_register(DPO_DVR,
+                 &dvr_dpo_vft,
+                 dvr_dpo_nodes);
+}
+
+/**
+ * @brief Interface DPO trace data
+ */
+typedef struct dvr_dpo_trace_t_
+{
+    u32 sw_if_index;
+} dvr_dpo_trace_t;
+
+always_inline uword
+dvr_dpo_inline (vlib_main_t * vm,
+                vlib_node_runtime_t * node,
+                vlib_frame_t * from_frame,
+                u8 is_ip6)
+{
+    u32 n_left_from, next_index, * from, * to_next;
+    ip_lookup_main_t *lm = (is_ip6?
+                            &ip6_main.lookup_main:
+                            &ip4_main.lookup_main);
+
+    from = vlib_frame_vector_args (from_frame);
+    n_left_from = from_frame->n_vectors;
+
+    next_index = node->cached_next_index;
+
+    while (n_left_from > 0)
+    {
+        u32 n_left_to_next;
+
+        vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
+
+        while (n_left_from >= 4 && n_left_to_next > 2)
+        {
+            const dvr_dpo_t *dd0, *dd1;
+            u32 bi0, ddi0, bi1, ddi1;
+            vlib_buffer_t *b0, *b1;
+            u32 next0, next1;
+            u8 len0, len1;
+
+            bi0 = from[0];
+            to_next[0] = bi0;
+            bi1 = from[1];
+            to_next[1] = bi1;
+            from += 2;
+            to_next += 2;
+            n_left_from -= 2;
+            n_left_to_next -= 2;
+            next0 = next1 = 0;
+
+            b0 = vlib_get_buffer (vm, bi0);
+            b1 = vlib_get_buffer (vm, bi1);
+
+            ddi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
+            ddi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
+            dd0 = dvr_dpo_get(ddi0);
+            dd1 = dvr_dpo_get(ddi1);
+
+            vnet_buffer(b0)->sw_if_index[VLIB_TX] = dd0->dd_sw_if_index;
+            vnet_buffer(b1)->sw_if_index[VLIB_TX] = dd1->dd_sw_if_index;
+
+            len0 = ((u8*)vlib_buffer_get_current(b0) -
+                    (u8*)ethernet_buffer_get_header(b0));
+            len1 = ((u8*)vlib_buffer_get_current(b1) -
+                    (u8*)ethernet_buffer_get_header(b1));
+            vnet_buffer(b0)->l2.l2_len = len0;
+            vnet_buffer(b1)->l2.l2_len = len1;
+            vnet_buffer(b0)->flags |= VNET_OPAQUE_F_IS_DVR;
+            vnet_buffer(b1)->flags |= VNET_OPAQUE_F_IS_DVR;
+
+            vlib_buffer_advance(b0, -len0);
+            vlib_buffer_advance(b1, -len1);
+
+            vnet_feature_arc_start (lm->output_feature_arc_index,
+                                    dd0->dd_sw_if_index, &next0, b0);
+            vnet_feature_arc_start (lm->output_feature_arc_index,
+                                    dd1->dd_sw_if_index, &next1, b1);
+
+            if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+            {
+                dvr_dpo_trace_t *tr0;
+
+                tr0 = vlib_add_trace (vm, node, b0, sizeof (*tr0));
+                tr0->sw_if_index = dd0->dd_sw_if_index;
+            }
+            if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
+            {
+                dvr_dpo_trace_t *tr1;
+
+                tr1 = vlib_add_trace (vm, node, b1, sizeof (*tr1));
+                tr1->sw_if_index = dd1->dd_sw_if_index;
+            }
+
+            vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next,
+                                            n_left_to_next, bi0, bi1,
+                                            next0, next1);
+        }
+
+        while (n_left_from > 0 && n_left_to_next > 0)
+        {
+            const dvr_dpo_t * dd0;
+            vlib_buffer_t * b0;
+            u32 bi0, ddi0;
+            u32 next0;
+            u8 len0;
+
+            bi0 = from[0];
+            to_next[0] = bi0;
+            from += 1;
+            to_next += 1;
+            n_left_from -= 1;
+            n_left_to_next -= 1;
+            next0 = 0;
+
+            b0 = vlib_get_buffer (vm, bi0);
+
+            ddi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
+            dd0 = dvr_dpo_get(ddi0);
+
+            vnet_buffer(b0)->sw_if_index[VLIB_TX] = dd0->dd_sw_if_index;
+
+            /*
+             * take that, rewind it back...
+             */
+            len0 = ((u8*)vlib_buffer_get_current(b0) -
+                    (u8*)ethernet_buffer_get_header(b0));
+            vnet_buffer(b0)->l2.l2_len = len0;
+            vnet_buffer(b0)->flags |= VNET_OPAQUE_F_IS_DVR;
+            vlib_buffer_advance(b0, -len0);
+
+            /*
+             * start processing the ipX output features
+             */
+            vnet_feature_arc_start(lm->output_feature_arc_index,
+                                   dd0->dd_sw_if_index, &next0, b0);
+
+            if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+            {
+                dvr_dpo_trace_t *tr;
+
+                tr = vlib_add_trace (vm, node, b0, sizeof (*tr));
+                tr->sw_if_index = dd0->dd_sw_if_index;
+            }
+
+            vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
+                                            n_left_to_next, bi0,
+                                            next0);
+        }
+        vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+    return from_frame->n_vectors;
+}
+
+static u8 *
+format_dvr_dpo_trace (u8 * s, va_list * args)
+{
+    CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+    CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+    dvr_dpo_trace_t * t = va_arg (*args, dvr_dpo_trace_t *);
+    u32 indent = format_get_indent (s);
+    s = format (s, "%U sw_if_index:%d",
+                format_white_space, indent,
+                t->sw_if_index);
+    return s;
+}
+
+static uword
+ip4_dvr_dpo (vlib_main_t * vm,
+             vlib_node_runtime_t * node,
+             vlib_frame_t * from_frame)
+{
+    return (dvr_dpo_inline(vm, node, from_frame, 0));
+}
+
+static uword
+ip6_dvr_dpo (vlib_main_t * vm,
+             vlib_node_runtime_t * node,
+             vlib_frame_t * from_frame)
+{
+    return (dvr_dpo_inline(vm, node, from_frame, 1));
+}
+
+VLIB_REGISTER_NODE (ip4_dvr_dpo_node) = {
+    .function = ip4_dvr_dpo,
+    .name = "ip4-dvr-dpo",
+    .vector_size = sizeof (u32),
+    .format_trace = format_dvr_dpo_trace,
+    .sibling_of = "ip4-rewrite",
+};
+VLIB_REGISTER_NODE (ip6_dvr_dpo_node) = {
+    .function = ip6_dvr_dpo,
+    .name = "ip6-dvr-dpo",
+    .vector_size = sizeof (u32),
+    .format_trace = format_dvr_dpo_trace,
+    .sibling_of = "ip6-rewrite",
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_dvr_dpo_node, ip4_dvr_dpo)
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_dvr_dpo_node, ip6_dvr_dpo)
+
+typedef enum dvr_reinject_next_t_
+{
+    DVR_REINJECT_OUTPUT = 0,
+} dvr_reinject_next_t;
+
+always_inline uword
+dvr_reinject_inline (vlib_main_t * vm,
+                     vlib_node_runtime_t * node,
+                     vlib_frame_t * from_frame)
+{
+    u32 n_left_from, next_index, * from, * to_next;
+
+    from = vlib_frame_vector_args (from_frame);
+    n_left_from = from_frame->n_vectors;
+
+    next_index = node->cached_next_index;
+
+    while (n_left_from > 0)
+    {
+        u32 n_left_to_next;
+
+        vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
+
+        while (n_left_from >= 4 && n_left_to_next > 2)
+        {
+            dvr_reinject_next_t next0, next1;
+            vlib_buffer_t *b0, *b1;
+            u32 bi0, bi1;
+
+            bi0 = from[0];
+            to_next[0] = bi0;
+            bi1 = from[1];
+            to_next[1] = bi1;
+            from += 2;
+            to_next += 2;
+            n_left_from -= 2;
+            n_left_to_next -= 2;
+
+            b0 = vlib_get_buffer (vm, bi0);
+            b1 = vlib_get_buffer (vm, bi1);
+
+            if (vnet_buffer(b0)->flags & VNET_OPAQUE_F_IS_DVR)
+                next0 = DVR_REINJECT_OUTPUT;
+            else
+                vnet_feature_next(vnet_buffer(b0)->sw_if_index[VLIB_TX],
+                                  &next0, b0);
+
+            if (vnet_buffer(b1)->flags & VNET_OPAQUE_F_IS_DVR)
+                next1 = DVR_REINJECT_OUTPUT;
+            else
+                vnet_feature_next(vnet_buffer(b1)->sw_if_index[VLIB_TX],
+                                  &next1, b1);
+
+            if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+            {
+                dvr_dpo_trace_t *tr0;
+
+                tr0 = vlib_add_trace (vm, node, b0, sizeof (*tr0));
+                tr0->sw_if_index = vnet_buffer(b0)->sw_if_index[VLIB_TX];
+            }
+            if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
+            {
+                dvr_dpo_trace_t *tr1;
+
+                tr1 = vlib_add_trace (vm, node, b1, sizeof (*tr1));
+                tr1->sw_if_index = vnet_buffer(b1)->sw_if_index[VLIB_TX];
+            }
+
+            vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next,
+                                            n_left_to_next, bi0, bi1,
+                                            next0, next1);
+        }
+
+        while (n_left_from > 0 && n_left_to_next > 0)
+        {
+            dvr_reinject_next_t next0;
+            vlib_buffer_t * b0;
+            u32 bi0;
+
+            bi0 = from[0];
+            to_next[0] = bi0;
+            from += 1;
+            to_next += 1;
+            n_left_from -= 1;
+            n_left_to_next -= 1;
+
+            b0 = vlib_get_buffer (vm, bi0);
+
+            if (vnet_buffer(b0)->flags & VNET_OPAQUE_F_IS_DVR)
+                next0 = DVR_REINJECT_OUTPUT;
+            else
+                vnet_feature_next(vnet_buffer(b0)->sw_if_index[VLIB_TX],
+                                  &next0, b0);
+
+            if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+            {
+                dvr_dpo_trace_t *tr;
+
+                tr = vlib_add_trace (vm, node, b0, sizeof (*tr));
+                tr->sw_if_index = vnet_buffer(b0)->sw_if_index[VLIB_TX];
+            }
+
+            vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
+                                            n_left_to_next, bi0, next0);
+        }
+        vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+    return from_frame->n_vectors;
+}
+
+static uword
+ip4_dvr_reinject (vlib_main_t * vm,
+                  vlib_node_runtime_t * node,
+                  vlib_frame_t * from_frame)
+{
+    return (dvr_reinject_inline(vm, node, from_frame));
+}
+
+static uword
+ip6_dvr_reinject (vlib_main_t * vm,
+                  vlib_node_runtime_t * node,
+                  vlib_frame_t * from_frame)
+{
+    return (dvr_reinject_inline(vm, node, from_frame));
+}
+
+VLIB_REGISTER_NODE (ip4_dvr_reinject_node) = {
+    .function = ip4_dvr_reinject,
+    .name = "ip4-dvr-reinject",
+    .vector_size = sizeof (u32),
+    .format_trace = format_dvr_dpo_trace,
+
+    .n_next_nodes = 1,
+    .next_nodes = {
+        [DVR_REINJECT_OUTPUT] = "l2-output",
+    },
+};
+
+VLIB_REGISTER_NODE (ip6_dvr_reinject_node) = {
+    .function = ip6_dvr_reinject,
+    .name = "ip6-dvr-reinject",
+    .vector_size = sizeof (u32),
+    .format_trace = format_dvr_dpo_trace,
+
+    .n_next_nodes = 1,
+    .next_nodes = {
+        [DVR_REINJECT_OUTPUT] = "l2-output",
+    },
+};
+
+VNET_FEATURE_INIT (ip4_dvr_reinject_feat_node, static) =
+{
+  .arc_name = "ip4-output",
+  .node_name = "ip4-dvr-reinject",
+  .runs_after = VNET_FEATURES ("nat44-in2out-output",
+                               "acl-plugin-out-ip4-fa"),
+};
+VNET_FEATURE_INIT (ip6_dvr_reinject_feat_node, static) =
+{
+  .arc_name = "ip6-output",
+  .node_name = "ip6-dvr-reinject",
+  .runs_after = VNET_FEATURES ("acl-plugin-out-ip6-fa"),
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_dvr_reinject_node, ip4_dvr_reinject)
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_dvr_reinject_node, ip6_dvr_reinject)
diff --git a/src/vnet/dpo/dvr_dpo.h b/src/vnet/dpo/dvr_dpo.h
new file mode 100644
index 00000000000..15fe113c596
--- /dev/null
+++ b/src/vnet/dpo/dvr_dpo.h
@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __DVR_DPO_H__
+#define __DVR_DPO_H__
+
+#include <vnet/dpo/dpo.h>
+
+/**
+ * @brief
+ * The DVR DPO. Used as the resolving object for a DVR route.
+ * This is used, in place of the usual L3 Adjacency, to retransmit
+ * the packet with the original L2 header intact but also to run L3 features.
+ * After running L3 features the packet is re-injected back into the L2 path
+ * so it can pick up the necessary VLAN tags of the egress interface.
+ * This re-injection is done with an output feature.
+ */
+typedef struct dvr_dpo_t_
+{
+    /**
+     * The Software interface index that the packets will output on
+     */
+    u32 dd_sw_if_index;
+
+    /**
+     * The protocol of packets using this DPO
+     */
+    dpo_proto_t dd_proto;
+
+    /**
+     * number of locks.
+     */
+    u16 dd_locks;
+} dvr_dpo_t;
+
+extern void dvr_dpo_add_or_lock (u32 sw_if_index,
+                                 dpo_proto_t dproto,
+                                 dpo_id_t *dpo);
+
+extern void dvr_dpo_module_init(void);
+
+/**
+ * @brief pool of all interface DPOs
+ */
+dvr_dpo_t *dvr_dpo_pool;
+
+static inline dvr_dpo_t *
+dvr_dpo_get (index_t index)
+{
+    return (pool_elt_at_index(dvr_dpo_pool, index));
+}
+
+#endif
diff --git a/src/vnet/dpo/l2_bridge_dpo.c b/src/vnet/dpo/l2_bridge_dpo.c
deleted file mode 100644
index fd92a61e960..00000000000
--- a/src/vnet/dpo/l2_bridge_dpo.c
+++ /dev/null
@@ -1,375 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <vnet/dpo/l2_bridge_dpo.h>
-#include <vnet/fib/fib_node.h>
-#include <vnet/ethernet/ethernet.h>
-
-/*
- * The 'DB' of L2 bridge DPOs.
- * There is only one per-interface, so this is a per-interface vector
- */
-static index_t *l2_bridge_dpo_db;
-
-static l2_bridge_dpo_t *
-l2_bridge_dpo_alloc (void)
-{
-    l2_bridge_dpo_t *l2b;
-
-    pool_get(l2_bridge_dpo_pool, l2b);
-
-    return (l2b);
-}
-
-static inline l2_bridge_dpo_t *
-l2_bridge_dpo_get_from_dpo (const dpo_id_t *dpo)
-{
-    ASSERT(DPO_L2_BRIDGE == dpo->dpoi_type);
-
-    return (l2_bridge_dpo_get(dpo->dpoi_index));
-}
-
-static inline index_t
-l2_bridge_dpo_get_index (l2_bridge_dpo_t *l2b)
-{
-    return (l2b - l2_bridge_dpo_pool);
-}
-
-static void
-l2_bridge_dpo_lock (dpo_id_t *dpo)
-{
-    l2_bridge_dpo_t *l2b;
-
-    l2b = l2_bridge_dpo_get_from_dpo(dpo);
-    l2b->l2b_locks++;
-}
-
-static void
-l2_bridge_dpo_unlock (dpo_id_t *dpo)
-{
-    l2_bridge_dpo_t *l2b;
-
-    l2b = l2_bridge_dpo_get_from_dpo(dpo);
-    l2b->l2b_locks--;
-
-    if (0 == l2b->l2b_locks)
-    {
-        l2_bridge_dpo_db[l2b->l2b_sw_if_index] = INDEX_INVALID;
-        pool_put(l2_bridge_dpo_pool, l2b);
-    }
-}
-
-/*
- * l2_bridge_dpo_add_or_lock
- *
- * Add/create and lock a new or lock an existing for the L2 Bridge
- * on the interface given
- */
-void
-l2_bridge_dpo_add_or_lock (u32 sw_if_index,
-                           dpo_id_t *dpo)
-{
-    l2_bridge_dpo_t *l2b;
-
-    vec_validate_init_empty(l2_bridge_dpo_db,
-                            sw_if_index,
-                            INDEX_INVALID);
-
-    if (INDEX_INVALID == l2_bridge_dpo_db[sw_if_index])
-    {
-        l2b = l2_bridge_dpo_alloc();
-
-        l2b->l2b_sw_if_index = sw_if_index;
-
-        l2_bridge_dpo_db[sw_if_index] =
-            l2_bridge_dpo_get_index(l2b);
-    }
-    else
-    {
-        l2b = l2_bridge_dpo_get(l2_bridge_dpo_db[sw_if_index]);
-    }
-
-    dpo_set(dpo, DPO_L2_BRIDGE, DPO_PROTO_ETHERNET, l2_bridge_dpo_get_index(l2b));
-}
-
-
-static clib_error_t *
-l2_bridge_dpo_interface_state_change (vnet_main_t * vnm,
-                                      u32 sw_if_index,
-                                      u32 flags)
-{
-    /*
-     */
-    return (NULL);
-}
-
-VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION(
-    l2_bridge_dpo_interface_state_change);
-
-/**
- * @brief Registered callback for HW interface state changes
- */
-static clib_error_t *
-l2_bridge_dpo_hw_interface_state_change (vnet_main_t * vnm,
-                                         u32 hw_if_index,
-                                         u32 flags)
-{
-    return (NULL);
-}
-
-VNET_HW_INTERFACE_LINK_UP_DOWN_FUNCTION(
-    l2_bridge_dpo_hw_interface_state_change);
-
-static clib_error_t *
-l2_bridge_dpo_interface_delete (vnet_main_t * vnm,
-                                u32 sw_if_index,
-                                u32 is_add)
-{
-    return (NULL);
-}
-
-VNET_SW_INTERFACE_ADD_DEL_FUNCTION(
-    l2_bridge_dpo_interface_delete);
-
-u8*
-format_l2_bridge_dpo (u8* s, va_list *ap)
-{
-    index_t index = va_arg(*ap, index_t);
-    CLIB_UNUSED(u32 indent) = va_arg(*ap, u32);
-    vnet_main_t * vnm = vnet_get_main();
-    l2_bridge_dpo_t *l2b = l2_bridge_dpo_get(index);
-
-    return (format(s, "l2-bridge-%U-dpo",
-                   format_vnet_sw_interface_name,
-                   vnm,
-                   vnet_get_sw_interface(vnm, l2b->l2b_sw_if_index)));
-}
-
-static void
-l2_bridge_dpo_mem_show (void)
-{
-    fib_show_memory_usage("L2-bridge",
-                          pool_elts(l2_bridge_dpo_pool),
-                          pool_len(l2_bridge_dpo_pool),
-                          sizeof(l2_bridge_dpo_t));
-}
-
-
-const static dpo_vft_t l2_bridge_dpo_vft = {
-    .dv_lock = l2_bridge_dpo_lock,
-    .dv_unlock = l2_bridge_dpo_unlock,
-    .dv_format = format_l2_bridge_dpo,
-    .dv_mem_show = l2_bridge_dpo_mem_show,
-};
-
-/**
- * @brief The per-protocol VLIB graph nodes that are assigned to a glean
- *        object.
- *
- * this means that these graph nodes are ones from which a glean is the
- * parent object in the DPO-graph.
- */
-const static char* const l2_bridge_dpo_l2_nodes[] =
-{
-    "l2-bridge-dpo",
-    NULL,
-};
-
-const static char* const * const l2_bridge_dpo_nodes[DPO_PROTO_NUM] =
-{
-    [DPO_PROTO_ETHERNET]  = l2_bridge_dpo_l2_nodes,
-};
-
-void
-l2_bridge_dpo_module_init (void)
-{
-    dpo_register(DPO_L2_BRIDGE,
-                 &l2_bridge_dpo_vft,
-                 l2_bridge_dpo_nodes);
-}
-
-/**
- * @brief Interface DPO trace data
- */
-typedef struct l2_bridge_dpo_trace_t_
-{
-    u32 sw_if_index;
-} l2_bridge_dpo_trace_t;
-
-typedef enum l2_bridge_dpo_next_t_
-{
-    L2_BRIDGE_DPO_DROP = 0,
-    L2_BRIDGE_DPO_OUTPUT = 1,
-} l2_bridge_dpo_next_t;
-
-always_inline uword
-l2_bridge_dpo_inline (vlib_main_t * vm,
-                      vlib_node_runtime_t * node,
-                      vlib_frame_t * from_frame)
-{
-    u32 n_left_from, next_index, * from, * to_next;
-
-    from = vlib_frame_vector_args (from_frame);
-    n_left_from = from_frame->n_vectors;
-
-    next_index = node->cached_next_index;
-
-    while (n_left_from > 0)
-    {
-        u32 n_left_to_next;
-
-        vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
-
-        while (n_left_from >= 4 && n_left_to_next > 2)
-        {
-            const l2_bridge_dpo_t *l2b0, *l2b1;
-            u32 bi0, l2bi0, bi1, l2bi1;
-            vlib_buffer_t *b0, *b1;
-            u8 len0, len1;
-
-            bi0 = from[0];
-            to_next[0] = bi0;
-            bi1 = from[1];
-            to_next[1] = bi1;
-            from += 2;
-            to_next += 2;
-            n_left_from -= 2;
-            n_left_to_next -= 2;
-
-            b0 = vlib_get_buffer (vm, bi0);
-            b1 = vlib_get_buffer (vm, bi1);
-
-            l2bi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
-            l2bi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
-            l2b0 = l2_bridge_dpo_get(l2bi0);
-            l2b1 = l2_bridge_dpo_get(l2bi1);
-
-            vnet_buffer(b0)->sw_if_index[VLIB_TX] = l2b0->l2b_sw_if_index;
-            vnet_buffer(b1)->sw_if_index[VLIB_TX] = l2b1->l2b_sw_if_index;
-
-            len0 = ((u8*)vlib_buffer_get_current(b0) -
-                    (u8*)ethernet_buffer_get_header(b0));
-            len1 = ((u8*)vlib_buffer_get_current(b1) -
-                    (u8*)ethernet_buffer_get_header(b1));
-            vnet_buffer(b0)->l2.l2_len = len0;
-            vnet_buffer(b1)->l2.l2_len = len1;
-
-            vlib_buffer_advance(b0, -len0);
-            vlib_buffer_advance(b1, -len1);
-
-            if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
-            {
-                l2_bridge_dpo_trace_t *tr0;
-
-                tr0 = vlib_add_trace (vm, node, b0, sizeof (*tr0));
-                tr0->sw_if_index = l2b0->l2b_sw_if_index;
-            }
-            if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
-            {
-                l2_bridge_dpo_trace_t *tr1;
-
-                tr1 = vlib_add_trace (vm, node, b1, sizeof (*tr1));
-                tr1->sw_if_index = l2b1->l2b_sw_if_index;
-            }
-
-            vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next,
-                                            n_left_to_next, bi0, bi1,
-                                            L2_BRIDGE_DPO_OUTPUT,
-                                            L2_BRIDGE_DPO_OUTPUT);
-        }
-
-        while (n_left_from > 0 && n_left_to_next > 0)
-        {
-            const l2_bridge_dpo_t * l2b0;
-            vlib_buffer_t * b0;
-            u32 bi0, l2bi0;
-            u8 len0;
-
-            bi0 = from[0];
-            to_next[0] = bi0;
-            from += 1;
-            to_next += 1;
-            n_left_from -= 1;
-            n_left_to_next -= 1;
-
-            b0 = vlib_get_buffer (vm, bi0);
-
-            l2bi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
-            l2b0 = l2_bridge_dpo_get(l2bi0);
-
-            vnet_buffer(b0)->sw_if_index[VLIB_TX] = l2b0->l2b_sw_if_index;
-
-            /*
-             * take that, rewind it back...
-             */
-            len0 = ((u8*)vlib_buffer_get_current(b0) -
-                    (u8*)ethernet_buffer_get_header(b0));
-            vnet_buffer(b0)->l2.l2_len = len0;
-            vlib_buffer_advance(b0, -len0);
-
-            if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
-            {
-                l2_bridge_dpo_trace_t *tr;
-
-                tr = vlib_add_trace (vm, node, b0, sizeof (*tr));
-                tr->sw_if_index = l2b0->l2b_sw_if_index;
-            }
-
-            vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
-                                            n_left_to_next, bi0,
-                                            L2_BRIDGE_DPO_OUTPUT);
-        }
-        vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-    return from_frame->n_vectors;
-}
-
-static u8 *
-format_l2_bridge_dpo_trace (u8 * s, va_list * args)
-{
-    CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
-    CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-    l2_bridge_dpo_trace_t * t = va_arg (*args, l2_bridge_dpo_trace_t *);
-    u32 indent = format_get_indent (s);
-    s = format (s, "%U sw_if_index:%d",
-                format_white_space, indent,
-                t->sw_if_index);
-    return s;
-}
-
-static uword
-l2_bridge_dpo_l2 (vlib_main_t * vm,
-                  vlib_node_runtime_t * node,
-                  vlib_frame_t * from_frame)
-{
-    return (l2_bridge_dpo_inline(vm, node, from_frame));
-}
-
-
-VLIB_REGISTER_NODE (l2_bridge_dpo_l2_node) = {
-    .function = l2_bridge_dpo_l2,
-    .name = "l2-bridge-dpo",
-    .vector_size = sizeof (u32),
-    .format_trace = format_l2_bridge_dpo_trace,
-
-    .n_next_nodes = 2,
-    .next_nodes = {
-        [L2_BRIDGE_DPO_DROP] = "error-drop",
-        [L2_BRIDGE_DPO_OUTPUT] = "l2-output",
-    },
-};
-
-VLIB_NODE_FUNCTION_MULTIARCH (l2_bridge_dpo_l2_node,
-                              l2_bridge_dpo_l2)
diff --git a/src/vnet/dpo/l2_bridge_dpo.h b/src/vnet/dpo/l2_bridge_dpo.h
deleted file mode 100644
index 0a20dd79335..00000000000
--- a/src/vnet/dpo/l2_bridge_dpo.h
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __L2_BRIDGE_DPO_H__
-#define __L2_BRIDGE_DPO_H__
-
-#include <vnet/dpo/dpo.h>
-
-/**
- * @brief
- * The data-path object representing an L2 bridge.
- * If a packet encounters an object of this type in the L3 data-path, it
- * is injected back into the L2 bridge.
- */
-typedef struct l2_bridge_dpo_t_
-{
-    /**
-     * The Software interface index that the packets will output on
-     */
-    u32 l2b_sw_if_index;
-
-    /**
-     * number of locks.
-     */
-    u16 l2b_locks;
-} l2_bridge_dpo_t;
-
-extern void l2_bridge_dpo_add_or_lock (u32 sw_if_index,
-                                       dpo_id_t *dpo);
-
-extern void l2_bridge_dpo_module_init(void);
-
-/**
- * @brief pool of all interface DPOs
- */
-l2_bridge_dpo_t *l2_bridge_dpo_pool;
-
-static inline l2_bridge_dpo_t *
-l2_bridge_dpo_get (index_t index)
-{
-    return (pool_elt_at_index(l2_bridge_dpo_pool, index));
-}
-
-#endif