Deprecate old buffer replication scheme

Change-Id: I1f54b994425c58776e1445c8d9fe142e7a644d3d
Signed-off-by: Damjan Marion <damarion@cisco.com>

11 files changed, 19 insertions, 624 deletions

diff --git a/src/plugins/dpdk/buffer.c b/src/plugins/dpdk/buffer.c
index 10646de68a7..edace2aaf61 100644
--- a/src/plugins/dpdk/buffer.c
+++ b/src/plugins/dpdk/buffer.c
@@ -280,30 +280,9 @@ static_always_inline void
 recycle_or_free (vlib_main_t * vm, vlib_buffer_main_t * bm, u32 bi,
 		 vlib_buffer_t * b)
 {
-  vlib_buffer_free_list_t *fl;
   u32 thread_index = vlib_get_thread_index ();
-  vlib_buffer_free_list_index_t fi;
-  fl = vlib_buffer_get_buffer_free_list (vm, b, &fi);
 
-  /* The only current use of this callback: multicast recycle */
-  if (PREDICT_FALSE (fl->buffers_added_to_freelist_function != 0))
-    {
-      int j;
-
-      vlib_buffer_add_to_free_list (vm, fl, bi,
-				    (b->flags & VLIB_BUFFER_RECYCLE) == 0);
-
-      for (j = 0; j < vec_len (vm->buffer_announce_list); j++)
-	{
-	  if (fl == vm->buffer_announce_list[j])
-	    goto already_announced;
-	}
-      vec_add1 (vm->buffer_announce_list, fl);
-    already_announced:
-      ;
-    }
-  else if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0))
-    dpdk_rte_pktmbuf_free (vm, thread_index, b, 1);
+  dpdk_rte_pktmbuf_free (vm, thread_index, b, 1);
 }
 
 static_always_inline void
@@ -314,7 +293,7 @@ vlib_buffer_free_inline (vlib_main_t * vm,
   vlib_buffer_t *bufp[n_buffers], **b = bufp;
   u32 thread_index = vlib_get_thread_index ();
   int i = 0;
-  u32 simple_mask = (VLIB_BUFFER_NON_DEFAULT_FREELIST | VLIB_BUFFER_RECYCLE |
+  u32 simple_mask = (VLIB_BUFFER_NON_DEFAULT_FREELIST |
 		     VLIB_BUFFER_NEXT_PRESENT);
   u32 n_left, *bi;
   u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
@@ -379,16 +358,6 @@ vlib_buffer_free_inline (vlib_main_t * vm,
       b += 1;
       n_left -= 1;
     }
-  if (vec_len (vm->buffer_announce_list))
-    {
-      vlib_buffer_free_list_t *fl;
-      for (i = 0; i < vec_len (vm->buffer_announce_list); i++)
-	{
-	  fl = vm->buffer_announce_list[i];
-	  fl->buffers_added_to_freelist_function (vm, fl);
-	}
-      _vec_len (vm->buffer_announce_list) = 0;
-    }
 }
 
 void
diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c
index b2c87c7930d..c38eec09d81 100644
--- a/src/plugins/dpdk/device/device.c
+++ b/src/plugins/dpdk/device/device.c
@@ -26,8 +26,7 @@
 
 #define foreach_dpdk_tx_func_error			\
   _(BAD_RETVAL, "DPDK tx function returned an error")	\
-  _(PKT_DROP, "Tx packet drops (dpdk tx failure)")	\
-  _(REPL_FAIL, "Tx packet drops (replication failure)")
+  _(PKT_DROP, "Tx packet drops (dpdk tx failure)")
 
 typedef enum
 {
@@ -65,48 +64,6 @@ dpdk_set_mac_address (vnet_hw_interface_t * hi, char *address)
     }
 }
 
-static struct rte_mbuf *
-dpdk_replicate_packet_mb (vlib_buffer_t * b)
-{
-  dpdk_main_t *dm = &dpdk_main;
-  struct rte_mbuf **mbufs = 0, *s, *d;
-  u8 nb_segs;
-  unsigned socket_id = rte_socket_id ();
-  int i;
-
-  ASSERT (dm->pktmbuf_pools[socket_id]);
-  s = rte_mbuf_from_vlib_buffer (b);
-  nb_segs = s->nb_segs;
-  vec_validate (mbufs, nb_segs - 1);
-
-  if (rte_pktmbuf_alloc_bulk (dm->pktmbuf_pools[socket_id], mbufs, nb_segs))
-    {
-      vec_free (mbufs);
-      return 0;
-    }
-
-  d = mbufs[0];
-  d->nb_segs = s->nb_segs;
-  d->data_len = s->data_len;
-  d->pkt_len = s->pkt_len;
-  d->data_off = s->data_off;
-  clib_memcpy (d->buf_addr, s->buf_addr, RTE_PKTMBUF_HEADROOM + s->data_len);
-
-  for (i = 1; i < nb_segs; i++)
-    {
-      d->next = mbufs[i];
-      d = mbufs[i];
-      s = s->next;
-      d->data_len = s->data_len;
-      clib_memcpy (d->buf_addr, s->buf_addr,
-		   RTE_PKTMBUF_HEADROOM + s->data_len);
-    }
-
-  d = mbufs[0];
-  vec_free (mbufs);
-  return d;
-}
-
 static void
 dpdk_tx_trace_buffer (dpdk_main_t * dm, vlib_node_runtime_t * node,
 		      dpdk_device_t * xd, u16 queue_id,
@@ -267,29 +224,6 @@ dpdk_prefetch_buffer (vlib_main_t * vm, struct rte_mbuf *mb)
 }
 
 static_always_inline void
-dpdk_buffer_recycle (vlib_main_t * vm, vlib_node_runtime_t * node,
-		     vlib_buffer_t * b, u32 bi, struct rte_mbuf **mbp)
-{
-  dpdk_main_t *dm = &dpdk_main;
-  struct rte_mbuf *mb_new;
-
-  if (PREDICT_FALSE (b->flags & VLIB_BUFFER_RECYCLE) == 0)
-    return;
-
-  mb_new = dpdk_replicate_packet_mb (b);
-  if (PREDICT_FALSE (mb_new == 0))
-    {
-      vlib_error_count (vm, node->node_index,
-			DPDK_TX_FUNC_ERROR_REPL_FAIL, 1);
-      b->flags |= VLIB_BUFFER_REPL_FAIL;
-    }
-  else
-    *mbp = mb_new;
-
-  vec_add1 (dm->recycle[vm->thread_index], bi);
-}
-
-static_always_inline void
 dpdk_buffer_tx_offload (dpdk_device_t * xd, vlib_buffer_t * b,
 			struct rte_mbuf *mb)
 {
@@ -454,29 +388,6 @@ VNET_DEVICE_CLASS_TX_FN (dpdk_device_class) (vlib_main_t * vm,
       n_left--;
     }
 
-  /* run inly if we have buffers to recycle */
-  if (PREDICT_FALSE (all_or_flags & VLIB_BUFFER_RECYCLE))
-    {
-      struct rte_mbuf **mb_old;
-      from = vlib_frame_vector_args (f);
-      n_left = n_packets;
-      mb_old = mb = ptd->mbufs;
-      while (n_left > 0)
-	{
-	  b[0] = vlib_buffer_from_rte_mbuf (mb[0]);
-	  dpdk_buffer_recycle (vm, node, b[0], from[0], &mb_old[0]);
-
-	  /* in case of REPL_FAIL we need to shift data */
-	  mb[0] = mb_old[0];
-
-	  if (PREDICT_TRUE ((b[0]->flags & VLIB_BUFFER_REPL_FAIL) == 0))
-	    mb++;
-	  mb_old++;
-	  from++;
-	  n_left--;
-	}
-    }
-
   /* transmit as many packets as possible */
   tx_pkts = n_packets = mb - ptd->mbufs;
   n_left = tx_burst_vector_internal (vm, xd, ptd->mbufs, n_packets);
@@ -503,14 +414,6 @@ VNET_DEVICE_CLASS_TX_FN (dpdk_device_class) (vlib_main_t * vm,
       }
   }
 
-  /* Recycle replicated buffers */
-  if (PREDICT_FALSE (vec_len (dm->recycle[thread_index])))
-    {
-      vlib_buffer_free (vm, dm->recycle[thread_index],
-			vec_len (dm->recycle[thread_index]));
-      _vec_len (dm->recycle[thread_index]) = 0;
-    }
-
   return tx_pkts;
 }
 
diff --git a/src/plugins/dpdk/device/dpdk.h b/src/plugins/dpdk/device/dpdk.h
index f75cd99e5d3..c5a9f94e458 100644
--- a/src/plugins/dpdk/device/dpdk.h
+++ b/src/plugins/dpdk/device/dpdk.h
@@ -404,9 +404,6 @@ typedef struct
   dpdk_device_and_queue_t **devices_by_hqos_cpu;
   dpdk_per_thread_data_t *per_thread_data;
 
-  /* per-thread recycle lists */
-  u32 **recycle;
-
   /* buffer flags template, configurable to enable/disable tcp / udp cksum */
   u32 buffer_flags_template;
 
diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c
index 23c12795361..0e9e6d6e9ad 100644
--- a/src/plugins/dpdk/device/init.c
+++ b/src/plugins/dpdk/device/init.c
@@ -1790,7 +1790,6 @@ dpdk_init (vlib_main_t * vm)
 {
   dpdk_main_t *dm = &dpdk_main;
   clib_error_t *error = 0;
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
 
   /* verify that structs are cacheline aligned */
   STATIC_ASSERT (offsetof (dpdk_device_t, cacheline0) == 0,
@@ -1811,8 +1810,6 @@ dpdk_init (vlib_main_t * vm)
   dm->conf->num_mbufs = dm->conf->num_mbufs ? dm->conf->num_mbufs : NB_MBUF;
   vec_add1 (dm->conf->eal_init_args, (u8 *) "vnet");
 
-  vec_validate (dm->recycle, tm->n_thread_stacks - 1);
-
   /* Default vlib_buffer_t flags, DISABLES tcp/udp checksumming... */
   dm->buffer_flags_template =
     (VLIB_BUFFER_TOTAL_LENGTH_VALID | VLIB_BUFFER_EXT_HDR_VALID
diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c
index 3f90f5a0860..a704efb40f4 100644
--- a/src/vlib/buffer.c
+++ b/src/vlib/buffer.c
@@ -573,50 +573,26 @@ recycle_or_free (vlib_main_t * vm, vlib_buffer_main_t * bm, u32 bi,
 {
   vlib_buffer_free_list_t *fl;
   vlib_buffer_free_list_index_t fi;
-  fl = vlib_buffer_get_buffer_free_list (vm, b, &fi);
+  u32 flags, next;
 
-  /* The only current use of this callback:
-   * multicast recycle */
-  if (PREDICT_FALSE (fl->buffers_added_to_freelist_function != 0))
-    {
-      int j;
+  fl = vlib_buffer_get_buffer_free_list (vm, b, &fi);
 
-      vlib_buffer_add_to_free_list (vm, fl, bi,
-				    (b->flags & VLIB_BUFFER_RECYCLE) == 0);
-      for (j = 0; j < vec_len (vm->buffer_announce_list); j++)
-	{
-	  if (fl == vm->buffer_announce_list[j])
-	    goto already_announced;
-	}
-      vec_add1 (vm->buffer_announce_list, fl);
-    already_announced:
-      ;
-    }
-  else
+  do
     {
-      if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0))
+      vlib_buffer_t *nb = vlib_get_buffer (vm, bi);
+      flags = nb->flags;
+      next = nb->next_buffer;
+      if (nb->n_add_refs)
+	nb->n_add_refs--;
+      else
 	{
-	  u32 flags, next;
-
-	  do
-	    {
-	      vlib_buffer_t *nb = vlib_get_buffer (vm, bi);
-	      flags = nb->flags;
-	      next = nb->next_buffer;
-	      if (nb->n_add_refs)
-		nb->n_add_refs--;
-	      else
-		{
-		  vlib_buffer_validate_alloc_free (vm, &bi, 1,
-						   VLIB_BUFFER_KNOWN_ALLOCATED);
-		  vlib_buffer_add_to_free_list (vm, fl, bi, 1);
-		}
-	      bi = next;
-	    }
-	  while (follow_buffer_next && (flags & VLIB_BUFFER_NEXT_PRESENT));
-
+	  vlib_buffer_validate_alloc_free (vm, &bi, 1,
+					   VLIB_BUFFER_KNOWN_ALLOCATED);
+	  vlib_buffer_add_to_free_list (vm, fl, bi, 1);
 	}
+      bi = next;
     }
+  while (follow_buffer_next && (flags & VLIB_BUFFER_NEXT_PRESENT));
 }
 
 static_always_inline void
@@ -673,17 +649,6 @@ vlib_buffer_free_inline (vlib_main_t * vm,
       recycle_or_free (vm, bm, buffers[i], b0, follow_buffer_next);
       i++;
     }
-
-  if (vec_len (vm->buffer_announce_list))
-    {
-      vlib_buffer_free_list_t *fl;
-      for (i = 0; i < vec_len (vm->buffer_announce_list); i++)
-	{
-	  fl = vm->buffer_announce_list[i];
-	  fl->buffers_added_to_freelist_function (vm, fl);
-	}
-      _vec_len (vm->buffer_announce_list) = 0;
-    }
 }
 
 static void
diff --git a/src/vlib/buffer.h b/src/vlib/buffer.h
index 9555cd7d224..651e7f0dc8f 100644
--- a/src/vlib/buffer.h
+++ b/src/vlib/buffer.h
@@ -67,11 +67,8 @@ typedef u8 vlib_buffer_free_list_index_t;
   _( 0, NON_DEFAULT_FREELIST, "non-default-fl")		\
   _( 1, IS_TRACED, 0)					\
   _( 2, NEXT_PRESENT, 0)				\
-  _( 3, IS_RECYCLED, "is-recycled")			\
-  _( 4, TOTAL_LENGTH_VALID, 0)				\
-  _( 5, REPL_FAIL, "repl-fail")				\
-  _( 6, RECYCLE, "recycle")				\
-  _( 7, EXT_HDR_VALID, "ext-hdr-valid")
+  _( 3, TOTAL_LENGTH_VALID, 0)				\
+  _( 4, EXT_HDR_VALID, "ext-hdr-valid")
 
 /* NOTE: only buffer generic flags should be defined here, please consider
    using user flags. i.e. src/vnet/buffer.h */
@@ -113,8 +110,6 @@ typedef struct
                 <br> VLIB_BUFFER_IS_TRACED: trace this buffer.
                 <br> VLIB_BUFFER_NEXT_PRESENT: this is a multi-chunk buffer.
                 <br> VLIB_BUFFER_TOTAL_LENGTH_VALID: as it says
-                <br> VLIB_BUFFER_REPL_FAIL: packet replication failure
-                <br> VLIB_BUFFER_RECYCLE: as it says
                 <br> VLIB_BUFFER_EXT_HDR_VALID: buffer contains valid external buffer manager header,
                 set to avoid adding it to a flow report
                 <br> VLIB_BUFFER_FLAG_USER(n): user-defined bit N
@@ -370,11 +365,6 @@ typedef struct vlib_buffer_free_list_t
 				struct vlib_buffer_free_list_t * fl,
 				u32 * buffers, u32 n_buffers);
 
-  /* Callback function to announce that buffers have been
-     added to the freelist */
-  void (*buffers_added_to_freelist_function)
-    (struct vlib_main_t * vm, struct vlib_buffer_free_list_t * fl);
-
   uword buffer_init_function_opaque;
 } __attribute__ ((aligned (16))) vlib_buffer_free_list_t;
 
diff --git a/src/vlib/main.h b/src/vlib/main.h
index 8b6f9478619..d21227a5fdf 100644
--- a/src/vlib/main.h
+++ b/src/vlib/main.h
@@ -112,9 +112,6 @@ typedef struct vlib_main_t
   /* Pool of buffer free lists. */
   vlib_buffer_free_list_t *buffer_free_list_pool;
 
-  /* List of free-lists needing Blue Light Special announcements */
-  vlib_buffer_free_list_t **buffer_announce_list;
-
   /* Allocate/free buffer memory for DMA transfers, descriptor rings, etc.
      buffer memory is guaranteed to be cache-aligned. */
 
diff --git a/src/vnet/CMakeLists.txt b/src/vnet/CMakeLists.txt
index 61945a41867..3dcb7efa799 100644
--- a/src/vnet/CMakeLists.txt
+++ b/src/vnet/CMakeLists.txt
@@ -37,7 +37,6 @@ list(APPEND VNET_SOURCES
   interface_output.c
   interface_stats.c
   misc.c
-  replication.c
 )
 
 list(APPEND VNET_HEADERS
@@ -56,7 +55,6 @@ list(APPEND VNET_HEADERS
   l3_types.h
   plugin/plugin.h
   pipeline.h
-  replication.h
   vnet.h
   vnet_all_api_h.h
   vnet_msg_enum.h
diff --git a/src/vnet/l2/l2_flood.c b/src/vnet/l2/l2_flood.c
index 8a635cec208..668301731a9 100644
--- a/src/vnet/l2/l2_flood.c
+++ b/src/vnet/l2/l2_flood.c
@@ -23,7 +23,6 @@
 #include <vnet/l2/l2_input.h>
 #include <vnet/l2/feat_bitmap.h>
 #include <vnet/l2/l2_bvi.h>
-#include <vnet/replication.h>
 #include <vnet/l2/l2_fib.h>
 
 #include <vppinfra/error.h>
diff --git a/src/vnet/replication.c b/src/vnet/replication.c
deleted file mode 100644
index 21f5485e6c7..00000000000
--- a/src/vnet/replication.c
+++ /dev/null
@@ -1,283 +0,0 @@
-/*
- * replication.c : packet replication
- *
- * Copyright (c) 2013 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <vlib/vlib.h>
-#include <vnet/vnet.h>
-#include <vppinfra/error.h>
-#include <vnet/ip/ip4_packet.h>
-#include <vnet/replication.h>
-
-
-replication_main_t replication_main;
-
-
-replication_context_t *
-replication_prep (vlib_main_t * vm,
-		  vlib_buffer_t * b0, u32 recycle_node_index, u32 l2_packet)
-{
-  replication_main_t *rm = &replication_main;
-  replication_context_t *ctx;
-  uword thread_index = vm->thread_index;
-  ip4_header_t *ip;
-  u32 ctx_id;
-
-  /* Allocate a context, reserve context 0 */
-  if (PREDICT_FALSE (rm->contexts[thread_index] == 0))
-    pool_get_aligned (rm->contexts[thread_index], ctx, CLIB_CACHE_LINE_BYTES);
-
-  pool_get_aligned (rm->contexts[thread_index], ctx, CLIB_CACHE_LINE_BYTES);
-  ctx_id = ctx - rm->contexts[thread_index];
-
-  /* Save state from vlib buffer */
-  ctx->saved_free_list_index = vlib_buffer_get_free_list_index (b0);
-  ctx->current_data = b0->current_data;
-  ctx->flags = b0->flags & VNET_BUFFER_FLAGS_VLAN_BITS;
-
-  /* Set up vlib buffer hooks */
-  b0->recycle_count = ctx_id;
-  vlib_buffer_set_free_list_index (b0, rm->recycle_list_index);
-  b0->flags |= VLIB_BUFFER_RECYCLE;
-
-  /* Save feature state */
-  ctx->recycle_node_index = recycle_node_index;
-
-  /* Save vnet state */
-  clib_memcpy (ctx->vnet_buffer, vnet_buffer (b0),
-	       sizeof (vnet_buffer_opaque_t));
-
-  /* Save packet contents */
-  ctx->l2_packet = l2_packet;
-  ip = (ip4_header_t *) vlib_buffer_get_current (b0);
-  if (l2_packet)
-    {
-      /* Save ethernet header */
-      ctx->l2_header[0] = ((u64 *) ip)[0];
-      ctx->l2_header[1] = ((u64 *) ip)[1];
-      ctx->l2_header[2] = ((u64 *) ip)[2];
-      /* set ip to the true ip header */
-      ip = (ip4_header_t *) (((u8 *) ip) + vnet_buffer (b0)->l2.l2_len);
-    }
-
-  /*
-   * Copy L3 fields.
-   * We need to save TOS for ip4 and ip6 packets.
-   * Fortunately the TOS field is
-   * in the first two bytes of both the ip4 and ip6 headers.
-   */
-  ctx->ip_tos = *((u16 *) (ip));
-
-  /*
-   * Save the ip4 checksum as well. We just blindly save the corresponding two
-   * bytes even for ip6 packets.
-   */
-  ctx->ip4_checksum = ip->checksum;
-
-  return ctx;
-}
-
-
-replication_context_t *
-replication_recycle (vlib_main_t * vm, vlib_buffer_t * b0, u32 is_last)
-{
-  replication_main_t *rm = &replication_main;
-  replication_context_t *ctx;
-  uword thread_index = vm->thread_index;
-  ip4_header_t *ip;
-
-  /* Get access to the replication context */
-  ctx = pool_elt_at_index (rm->contexts[thread_index], b0->recycle_count);
-
-  /* Restore vnet buffer state */
-  clib_memcpy (vnet_buffer (b0), ctx->vnet_buffer,
-	       sizeof (vnet_buffer_opaque_t));
-
-  /* Restore the vlan flags */
-  b0->flags &= ~VNET_BUFFER_FLAGS_VLAN_BITS;
-  b0->flags |= ctx->flags;
-
-  /* Restore the packet start (current_data) and length */
-  vlib_buffer_advance (b0, ctx->current_data - b0->current_data);
-
-  /* Restore packet contents */
-  ip = (ip4_header_t *) vlib_buffer_get_current (b0);
-  if (ctx->l2_packet)
-    {
-      /* Restore ethernet header */
-      ((u64 *) ip)[0] = ctx->l2_header[0];
-      ((u64 *) ip)[1] = ctx->l2_header[1];
-      ((u64 *) ip)[2] = ctx->l2_header[2];
-      /* set ip to the true ip header */
-      ip = (ip4_header_t *) (((u8 *) ip) + vnet_buffer (b0)->l2.l2_len);
-    }
-
-  // Restore L3 fields
-  *((u16 *) (ip)) = ctx->ip_tos;
-  ip->checksum = ctx->ip4_checksum;
-
-  if (is_last)
-    {
-      /*
-       * This is the last replication in the list.
-       * Restore original buffer free functionality.
-       */
-      vlib_buffer_set_free_list_index (b0, ctx->saved_free_list_index);
-      b0->flags &= ~VLIB_BUFFER_RECYCLE;
-
-      /* Free context back to its pool */
-      pool_put (rm->contexts[thread_index], ctx);
-    }
-
-  return ctx;
-}
-
-
-
-/*
- * fish pkts back from the recycle queue/freelist
- * un-flatten the context chains
- */
-static void
-replication_recycle_callback (vlib_main_t * vm, vlib_buffer_free_list_t * fl)
-{
-  vlib_frame_t *f = 0;
-  u32 n_left_from;
-  u32 n_left_to_next = 0;
-  u32 n_this_frame = 0;
-  u32 *from;
-  u32 *to_next = 0;
-  u32 bi0, pi0;
-  vlib_buffer_t *b0;
-  int i;
-  replication_main_t *rm = &replication_main;
-  replication_context_t *ctx;
-  u32 feature_node_index = 0;
-  uword thread_index = vm->thread_index;
-
-  /*
-   * All buffers in the list are destined to the same recycle node.
-   * Pull the recycle node index from the first buffer.
-   * Note: this could be sped up if the node index were stuffed into
-   * the freelist itself.
-   */
-  if (vec_len (fl->buffers) > 0)
-    {
-      bi0 = fl->buffers[0];
-      b0 = vlib_get_buffer (vm, bi0);
-      ctx = pool_elt_at_index (rm->contexts[thread_index], b0->recycle_count);
-      feature_node_index = ctx->recycle_node_index;
-    }
-
-  /* buffers */
-  for (i = 0; i < 2; i++)
-    {
-      if (i == 0)
-	{
-	  from = fl->buffers;
-	  n_left_from = vec_len (from);
-	}
-
-      while (n_left_from > 0)
-	{
-	  if (PREDICT_FALSE (n_left_to_next == 0))
-	    {
-	      if (f)
-		{
-		  f->n_vectors = n_this_frame;
-		  vlib_put_frame_to_node (vm, feature_node_index, f);
-		}
-
-	      f = vlib_get_frame_to_node (vm, feature_node_index);
-	      to_next = vlib_frame_vector_args (f);
-	      n_left_to_next = VLIB_FRAME_SIZE;
-	      n_this_frame = 0;
-	    }
-
-	  bi0 = from[0];
-	  if (PREDICT_TRUE (n_left_from > 1))
-	    {
-	      pi0 = from[1];
-	      vlib_prefetch_buffer_with_index (vm, pi0, LOAD);
-	    }
-
-	  b0 = vlib_get_buffer (vm, bi0);
-
-	  /* Mark that this buffer was just recycled */
-	  b0->flags |= VLIB_BUFFER_IS_RECYCLED;
-
-#if (CLIB_DEBUG > 0)
-	  if (buffer_main.callbacks_registered == 0)
-	    vlib_buffer_set_known_state (bi0, VLIB_BUFFER_KNOWN_ALLOCATED);
-#endif
-
-	  /* If buffer is traced, mark frame as traced */
-	  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
-	    f->flags |= VLIB_FRAME_TRACE;
-
-	  to_next[0] = bi0;
-
-	  from++;
-	  to_next++;
-	  n_this_frame++;
-	  n_left_to_next--;
-	  n_left_from--;
-	}
-    }
-
-  vec_reset_length (fl->buffers);
-
-  if (f)
-    {
-      ASSERT (n_this_frame);
-      f->n_vectors = n_this_frame;
-      vlib_put_frame_to_node (vm, feature_node_index, f);
-    }
-}
-
-clib_error_t *
-replication_init (vlib_main_t * vm)
-{
-  replication_main_t *rm = &replication_main;
-  vlib_buffer_free_list_t *fl;
-  __attribute__ ((unused)) replication_context_t *ctx;
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
-
-  rm->vlib_main = vm;
-  rm->vnet_main = vnet_get_main ();
-  rm->recycle_list_index =
-    vlib_buffer_create_free_list (vm, 1024 /* fictional */ ,
-				  "replication-recycle");
-
-  fl = pool_elt_at_index (vm->buffer_free_list_pool, rm->recycle_list_index);
-
-  fl->buffers_added_to_freelist_function = replication_recycle_callback;
-
-  /* Verify the replication context is the expected size */
-  ASSERT (sizeof (replication_context_t) == 128);	/* 2 cache lines */
-
-  vec_validate (rm->contexts, tm->n_vlib_mains - 1);
-  return 0;
-}
-
-VLIB_INIT_FUNCTION (replication_init);
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/replication.h b/src/vnet/replication.h
deleted file mode 100644
index 6350aedfe7f..00000000000
--- a/src/vnet/replication.h
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * replication.h : packet replication
- *
- * Copyright (c) 2013 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef included_replication_h
-#define included_replication_h
-
-
-#include <vlib/vlib.h>
-#include <vnet/vnet.h>
-#include <vnet/replication.h>
-
-
-typedef struct
-{
-  /* The entire vnet buffer header restored for each replica */
-  u8 vnet_buffer[40];		/* 16B aligned to allow vector unit copy */
-  u8 reserved[24];		/* space for future expansion of vnet buffer header */
-
-  /* feature state used during this replication */
-  u64 feature_replicas;		/* feature's id for its set of replicas */
-  u32 feature_counter;		/* feature's current index into set of replicas */
-  u32 recycle_node_index;	/* feature's recycle node index */
-
-  /*
-   * data saved from the start of replication and restored
-   * at the end of replication
-   */
-  vlib_buffer_free_list_index_t saved_free_list_index;	/* from vlib buffer */
-
-  /* data saved from the original packet and restored for each replica */
-  u64 l2_header[3];		/*  24B (must be at least 22B for l2 packets) */
-  u32 flags;			/* vnet buffer flags */
-  u16 ip_tos;			/* v4 and v6 */
-  u16 ip4_checksum;		/* needed for v4 only */
-
-  /* data saved from the vlib buffer header and restored for each replica */
-  i16 current_data;		/* offset of first byte of packet in packet data */
-  u8 pad[2];			/* to 64B */
-  u8 l2_packet;			/* flag for l2 vs l3 packet data */
-
-} replication_context_t;	/* 128B */
-
-
-typedef struct
-{
-
-  u32 recycle_list_index;
-
-  /* per-thread pools of replication contexts */
-  replication_context_t **contexts;
-
-  vlib_main_t *vlib_main;
-  vnet_main_t *vnet_main;
-
-} replication_main_t;
-
-
-extern replication_main_t replication_main;
-
-
-/* Return 1 if this buffer just came from the replication recycle handler. */
-always_inline u32
-replication_is_recycled (vlib_buffer_t * b0)
-{
-  return b0->flags & VLIB_BUFFER_IS_RECYCLED;
-}
-
-/*
- * Clear the recycle flag. If buffer came from the replication recycle
- * handler, this flag must be cleared before the packet is transmitted again.
- */
-always_inline void
-replication_clear_recycled (vlib_buffer_t * b0)
-{
-  b0->flags &= ~VLIB_BUFFER_IS_RECYCLED;
-}
-
-/*
- * Return the active replication context if this buffer has
- * been recycled, otherwise return 0. (Note that this essentially
- * restricts access to the replication context to the replication
- * feature's prep and recycle nodes.)
- */
-always_inline replication_context_t *
-replication_get_ctx (vlib_buffer_t * b0)
-{
-  replication_main_t *rm = &replication_main;
-
-  return replication_is_recycled (b0) ?
-    pool_elt_at_index (rm->contexts[vlib_get_thread_index ()],
-		       b0->recycle_count) : 0;
-}
-
-/* Prefetch the replication context for this buffer, if it exists */
-always_inline void
-replication_prefetch_ctx (vlib_buffer_t * b0)
-{
-  replication_context_t *ctx = replication_get_ctx (b0);
-
-  if (ctx)
-    {
-      CLIB_PREFETCH (ctx, (2 * CLIB_CACHE_LINE_BYTES), STORE);
-    }
-}
-
-replication_context_t *replication_prep (vlib_main_t * vm,
-					 vlib_buffer_t * b0,
-					 u32 recycle_node_index,
-					 u32 l2_packet);
-
-replication_context_t *replication_recycle (vlib_main_t * vm,
-					    vlib_buffer_t * b0, u32 is_last);
-
-
-#endif
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */