af_xdp: use desc offset on rx

Instead of pre-programming the data offset on rx, use offset passed in
the descriptor. This is more robust and future-proof.

Type: fix

Change-Id: I2bd910d92b8b03d17be5be85a24108be711dc7b9
Signed-off-by: Benoît Ganne <bganne@cisco.com>

2 files changed, 26 insertions, 41 deletions

diff --git a/src/plugins/af_xdp/device.c b/src/plugins/af_xdp/device.c
index 03a3053e449..7a10bce4290 100644
--- a/src/plugins/af_xdp/device.c
+++ b/src/plugins/af_xdp/device.c
@@ -215,18 +215,7 @@ af_xdp_create_queue (vlib_main_t *vm, af_xdp_create_if_args_t *args,
   umem_config.comp_size = args->txq_size;
   umem_config.frame_size =
     sizeof (vlib_buffer_t) + vlib_buffer_get_default_data_size (vm);
-  /*
-   * Note about headroom: for some reasons, there seem to be a discrepency
-   * between 0-copy and copy mode:
-   *   - 0-copy: XDP_PACKET_HEADROOM will be added to the user headroom
-   *   - copy: nothing is added to the user headroom
-   * We privileged 0-copy and set headroom so that frame_headroom +
-   * XDP_PACKET_HEADROOM == sizeof(vlib_buffer_t), ie data will correctly
-   * point to vlib_buffer_t->data for 0-copy. In copy mode, we have to add
-   * XDP_PACKET_HEADROOM to desc offset during refill.
-   */
-  STATIC_ASSERT (sizeof (vlib_buffer_t) >= XDP_PACKET_HEADROOM, "wrong size");
-  umem_config.frame_headroom = sizeof (vlib_buffer_t) - XDP_PACKET_HEADROOM;
+  umem_config.frame_headroom = sizeof (vlib_buffer_t);
   umem_config.flags = XDP_UMEM_UNALIGNED_CHUNK_FLAG;
   if (xsk_umem__create
       (umem, uword_to_pointer (vm->buffer_main->buffer_mem_start, void *),
diff --git a/src/plugins/af_xdp/input.c b/src/plugins/af_xdp/input.c
index 98d841b77a0..4f3ac5725a4 100644
--- a/src/plugins/af_xdp/input.c
+++ b/src/plugins/af_xdp/input.c
@@ -106,8 +106,7 @@ af_xdp_device_input_refill_db (vlib_main_t * vm,
 static_always_inline void
 af_xdp_device_input_refill_inline (vlib_main_t *vm,
 				   const vlib_node_runtime_t *node,
-				   af_xdp_device_t *ad, af_xdp_rxq_t *rxq,
-				   const int copy)
+				   af_xdp_device_t *ad, af_xdp_rxq_t *rxq)
 {
   __u64 *fill;
   const u32 size = rxq->fq.size;
@@ -133,13 +132,7 @@ af_xdp_device_input_refill_inline (vlib_main_t *vm,
   n = clib_min (n_alloc, size - (idx & mask));
   n_wrap = n_alloc - n;
 
-  /*
-   * Note about headroom: for some reasons, there seem to be a discrepency
-   * between 0-copy and copy mode. See
-   * src/plugins/af_xdp/device.c:af_xdp_create_queue()
-   */
-#define bi2addr(bi)                                                           \
-  (((bi) << CLIB_LOG2_CACHE_LINE_BYTES) + (copy ? XDP_PACKET_HEADROOM : 0))
+#define bi2addr(bi) ((bi) << CLIB_LOG2_CACHE_LINE_BYTES)
 
 wrap_around:
 
@@ -208,27 +201,30 @@ af_xdp_device_input_ethernet (vlib_main_t * vm, vlib_node_runtime_t * node,
 }
 
 static_always_inline u32
-af_xdp_device_input_bufs (vlib_main_t * vm, const af_xdp_device_t * ad,
-			  af_xdp_rxq_t * rxq, u32 * bis, const u32 n_rx,
-			  vlib_buffer_t * bt, u32 idx, const int copy)
+af_xdp_device_input_bufs (vlib_main_t *vm, const af_xdp_device_t *ad,
+			  af_xdp_rxq_t *rxq, u32 *bis, const u32 n_rx,
+			  vlib_buffer_t *bt, u32 idx)
 {
   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
+  u16 offs[VLIB_FRAME_SIZE], *off = offs;
   u16 lens[VLIB_FRAME_SIZE], *len = lens;
   const u32 mask = rxq->rx.mask;
   u32 n = n_rx, *bi = bis, bytes = 0;
 
-#define addr2bi(addr)                                                         \
-  (((addr) - (copy ? XDP_PACKET_HEADROOM : 0)) >> CLIB_LOG2_CACHE_LINE_BYTES)
+#define addr2bi(addr) ((addr) >> CLIB_LOG2_CACHE_LINE_BYTES)
 
   while (n >= 1)
     {
       const struct xdp_desc *desc = xsk_ring_cons__rx_desc (&rxq->rx, idx);
-      bi[0] = addr2bi (xsk_umem__extract_addr (desc->addr));
+      const u64 addr = desc->addr;
+      bi[0] = addr2bi (xsk_umem__extract_addr (addr));
       ASSERT (vlib_buffer_is_known (vm, bi[0]) ==
 	      VLIB_BUFFER_KNOWN_ALLOCATED);
+      off[0] = xsk_umem__extract_offset (addr) - sizeof (vlib_buffer_t);
       len[0] = desc->len;
       idx = (idx + 1) & mask;
       bi += 1;
+      off += 1;
       len += 1;
       n -= 1;
     }
@@ -236,27 +232,33 @@ af_xdp_device_input_bufs (vlib_main_t * vm, const af_xdp_device_t * ad,
   vlib_get_buffers (vm, bis, bufs, n_rx);
 
   n = n_rx;
+  off = offs;
   len = lens;
 
   while (n >= 8)
     {
       vlib_prefetch_buffer_header (b[4], LOAD);
       vlib_buffer_copy_template (b[0], bt);
+      b[0]->current_data = off[0];
       bytes += b[0]->current_length = len[0];
 
       vlib_prefetch_buffer_header (b[5], LOAD);
       vlib_buffer_copy_template (b[1], bt);
+      b[1]->current_data = off[1];
       bytes += b[1]->current_length = len[1];
 
       vlib_prefetch_buffer_header (b[6], LOAD);
       vlib_buffer_copy_template (b[2], bt);
+      b[2]->current_data = off[2];
       bytes += b[2]->current_length = len[2];
 
       vlib_prefetch_buffer_header (b[7], LOAD);
       vlib_buffer_copy_template (b[3], bt);
+      b[3]->current_data = off[3];
       bytes += b[3]->current_length = len[3];
 
       b += 4;
+      off += 4;
       len += 4;
       n -= 4;
     }
@@ -264,8 +266,10 @@ af_xdp_device_input_bufs (vlib_main_t * vm, const af_xdp_device_t * ad,
   while (n >= 1)
     {
       vlib_buffer_copy_template (b[0], bt);
+      b[0]->current_data = off[0];
       bytes += b[0]->current_length = len[0];
       b += 1;
+      off += 1;
       len += 1;
       n -= 1;
     }
@@ -275,9 +279,8 @@ af_xdp_device_input_bufs (vlib_main_t * vm, const af_xdp_device_t * ad,
 }
 
 static_always_inline uword
-af_xdp_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
-			    vlib_frame_t * frame, af_xdp_device_t * ad,
-			    u16 qid, const int copy)
+af_xdp_device_input_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
+			    vlib_frame_t *frame, af_xdp_device_t *ad, u16 qid)
 {
   vnet_main_t *vnm = vnet_get_main ();
   af_xdp_rxq_t *rxq = vec_elt_at_index (ad->rxqs, qid);
@@ -299,8 +302,7 @@ af_xdp_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
   vlib_get_new_next_frame (vm, node, next_index, to_next, n_left_to_next);
 
   n_rx_bytes =
-    af_xdp_device_input_bufs (vm, ad, rxq, to_next, n_rx_packets, &bt, idx,
-			      copy);
+    af_xdp_device_input_bufs (vm, ad, rxq, to_next, n_rx_packets, &bt, idx);
   af_xdp_device_input_ethernet (vm, node, next_index, ad->sw_if_index,
 				ad->hw_if_index);
 
@@ -315,7 +317,7 @@ af_xdp_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
      ad->hw_if_index, n_rx_packets, n_rx_bytes);
 
 refill:
-  af_xdp_device_input_refill_inline (vm, node, ad, rxq, copy);
+  af_xdp_device_input_refill_inline (vm, node, ad, rxq);
 
   return n_rx_packets;
 }
@@ -334,12 +336,7 @@ VLIB_NODE_FN (af_xdp_input_node) (vlib_main_t * vm,
       af_xdp_device_t *ad = vec_elt_at_index (am->devices, p->dev_instance);
       if ((ad->flags & AF_XDP_DEVICE_F_ADMIN_UP) == 0)
 	continue;
-      if (PREDICT_TRUE (ad->flags & AF_XDP_DEVICE_F_ZEROCOPY))
-	n_rx += af_xdp_device_input_inline (vm, node, frame, ad, p->queue_id,
-					    /* copy */ 0);
-      else
-	n_rx += af_xdp_device_input_inline (vm, node, frame, ad, p->queue_id,
-					    /* copy */ 1);
+      n_rx += af_xdp_device_input_inline (vm, node, frame, ad, p->queue_id);
     }
 
   return n_rx;
@@ -352,8 +349,7 @@ af_xdp_device_input_refill (af_xdp_device_t *ad)
   vlib_main_t *vm = vlib_get_main ();
   af_xdp_rxq_t *rxq;
   vec_foreach (rxq, ad->rxqs)
-    af_xdp_device_input_refill_inline (
-      vm, 0, ad, rxq, 0 == (ad->flags & AF_XDP_DEVICE_F_ZEROCOPY));
+    af_xdp_device_input_refill_inline (vm, 0, ad, rxq);
 }
 #endif /* CLIB_MARCH_VARIANT */