Add jumbo frames support to non-dpdk vhost interfaces.

This code provided inter-VM (2 cores per VM) throughput of
22Gbps using iperf through VPP (1 core) with 9k frames.
With the same setup and pktgen running on both sides, it
reached 5Mpps with no packets drop (Equivalent to before the patch).
During the tests the average vector length was about 1, which
likely means that VPP is not the bottleneck.

The patch also includes some generic functions for vlib buffers
allowing for chained buffer construction whether or not DPDK is enabled.

Change-Id: Icfd1803e84b2b4578f305ab730576211f6242d6a
Signed-off-by: Pierre Pfister <ppfister@cisco.com>

3 files changed, 165 insertions, 0 deletions

diff --git a/vlib/vlib/buffer.c b/vlib/vlib/buffer.c
index 4463f7fdb4f..332b43044d3 100644
--- a/vlib/vlib/buffer.c
+++ b/vlib/vlib/buffer.c
@@ -1220,6 +1220,34 @@ u32 vlib_buffer_add_data (vlib_main_t * vm,
   return bi;
 }
 
+u16
+vlib_buffer_chain_append_data_with_alloc(vlib_main_t *vm,
+                             u32 free_list_index,
+                             vlib_buffer_t *first,
+                             vlib_buffer_t **last,
+                             void * data, u16 data_len) {
+  vlib_buffer_t *l = *last;
+  u32 n_buffer_bytes = vlib_buffer_free_list_buffer_size (vm, free_list_index);
+  u16 copied = 0;
+  ASSERT(n_buffer_bytes >= l->current_length + l->current_data);
+  while (data_len) {
+    u16 max = n_buffer_bytes - l->current_length - l->current_data;
+    if (max == 0) {
+      if (1 != vlib_buffer_alloc_from_free_list (vm, &l->next_buffer, 1, free_list_index))
+        return copied;
+      *last = l = vlib_buffer_chain_buffer(vm, first, l, l->next_buffer);
+      max = n_buffer_bytes - l->current_length - l->current_data;
+    }
+
+    u16 len = (data_len > max)?max:data_len;
+    memcpy(vlib_buffer_get_current (l) + l->current_length, data + copied, len);
+    vlib_buffer_chain_increase_length(first, l, len);
+    data_len -= len;
+    copied += len;
+  }
+  return copied;
+}
+
 static void vlib_serialize_tx (serialize_main_header_t * m, serialize_stream_t * s)
 {
   vlib_main_t * vm;
diff --git a/vlib/vlib/buffer_funcs.h b/vlib/vlib/buffer_funcs.h
index 452cdcb26a7..2ea506a094e 100644
--- a/vlib/vlib/buffer_funcs.h
+++ b/vlib/vlib/buffer_funcs.h
@@ -42,6 +42,19 @@
 
 #include <vppinfra/hash.h>
 
+#if DPDK == 1
+#undef always_inline // dpdk and clib use conflicting always_inline macros.
+#include <rte_config.h>
+#include <rte_mbuf.h>
+#include <rte_memcpy.h>
+
+#if CLIB_DEBUG > 0
+#define always_inline static inline
+#else
+#define always_inline static inline __attribute__ ((__always_inline__))
+#endif
+#endif
+
 /** \file
     vlib buffer access methods.
 */
@@ -398,6 +411,102 @@ u32 vlib_buffer_add_data (vlib_main_t * vm,
 			  u32 buffer_index,
 			  void * data, u32 n_data_bytes);
 
+/*
+ * vlib_buffer_chain_* functions provide a way to create long buffers.
+ * When DPDK is enabled, the 'hidden' DPDK header is taken care of transparently.
+ */
+
+/* Initializes the buffer as an empty packet with no chained buffers. */
+always_inline void
+vlib_buffer_chain_init(vlib_buffer_t *first)
+{
+  first->total_length_not_including_first_buffer = 0;
+  first->current_length = 0;
+  first->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
+  first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
+#if DPDK == 1
+  (((struct rte_mbuf *) first) - 1)->nb_segs = 1;
+  (((struct rte_mbuf *) first) - 1)->next = 0;
+  (((struct rte_mbuf *) first) - 1)->pkt_len = 0;
+  (((struct rte_mbuf *) first) - 1)->data_len = 0;
+  (((struct rte_mbuf *) first) - 1)->data_off = RTE_PKTMBUF_HEADROOM + first->current_data;
+#endif
+}
+
+/* The provided next_bi buffer index is appended to the end of the packet. */
+always_inline vlib_buffer_t *
+vlib_buffer_chain_buffer(vlib_main_t *vm,
+                    vlib_buffer_t *first,
+                    vlib_buffer_t *last,
+                    u32 next_bi)
+{
+  vlib_buffer_t *next_buffer = vlib_get_buffer(vm, next_bi);
+  last->next_buffer = next_bi;
+  last->flags |= VLIB_BUFFER_NEXT_PRESENT;
+  next_buffer->current_length = 0;
+  next_buffer->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
+#if DPDK == 1
+  (((struct rte_mbuf *) first) - 1)->nb_segs++;
+  (((struct rte_mbuf *) last) - 1)->next = (((struct rte_mbuf *) next_buffer) - 1);
+  (((struct rte_mbuf *) next_buffer) - 1)->data_len = 0;
+  (((struct rte_mbuf *) next_buffer) - 1)->data_off = RTE_PKTMBUF_HEADROOM + next_buffer->current_data;
+  (((struct rte_mbuf *) next_buffer) - 1)->next = 0;
+#endif
+  return next_buffer;
+}
+
+/* Increases or decreases the packet length.
+ * It does not allocate or deallocate new buffers.
+ * Therefore, the added length must be compatible
+ * with the last buffer. */
+always_inline void
+vlib_buffer_chain_increase_length(vlib_buffer_t *first,
+                             vlib_buffer_t *last,
+                             i32 len)
+{
+  last->current_length += len;
+  if (first != last)
+    first->total_length_not_including_first_buffer += len;
+#if DPDK == 1
+  (((struct rte_mbuf *) first) - 1)->pkt_len += len;
+  (((struct rte_mbuf *) last) - 1)->data_len += len;
+#endif
+}
+
+/* Copy data to the end of the packet and increases its length.
+ * It does not allocate new buffers.
+ * Returns the number of copied bytes. */
+always_inline u16
+vlib_buffer_chain_append_data(vlib_main_t *vm,
+                             u32 free_list_index,
+                             vlib_buffer_t *first,
+                             vlib_buffer_t *last,
+                             void *data, u16 data_len)
+{
+  u32 n_buffer_bytes = vlib_buffer_free_list_buffer_size (vm, free_list_index);
+  ASSERT(n_buffer_bytes >= last->current_length + last->current_data);
+  u16 len = clib_min(data_len, n_buffer_bytes - last->current_length - last->current_data);
+#if DPDK == 1
+  rte_memcpy(vlib_buffer_get_current (last) + last->current_length, data, len);
+#else
+  memcpy(vlib_buffer_get_current (last) + last->current_length, data, len);
+#endif
+  vlib_buffer_chain_increase_length(first, last, len);
+  return len;
+}
+
+/* Copy data to the end of the packet and increases its length.
+ * Allocates additional buffers from the free list if necessary.
+ * Returns the number of copied bytes.
+ * 'last' value is modified whenever new buffers are allocated and
+ * chained and points to the last buffer in the chain. */
+u16
+vlib_buffer_chain_append_data_with_alloc(vlib_main_t *vm,
+                             u32 free_list_index,
+                             vlib_buffer_t *first,
+                             vlib_buffer_t **last,
+                             void * data, u16 data_len);
+
 format_function_t format_vlib_buffer, format_vlib_buffer_and_data, format_vlib_buffer_contents;
 
 typedef struct {
diff --git a/vlib/vlib/dpdk_buffer.c b/vlib/vlib/dpdk_buffer.c
index 9db84a16d38..145720dd7a4 100644
--- a/vlib/vlib/dpdk_buffer.c
+++ b/vlib/vlib/dpdk_buffer.c
@@ -882,6 +882,34 @@ u32 vlib_buffer_add_data (vlib_main_t * vm,
   return bi;
 }
 
+u16
+vlib_buffer_chain_append_data_with_alloc(vlib_main_t *vm,
+                             u32 free_list_index,
+                             vlib_buffer_t *first,
+                             vlib_buffer_t **last,
+                             void * data, u16 data_len) {
+  vlib_buffer_t *l = *last;
+  u32 n_buffer_bytes = vlib_buffer_free_list_buffer_size (vm, free_list_index);
+  u16 copied = 0;
+  ASSERT(n_buffer_bytes >= l->current_length + l->current_data);
+  while (data_len) {
+    u16 max = n_buffer_bytes - l->current_length - l->current_data;
+    if (max == 0) {
+      if (1 != vlib_buffer_alloc_from_free_list (vm, &l->next_buffer, 1, free_list_index))
+        return copied;
+      *last = l = vlib_buffer_chain_buffer(vm, first, l, l->next_buffer);
+      max = n_buffer_bytes - l->current_length - l->current_data;
+    }
+
+    u16 len = (data_len > max)?max:data_len;
+    rte_memcpy(vlib_buffer_get_current (l) + l->current_length, data + copied, len);
+    vlib_buffer_chain_increase_length(first, l, len);
+    data_len -= len;
+    copied += len;
+  }
+  return copied;
+}
+
 clib_error_t *
 vlib_buffer_pool_create(vlib_main_t * vm, unsigned num_mbufs,
                         unsigned mbuf_size, unsigned socket_id)