summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDamjan Marion <damarion@cisco.com>2018-06-03 14:29:42 +0200
committerDamjan Marion <damarion@cisco.com>2018-06-03 14:46:12 +0200
commit65165072d49c5a6fdb4b29bbed30d633bdf051c8 (patch)
tree355adce8f26866af257bce8ea8132c1da721af36
parent8855386411af888e47c60645daa1fe6081fa56e1 (diff)
dpdk: buffer free optimizations
~5 clocks/packet improvement... Change-Id: I1a78fa24dcd1b3ab7f45e10b9ded50f79517114a Signed-off-by: Damjan Marion <damarion@cisco.com>
Diffstat
-rw-r--r--src/plugins/dpdk/buffer.c137
1 files changed, 61 insertions, 76 deletions
diff --git a/src/plugins/dpdk/buffer.c b/src/plugins/dpdk/buffer.c
index 78d5becad78..31e8eee25a9 100644
--- a/src/plugins/dpdk/buffer.c
+++ b/src/plugins/dpdk/buffer.c
@@ -89,11 +89,6 @@ typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
struct rte_mbuf **mbuf_alloc_list;
- struct rte_mbuf ***mbuf_pending_free_list;
-
- /* cached last pool */
- struct rte_mempool *last_pool;
- u8 last_buffer_pool_index;
} dpdk_buffer_per_thread_data;
typedef struct
@@ -105,14 +100,11 @@ typedef struct
dpdk_buffer_main_t dpdk_buffer_main;
static_always_inline void
-dpdk_rte_pktmbuf_free (vlib_main_t * vm, u32 thread_index, vlib_buffer_t * b)
+dpdk_rte_pktmbuf_free (vlib_main_t * vm, u32 thread_index, vlib_buffer_t * b,
+ int maybe_next)
{
- vlib_buffer_t *hb = b;
- dpdk_buffer_main_t *dbm = &dpdk_buffer_main;
- dpdk_buffer_per_thread_data *d = vec_elt_at_index (dbm->ptd, thread_index);
struct rte_mbuf *mb;
u32 next, flags;
- mb = rte_mbuf_from_vlib_buffer (hb);
next:
flags = b->flags;
@@ -125,22 +117,10 @@ next:
b->n_add_refs = 0;
}
- mb = rte_pktmbuf_prefree_seg (mb);
- if (mb)
- {
- if (mb->pool != d->last_pool)
- {
- d->last_pool = mb->pool;
- dpdk_mempool_private_t *privp = rte_mempool_get_priv (d->last_pool);
- d->last_buffer_pool_index = privp->buffer_pool_index;
- vec_validate_aligned (d->mbuf_pending_free_list,
- d->last_buffer_pool_index,
- CLIB_CACHE_LINE_BYTES);
- }
- vec_add1 (d->mbuf_pending_free_list[d->last_buffer_pool_index], mb);
- }
+ if ((mb = rte_pktmbuf_prefree_seg (mb)))
+ rte_mempool_put (mb->pool, mb);
- if (flags & VLIB_BUFFER_NEXT_PRESENT)
+ if (maybe_next && (flags & VLIB_BUFFER_NEXT_PRESENT))
{
b = vlib_get_buffer (vm, next);
goto next;
@@ -158,7 +138,7 @@ del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f)
for (i = 0; i < vec_len (f->buffers); i++)
{
b = vlib_get_buffer (vm, f->buffers[i]);
- dpdk_rte_pktmbuf_free (vm, thread_index, b);
+ dpdk_rte_pktmbuf_free (vm, thread_index, b, 1);
}
vec_free (f->name);
@@ -288,13 +268,11 @@ CLIB_MULTIARCH_FN (dpdk_buffer_fill_free_list) (vlib_main_t * vm,
}
static_always_inline void
-dpdk_prefetch_buffer_by_index (vlib_main_t * vm, u32 bi)
+dpdk_prefetch_buffer (vlib_buffer_t * b)
{
- vlib_buffer_t *b;
struct rte_mbuf *mb;
- b = vlib_get_buffer (vm, bi);
mb = rte_mbuf_from_vlib_buffer (b);
- CLIB_PREFETCH (mb, 2 * CLIB_CACHE_LINE_BYTES, STORE);
+ CLIB_PREFETCH (mb, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD);
}
@@ -324,11 +302,8 @@ recycle_or_free (vlib_main_t * vm, vlib_buffer_main_t * bm, u32 bi,
already_announced:
;
}
- else
- {
- if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0))
- dpdk_rte_pktmbuf_free (vm, thread_index, b);
- }
+ else if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0))
+ dpdk_rte_pktmbuf_free (vm, thread_index, b, 1);
}
static_always_inline void
@@ -336,11 +311,12 @@ vlib_buffer_free_inline (vlib_main_t * vm,
u32 * buffers, u32 n_buffers, u32 follow_buffer_next)
{
vlib_buffer_main_t *bm = &buffer_main;
- dpdk_buffer_main_t *dbm = &dpdk_buffer_main;
- vlib_buffer_t *b0, *b1, *b2, *b3;
+ vlib_buffer_t *bufp[n_buffers], **b = bufp;
u32 thread_index = vlib_get_thread_index ();
- dpdk_buffer_per_thread_data *d = vec_elt_at_index (dbm->ptd, thread_index);
int i = 0;
+ u32 simple_mask = (VLIB_BUFFER_NON_DEFAULT_FREELIST | VLIB_BUFFER_RECYCLE |
+ VLIB_BUFFER_NEXT_PRESENT);
+ u32 n_left, *bi;
u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
u32 follow_buffer_next);
@@ -352,36 +328,56 @@ vlib_buffer_free_inline (vlib_main_t * vm,
if (!n_buffers)
return;
- while (i + 7 < n_buffers)
+ n_left = n_buffers;
+ bi = buffers;
+ b = bufp;
+ vlib_get_buffers (vm, bi, b, n_buffers);
+
+ while (n_left >= 4)
{
- dpdk_prefetch_buffer_by_index (vm, buffers[i + 4]);
- dpdk_prefetch_buffer_by_index (vm, buffers[i + 5]);
- dpdk_prefetch_buffer_by_index (vm, buffers[i + 6]);
- dpdk_prefetch_buffer_by_index (vm, buffers[i + 7]);
-
- b0 = vlib_get_buffer (vm, buffers[i]);
- b1 = vlib_get_buffer (vm, buffers[i + 1]);
- b2 = vlib_get_buffer (vm, buffers[i + 2]);
- b3 = vlib_get_buffer (vm, buffers[i + 3]);
-
- VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
- VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b1);
- VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b2);
- VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b3);
-
- recycle_or_free (vm, bm, buffers[i], b0);
- recycle_or_free (vm, bm, buffers[i + 1], b1);
- recycle_or_free (vm, bm, buffers[i + 2], b2);
- recycle_or_free (vm, bm, buffers[i + 3], b3);
-
- i += 4;
+ u32 or_flags;
+ vlib_buffer_t **p;
+
+ if (n_left < 16)
+ goto no_prefetch;
+
+ p = b + 12;
+ dpdk_prefetch_buffer (p[0]);
+ dpdk_prefetch_buffer (p[1]);
+ dpdk_prefetch_buffer (p[2]);
+ dpdk_prefetch_buffer (p[3]);
+ no_prefetch:
+
+ for (i = 0; i < 4; i++)
+ VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[i]);
+
+ or_flags = b[0]->flags | b[1]->flags | b[2]->flags | b[3]->flags;
+
+ if (or_flags & simple_mask)
+ {
+ recycle_or_free (vm, bm, bi[0], b[0]);
+ recycle_or_free (vm, bm, bi[1], b[1]);
+ recycle_or_free (vm, bm, bi[2], b[2]);
+ recycle_or_free (vm, bm, bi[3], b[3]);
+ }
+ else
+ {
+ dpdk_rte_pktmbuf_free (vm, thread_index, b[0], 0);
+ dpdk_rte_pktmbuf_free (vm, thread_index, b[1], 0);
+ dpdk_rte_pktmbuf_free (vm, thread_index, b[2], 0);
+ dpdk_rte_pktmbuf_free (vm, thread_index, b[3], 0);
+ }
+ bi += 4;
+ b += 4;
+ n_left -= 4;
}
- while (i < n_buffers)
+ while (n_left)
{
- b0 = vlib_get_buffer (vm, buffers[i]);
- VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
- recycle_or_free (vm, bm, buffers[i], b0);
- i++;
+ VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]);
+ recycle_or_free (vm, bm, bi[0], b[0]);
+ bi += 1;
+ b += 1;
+ n_left -= 1;
}
if (vec_len (vm->buffer_announce_list))
{
@@ -393,17 +389,6 @@ vlib_buffer_free_inline (vlib_main_t * vm,
}
_vec_len (vm->buffer_announce_list) = 0;
}
-
- vec_foreach_index (i, d->mbuf_pending_free_list)
- {
- int len = vec_len (d->mbuf_pending_free_list[i]);
- if (len)
- {
- rte_mempool_put_bulk (d->mbuf_pending_free_list[i][len - 1]->pool,
- (void *) d->mbuf_pending_free_list[i], len);
- vec_reset_length (d->mbuf_pending_free_list[i]);
- }
- }
}
void
olor: #0000DD; font-weight: bold } /* Literal.Number.Hex */ .highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */ .highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */ .highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */ .highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */ .highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */ .highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */ .highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */ .highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */ .highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */ .highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */ .highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */ .highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */ .highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */ .highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */ .highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */ .highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */ .highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */ .highlight .vc { color: #336699 } /* Name.Variable.Class */ .highlight .vg { color: #dd7700 } /* Name.Variable.Global */ .highlight .vi { color: #3333bb } /* Name.Variable.Instance */ .highlight .vm { color: #336699 } /* Name.Variable.Magic */ .highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */ } 
/*
  macros.c - a simple macro expander

  Copyright (c) 2010, 2014 Cisco and/or its affiliates.

  * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
*/

#include <vppinfra/macros.h>

static inline int
macro_isalnum (i8 c)
{
  if ((c >= 'A' && c <= 'Z')
      || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || (c == '_'))
    return 1;
  return 0;
}

static i8 *
builtin_eval (macro_main_t * mm, i8 * varname, i32 complain)
{
  uword *p;
  i8 *(*fp) (macro_main_t *, i32);

  p = hash_get_mem (mm->the_builtin_eval_hash, varname);
  if (p == 0)
    return 0;
  fp = (void *) (p[0]);
  return (*fp) (mm, complain);
}

int
clib_macro_unset (macro_main_t * mm, char *name)
{
  hash_pair_t *p;
  u8 *key, *value;

  p = hash_get_pair (mm->the_value_table_hash, name);

  if (p == 0)
    return 1;

  key = (u8 *) (p->key);
  value = (u8 *) (p->value[0]);
  hash_unset_mem (mm->the_value_table_hash, name);

  vec_free (value);
  vec_free (key);
  return 0;
}

int
clib_macro_set_value (macro_main_t * mm, char *name, char *value)
{
  u8 *key_copy, *value_copy;
  int rv;

  rv = clib_macro_unset (mm, name);

  key_copy = format (0, "%s%c", name, 0);
  value_copy = format (0, "%s%c", value, 0);

  hash_set_mem (mm->the_value_table_hash, key_copy, value_copy);
  return rv;
}

i8 *
clib_macro_get_value (macro_main_t * mm, char *name)
{
  uword *p;

  p = hash_get_mem (mm->the_value_table_hash, name);
  if (p)
    return (i8 *) (p[0]);
  else
    return 0;
}

/*
 * eval: takes a string, returns a vector.
 * looks up $foobar in the variable table.
 */
i8 *
clib_macro_eval (macro_main_t * mm, i8 * s, i32 complain)
{
  i8 *rv = 0;
  i8 *varname, *varvalue;
  i8 *ts;

  while (*s)
    {
      switch (*s)
	{
	case '\\':
	  s++;
	  /* fallthrough */

	default:
	  vec_add1 (rv, *s);
	  s++;
	  break;

	case '$':
	  s++;
	  varname = 0;
	  /*
	   * Make vector with variable name in it.
	   */
	  while (*s && (macro_isalnum (*s) || (*s == '_') || (*s == '(')))
	    {

	      /* handle $(foo) */
	      if (*s == '(')
		{
		  s++;		/* skip '(' */
		  while (*s && *s != ')')
		    {
		      vec_add1 (varname, *s);
		      s++;
		    }
		  if (*s)
		    s++;	/* skip ')' */
		  break;
		}
	      vec_add1 (varname, *s);
	      s++;
	    }
	  /* null terminate */
	  vec_add1 (varname, 0);
	  /* Look for a builtin, e.g. $my_hostname */
	  if (!(varvalue = builtin_eval (mm, varname, complain)))
	    {
	      /* Look in value table */
	      if (!varvalue)
		{
		  i8 *tmp = clib_macro_get_value (mm, (char *) varname);
		  if (tmp)
		    varvalue = (i8 *) format (0, "%s%c", tmp, 0);
		}
#ifdef CLIB_UNIX
	      /* Look in environment. */
	      if (!varvalue)
		{
		  char *tmp = getenv ((char *) varname);
		  if (tmp)
		    varvalue = (i8 *) format (0, "%s%c", tmp, 0);
		}
#endif /* CLIB_UNIX */
	    }
	  if (varvalue)
	    {
	      /* recursively evaluate */
	      ts = clib_macro_eval (mm, varvalue, complain);
	      vec_free (varvalue);
	      /* add results to answer */
	      vec_append (rv, ts);
	      /* Remove NULL termination or the results are sad */
	      _vec_len (rv) = vec_len (rv) - 1;
	      vec_free (ts);
	    }
	  else
	    {
	      if (complain)
		clib_warning ("Undefined Variable Reference: %s\n", varname);
	      vec_append (rv, format (0, "UNSET "));
	      _vec_len (rv) = vec_len (rv) - 1;

	    }
	  vec_free (varname);
	}
    }
  vec_add1 (rv, 0);
  return (rv);
}

/*
 * eval: takes a string, returns a vector.
 * looks up $foobar in the variable table.
 */
i8 *
clib_macro_eval_dollar (macro_main_t * mm, i8 * s, i32 complain)
{
  i8 *s2;
  i8 *rv;

  s2 = (i8 *) format (0, "$(%s)%c", s, 0);
  rv = clib_macro_eval (mm, s2, complain);
  vec_free (s2);
  return (rv);
}

void
clib_macro_add_builtin (macro_main_t * mm, char *name, void *eval_fn)
{
  hash_set_mem (mm->the_builtin_eval_hash, name, (uword) eval_fn);
}

#ifdef CLIB_UNIX
static i8 *
eval_hostname (macro_main_t * mm, i32 complain)
{
  char tmp[128];
  if (gethostname (tmp, sizeof (tmp)))
    return ((i8 *) format (0, "gethostname-error%c", 0));
  return ((i8 *) format (0, "%s%c", tmp, 0));
}
#endif

void
clib_macro_init (macro_main_t * mm)
{
  if (mm->the_builtin_eval_hash != 0)
    {
      clib_warning ("mm %p already initialized", mm);
      return;
    }

  mm->the_builtin_eval_hash = hash_create_string (0, sizeof (uword));
  mm->the_value_table_hash = hash_create_string (0, sizeof (uword));

#ifdef CLIB_UNIX
  hash_set_mem (mm->the_builtin_eval_hash, "hostname", (uword) eval_hostname);
#endif
}

void
clib_macro_free (macro_main_t * mm)
{
  hash_pair_t *p;
  u8 **strings_to_free = 0;
  int i;

  hash_free (mm->the_builtin_eval_hash);

  /* *INDENT-OFF* */
  hash_foreach_pair (p, mm->the_value_table_hash,
  ({
    vec_add1 (strings_to_free, (u8 *) (p->key));
    vec_add1 (strings_to_free, (u8 *) (p->value[0]));
  }));
  /* *INDENT-ON* */

  for (i = 0; i < vec_len (strings_to_free); i++)
    vec_free (strings_to_free[i]);
  vec_free (strings_to_free);
  hash_free (mm->the_value_table_hash);
}

/*
 * fd.io coding-style-patch-verification: ON
 *
 * Local Variables:
 * eval: (c-set-style "gnu")
 * End:
 */