summaryrefslogtreecommitdiffstats
path: root/src/plugins/af_xdp/device.c
diff options
context:
space:
mode:
authorBenoît Ganne <bganne@cisco.com>2021-08-25 16:53:22 +0200
committerDamjan Marion <dmarion@me.com>2021-08-26 14:44:39 +0000
commit4fe2f4c29e3672f38038d34f177a9ccbff18d9ad (patch)
tree6ba49d054ca502484db4f8f7cc1bc470df1bcd8d /src/plugins/af_xdp/device.c
parentad8b82d8394e7e711b2c1953b3c273aab4920355 (diff)
af_xdp: fix stale rx/tx pointers in xsk objects
xsk objects keep pointers to the rx and tx objects. If we re-allocate the rx and tx vectors after initializing the associated xsk object, the pointers in the xsk object will be staled. To avoid this, we allocate the vectors to the max expected size instead of growing them. Type: fix Change-Id: If30433a28c186787d66c12dbab34bf210c95b519 Signed-off-by: Benoît Ganne <bganne@cisco.com>
Diffstat (limited to 'src/plugins/af_xdp/device.c')
-rw-r--r--src/plugins/af_xdp/device.c57
1 files changed, 47 insertions, 10 deletions
diff --git a/src/plugins/af_xdp/device.c b/src/plugins/af_xdp/device.c
index 2600170704b..c82b700bbea 100644
--- a/src/plugins/af_xdp/device.c
+++ b/src/plugins/af_xdp/device.c
@@ -17,7 +17,10 @@
#include <stdio.h>
#include <net/if.h>
+#include <sys/ioctl.h>
+#include <linux/ethtool.h>
#include <linux/if_link.h>
+#include <linux/sockios.h>
#include <bpf/libbpf.h>
#include <vlib/vlib.h>
#include <vlib/unix/unix.h>
@@ -188,16 +191,9 @@ af_xdp_create_queue (vlib_main_t *vm, af_xdp_create_if_args_t *args,
const int is_rx = qid < ad->rxq_num;
const int is_tx = qid < ad->txq_num;
- vec_validate_aligned (ad->umem, qid, CLIB_CACHE_LINE_BYTES);
umem = vec_elt_at_index (ad->umem, qid);
-
- vec_validate_aligned (ad->xsk, qid, CLIB_CACHE_LINE_BYTES);
xsk = vec_elt_at_index (ad->xsk, qid);
-
- vec_validate_aligned (ad->rxqs, qid, CLIB_CACHE_LINE_BYTES);
rxq = vec_elt_at_index (ad->rxqs, qid);
-
- vec_validate_aligned (ad->txqs, qid, CLIB_CACHE_LINE_BYTES);
txq = vec_elt_at_index (ad->txqs, qid);
/*
@@ -321,6 +317,31 @@ af_xdp_get_numa (const char *ifname)
return numa;
}
+static void
+af_xdp_get_q_count (const char *ifname, int *rxq_num, int *txq_num)
+{
+ struct ethtool_channels ec = { .cmd = ETHTOOL_GCHANNELS };
+ struct ifreq ifr = { .ifr_data = (void *) &ec };
+ int fd, err;
+
+ *rxq_num = *txq_num = 1;
+
+ fd = socket (AF_INET, SOCK_DGRAM, 0);
+ if (fd < 0)
+ return;
+
+ snprintf (ifr.ifr_name, sizeof (ifr.ifr_name), "%s", ifname);
+ err = ioctl (fd, SIOCETHTOOL, &ifr);
+
+ close (fd);
+
+ if (err)
+ return;
+
+ *rxq_num = clib_max (ec.combined_count, ec.rx_count);
+ *txq_num = clib_max (ec.combined_count, ec.tx_count);
+}
+
static clib_error_t *
af_xdp_device_rxq_read_ready (clib_file_t * f)
{
@@ -375,8 +396,7 @@ af_xdp_create_if (vlib_main_t * vm, af_xdp_create_if_args_t * args)
args->rxq_size = args->rxq_size ? args->rxq_size : 2 * VLIB_FRAME_SIZE;
args->txq_size = args->txq_size ? args->txq_size : 2 * VLIB_FRAME_SIZE;
- rxq_num = args->rxq_num ? args->rxq_num : 1;
- txq_num = tm->n_vlib_mains;
+ args->rxq_num = args->rxq_num ? args->rxq_num : 1;
if (!args->linux_ifname)
{
@@ -397,6 +417,17 @@ af_xdp_create_if (vlib_main_t * vm, af_xdp_create_if_args_t * args)
goto err0;
}
+ af_xdp_get_q_count (args->linux_ifname, &rxq_num, &txq_num);
+ if (args->rxq_num > rxq_num && AF_XDP_NUM_RX_QUEUES_ALL != args->rxq_num)
+ {
+ args->rv = VNET_API_ERROR_INVALID_VALUE;
+ args->error = clib_error_create ("too many rxq requested (%d > %d)",
+ args->rxq_num, rxq_num);
+ goto err0;
+ }
+ rxq_num = clib_min (rxq_num, args->rxq_num);
+ txq_num = clib_min (txq_num, tm->n_vlib_mains);
+
pool_get_zero (am->devices, ad);
if (tm->n_vlib_mains > 1 &&
@@ -412,6 +443,12 @@ af_xdp_create_if (vlib_main_t * vm, af_xdp_create_if_args_t * args)
q_num = clib_max (rxq_num, txq_num);
ad->rxq_num = rxq_num;
ad->txq_num = txq_num;
+
+ vec_validate_aligned (ad->umem, q_num - 1, CLIB_CACHE_LINE_BYTES);
+ vec_validate_aligned (ad->xsk, q_num - 1, CLIB_CACHE_LINE_BYTES);
+ vec_validate_aligned (ad->rxqs, q_num - 1, CLIB_CACHE_LINE_BYTES);
+ vec_validate_aligned (ad->txqs, q_num - 1, CLIB_CACHE_LINE_BYTES);
+
for (i = 0; i < q_num; i++)
{
if (af_xdp_create_queue (vm, args, ad, i))
@@ -433,7 +470,7 @@ af_xdp_create_if (vlib_main_t * vm, af_xdp_create_if_args_t * args)
ad->rxq_num = clib_min (i, rxq_num);
ad->txq_num = clib_min (i, txq_num);
- if (i < rxq_num && AF_XDP_NUM_RX_QUEUES_ALL != rxq_num)
+ if (i < rxq_num && AF_XDP_NUM_RX_QUEUES_ALL != args->rxq_num)
{
ad->rxq_num = ad->txq_num = 0;
goto err1; /* failed creating requested rxq: fatal error, bailing
e } /* Name.Attribute */ .highlight .nb { color: #f8f8f2 } /* Name.Builtin */ .highlight .nc { color: #a6e22e } /* Name.Class */ .highlight .no { color: #66d9ef } /* Name.Constant */ .highlight .nd { color: #a6e22e } /* Name.Decorator */ .highlight .ni { color: #f8f8f2 } /* Name.Entity */ .highlight .ne { color: #a6e22e } /* Name.Exception */ .highlight .nf { color: #a6e22e } /* Name.Function */ .highlight .nl { color: #f8f8f2 } /* Name.Label */ .highlight .nn { color: #f8f8f2 } /* Name.Namespace */ .highlight .nx { color: #a6e22e } /* Name.Other */ .highlight .py { color: #f8f8f2 } /* Name.Property */ .highlight .nt { color: #f92672 } /* Name.Tag */ .highlight .nv { color: #f8f8f2 } /* Name.Variable */ .highlight .ow { color: #f92672 } /* Operator.Word */ .highlight .w { color: #f8f8f2 } /* Text.Whitespace */ .highlight .mb { color: #ae81ff } /* Literal.Number.Bin */ .highlight .mf { color: #ae81ff } /* Literal.Number.Float */ .highlight .mh { color: #ae81ff } /* Literal.Number.Hex */ .highlight .mi { color: #ae81ff } /* Literal.Number.Integer */ .highlight .mo { color: #ae81ff } /* Literal.Number.Oct */ .highlight .sa { color: #e6db74 } /* Literal.String.Affix */ .highlight .sb { color: #e6db74 } /* Literal.String.Backtick */ .highlight .sc { color: #e6db74 } /* Literal.String.Char */ .highlight .dl { color: #e6db74 } /* Literal.String.Delimiter */ .highlight .sd { color: #e6db74 } /* Literal.String.Doc */ .highlight .s2 { color: #e6db74 } /* Literal.String.Double */ .highlight .se { color: #ae81ff } /* Literal.String.Escape */ .highlight .sh { color: #e6db74 } /* Literal.String.Heredoc */ .highlight .si { color: #e6db74 } /* Literal.String.Interpol */ .highlight .sx { color: #e6db74 } /* Literal.String.Other */ .highlight .sr { color: #e6db74 } /* Literal.String.Regex */ .highlight .s1 { color: #e6db74 } /* Literal.String.Single */ .highlight .ss { color: #e6db74 } /* Literal.String.Symbol */ .highlight .bp { color: #f8f8f2 } /* Name.Builtin.Pseudo */ .highlight .fm { color: #a6e22e } /* Name.Function.Magic */ .highlight .vc { color: #f8f8f2 } /* Name.Variable.Class */ .highlight .vg { color: #f8f8f2 } /* Name.Variable.Global */ .highlight .vi { color: #f8f8f2 } /* Name.Variable.Instance */ .highlight .vm { color: #f8f8f2 } /* Name.Variable.Magic */ .highlight .il { color: #ae81ff } /* Literal.Number.Integer.Long */ } @media (prefers-color-scheme: light) { .highlight .hll { background-color: #ffffcc } .highlight .c { color: #888888 } /* Comment */ .highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */ .highlight .k { color: #008800; font-weight: bold } /* Keyword */ .highlight .ch { color: #888888 } /* Comment.Hashbang */ .highlight .cm { color: #888888 } /* Comment.Multiline */ .highlight .cp { color: #cc0000; font-weight: bold } /* Comment.Preproc */ .highlight .cpf { color: #888888 } /* Comment.PreprocFile */ .highlight .c1 { color: #888888 } /* Comment.Single */ .highlight .cs { color: #cc0000; font-weight: bold; background-color: #fff0f0 } /* Comment.Special */ .highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */ .highlight .ge { font-style: italic } /* Generic.Emph */ .highlight .gr { color: #aa0000 } /* Generic.Error */ .highlight .gh { color: #333333 } /* Generic.Heading */ .highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */ .highlight .go { color: #888888 } /* Generic.Output */ .highlight .gp { color: #555555 } /* Generic.Prompt */ .highlight .gs { font-weight: bold } /* Generic.Strong */ .highlight .gu { color: #666666 } /* Generic.Subheading */ .highlight .gt { color: #aa0000 } /* Generic.Traceback */ .highlight .kc { color: #008800; font-weight: bold } /* Keyword.Constant */ .highlight .kd { color: #008800; font-weight: bold } /* Keyword.Declaration */ .highlight .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */ .highlight .kp { color: #008800 } /* Keyword.Pseudo */ .highlight .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */ .highlight .kt { color: #888888; font-weight: bold } /* Keyword.Type */ .highlight .m { color: #0000DD; font-weight: bold } /* Literal.Number */ .highlight .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */ .highlight .na { color: #336699 } /* Name.Attribute */ .highlight .nb { color: #003388 } /* Name.Builtin */ .highlight .nc { color: #bb0066; font-weight: bold } /* Name.Class */ .highlight .no { color: #003366; font-weight: bold } /* Name.Constant */ .highlight .nd { color: #555555 } /* Name.Decorator */ .highlight .ne { color: #bb0066; font-weight: bold } /* Name.Exception */ .highlight .nf { color: #0066bb; font-weight: bold } /* Name.Function */ .highlight .nl { color: #336699; font-style: italic } /* Name.Label */ .highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */ .highlight .py { color: #336699; font-weight: bold } /* Name.Property */ .highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */ .highlight .nv { color: #336699 } /* Name.Variable */ .highlight .ow { color: #008800 } /* Operator.Word */ .highlight .w { color: #bbbbbb } /* Text.Whitespace */ .highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */ .highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */ .highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */ .highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */ .highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */ .highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */ .highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */ .highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */ .highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */ .highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */ .highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */ .highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */ .highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */ .highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */ .highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */ .highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */ .highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */ .highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */ .highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */ .highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */ .highlight .vc { color: #336699 } /* Name.Variable.Class */ .highlight .vg { color: #dd7700 } /* Name.Variable.Global */ .highlight .vi { color: #3333bb } /* Name.Variable.Instance */ .highlight .vm { color: #336699 } /* Name.Variable.Magic */ .highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */ } 
/*
 * Copyright (c) 2019 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <vlib/punt.h>

#define foreach_punt_error                     \
  _(DISPATCHED, "dispatched")                  \
  _(NO_REASON, "No such punt reason")          \
  _(NO_REG, "No registrations")                \
  _(REP_FAIL, "Replication Failure")

typedef enum punt_error_t_
{
#define _(v,s) PUNT_ERROR_##v,
  foreach_punt_error
#undef _
    PUNT_N_ERRORS,
} punt_error_t;

static char *punt_error_strings[] = {
#define _(v,s) [PUNT_ERROR_##v] = s,
  foreach_punt_error
#undef _
};

typedef enum punt_next_t_
{
  PUNT_NEXT_DROP,
  PUNT_N_NEXT,
} punt_next_t;

typedef struct punt_trace_t_
{
  vlib_punt_reason_t pt_reason;
} punt_trace_t;

/**
 * Per-thread clone vectors
 */
#ifndef CLIB_MARCH_VARIANT
u32 **punt_clones;
#else
extern u32 **punt_clones;
#endif

static u8 *
format_punt_trace (u8 * s, va_list * args)
{
  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  punt_trace_t *t = va_arg (*args, punt_trace_t *);

  s = format (s, "reason: %U", format_vlib_punt_reason, t->pt_reason);

  return s;
}

always_inline u32
punt_replicate (vlib_main_t * vm,
		vlib_node_runtime_t * node,
		u32 thread_index,
		vlib_buffer_t * b0,
		u32 bi0,
		vlib_punt_reason_t pr0,
		u32 * next_index,
		u32 * n_left_to_next, u32 ** to_next, u32 * n_dispatched)
{
  /* multiple clients => replicate a copy to each */
  u16 n_clones0, n_cloned0, clone0;
  u32 ci0, next0;

  n_clones0 = vec_len (punt_dp_db[pr0]);
  vec_validate (punt_clones[thread_index], n_clones0);

  n_cloned0 = vlib_buffer_clone (vm, bi0,
				 punt_clones[thread_index],
				 n_clones0, 2 * CLIB_CACHE_LINE_BYTES);

  if (PREDICT_FALSE (n_cloned0 != n_clones0))
    {
      b0->error = node->errors[PUNT_ERROR_REP_FAIL];
    }

  for (clone0 = 1; clone0 < n_cloned0; clone0++)
    {
      ci0 = punt_clones[thread_index][clone0];

      *to_next[0] = ci0;
      *to_next += 1;
      *n_left_to_next -= 1;

      next0 = punt_dp_db[pr0][clone0];

      if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
	{
	  vlib_buffer_t *c0;
	  punt_trace_t *t;

	  c0 = vlib_get_buffer (vm, ci0);
	  t = vlib_add_trace (vm, node, c0, sizeof (*t));
	  t->pt_reason = pr0;
	}

      vlib_validate_buffer_enqueue_x1 (vm, node, *next_index,
				       *to_next, *n_left_to_next, ci0, next0);

      /* replications here always go to different next-nodes
       * so there's no need to check if the to_next frame
       * is full */
    }
  *n_dispatched = *n_dispatched + n_cloned0;

  /* The original buffer is the first clone */
  next0 = punt_dp_db[pr0][0];
  /*
   * Note: the original buffer is enqueued in punt_dispatch_node.
   * Don't do it here.
   *
   * *to_next[0] = bi0;
   */
  return next0;
}

always_inline u32
punt_dispatch_one (vlib_main_t * vm,
		   vlib_node_runtime_t * node,
		   vlib_combined_counter_main_t * cm,
		   u32 thread_index,
		   u32 bi0,
		   u32 * next_index,
		   u32 * n_left_to_next, u32 ** to_next, u32 * n_dispatched)
{
  vlib_punt_reason_t pr0;
  vlib_buffer_t *b0;
  u32 next0;

  b0 = vlib_get_buffer (vm, bi0);
  pr0 = b0->punt_reason;

  if (PREDICT_FALSE (pr0 >= vec_len (punt_dp_db)))
    {
      b0->error = node->errors[PUNT_ERROR_NO_REASON];
      next0 = PUNT_NEXT_DROP;
    }
  else
    {
      vlib_increment_combined_counter
	(cm, thread_index, pr0, 1, vlib_buffer_length_in_chain (vm, b0));

      if (PREDICT_TRUE (1 == vec_len (punt_dp_db[pr0])))
	{
	  /*
	   * one registered client => give it the packet
	   * This is the most likely outcome.
	   */
	  next0 = punt_dp_db[pr0][0];
	  *n_dispatched = *n_dispatched + 1;
	}
      else if (0 == vec_len (punt_dp_db[pr0]))
	{
	  /* no registered clients => drop */
	  next0 = PUNT_NEXT_DROP;
	  b0->error = node->errors[PUNT_ERROR_NO_REG];
	}
      else
	{
	  /*
	   * multiple registered clients => replicate
	   */
	  next0 = punt_replicate (vm, node, thread_index, b0, bi0, pr0,
				  next_index, n_left_to_next, to_next,
				  n_dispatched);
	}
    }

  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
    {
      punt_trace_t *t;

      t = vlib_add_trace (vm, node, b0, sizeof (*t));
      t->pt_reason = pr0;
    }

  return (next0);
}

VLIB_NODE_FN (punt_dispatch_node) (vlib_main_t * vm,
				   vlib_node_runtime_t * node,
				   vlib_frame_t * frame)
{
  u32 n_left_from, *from, *to_next, next_index, thread_index;
  vlib_combined_counter_main_t *cm;
  u32 n_dispatched;

  cm = &punt_counters;
  from = vlib_frame_vector_args (frame);
  n_left_from = frame->n_vectors;
  next_index = node->cached_next_index;
  thread_index = vlib_get_thread_index ();
  n_dispatched = 0;

  while (n_left_from > 0)
    {
      u32 n_left_to_next;

      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);

      while (n_left_from > 4 && n_left_to_next > 2)
	{
	  punt_next_t next0, next1;
	  u32 bi0, bi1;

	  {
	    vlib_buffer_t *b2, *b3;

	    b2 = vlib_get_buffer (vm, from[2]);
	    b3 = vlib_get_buffer (vm, from[3]);

	    vlib_prefetch_buffer_header (b2, LOAD);
	    vlib_prefetch_buffer_header (b3, LOAD);
	  }

	  bi0 = to_next[0] = from[0];
	  bi1 = to_next[1] = from[1];
	  from += 2;
	  n_left_from -= 2;

	  next0 = punt_dispatch_one (vm, node, cm, thread_index, bi0,
				     &next_index, &n_left_to_next,
				     &to_next, &n_dispatched);
	  next1 = punt_dispatch_one (vm, node, cm, thread_index, bi1,
				     &next_index, &n_left_to_next,
				     &to_next, &n_dispatched);

	  to_next += 2;
	  n_left_to_next -= 2;

	  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
					   to_next, n_left_to_next,
					   bi0, bi1, next0, next1);
	}
      while (n_left_from > 0 && n_left_to_next > 0)
	{
	  punt_next_t next0;
	  u32 bi0;

	  bi0 = to_next[0] = from[0];
	  from += 1;
	  n_left_from -= 1;

	  next0 = punt_dispatch_one (vm, node, cm, thread_index, bi0,
				     &next_index, &n_left_to_next,
				     &to_next, &n_dispatched);

	  to_next += 1;
	  n_left_to_next -= 1;

	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
					   to_next, n_left_to_next,
					   bi0, next0);
	}
      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
    }

  vlib_node_increment_counter (vm, node->node_index,
			       PUNT_ERROR_DISPATCHED, n_dispatched);

  return frame->n_vectors;
}

/* *INDENT-OFF* */
VLIB_REGISTER_NODE (punt_dispatch_node) = {
  .name = "punt-dispatch",
  .vector_size = sizeof (u32),
  .format_trace = format_punt_trace,
  .n_errors = PUNT_N_ERRORS,
  .error_strings = punt_error_strings,
  .n_next_nodes = PUNT_N_NEXT,
  .next_nodes = {
    [PUNT_NEXT_DROP] = "drop",
  },
};

/* *INDENT-ON* */

#ifndef CLIB_MARCH_VARIANT
clib_error_t *
punt_node_init (vlib_main_t * vm)
{
  vec_validate (punt_clones, vlib_num_workers ());

  return NULL;
}

VLIB_INIT_FUNCTION (punt_node_init);
#endif

/*
 * fd.io coding-style-patch-verification: ON
 *
 * Local Variables:
 * eval: (c-set-style "gnu")
 * End:
 */