summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
authorarikachen <eaglesora@gmail.com>2021-11-10 08:51:51 +0000
committerarikachen <eaglesora@gmail.com>2021-11-19 02:10:11 +0000
commitad80663eb3fd954f42607168ad4babb91cb0edcc (patch)
tree7fb5a6d49f91e84e8c15a3dbfa612a02cfbfa2f7 /src
parent57bdb26ba42c3fc6da601d007b27fa5c3f09dd96 (diff)
af_xdp: integrate with new tx infra
Type: improvement Signed-off-by: arikachen <eaglesora@gmail.com> Signed-off-by: Benoît Ganne <bganne@cisco.com> Change-Id: If8d57bcf033864935bd5e3a9912b2c1a7c712f44
Diffstat (limited to 'src')
-rw-r--r--src/plugins/af_xdp/af_xdp.h4
-rw-r--r--src/plugins/af_xdp/device.c98
-rw-r--r--src/plugins/af_xdp/output.c12
3 files changed, 83 insertions, 31 deletions
diff --git a/src/plugins/af_xdp/af_xdp.h b/src/plugins/af_xdp/af_xdp.h
index 85c8fee545d..84fc65f7674 100644
--- a/src/plugins/af_xdp/af_xdp.h
+++ b/src/plugins/af_xdp/af_xdp.h
@@ -86,6 +86,10 @@ typedef struct
struct xsk_ring_prod tx;
struct xsk_ring_cons cq;
int xsk_fd;
+
+ /* fields below are accessed in control-plane only (cold) */
+
+ u32 queue_index;
} af_xdp_txq_t;
typedef struct
diff --git a/src/plugins/af_xdp/device.c b/src/plugins/af_xdp/device.c
index 0b39280c569..d27901b70b3 100644
--- a/src/plugins/af_xdp/device.c
+++ b/src/plugins/af_xdp/device.c
@@ -30,6 +30,7 @@
#include <vppinfra/unix.h>
#include <vnet/ethernet/ethernet.h>
#include <vnet/interface/rx_queue_funcs.h>
+#include <vnet/interface/tx_queue_funcs.h>
#include "af_xdp.h"
af_xdp_main_t af_xdp_main;
@@ -315,6 +316,7 @@ af_xdp_create_queue (vlib_main_t *vm, af_xdp_create_if_args_t *args,
if (is_tx)
{
txq->xsk_fd = fd;
+ clib_spinlock_init (&txq->lock);
if (is_rx && (ad->flags & AF_XDP_DEVICE_F_SYSCALL_LOCK))
{
/* This is a shared rx+tx queue and we need to lock before syscalls.
@@ -432,6 +434,72 @@ af_xdp_device_set_rxq_mode (const af_xdp_device_t *ad, af_xdp_rxq_t *rxq,
return 0;
}
+static u32
+af_xdp_find_rxq_for_thread (vnet_main_t *vnm, const af_xdp_device_t *ad,
+ const u32 thread)
+{
+ u32 i;
+ for (i = 0; i < ad->rxq_num; i++)
+ {
+ const u32 qid = vec_elt (ad->rxqs, i).queue_index;
+ const u32 tid = vnet_hw_if_get_rx_queue (vnm, qid)->thread_index;
+ if (tid == thread)
+ return i;
+ }
+ return ~0;
+}
+
+static clib_error_t *
+af_xdp_finalize_queues (vnet_main_t *vnm, af_xdp_device_t *ad,
+ const int n_vlib_mains)
+{
+ clib_error_t *err = 0;
+ int i;
+
+ for (i = 0; i < ad->rxq_num; i++)
+ {
+ af_xdp_rxq_t *rxq = vec_elt_at_index (ad->rxqs, i);
+ rxq->queue_index = vnet_hw_if_register_rx_queue (
+ vnm, ad->hw_if_index, i, VNET_HW_IF_RXQ_THREAD_ANY);
+ u8 *desc = format (0, "%U rxq %d", format_af_xdp_device_name,
+ ad->dev_instance, i);
+ clib_file_t f = {
+ .file_descriptor = rxq->xsk_fd,
+ .private_data = rxq->queue_index,
+ .read_function = af_xdp_device_rxq_read_ready,
+ .description = desc,
+ };
+ rxq->file_index = clib_file_add (&file_main, &f);
+ vnet_hw_if_set_rx_queue_file_index (vnm, rxq->queue_index,
+ rxq->file_index);
+ err = af_xdp_device_set_rxq_mode (ad, rxq, AF_XDP_RXQ_MODE_POLLING);
+ if (err)
+ return err;
+ }
+
+ for (i = 0; i < ad->txq_num; i++)
+ vec_elt (ad->txqs, i).queue_index =
+ vnet_hw_if_register_tx_queue (vnm, ad->hw_if_index, i);
+
+ /* We set the rxq and txq of the same queue pair on the same thread
+ * by default to avoid locking because of the syscall lock. */
+ int last_qid = clib_min (ad->rxq_num, ad->txq_num - 1);
+ for (i = 0; i < n_vlib_mains; i++)
+ {
+ /* search for the 1st rxq assigned on this thread, if any */
+ u32 qid = af_xdp_find_rxq_for_thread (vnm, ad, i);
+ /* if this rxq is combined with a txq, use it. Otherwise, we'll
+ * assign txq in a round-robin fashion. We start from the 1st txq
+ * not shared with a rxq if possible... */
+ qid = qid < ad->txq_num ? qid : (last_qid++ % ad->txq_num);
+ vnet_hw_if_tx_queue_assign_thread (
+ vnm, vec_elt (ad->txqs, qid).queue_index, i);
+ }
+
+ vnet_hw_if_update_runtime_data (vnm, ad->hw_if_index);
+ return 0;
+}
+
void
af_xdp_create_if (vlib_main_t * vm, af_xdp_create_if_args_t * args)
{
@@ -556,13 +624,6 @@ af_xdp_create_if (vlib_main_t * vm, af_xdp_create_if_args_t * args)
goto err2;
}
- if (ad->txq_num < tm->n_vlib_mains)
- {
- /* initialize lock for shared txq */
- for (i = 0; i < ad->txq_num; i++)
- clib_spinlock_init (&vec_elt (ad->txqs, i).lock);
- }
-
ad->dev_instance = ad - am->devices;
ad->per_interface_next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
ad->pool =
@@ -603,28 +664,13 @@ af_xdp_create_if (vlib_main_t * vm, af_xdp_create_if_args_t * args)
vnet_hw_if_set_input_node (vnm, ad->hw_if_index, af_xdp_input_node.index);
- for (i = 0; i < ad->rxq_num; i++)
+ args->error = af_xdp_finalize_queues (vnm, ad, tm->n_vlib_mains);
+ if (args->error)
{
- af_xdp_rxq_t *rxq = vec_elt_at_index (ad->rxqs, i);
- rxq->queue_index = vnet_hw_if_register_rx_queue (
- vnm, ad->hw_if_index, i, VNET_HW_IF_RXQ_THREAD_ANY);
- u8 *desc = format (0, "%U rxq %d", format_af_xdp_device_name,
- ad->dev_instance, i);
- clib_file_t f = {
- .file_descriptor = rxq->xsk_fd,
- .private_data = rxq->queue_index,
- .read_function = af_xdp_device_rxq_read_ready,
- .description = desc,
- };
- rxq->file_index = clib_file_add (&file_main, &f);
- vnet_hw_if_set_rx_queue_file_index (vnm, rxq->queue_index,
- rxq->file_index);
- if (af_xdp_device_set_rxq_mode (ad, rxq, AF_XDP_RXQ_MODE_POLLING))
- goto err2;
+ args->rv = VNET_API_ERROR_SYSCALL_ERROR_7;
+ goto err2;
}
- vnet_hw_if_update_runtime_data (vnm, ad->hw_if_index);
-
/* buffer template */
vec_validate_aligned (ad->buffer_template, 1, CLIB_CACHE_LINE_BYTES);
ad->buffer_template->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
diff --git a/src/plugins/af_xdp/output.c b/src/plugins/af_xdp/output.c
index c5b73f98894..d1500a63cbb 100644
--- a/src/plugins/af_xdp/output.c
+++ b/src/plugins/af_xdp/output.c
@@ -215,9 +215,9 @@ VNET_DEVICE_CLASS_TX_FN (af_xdp_device_class) (vlib_main_t * vm,
af_xdp_main_t *rm = &af_xdp_main;
vnet_interface_output_runtime_t *ord = (void *) node->runtime_data;
af_xdp_device_t *ad = pool_elt_at_index (rm->devices, ord->dev_instance);
- u32 thread_index = vm->thread_index;
- af_xdp_txq_t *txq = vec_elt_at_index (
- ad->txqs, (thread_index - 1 + ad->txq_num) % ad->txq_num);
+ const vnet_hw_if_tx_frame_t *tf = vlib_frame_scalar_args (frame);
+ const int shared_queue = tf->shared_queue;
+ af_xdp_txq_t *txq = vec_elt_at_index (ad->txqs, tf->queue_id);
u32 *from;
u32 n, n_tx;
int i;
@@ -225,7 +225,8 @@ VNET_DEVICE_CLASS_TX_FN (af_xdp_device_class) (vlib_main_t * vm,
from = vlib_frame_vector_args (frame);
n_tx = frame->n_vectors;
- clib_spinlock_lock_if_init (&txq->lock);
+ if (shared_queue)
+ clib_spinlock_lock (&txq->lock);
for (i = 0, n = 0; i < AF_XDP_TX_RETRIES && n < n_tx; i++)
{
@@ -238,7 +239,8 @@ VNET_DEVICE_CLASS_TX_FN (af_xdp_device_class) (vlib_main_t * vm,
af_xdp_device_output_tx_db (vm, node, ad, txq, n);
- clib_spinlock_unlock_if_init (&txq->lock);
+ if (shared_queue)
+ clib_spinlock_unlock (&txq->lock);
if (PREDICT_FALSE (n != n_tx))
{
.vi { color: #3333bb } /* Name.Variable.Instance */ .highlight .vm { color: #336699 } /* Name.Variable.Magic */ .highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */ } 
/*
 * Copyright (c) 2017 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#define _GNU_SOURCE

#include <vlib/vlib.h>
#include <vnet/bonding/node.h>
#include <lacp/node.h>

/*
 *  LACP State = DETACHED
 */
static lacp_fsm_state_t lacp_mux_state_detached[] = {
  {LACP_ACTION_DETACHED, LACP_MUX_STATE_DETACHED},	// event 0 BEGIN
  {LACP_ACTION_WAITING, LACP_MUX_STATE_WAITING},	// event 1 SELECTED
  {LACP_ACTION_WAITING, LACP_MUX_STATE_WAITING},	// event 2 STANDBY
  {LACP_ACTION_DETACHED, LACP_MUX_STATE_DETACHED},	// event 3 UNSELECTED
  {LACP_ACTION_DETACHED, LACP_MUX_STATE_DETACHED},	// event 4 READY
  {LACP_ACTION_DETACHED, LACP_MUX_STATE_DETACHED},	// event 5 SYNC
};

/*
 *  LACP State = WAITING
 */
static lacp_fsm_state_t lacp_mux_state_waiting[] = {
  {LACP_ACTION_DETACHED, LACP_MUX_STATE_DETACHED},	// event 0 BEGIN
  {LACP_ACTION_WAITING, LACP_MUX_STATE_WAITING},	// event 1 SELECTED
  {LACP_ACTION_WAITING, LACP_MUX_STATE_WAITING},	// event 2 STANDBY
  {LACP_ACTION_DETACHED, LACP_MUX_STATE_DETACHED},	// event 3 UNSELECTED
  {LACP_ACTION_ATTACHED, LACP_MUX_STATE_ATTACHED},	// event 4 READY
  {LACP_ACTION_WAITING, LACP_MUX_STATE_WAITING},	// event 5 SYNC
};

/*
 *  LACP State = ATTACHED
 */
static lacp_fsm_state_t lacp_mux_state_attached[] = {
  {LACP_ACTION_DETACHED, LACP_MUX_STATE_DETACHED},	// event 0 BEGIN
  {LACP_ACTION_ATTACHED, LACP_MUX_STATE_ATTACHED},	// event 1 SELECTED
  {LACP_ACTION_DETACHED, LACP_MUX_STATE_DETACHED},	// event 2 STANDBY
  {LACP_ACTION_DETACHED, LACP_MUX_STATE_DETACHED},	// event 3 UNSELECTED
  {LACP_ACTION_ATTACHED, LACP_MUX_STATE_ATTACHED},	// event 4 READY
  {LACP_ACTION_COLLECTING_DISTRIBUTING, LACP_MUX_STATE_COLLECTING_DISTRIBUTING},	// event 5_SYNC
};

/*
 *  LACP State = COLLECTING_DISTRIBUTING
 */
static lacp_fsm_state_t lacp_mux_state_collecting_distributing[] = {
  {LACP_ACTION_DETACHED, LACP_MUX_STATE_DETACHED},	// event 0 BEGIN
  {LACP_ACTION_COLLECTING_DISTRIBUTING, LACP_MUX_STATE_COLLECTING_DISTRIBUTING},	// event 1 SELECTED
  {LACP_ACTION_COLLECTING_DISTRIBUTING, LACP_MUX_STATE_COLLECTING_DISTRIBUTING},	// event 2 STANDBY
  {LACP_ACTION_ATTACHED, LACP_MUX_STATE_ATTACHED},	// event 3 UNSELECTED
  {LACP_ACTION_COLLECTING_DISTRIBUTING, LACP_MUX_STATE_COLLECTING_DISTRIBUTING},	// event 4 READY
  {LACP_ACTION_COLLECTING_DISTRIBUTING, LACP_MUX_STATE_COLLECTING_DISTRIBUTING},	// event 5 SYNC
};

static lacp_fsm_machine_t lacp_mux_fsm_table[] = {
  {lacp_mux_state_detached},
  {lacp_mux_state_waiting},
  {lacp_mux_state_attached},
  {lacp_mux_state_collecting_distributing},
};

lacp_machine_t lacp_mux_machine = {
  lacp_mux_fsm_table,
  lacp_mux_debug_func,
};

static void
lacp_detach_mux_from_aggregator (vlib_main_t * vm, slave_if_t * sif)
{
  sif->actor.state &= ~LACP_STATE_SYNCHRONIZATION;
  sif->ready = 0;
  sif->ready_n = 0;
}

static void
lacp_attach_mux_to_aggregator (vlib_main_t * vm, slave_if_t * sif)
{
  sif->actor.state |= LACP_STATE_SYNCHRONIZATION;
}

int
lacp_mux_action_detached (void *p1, void *p2)
{
  vlib_main_t *vm = p1;
  slave_if_t *sif = p2;
  lacp_main_t *lm = &lacp_main;

  lacp_detach_mux_from_aggregator (vm, sif);
  sif->actor.state &= ~LACP_STATE_COLLECTING;
  bond_disable_collecting_distributing (vm, sif);
  sif->actor.state &= ~LACP_STATE_DISTRIBUTING;
  sif->ntt = 1;
  lacp_start_periodic_timer (lm->vlib_main, sif, 0);

  if (sif->selected == LACP_PORT_SELECTED)
    lacp_machine_dispatch (&lacp_mux_machine, vm, sif,
			   LACP_MUX_EVENT_SELECTED, &sif->mux_state);

  if (sif->selected == LACP_PORT_STANDBY)
    lacp_machine_dispatch (&lacp_mux_machine, vm, sif, LACP_MUX_EVENT_STANDBY,
			   &sif->mux_state);

  return 0;
}

int
lacp_mux_action_attached (void *p1, void *p2)
{
  vlib_main_t *vm = p1;
  slave_if_t *sif = p2;
  lacp_main_t *lm = &lacp_main;

  lacp_attach_mux_to_aggregator (vm, sif);
  sif->actor.state &= ~LACP_STATE_COLLECTING;
  bond_disable_collecting_distributing (vm, sif);
  sif->actor.state &= ~LACP_STATE_DISTRIBUTING;
  sif->ntt = 1;
  lacp_start_periodic_timer (lm->vlib_main, sif, 0);

  if ((sif->selected == LACP_PORT_UNSELECTED) ||
      (sif->selected == LACP_PORT_STANDBY))
    lacp_machine_dispatch (&lacp_mux_machine, vm, sif,
			   LACP_MUX_EVENT_UNSELECTED, &sif->mux_state);

  if ((sif->selected == LACP_PORT_SELECTED) &&
      (sif->partner.state & LACP_STATE_SYNCHRONIZATION))
    lacp_machine_dispatch (&lacp_mux_machine, vm, sif, LACP_MUX_EVENT_SYNC,
			   &sif->mux_state);
  return 0;
}

int
lacp_mux_action_waiting (void *p1, void *p2)
{
  vlib_main_t *vm = p1;
  slave_if_t *sif = p2;
  lacp_main_t *lm = &lacp_main;

  if (!lacp_timer_is_running (sif->wait_while_timer))
    lacp_start_wait_while_timer (lm->vlib_main, sif,
				 LACP_AGGREGATE_WAIT_TIME);

  if ((sif->selected == LACP_PORT_SELECTED) && sif->ready)
    lacp_machine_dispatch (&lacp_mux_machine, vm, sif,
			   LACP_MUX_EVENT_READY, &sif->mux_state);

  if (sif->selected == LACP_PORT_UNSELECTED)
    lacp_machine_dispatch (&lacp_mux_machine, vm, sif,
			   LACP_MUX_EVENT_UNSELECTED, &sif->mux_state);

  return 0;
}

int
lacp_mux_action_collecting_distributing (void *p1, void *p2)
{
  vlib_main_t *vm = p1;
  slave_if_t *sif = p2;
  lacp_main_t *lm = &lacp_main;

  sif->actor.state |= LACP_STATE_SYNCHRONIZATION | LACP_STATE_COLLECTING |
    LACP_STATE_DISTRIBUTING;
  bond_enable_collecting_distributing (vm, sif);
  sif->ntt = 1;
  lacp_start_periodic_timer (lm->vlib_main, sif, 0);
  if ((sif->selected == LACP_PORT_UNSELECTED) ||
      (sif->selected == LACP_PORT_STANDBY) ||
      !(sif->partner.state & LACP_STATE_SYNCHRONIZATION))
    lacp_machine_dispatch (&lacp_mux_machine, vm, sif,
			   LACP_MUX_EVENT_UNSELECTED, &sif->mux_state);


  return 0;
}

static u8 *
format_mux_event (u8 * s, va_list * args)
{
  static lacp_event_struct lacp_mux_event_array[] = {
#define _(b, s, n) {.bit = b, .str = #s, },
    foreach_lacp_mux_event
#undef _
    {.str = NULL}
  };
  int e = va_arg (*args, int);
  lacp_event_struct *event_entry = lacp_mux_event_array;

  if (e >= (sizeof (lacp_mux_event_array) / sizeof (*event_entry)))
    s = format (s, "Bad event %d", e);
  else
    s = format (s, "%s", event_entry[e].str);

  return s;
}

void
lacp_mux_debug_func (slave_if_t * sif, int event, int state,
		     lacp_fsm_state_t * transition)
{
  vlib_worker_thread_t *w = vlib_worker_threads + os_get_thread_index ();
  /* *INDENT-OFF* */
  ELOG_TYPE_DECLARE (e) =
    {
      .format = "%s",
      .format_args = "T4",
    };
  /* *INDENT-ON* */
  struct
  {
    u32 event;
  } *ed = 0;

  ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e, w->elog_track);
  ed->event =
    elog_string (&vlib_global_main.elog_main, "%U-MUX: %U, %U->%U%c",
		 format_vnet_sw_if_index_name, vnet_get_main (),
		 sif->sw_if_index, format_mux_event, event,
		 format_mux_sm_state, state, format_mux_sm_state,
		 transition->next_state, 0);
}

void
lacp_init_mux_machine (vlib_main_t * vm, slave_if_t * sif)
{
  lacp_machine_dispatch (&lacp_mux_machine, vm, sif, LACP_MUX_EVENT_BEGIN,
			 &sif->mux_state);
}

/*
 * fd.io coding-style-patch-verification: ON
 *
 * Local Variables:
 * eval: (c-set-style "gnu")
 * End:
 */