#include #include #include #include #include #include #include typedef struct { /** Handle returned from tw_start_timer */ u32 stop_timer_handle; /** Test item should expire at this clock tick */ u64 expected_to_expire; } tw_timer_test_elt_t; typedef struct { /** Pool of test objects */ tw_timer_test_elt_t *test_elts; /** The single-wheel */ tw_timer_wheel_2t_1w_2048sl_t single_wheel; /** The double-wheel */ tw_timer_wheel_16t_2w_512sl_t double_wheel; /* The triple wheel */ tw_timer_wheel_4t_3w_256sl_t triple_wheel; /* The triple wheel with overflow vector */ tw_timer_wheel_1t_3w_1024sl_ov_t triple_ov_wheel; /** random number seed */ u64 seed; /** number of timers */ u32 ntimers; /** number of "churn" iterations */ u32 niter; /** number of clock ticks per churn iteration */ u32 ticks_per_iter; /** cpu timer */ clib_time_t clib_time; } tw_timer_test_main_t; tw_timer_test_main_t tw_timer_test_main; static void run_single_wheel (tw_timer_wheel_2t_1w_2048sl_t * tw, u32 n_ticks) { u32 i; f64 now = tw->last_run_time + 1.01; for (i = 0; i < n_ticks; i++) { tw_timer_expire_timers_2t_1w_2048sl (tw, now); now += 1.01; } } static void run_double_wheel (tw_timer_wheel_16t_2w_512sl_t * tw, u32 n_ticks) { u32 i; f64 now = tw->last_run_time + 1.01; for (i = 0; i < n_ticks; i++) { tw_timer_expire_timers_16t_2w_512sl (tw, now); now += 1.01; } } static void run_triple_wheel (tw_timer_wheel_4t_3w_256sl_t * tw, u32 n_ticks) { u32 i; f64 now = tw->last_run_time + 1.01; for (i = 0; i < n_ticks; i++) { tw_timer_expire_timers_4t_3w_256sl (tw, now); now += 1.01; } } static void run_triple_ov_wheel (tw_timer_wheel_1t_3w_1024sl_ov_t * tw, u32 n_ticks) { u32 i; f64 now = tw->last_run_time + 1.01; for (i = 0; i < n_ticks; i++) { tw_timer_expire_timers_1t_3w_1024sl_ov (tw, now); now += 1.01; } } static void expired_timer_single_callback (u32 * expired_timers) { int i; u32 pool_index, timer_id; tw_timer_test_elt_t *e; tw_timer_test_main_t *tm = &tw_timer_test_main; for (i = 0; i < vec_len (expired_timers); i++) { pool_index = expired_timers[i] & 0x7FFFFFFF; timer_id = expired_timers[i] >> 31; ASSERT (timer_id == 1); e = pool_elt_at_index (tm->test_elts, pool_index); if (e->expected_to_expire != tm->single_wheel.current_tick) { fformat (stdout, "[%d] expired at %lld not %lld\n", e - tm->test_elts, tm->single_wheel.current_tick, e->expected_to_expire); } pool_put (tm->test_elts, e); } } static void expired_timer_double_callback (u32 * expired_timers) { int i; u32 pool_index, timer_id; tw_timer_test_elt_t *e; tw_timer_test_main_t *tm = &tw_timer_test_main; for (i = 0; i < vec_len (expired_timers); i++) { pool_index = expired_timers[i] & 0x0FFFFFFF; timer_id = expired_timers[i] >> 28; ASSERT (timer_id == 14); e = pool_elt_at_index (tm->test_elts, pool_index); if (e->expected_to_expire != tm->double_wheel.current_tick) { fformat (stdout, "[%d] expired at %lld not %lld\n", e - tm->test_elts, tm->double_wheel.current_tick, e->expected_to_expire); } pool_put (tm->test_elts, e); } } static void expired_timer_triple_callback (u32 * expired_timers) { int i; u32 pool_index, timer_id; tw_timer_test_elt_t *e; tw_timer_test_main_t *tm = &tw_timer_test_main; for (i = 0; i < vec_len (expired_timers); i++) { pool_index = expired_timers[i] & 0x3FFFFFFF; timer_id = expired_timers[i] >> 30; ASSERT (timer_id == 3); e = pool_elt_at_index (tm->test_elts, pool_index); if (e->expected_to_expire != tm->triple_wheel.current_tick) { fformat (stdout, "[%d] expired at %lld not %lld\n", e - tm->test_elts, tm->triple_wheel.current_tick, e->expected_to_expire); } pool_put (tm->test_elts, e); } } static void expired_timer_triple_ov_callback (u32 * expired_timers) { int i; u32 pool_index; tw_timer_test_elt_t *e; tw_timer_test_main_t *tm = &tw_timer_test_main; for (i = 0; i < vec_len (expired_timers); i++) { pool_index = expired_timers[i]; e = pool_elt_at_index (tm->test_elts, pool_index); if (e->expected_to_expire != tm->triple_ov_wheel.current_tick) { fformat (stdout, "[%d] expired at %lld not %lld\n", e - tm->test_elts, tm->triple_ov_wheel.current_tick, e->expected_to_expire); } pool_put (tm->test_elts, e); } } static clib_error_t * test2_single (tw_timer_test_main_t * tm) { u32 i, j; tw_timer_test_elt_t *e; u32 initial_wheel_offset; u64 expiration_time; u32 max_expiration_time = 0; u32 *deleted_indices = 0; u32 adds = 0, deletes = 0; f64 before, after; clib_time_init (&tm->clib_time); tw_timer_wheel_init_2t_1w_2048sl (&tm->single_wheel, expired_timer_single_callback, 1.0 /* timer interval */ , ~0); /* Prime offset */ initial_wheel_offset = 757; run_single_wheel (&tm->single_wheel, initial_wheel_offset); fformat (stdout, "initial wheel time %d, fast index %d\n", tm->single_wheel.current_tick, tm->single_wheel.current_index[TW_TIMER_RING_FAST]); initial_wheel_offset = tm->single_wheel.current_tick; fformat (stdout, "test %d timers, %d iter, %d ticks per iter, 0x%llx seed\n", tm->ntimers, tm->niter, tm->ticks_per_iter, tm->seed); before = clib_time_now (&tm->clib_time); /* Prime the pump */ for (i = 0; i < tm->ntimers; i++) { pool_get (tm->test_elts, e); clib_memset (e, 0, sizeof (*e)); do { expiration_time = random_u64 (&tm->seed) & (2047); } while (expiration_time == 0); if (expiration_time > max_expiration_time) max_expiration_time = expiration_time; e->expected_to_expire = expiration_time + initial_wheel_offset; e->stop_timer_handle = tw_timer_start_2t_1w_2048sl (&tm->single_wheel, e - tm->test_elts, 1 /* timer id */ , expiration_time); } adds += i; for (i = 0; i < tm->niter; i++) { run_single_wheel (&tm->single_wheel, tm->ticks_per_iter); j = 0; vec_reset_length (deleted_indices); /* *INDENT-OFF* */ pool_foreach (e, tm->test_elts, ({ tw_timer_stop_2t_1w_2048sl (&tm->single_wheel, e->stop_timer_handle); vec_add1 (deleted_indices, e - tm->test_elts); if (++j >= tm->ntimers / 4) goto del_and_re_add; })); /* *INDENT-ON* */ del_and_re_add: for (j = 0; j < vec_len (deleted_indices); j++) { pool_put_index (tm->test_elts, deleted_indices[j]); } deletes += j; for (j = 0; j < tm->ntimers / 4; j++) { pool_get (tm->test_elts, e); clib_memset (e, 0, sizeof (*e)); do { expiration_time = random_u64 (&tm->seed) & (2047); } while (expiration_time == 0); if (expiration_time > max_expiration_time) max_expiration_time = expiration_time; e->expected_to_expire = expiration_time + tm->single_wheel.current_tick; e->stop_timer_handle = tw_timer_start_2t_1w_2048sl (&tm->single_wheel, e - tm->test_elts, 1 /* timer id */ , expiration_time); } adds += j; } vec_free (deleted_indices); run_single_wheel (&tm->single_wheel, max_expiration_time + 1); after = clib_time_now (&tm->clib_time); fformat (stdout, "%d adds, %d deletes, %d ticks\n", adds, deletes, tm->single_wheel.current_tick); fformat (stdout, "test ran %.2f seconds, %.2f ops/second\n", (after - before), ((f64) adds + (f64) deletes + (f64) tm->single_wheel.current_tick) / (after - before)); if (pool_elts (tm->test_elts)) fformat (stdout, "Note: %d elements remain in pool\n", pool_elts (tm->test_elts)); /* *INDENT-OFF* */ pool_foreach (e, tm->test_elts, ({ fformat (stdout, "[%d] expected to expire %d\n", e - tm->test_elts, e->expected_to_expire); })); /* *INDENT-ON* */ pool_free (tm->test_elts); tw_timer_wheel_free_2t_1w_2048sl (&tm->single_wheel); return 0; } static clib_error_t * test2_double (tw_timer_test_main_t * tm) { u32 i, j; tw_timer_test_elt_t *e; u32 initial_wheel_offset; u32 expiration_time; u32 max_expiration_time = 0; u32 *deleted_indices = 0; u32 adds = 0, deletes = 0; f64 before, after; clib_time_init (&tm->clib_time); tw_timer_wheel_init_16t_2w_512sl (&tm->double_wheel, expired_timer_double_callback, 1.0 /* timer interval */ , ~0); /* Prime offset */ initial_wheel_offset = 7577; run_double_wheel (&tm->double_wheel, initial_wheel_offset); fformat (stdout, "initial wheel time %d, fast index %d slow index %d\n", tm->double_wheel.current_tick, tm->double_wheel.current_index[TW_TIMER_RING_FAST], tm->double_wheel.current_index[TW_TIMER_RING_SLOW]); initial_wheel_offset = tm->double_wheel.current_tick; fformat (stdout, "test %d timers, %d iter, %d ticks per iter, 0x%llx seed\n", tm->ntimers, tm->niter, tm->ticks_per_iter, tm->seed); before = clib_time_now (&tm->clib_time); /* Prime the pump */ for (i = 0; i < tm->ntimers; i++) { pool_get (tm->test_elts, e); clib_memset (e, 0, sizeof (*e)); do { expiration_time = random_u64 (&tm->seed) & ((1 << 17) - 1); } while (expiration_time == 0); if (expiration_time > max_expiration_time) max_expiration_time = expiration_time; e->expected_to_expire = expiration_time + initial_wheel_offset; e->stop_timer_handle = tw_timer_start_16t_2w_512sl (&tm->double_wheel, e - tm->test_elts, 14 /* timer id */ , expiration_time); } adds += i; for (i = 0; i < tm->niter; i++) { run_double_wheel (&tm->double_wheel, tm->ticks_per_iter); j = 0; vec_reset_length (deleted_indices); /* *INDENT-OFF* */ pool_foreach (e, tm->test_elts, ({ tw_timer_stop_16t_2w_512sl (&tm->double_wheel, e->stop_timer_handle); vec_add1 (deleted_indices, e - tm->test_elts); if (++j >= tm->ntimers / 4) goto del_and_re_add; })); /* *INDENT-ON* */ del_and_re_add: for (j = 0; j < vec_len (deleted_indices); j++) pool_put_index (tm->test_elts, deleted_indices[j]); deletes += j; for (j = 0; j < tm->ntimers / 4; j++) { pool_get (tm->test_elts, e); clib_memset (e, 0, sizeof (*e)); do { expiration_time = random_u64 (&tm->seed) & ((1 << 17) - 1); } while (expiration_time == 0); if (expiration_time > max_expiration_time) max_expiration_time = expiration_time; e->expected_to_expire = expiration_time + tm->double_wheel.current_tick; e->stop_timer_handle = tw_timer_start_16t_2w_512sl (&tm->double_wheel, e - tm->test_elts, 14 /* timer id */ , expiration_time); } adds += j; } vec_free (deleted_indices); run_double_wheel (&tm->double_wheel, max_expiration_time + 1); after = clib_time_now (&tm->clib_time); fformat (stdout, "%d adds, %d deletes, %d ticks\n", adds, deletes, tm->double_wheel.current_tick); fformat (stdout, "test ran %.2f seconds, %.2f ops/second\n", (after - before), ((f64) adds + (f64) deletes + (f64) tm->double_wheel.current_tick) / (after - before)); if (pool_elts (tm->test_elts)) fformat (stdout, "Note: %d elements remain in pool\n", pool_elts (tm->test_elts)); /* *INDENT-OFF* */ pool_foreach (e, tm->test_elts, ({ fformat (stdout, "[%d] expected to expire %d\n", e - tm->test_elts, e->expected_to_expire); })); /* *INDENT-ON* */ pool_free (tm->test_elts); tw_timer_wheel_free_16t_2w_512sl (&tm->double_wheel); return 0; } static u32 get_expiration_time (tw_timer_test_main_t * tm) { u32 expiration_time; do { expiration_time = random_u64 (&tm->seed) & ((1 << 17) - 1); } while (expiration_time == 0); return expiration_time; } static clib_error_t * test2_double_updates (tw_timer_test_main_t * tm) { u32 i, j; tw_timer_test_elt_t *e; u32 initial_wheel_offset; u32 expiration_time; u32 max_expiration_time = 0, updates = 0; f64 before, after; clib_time_init (&tm->clib_time); tw_timer_wheel_init_16t_2w_512sl (&tm->double_wheel, expired_timer_double_callback, 1.0 /* timer interval */ , ~0); /* Prime offset */ initial_wheel_offset = 7577; run_double_wheel (&tm->double_wheel, initial_wheel_offset); fformat (stdout, "initial wheel time %d, fast index %d slow index %d\n", tm->double_wheel.current_tick, tm->double_wheel.current_index[TW_TIMER_RING_FAST], tm->double_wheel.current_index[TW_TIMER_RING_SLOW]); initial_wheel_offset = tm->double_wheel.current_tick; fformat (stdout, "test %d timers, %d iter, %d ticks per iter, 0x%llx seed\n", tm->ntimers, tm->niter, tm->ticks_per_iter, tm->seed); before = clib_time_now (&tm->clib_time); /* Prime the pump */ for (i = 0; i < tm->ntimers; i++) { pool_get (tm->test_elts, e); clib_memset (e, 0, sizeof (*e)); expiration_time = get_expiration_time (tm); max_expiration_time = clib_max (expiration_time, max_expiration_time); e->expected_to_expire = expiration_time + initial_wheel_offset; e->stop_timer_handle = tw_timer_start_16t_2w_512sl (&tm->double_wheel, e - tm->test_elts, 14 /* timer id */ , expiration_time); } for (i = 0; i < tm->niter; i++) { run_double_wheel (&tm->double_wheel, tm->ticks_per_iter); j = 0; /* *INDENT-OFF* */ pool_foreach (e, tm->test_elts, ({ expiration_time = get_expiration_time (tm); max_expiration_time = clib_max (expiration_time, max_expiration_time); e->expected_to_expire = expiration_time + tm->double_wheel.current_tick; tw_timer_update_16t_2w_512sl (&tm->double_wheel, e->stop_timer_handle, expiration_time); if (++j >= tm->ntimers / 4) goto done; })); /* *INDENT-ON* */ done: updates += j; } run_double_wheel (&tm->double_wheel, max_expiration_time + 1); after = clib_time_now (&tm->clib_time); fformat (stdout, "%d updates, %d ticks\n", updates, tm->double_wheel.current_tick); fformat (stdout, "test ran %.2f seconds, %.2f ops/second\n", (after - before), ((f64) updates + (f64) tm->double_wheel.current_tick) / (after - before)); if (pool_elts (tm->test_elts)) fformat (stdout, "Note: %d elements remain in pool\n", pool_elts (tm->test_elts)); /* *INDENT-OFF* */ pool_foreach (e, tm->test_elts, ({ fformat (stdout, "[%d] expected to expire %d\n", e - tm->test_elts, e->expected_to_expire); })); /* *INDENT-ON* */ pool_free (tm->test_elts); tw_timer_wheel_free_16t_2w_512sl (&tm->double_wheel); return 0; } static clib_error_t * test2_triple (tw_timer_test_main_t * tm) { u32 i, j; tw_timer_test_elt_t *e; u32 initial_wheel_offset = 0; u32 expiration_time; u32 max_expiration_time = 0; u32 *deleted_indices = 0; u32 adds = 0, deletes = 0; f64 before, after; clib_time_init (&tm->clib_time); tw_timer_wheel_init_4t_3w_256sl (&tm->triple_wheel, expi
/*-
 *   BSD LICENSE
 *
 *   Copyright 2015 6WIND S.A.
 *   Copyright 2015 Mellanox.
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of 6WIND S.A. nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <stddef.h>
#include <stdint.h>
#include <errno.h>
#include <string.h>
#include <assert.h>

/* Verbs header. */
/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
#ifdef PEDANTIC
#pragma GCC diagnostic ignored "-pedantic"
#endif
#include <infiniband/verbs.h>
#ifdef PEDANTIC
#pragma GCC diagnostic error "-pedantic"
#endif

/* DPDK headers don't like -pedantic. */
#ifdef PEDANTIC
#pragma GCC diagnostic ignored "-pedantic"
#endif
#include <rte_malloc.h>
#include <rte_ethdev.h>
#ifdef PEDANTIC
#pragma GCC diagnostic error "-pedantic"
#endif

#include "mlx5.h"
#include "mlx5_rxtx.h"

/**
 * Get a RSS configuration hash key.
 *
 * @param priv
 *   Pointer to private structure.
 * @param rss_hf
 *   RSS hash functions configuration must be retrieved for.
 *
 * @return
 *   Pointer to a RSS configuration structure or NULL if rss_hf cannot
 *   be matched.
 */
static struct rte_eth_rss_conf *
rss_hash_get(struct priv *priv, uint64_t rss_hf)
{
	unsigned int i;

	for (i = 0; (i != hash_rxq_init_n); ++i) {
		uint64_t dpdk_rss_hf = hash_rxq_init[i].dpdk_rss_hf;

		if (!(dpdk_rss_hf & rss_hf))
			continue;
		return (*priv->rss_conf)[i];
	}
	return NULL;
}

/**
 * Register a RSS key.
 *
 * @param priv
 *   Pointer to private structure.
 * @param key
 *   Hash key to register.
 * @param key_len
 *   Hash key length in bytes.
 * @param rss_hf
 *   RSS hash functions the provided key applies to.
 *
 * @return
 *   0 on success, errno value on failure.
 */
int
rss_hash_rss_conf_new_key(struct priv *priv, const uint8_t *key,
			  unsigned int key_len, uint64_t rss_hf)
{
	unsigned int i;

	for (i = 0; (i != hash_rxq_init_n); ++i) {
		struct rte_eth_rss_conf *rss_conf;
		uint64_t dpdk_rss_hf = hash_rxq_init[i].dpdk_rss_hf;

		if (!(dpdk_rss_hf & rss_hf))
			continue;
		rss_conf = rte_realloc((*priv->rss_conf)[i],
				       (sizeof(*rss_conf) + key_len),
				       0);
		if (!rss_conf)
			return ENOMEM;
		rss_conf->rss_key = (void *)(rss_conf + 1);
		rss_conf->rss_key_len = key_len;
		rss_conf->rss_hf = dpdk_rss_hf;
		memcpy(rss_conf->rss_key, key, key_len);
		(*priv->rss_conf)[i] = rss_conf;
	}
	return 0;
}

/**
 * DPDK callback to update the RSS hash configuration.
 *
 * @param dev
 *   Pointer to Ethernet device structure.
 * @param[in] rss_conf
 *   RSS configuration data.
 *
 * @return
 *   0 on success, negative errno value on failure.
 */
int
mlx5_rss_hash_update(struct rte_eth_dev *dev,
		     struct rte_eth_rss_conf *rss_conf)
{
	struct priv *priv = dev->data->dev_private;
	int err = 0;

	priv_lock(priv);

	assert(priv->rss_conf != NULL);

	/* Apply configuration. */
	if (rss_conf->rss_key)
		err = rss_hash_rss_conf_new_key(priv,
						rss_conf->rss_key,
						rss_conf->rss_key_len,
						rss_conf->rss_hf);
	/* Store protocols for which RSS is enabled. */
	priv->rss_hf = rss_conf->rss_hf;
	priv_unlock(priv);
	assert(err >= 0);
	return -err;
}

/**
 * DPDK callback to get the RSS hash configuration.
 *
 * @param dev
 *   Pointer to Ethernet device structure.
 * @param[in, out] rss_conf
 *   RSS configuration data.
 *
 * @return
 *   0 on success, negative errno value on failure.
 */
int
mlx5_rss_hash_conf_get(struct rte_eth_dev *dev,
		       struct rte_eth_rss_conf *rss_conf)
{
	struct priv *priv