summaryrefslogtreecommitdiffstats
path: root/lib/librte_latencystats/rte_latencystats.c
blob: 5715549e46ee90c81e1c2d12852f920c4911a8d4 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2018 Intel Corporation
 */

#include <unistd.h>
#include <sys/types.h>
#include <stdbool.h>
#include <math.h>

#include <rte_mbuf.h>
#include <rte_log.h>
#include <rte_cycles.h>
#include <rte_ethdev.h>
#include <rte_metrics.h>
#include <rte_memzone.h>
#include <rte_lcore.h>

#include "rte_latencystats.h"

/** Nano seconds per second */
#define NS_PER_SEC 1E9

/** Clock cycles per nano second */
static uint64_t
latencystat_cycles_per_ns(void)
{
	return rte_get_timer_hz() / NS_PER_SEC;
}

/* Macros for printing using RTE_LOG */
#define RTE_LOGTYPE_LATENCY_STATS RTE_LOGTYPE_USER1

static const char *MZ_RTE_LATENCY_STATS = "rte_latencystats";
static int latency_stats_index;
static uint64_t samp_intvl;
static uint64_t timer_tsc;
static uint64_t prev_tsc;

struct rte_latency_stats {
	float min_latency; /**< Minimum latency in nano seconds */
	float avg_latency; /**< Average latency in nano seconds */
	float max_latency; /**< Maximum latency in nano seconds */
	float jitter; /** Latency variation */
};

static struct rte_latency_stats *glob_stats;

struct rxtx_cbs {
	const struct rte_eth_rxtx_callback *cb;
};

static struct rxtx_cbs rx_cbs[RTE_MAX_ETHPORTS][RTE_MAX_QUEUES_PER_PORT];
static struct rxtx_cbs tx_cbs[RTE_MAX_ETHPORTS][RTE_MAX_QUEUES_PER_PORT];

struct latency_stats_nameoff {
	char name[RTE_ETH_XSTATS_NAME_SIZE];
	unsigned int offset;
};

static const struct latency_stats_nameoff lat_stats_strings[] = {
	{"min_latency_ns", offsetof(struct rte_latency_stats, min_latency)},
	{"avg_latency_ns", offsetof(struct rte_latency_stats, avg_latency)},
	{"max_latency_ns", offsetof(struct rte_latency_stats, max_latency)},
	{"jitter_ns", offsetof(struct rte_latency_stats, jitter)},
};

#define NUM_LATENCY_STATS (sizeof(lat_stats_strings) / \
				sizeof(lat_stats_strings[0]))

int32_t
rte_latencystats_update(void)
{
	unsigned int i;
	float *stats_ptr = NULL;
	uint64_t values[NUM_LATENCY_STATS] = {0};
	int ret;

	for (i = 0; i < NUM_LATENCY_STATS; i++) {
		stats_ptr = RTE_PTR_ADD(glob_stats,
				lat_stats_strings[i].offset);
		values[i] = (uint64_t)floor((*stats_ptr)/
				latencystat_cycles_per_ns());
	}

	ret = rte_metrics_update_values(RTE_METRICS_GLOBAL,
					latency_stats_index,
					values, NUM_LATENCY_STATS);
	if (ret < 0)
		RTE_LOG(INFO, LATENCY_STATS, "Failed to push the stats\n");

	return ret;
}

static void
rte_latencystats_fill_values(struct rte_metric_value *values)
{
	unsigned int i;
	float *stats_ptr = NULL;

	for (i = 0; i < NUM_LATENCY_STATS; i++) {
		stats_ptr = RTE_PTR_ADD(glob_stats,
				lat_stats_strings[i].offset);
		values[i].key = i;
		values[i].value = (uint64_t)floor((*stats_ptr)/
						latencystat_cycles_per_ns());
	}
}

static uint16_t
add_time_stamps(uint16_t pid __rte_unused,
		uint16_t qid __rte_unused,
		struct rte_mbuf **pkts,
		uint16_t nb_pkts,
		uint16_t max_pkts __rte_unused,
		void *user_cb __rte_unused)
{
	unsigned int i;
	uint64_t diff_tsc, now;

	/*
	 * For every sample interval,
	 * time stamp is marked on one received packet.
	 */
	now = rte_rdtsc();
	for (i = 0; i < nb_pkts; i++) {
		diff_tsc = now - prev_tsc;
		timer_tsc += diff_tsc;

		if ((pkts[i]->ol_flags & PKT_RX_TIMESTAMP) == 0
				&& (timer_tsc >= samp_intvl)) {
			pkts[i]->timestamp = now;
			pkts[i]->ol_flags |= PKT_RX_TIMESTAMP;
			timer_tsc = 0;
		}
		prev_tsc = now;
		now = rte_rdtsc();
	}

	return nb_pkts;
}

static uint16_t
calc_latency(uint16_t pid __rte_unused,
		uint16_t qid __rte_unused,
		struct rte_mbuf **pkts,
		uint16_t nb_pkts,
		void *_ __rte_unused)
{
	unsigned int i, cnt = 0;
	uint64_t now;
	float latency[nb_pkts];
	static float prev_latency;
	/*
	 * Alpha represents degree of weighting decrease in EWMA,
	 * a constant smoothing factor between 0 and 1. The value
	 * is used below for measuring average latency.
	 */
	const float alpha = 0.2;

	now = rte_rdtsc();
	for (i = 0; i < nb_pkts; i++) {
		if (pkts[i]->ol_flags & PKT_RX_TIMESTAMP)
			latency[cnt++] = now - pkts[i]->timestamp;
	}

	for (i = 0; i < cnt; i++) {
		/*
		 * The jitter is calculated as statistical mean of interpacket
		 * delay variation. The "jitter estimate" is computed by taking
		 * the absolute values of the ipdv sequence and applying an
		 * exponential filter with parameter 1/16 to generate the
		 * estimate. i.e J=J+(|D(i-1,i)|-J)/16. Where J is jitter,
		 * D(i-1,i) is difference in latency of two consecutive packets
		 * i-1 and i.
		 * Reference: Calculated as per RFC 5481, sec 4.1,
		 * RFC 3393 sec 4.5, RFC 1889 sec.
		 */
		glob_stats->jitter +=  (fabsf(prev_latency - latency[i])
					- glob_stats->jitter)/16;
		if (glob_stats->min_latency == 0)
			glob_stats->min_latency = latency[i];
		else if (latency[i] < glob_stats->min_latency)
			glob_stats->min_latency = latency[i];
		else if (latency[i] > glob_stats->max_latency)
			glob_stats->max_latency = latency[i];
		/*
		 * The average latency is measured using exponential moving
		 * average, i.e. using EWMA
		 * https://en.wikipedia.org/wiki/Moving_average
		 */
		glob_stats->avg_latency +=
			alpha * (latency[i] - glob_stats->avg_latency);
		prev_latency = latency[i];
	}

	return nb_pkts;
}

int
rte_latencystats_init(uint64_t app_samp_intvl,
		rte_latency_stats_flow_type_fn user_cb)
{
	unsigned int i;
	uint16_t pid;
	uint16_t qid;
	struct rxtx_cbs *cbs = NULL;
	const char *ptr_strings[NUM_LATENCY_STATS] = {0};
	const struct rte_memzone *mz = NULL;
	const unsigned int flags = 0;

	if (rte_memzone_lookup(MZ_RTE_LATENCY_STATS))
		return -EEXIST;

	/** Allocate stats in shared memory fo multi process support */
	mz = rte_memzone_reserve(MZ_RTE_LATENCY_STATS, sizeof(*glob_stats),
					rte_socket_id(), flags);
	if (mz == NULL) {
		RTE_LOG(ERR, LATENCY_STATS, "Cannot reserve memory: %s:%d\n",
			__func__, __LINE__);
		return -ENOMEM;
	}

	glob_stats = mz->addr;
	samp_intvl = app_samp_intvl * latencystat_cycles_per_ns();

	/** Register latency stats with stats library */
	for (i = 0; i < NUM_LATENCY_STATS; i++)
		ptr_strings[i] = lat_stats_strings[i].name;

	latency_stats_index = rte_metrics_reg_names(ptr_strings,
							NUM_LATENCY_STATS);
	if (latency_stats_index < 0) {
		RTE_LOG(DEBUG, LATENCY_STATS,
			"Failed to register latency stats names\n");
		return -1;
	}

	/** Register Rx/Tx callbacks */
	RTE_ETH_FOREACH_DEV(pid) {
		struct rte_eth_dev_info dev_info;
		rte_eth_dev_info_get(pid, &dev_info);
		for (qid = 0; qid < dev_info.nb_rx_queues; qid++) {
			cbs = &rx_cbs[pid][qid];
			cbs->cb = rte_eth_add_first_rx_callback(pid, qid,
					add_time_stamps, user_cb);
			if (!cbs->cb)
				RTE_LOG(INFO, LATENCY_STATS, "Failed to "
					"register Rx callback for pid=%d, "
					"qid=%d\n", pid, qid);
		}
		for (qid = 0; qid < dev_info.nb_tx_queues; qid++) {
			cbs = &tx_cbs[pid][qid];
			cbs->cb =  rte_eth_add_tx_callback(pid, qid,
					calc_latency, user_cb);
			if (!cbs->cb)
				RTE_LOG(INFO, LATENCY_STATS, "Failed to "
					"register Tx callback for pid=%d, "
					"qid=%d\n", pid, qid);
		}
	}
	return 0;
}

int
rte_latencystats_uninit(void)
{
	uint16_t pid;
	uint16_t qid;
	int ret = 0;
	struct rxtx_cbs *cbs = NULL;
	const struct rte_memzone *mz = NULL;

	/** De register Rx/Tx callbacks */
	RTE_ETH_FOREACH_DEV(pid) {
		struct rte_eth_dev_info dev_info;
		rte_eth_dev_info_get(pid, &dev_info);
		for (qid = 0; qid < dev_info.nb_rx_queues; qid++) {
			cbs = &rx_cbs[pid][qid];
			ret = rte_eth_remove_rx_callback(pid, qid, cbs->cb);
			if (ret)
				RTE_LOG(INFO, LATENCY_STATS, "failed to "
					"remove Rx callback for pid=%d, "
					"qid=%d\n", pid, qid);
		}
		for (qid = 0; qid < dev_info.nb_tx_queues; qid++) {
			cbs = &tx_cbs[pid][qid];
			ret = rte_eth_remove_tx_callback(pid, qid, cbs->cb);
			if (ret)
				RTE_LOG(INFO, LATENCY_STATS, "failed to "
					"remove Tx callback for pid=%d, "
					"qid=%d\n", pid, qid);
		}
	}

	/* free up the memzone */
	mz = rte_memzone_lookup(MZ_RTE_LATENCY_STATS);
	if (mz)
		rte_memzone_free(mz);

	return 0;
}

int
rte_latencystats_get_names(struct rte_metric_name *names, uint16_t size)
{
	unsigned int i;

	if (names == NULL || size < NUM_LATENCY_STATS)
		return NUM_LATENCY_STATS;

	for (i = 0; i < NUM_LATENCY_STATS; i++)
		snprintf(names[i].name, sizeof(names[i].name),
				"%s", lat_stats_strings[i].name);

	return NUM_LATENCY_STATS;
}

int
rte_latencystats_get(struct rte_metric_value *values, uint16_t size)
{
	if (size < NUM_LATENCY_STATS || values == NULL)
		return NUM_LATENCY_STATS;

	if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
		const struct rte_memzone *mz;
		mz = rte_memzone_lookup(MZ_RTE_LATENCY_STATS);
		if (mz == NULL) {
			RTE_LOG(ERR, LATENCY_STATS,
				"Latency stats memzone not found\n");
			return -ENOMEM;
		}
		glob_stats =  mz->addr;
	}

	/* Retrieve latency stats */
	rte_latencystats_fill_values(values);

	return NUM_LATENCY_STATS;
}