summaryrefslogtreecommitdiffstats
path: root/drivers/net/failsafe/failsafe_ether.c
blob: 178316521a29491749bbe4e3829cf10db161ca26 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright 2017 6WIND S.A.
 * Copyright 2017 Mellanox Technologies, Ltd
 */

#include <unistd.h>

#include <rte_flow.h>
#include <rte_flow_driver.h>
#include <rte_cycles.h>

#include "failsafe_private.h"

/** Print a message out of a flow error. */
static int
fs_flow_complain(struct rte_flow_error *error)
{
	static const char *const errstrlist[] = {
		[RTE_FLOW_ERROR_TYPE_NONE] = "no error",
		[RTE_FLOW_ERROR_TYPE_UNSPECIFIED] = "cause unspecified",
		[RTE_FLOW_ERROR_TYPE_HANDLE] = "flow rule (handle)",
		[RTE_FLOW_ERROR_TYPE_ATTR_GROUP] = "group field",
		[RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY] = "priority field",
		[RTE_FLOW_ERROR_TYPE_ATTR_INGRESS] = "ingress field",
		[RTE_FLOW_ERROR_TYPE_ATTR_EGRESS] = "egress field",
		[RTE_FLOW_ERROR_TYPE_ATTR] = "attributes structure",
		[RTE_FLOW_ERROR_TYPE_ITEM_NUM] = "pattern length",
		[RTE_FLOW_ERROR_TYPE_ITEM] = "specific pattern item",
		[RTE_FLOW_ERROR_TYPE_ACTION_NUM] = "number of actions",
		[RTE_FLOW_ERROR_TYPE_ACTION] = "specific action",
	};
	const char *errstr;
	char buf[32];
	int err = rte_errno;

	if ((unsigned int)error->type >= RTE_DIM(errstrlist) ||
			!errstrlist[error->type])
		errstr = "unknown type";
	else
		errstr = errstrlist[error->type];
	ERROR("Caught error type %d (%s): %s%s\n",
		error->type, errstr,
		error->cause ? (snprintf(buf, sizeof(buf), "cause: %p, ",
				error->cause), buf) : "",
		error->message ? error->message : "(no stated reason)");
	return -err;
}

static int
eth_dev_flow_isolate_set(struct rte_eth_dev *dev,
			 struct sub_device *sdev)
{
	struct rte_flow_error ferror;
	int ret;

	if (!PRIV(dev)->flow_isolated) {
		DEBUG("Flow isolation already disabled");
	} else {
		DEBUG("Enabling flow isolation");
		ret = rte_flow_isolate(PORT_ID(sdev),
				       PRIV(dev)->flow_isolated,
				       &ferror);
		if (ret) {
			fs_flow_complain(&ferror);
			return ret;
		}
	}
	return 0;
}

static int
fs_eth_dev_conf_apply(struct rte_eth_dev *dev,
		struct sub_device *sdev)
{
	struct rte_eth_dev *edev;
	struct rte_vlan_filter_conf *vfc1;
	struct rte_vlan_filter_conf *vfc2;
	struct rte_flow *flow;
	struct rte_flow_error ferror;
	uint32_t i;
	int ret;

	edev = ETH(sdev);
	/* RX queue setup */
	for (i = 0; i < dev->data->nb_rx_queues; i++) {
		struct rxq *rxq;

		rxq = dev->data->rx_queues[i];
		ret = rte_eth_rx_queue_setup(PORT_ID(sdev), i,
				rxq->info.nb_desc, rxq->socket_id,
				&rxq->info.conf, rxq->info.mp);
		if (ret) {
			ERROR("rx_queue_setup failed");
			return ret;
		}
	}
	/* TX queue setup */
	for (i = 0; i < dev->data->nb_tx_queues; i++) {
		struct txq *txq;

		txq = dev->data->tx_queues[i];
		ret = rte_eth_tx_queue_setup(PORT_ID(sdev), i,
				txq->info.nb_desc, txq->socket_id,
				&txq->info.conf);
		if (ret) {
			ERROR("tx_queue_setup failed");
			return ret;
		}
	}
	/* dev_link.link_status */
	if (dev->data->dev_link.link_status !=
	    edev->data->dev_link.link_status) {
		DEBUG("Configuring link_status");
		if (dev->data->dev_link.link_status)
			ret = rte_eth_dev_set_link_up(PORT_ID(sdev));
		else
			ret = rte_eth_dev_set_link_down(PORT_ID(sdev));
		if (ret) {
			ERROR("Failed to apply link_status");
			return ret;
		}
	} else {
		DEBUG("link_status already set");
	}
	/* promiscuous */
	if (dev->data->promiscuous != edev->data->promiscuous) {
		DEBUG("Configuring promiscuous");
		if (dev->data->promiscuous)
			rte_eth_promiscuous_enable(PORT_ID(sdev));
		else
			rte_eth_promiscuous_disable(PORT_ID(sdev));
	} else {
		DEBUG("promiscuous already set");
	}
	/* all_multicast */
	if (dev->data->all_multicast != edev->data->all_multicast) {
		DEBUG("Configuring all_multicast");
		if (dev->data->all_multicast)
			rte_eth_allmulticast_enable(PORT_ID(sdev));
		else
			rte_eth_allmulticast_disable(PORT_ID(sdev));
	} else {
		DEBUG("all_multicast already set");
	}
	/* MTU */
	if (dev->data->mtu != edev->data->mtu) {
		DEBUG("Configuring MTU");
		ret = rte_eth_dev_set_mtu(PORT_ID(sdev), dev->data->mtu);
		if (ret) {
			ERROR("Failed to apply MTU");
			return ret;
		}
	} else {
		DEBUG("MTU already set");
	}
	/* default MAC */
	DEBUG("Configuring default MAC address");
	ret = rte_eth_dev_default_mac_addr_set(PORT_ID(sdev),
			&dev->data->mac_addrs[0]);
	if (ret) {
		ERROR("Setting default MAC address failed");
		return ret;
	}
	/* additional MAC */
	if (PRIV(dev)->nb_mac_addr > 1)
		DEBUG("Configure additional MAC address%s",
			(PRIV(dev)->nb_mac_addr > 2 ? "es" : ""));
	for (i = 1; i < PRIV(dev)->nb_mac_addr; i++) {
		struct ether_addr *ea;

		ea = &dev->data->mac_addrs[i];
		ret = rte_eth_dev_mac_addr_add(PORT_ID(sdev), ea,
				PRIV(dev)->mac_addr_pool[i]);
		if (ret) {
			char ea_fmt[ETHER_ADDR_FMT_SIZE];

			ether_format_addr(ea_fmt, ETHER_ADDR_FMT_SIZE, ea);
			ERROR("Adding MAC address %s failed", ea_fmt);
			return ret;
		}
	}
	/*
	 * Propagate multicast MAC addresses to sub-devices,
	 * if non zero number of addresses is set.
	 * The condition is required to avoid breakage of failsafe
	 * for sub-devices which do not support the operation
	 * if the feature is really not used.
	 */
	if (PRIV(dev)->nb_mcast_addr > 0) {
		DEBUG("Configuring multicast MAC addresses");
		ret = rte_eth_dev_set_mc_addr_list(PORT_ID(sdev),
						   PRIV(dev)->mcast_addrs,
						   PRIV(dev)->nb_mcast_addr);
		if (ret) {
			ERROR("Failed to apply multicast MAC addresses");
			return ret;
		}
	}
	/* VLAN filter */
	vfc1 = &dev->data->vlan_filter_conf;
	vfc2 = &edev->data->vlan_filter_conf;
	if (memcmp(vfc1, vfc2, sizeof(struct rte_vlan_filter_conf))) {
		uint64_t vbit;
		uint64_t ids;
		size_t i;
		uint16_t vlan_id;

		DEBUG("Configuring VLAN filter");
		for (i = 0; i < RTE_DIM(vfc1->ids); i++) {
			if (vfc1->ids[i] == 0)
				continue;
			ids = vfc1->ids[i];
			while (ids) {
				vlan_id = 64 * i;
				/* count trailing zeroes */
				vbit = ~ids & (ids - 1);
				/* clear least significant bit set */
				ids ^= (ids ^ (ids - 1)) ^ vbit;
				for (; vbit; vlan_id++)
					vbit >>= 1;
				ret = rte_eth_dev_vlan_filter(
					PORT_ID(sdev), vlan_id, 1);
				if (ret) {
					ERROR("Failed to apply VLAN filter %hu",
						vlan_id);
					return ret;
				}
			}
		}
	} else {
		DEBUG("VLAN filter already set");
	}
	/* rte_flow */
	if (TAILQ_EMPTY(&PRIV(dev)->flow_list)) {
		DEBUG("rte_flow already set");
	} else {
		DEBUG("Resetting rte_flow configuration");
		ret = rte_flow_flush(PORT_ID(sdev), &ferror);
		if (ret) {
			fs_flow_complain(&ferror);
			return ret;
		}
		i = 0;
		rte_errno = 0;
		DEBUG("Configuring rte_flow");
		TAILQ_FOREACH(flow, &PRIV(dev)->flow_list, next) {
			DEBUG("Creating flow #%" PRIu32, i++);
			flow->flows[SUB_ID(sdev)] =
				rte_flow_create(PORT_ID(sdev),
						flow->rule.attr,
						flow->rule.pattern,
						flow->rule.actions,
						&ferror);
			ret = rte_errno;
			if (ret)
				break;
		}
		if (ret) {
			fs_flow_complain(&ferror);
			return ret;
		}
	}
	return 0;
}

static void
fs_dev_remove(struct sub_device *sdev)
{
	int ret;

	if (sdev == NULL)
		return;
	switch (sdev->state) {
	case DEV_STARTED:
		failsafe_rx_intr_uninstall_subdevice(sdev);
		rte_eth_dev_stop(PORT_ID(sdev));
		sdev->state = DEV_ACTIVE;
		/* fallthrough */
	case DEV_ACTIVE:
		failsafe_eth_dev_unregister_callbacks(sdev);
		rte_eth_dev_close(PORT_ID(sdev));
		sdev->state = DEV_PROBED;
		/* fallthrough */
	case DEV_PROBED:
		ret = rte_dev_remove(sdev->dev);
		if (ret) {
			ERROR("Bus detach failed for sub_device %u",
			      SUB_ID(sdev));
		} else {
			rte_eth_dev_release_port(ETH(sdev));
		}
		sdev->state = DEV_PARSED;
		/* fallthrough */
	case DEV_PARSED:
	case DEV_UNDEFINED:
		sdev->state = DEV_UNDEFINED;
		/* the end */
		break;
	}
	sdev->remove = 0;
	failsafe_hotplug_alarm_install(sdev->fs_dev);
}

static void
fs_dev_stats_save(struct sub_device *sdev)
{
	struct rte_eth_stats stats;
	int err;

	/* Attempt to read current stats. */
	err = rte_eth_stats_get(PORT_ID(sdev), &stats);
	if (err) {
		uint64_t timestamp = sdev->stats_snapshot.timestamp;

		WARN("Could not access latest statistics from sub-device %d.\n",
			 SUB_ID(sdev));
		if (timestamp != 0)
			WARN("Using latest snapshot taken before %"PRIu64" seconds.\n",
				 (rte_rdtsc() - timestamp) / rte_get_tsc_hz());
	}
	failsafe_stats_increment(&PRIV(sdev->fs_dev)->stats_accumulator,
			err ? &sdev->stats_snapshot.stats : &stats);
	memset(&sdev->stats_snapshot, 0, sizeof(sdev->stats_snapshot));
}

static inline int
fs_rxtx_clean(struct sub_device *sdev)
{
	uint16_t i;

	for (i = 0; i < ETH(sdev)->data->nb_rx_queues; i++)
		if (FS_ATOMIC_RX(sdev, i))
			return 0;
	for (i = 0; i < ETH(sdev)->data->nb_tx_queues; i++)
		if (FS_ATOMIC_TX(sdev, i))
			return 0;
	return 1;
}

void
failsafe_eth_dev_unregister_callbacks(struct sub_device *sdev)
{
	int ret;

	if (sdev == NULL)
		return;
	if (sdev->rmv_callback) {
		ret = rte_eth_dev_callback_unregister(PORT_ID(sdev),
						RTE_ETH_EVENT_INTR_RMV,
						failsafe_eth_rmv_event_callback,
						sdev);
		if (ret)
			WARN("Failed to unregister RMV callback for sub_device"
			     " %d", SUB_ID(sdev));
		sdev->rmv_callback = 0;
	}
	if (sdev->lsc_callback) {
		ret = rte_eth_dev_callback_unregister(PORT_ID(sdev),
						RTE_ETH_EVENT_INTR_LSC,
						failsafe_eth_lsc_event_callback,
						sdev);
		if (ret)
			WARN("Failed to unregister LSC callback for sub_device"
			     " %d", SUB_ID(sdev));
		sdev->lsc_callback = 0;
	}
}

void
failsafe_dev_remove(struct rte_eth_dev *dev)
{
	struct sub_device *sdev;
	uint8_t i;

	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
		if (sdev->remove && fs_rxtx_clean(sdev)) {
			if (fs_lock(dev, 1) != 0)
				return;
			fs_dev_stats_save(sdev);
			fs_dev_remove(sdev);
			fs_unlock(dev, 1);
		}
}

static int
failsafe_eth_dev_rx_queues_sync(struct rte_eth_dev *dev)
{
	struct rxq *rxq;
	int ret;
	uint16_t i;

	for (i = 0; i < dev->data->nb_rx_queues; i++) {
		rxq = dev->data->rx_queues[i];

		if (rxq->info.conf.rx_deferred_start &&
		    dev->data->rx_queue_state[i] ==
						RTE_ETH_QUEUE_STATE_STARTED) {
			/*
			 * The subdevice Rx queue does not launch on device
			 * start if deferred start flag is set. It needs to be
			 * started manually in case an appropriate failsafe Rx
			 * queue has been started earlier.
			 */
			ret = dev->dev_ops->rx_queue_start(dev, i);
			if (ret) {
				ERROR("Could not synchronize Rx queue %d", i);
				return ret;
			}
		} else if (dev->data->rx_queue_state[i] ==
						RTE_ETH_QUEUE_STATE_STOPPED) {
			/*
			 * The subdevice Rx queue needs to be stopped manually
			 * in case an appropriate failsafe Rx queue has been
			 * stopped earlier.
			 */
			ret = dev->dev_ops->rx_queue_stop(dev, i);
			if (ret) {
				ERROR("Could not synchronize Rx queue %d", i);
				return ret;
			}
		}
	}
	return 0;
}

static int
failsafe_eth_dev_tx_queues_sync(struct rte_eth_dev *dev)
{
	struct txq *txq;
	int ret;
	uint16_t i;

	for (i = 0; i < dev->data->nb_tx_queues; i++) {
		txq = dev->data->tx_queues[i];

		if (txq->info.conf.tx_deferred_start &&
		    dev->data->tx_queue_state[i] ==
						RTE_ETH_QUEUE_STATE_STARTED) {
			/*
			 * The subdevice Tx queue does not launch on device
			 * start if deferred start flag is set. It needs to be
			 * started manually in case an appropriate failsafe Tx
			 * queue has been started earlier.
			 */
			ret = dev->dev_ops->tx_queue_start(dev, i);
			if (ret) {
				ERROR("Could not synchronize Tx queue %d", i);
				return ret;
			}
		} else if (dev->data->tx_queue_state[i] ==
						RTE_ETH_QUEUE_STATE_STOPPED) {
			/*
			 * The subdevice Tx queue needs to be stopped manually
			 * in case an appropriate failsafe Tx queue has been
			 * stopped earlier.
			 */
			ret = dev->dev_ops->tx_queue_stop(dev, i);
			if (ret) {
				ERROR("Could not synchronize Tx queue %d", i);
				return ret;
			}
		}
	}
	return 0;
}

int
failsafe_eth_dev_state_sync(struct rte_eth_dev *dev)
{
	struct sub_device *sdev;
	uint32_t inactive;
	int ret;
	uint8_t i;

	if (PRIV(dev)->state < DEV_PARSED)
		return 0;

	ret = failsafe_args_parse_subs(dev);
	if (ret)
		goto err_remove;

	if (PRIV(dev)->state < DEV_PROBED)
		return 0;
	ret = failsafe_eal_init(dev);
	if (ret)
		goto err_remove;
	if (PRIV(dev)->state < DEV_ACTIVE)
		return 0;
	inactive = 0;
	FOREACH_SUBDEV(sdev, i, dev) {
		if (sdev->state == DEV_PROBED) {
			inactive |= UINT32_C(1) << i;
			ret = eth_dev_flow_isolate_set(dev, sdev);
			if (ret) {
				ERROR("Could not apply configuration to sub_device %d",
				      i);
				goto err_remove;
			}
		}
	}
	ret = dev->dev_ops->dev_configure(dev);
	if (ret)
		goto err_remove;
	FOREACH_SUBDEV(sdev, i, dev) {
		if (inactive & (UINT32_C(1) << i)) {
			ret = fs_eth_dev_conf_apply(dev, sdev);
			if (ret) {
				ERROR("Could not apply configuration to sub_device %d",
				      i);
				goto err_remove;
			}
		}
	}
	/*
	 * If new devices have been configured, check if
	 * the link state has changed.
	 */
	if (inactive)
		dev->dev_ops->link_update(dev, 1);
	if (PRIV(dev)->state < DEV_STARTED)
		return 0;
	ret = dev->dev_ops->dev_start(dev);
	if (ret)
		goto err_remove;
	ret = failsafe_eth_dev_rx_queues_sync(dev);
	if (ret)
		goto err_remove;
	ret = failsafe_eth_dev_tx_queues_sync(dev);
	if (ret)
		goto err_remove;
	return 0;
err_remove:
	FOREACH_SUBDEV(sdev, i, dev)
		if (sdev->state != PRIV(dev)->state)
			sdev->remove = 1;
	return ret;
}

void
failsafe_stats_increment(struct rte_eth_stats *to, struct rte_eth_stats *from)
{
	uint32_t i;

	RTE_ASSERT(to != NULL && from != NULL);
	to->ipackets += from->ipackets;
	to->opackets += from->opackets;
	to->ibytes += from->ibytes;
	to->obytes += from->obytes;
	to->imissed += from->imissed;
	to->ierrors += from->ierrors;
	to->oerrors += from->oerrors;
	to->rx_nombuf += from->rx_nombuf;
	for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
		to->q_ipackets[i] += from->q_ipackets[i];
		to->q_opackets[i] += from->q_opackets[i];
		to->q_ibytes[i] += from->q_ibytes[i];
		to->q_obytes[i] += from->q_obytes[i];
		to->q_errors[i] += from->q_errors[i];
	}
}

int
failsafe_eth_rmv_event_callback(uint16_t port_id __rte_unused,
				enum rte_eth_event_type event __rte_unused,
				void *cb_arg, void *out __rte_unused)
{
	struct sub_device *sdev = cb_arg;

	fs_lock(sdev->fs_dev, 0);
	/* Switch as soon as possible tx_dev. */
	fs_switch_dev(sdev->fs_dev, sdev);
	/* Use safe bursts in any case. */
	failsafe_set_burst_fn(sdev->fs_dev, 1);
	/*
	 * Async removal, the sub-PMD will try to unregister
	 * the callback at the source of the current thread context.
	 */
	sdev->remove = 1;
	fs_unlock(sdev->fs_dev, 0);
	return 0;
}

int
failsafe_eth_lsc_event_callback(uint16_t port_id __rte_unused,
				enum rte_eth_event_type event __rte_unused,
				void *cb_arg, void *out __rte_unused)
{
	struct rte_eth_dev *dev = cb_arg;
	int ret;

	ret = dev->dev_ops->link_update(dev, 0);
	/* We must pass on the LSC event */
	if (ret)
		return _rte_eth_dev_callback_process(dev,
						     RTE_ETH_EVENT_INTR_LSC,
						     NULL);
	else
		return 0;
}

/* Take sub-device ownership before it becomes exposed to the application. */
int
failsafe_eth_new_event_callback(uint16_t port_id,
				enum rte_eth_event_type event __rte_unused,
				void *cb_arg, void *out __rte_unused)
{
	struct rte_eth_dev *fs_dev = cb_arg;
	struct sub_device *sdev;
	struct rte_eth_dev *dev = &rte_eth_devices[port_id];
	uint8_t i;

	FOREACH_SUBDEV_STATE(sdev, i, fs_dev, DEV_PARSED) {
		if (sdev->state >= DEV_PROBED)
			continue;
		if (strcmp(sdev->devargs.name, dev->device->name) != 0)
			continue;
		rte_eth_dev_owner_set(port_id, &PRIV(fs_dev)->my_owner);
		/* The actual owner will be checked after the port probing. */
		break;
	}
	return 0;
}