summaryrefslogtreecommitdiffstats
path: root/drivers/net/virtio/virtio_rxtx.h
blob: 54f1e849b04536896c49ccdf2fd72a9127796575 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
/*-
 *   BSD LICENSE
 *
 *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
 *   All rights reserved.
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef _VIRTIO_RXTX_H_
#define _VIRTIO_RXTX_H_

#define RTE_PMD_VIRTIO_RX_MAX_BURST 64

struct virtnet_stats {
	uint64_t	packets;
	uint64_t	bytes;
	uint64_t	errors;
	uint64_t	multicast;
	uint64_t	broadcast;
	/* Size bins in array as RFC 2819, undersized [0], 64 [1], etc */
	uint64_t	size_bins[8];
};

struct virtnet_rx {
	struct virtqueue *vq;
	/* dummy mbuf, for wraparound when processing RX ring. */
	struct rte_mbuf fake_mbuf;
	uint64_t mbuf_initializer; /**< value to init mbufs. */
	struct rte_mempool *mpool; /**< mempool for mbuf allocation */

	uint16_t queue_id;   /**< DPDK queue index. */
	uint16_t port_id;     /**< Device port identifier. */

	/* Statistics */
	struct virtnet_stats stats;

	const struct rte_memzone *mz; /**< mem zone to populate RX ring. */
};

struct virtnet_tx {
	struct virtqueue *vq;
	/**< memzone to populate hdr. */
	const struct rte_memzone *virtio_net_hdr_mz;
	rte_iova_t virtio_net_hdr_mem;   /**< hdr for each xmit packet */

	uint16_t    queue_id;            /**< DPDK queue index. */
	uint16_t    port_id;             /**< Device port identifier. */

	/* Statistics */
	struct virtnet_stats stats;

	const struct rte_memzone *mz;    /**< mem zone to populate TX ring. */
};

struct virtnet_ctl {
	struct virtqueue *vq;
	/**< memzone to populate hdr. */
	const struct rte_memzone *virtio_net_hdr_mz;
	rte_iova_t virtio_net_hdr_mem;  /**< hdr for each xmit packet */
	uint16_t port_id;               /**< Device port identifier. */
	const struct rte_memzone *mz;   /**< mem zone to populate CTL ring. */
};

int virtio_rxq_vec_setup(struct virtnet_rx *rxvq);

int virtqueue_enqueue_recv_refill_simple(struct virtqueue *vq,
	struct rte_mbuf *m);

#endif /* _VIRTIO_RXTX_H_ */
ic.Subheading */ .highlight .gt { color: #aa0000 } /* Generic.Traceback */ .highlight .kc { color: #008800; font-weight: bold } /* Keyword.Constant */ .highlight .kd { color: #008800; font-weight: bold } /* Keyword.Declaration */ .highlight .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */ .highlight .kp { color: #008800 } /* Keyword.Pseudo */ .highlight .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */ .highlight .kt { color: #888888; font-weight: bold } /* Keyword.Type */ .highlight .m { color: #0000DD; font-weight: bold } /* Literal.Number */ .highlight .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */ .highlight .na { color: #336699 } /* Name.Attribute */ .highlight .nb { color: #003388 } /* Name.Builtin */ .highlight .nc { color: #bb0066; font-weight: bold } /* Name.Class */ .highlight .no { color: #003366; font-weight: bold } /* Name.Constant */ .highlight .nd { color: #555555 } /* Name.Decorator */ .highlight .ne { color: #bb0066; font-weight: bold } /* Name.Exception */ .highlight .nf { color: #0066bb; font-weight: bold } /* Name.Function */ .highlight .nl { color: #336699; font-style: italic } /* Name.Label */ .highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */ .highlight .py { color: #336699; font-weight: bold } /* Name.Property */ .highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */ .highlight .nv { color: #336699 } /* Name.Variable */ .highlight .ow { color: #008800 } /* Operator.Word */ .highlight .w { color: #bbbbbb } /* Text.Whitespace */ .highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */ .highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */ .highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */ .highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */ .highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */ .highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */ .highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */ .highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */ .highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */ .highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */ .highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */ .highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */ .highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */ .highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */ .highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */ .highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */ .highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */ .highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */ .highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */ .highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */ .highlight .vc { color: #336699 } /* Name.Variable.Class */ .highlight .vg { color: #dd7700 } /* Name.Variable.Global */ .highlight .vi { color: #3333bb } /* Name.Variable.Instance */ .highlight .vm { color: #336699 } /* Name.Variable.Magic */ .highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */ }
/*-
 * BSD LICENSE
 *
 * Copyright (c) 2015-2017 Atomic Rules LLC
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * * Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in
 * the documentation and/or other materials provided with the
 * distribution.
 * * Neither the name of copyright holder nor the names of its
 * contributors may be used to endorse or promote products derived
 * from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <unistd.h>

#include "ark_ethdev_tx.h"
#include "ark_global.h"
#include "ark_mpu.h"
#include "ark_ddm.h"
#include "ark_logs.h"

#define ARK_TX_META_SIZE   32
#define ARK_TX_META_OFFSET (RTE_PKTMBUF_HEADROOM - ARK_TX_META_SIZE)
#define ARK_TX_MAX_NOCHAIN (RTE_MBUF_DEFAULT_DATAROOM)


/* ************************************************************************* */
struct ark_tx_queue {
	struct ark_tx_meta *meta_q;
	struct rte_mbuf **bufs;

	/* handles for hw objects */
	struct ark_mpu_t *mpu;
	struct ark_ddm_t *ddm;

	/* Stats HW tracks bytes and packets, need to count send errors */
	uint64_t tx_errors;

	uint32_t queue_size;
	uint32_t queue_mask;

	/* 3 indexes to the paired data rings. */
	uint32_t prod_index;		/* where to put the next one */
	uint32_t free_index;		/* mbuf has been freed */

	/* The queue Id is used to identify the HW Q */
	uint16_t phys_qid;
	/* The queue Index within the dpdk device structures */
	uint16_t queue_index;

	uint32_t pad[1];

	/* second cache line - fields only used in slow path */
	MARKER cacheline1 __rte_cache_min_aligned;
	uint32_t cons_index;		/* hw is done, can be freed */
} __rte_cache_aligned;

/* Forward declarations */
static uint32_t eth_ark_tx_jumbo(struct ark_tx_queue *queue,
				 struct rte_mbuf *mbuf);
static int eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue);
static void free_completed_tx(struct ark_tx_queue *queue);

static inline void
ark_tx_hw_queue_stop(struct ark_tx_queue *queue)
{
	ark_mpu_stop(queue->mpu);
}

/* ************************************************************************* */
static inline void
eth_ark_tx_meta_from_mbuf(struct ark_tx_meta *meta,
			  const struct rte_mbuf *mbuf,
			  uint8_t flags)
{
	meta->physaddr = rte_mbuf_data_iova(mbuf);
	meta->delta_ns = 0;
	meta->data_len = rte_pktmbuf_data_len(mbuf);
	meta->flags = flags;
}

/* ************************************************************************* */
uint16_t
eth_ark_xmit_pkts_noop(void *vtxq __rte_unused,
		       struct rte_mbuf **tx_pkts __rte_unused,
		       uint16_t nb_pkts __rte_unused)
{
	return 0;
}

/* ************************************************************************* */
uint16_t
eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
{
	struct ark_tx_queue *queue;
	struct rte_mbuf *mbuf;
	struct ark_tx_meta *meta;

	uint32_t idx;
	uint32_t prod_index_limit;
	int stat;
	uint16_t nb;

	queue = (struct ark_tx_queue *)vtxq;

	/* free any packets after the HW is done with them */
	free_completed_tx(queue);

	prod_index_limit = queue->queue_size + queue->free_index;

	for (nb = 0;
	     (nb < nb_pkts) && (queue->prod_index != prod_index_limit);
	     ++nb) {
		mbuf = tx_pkts[nb];

		if (ARK_TX_PAD_TO_60) {
			if (unlikely(rte_pktmbuf_pkt_len(mbuf) < 60)) {
				/* this packet even if it is small can be split,
				 * be sure to add to the end mbuf
				 */
				uint16_t to_add =
					60 - rte_pktmbuf_pkt_len(mbuf);
				char *appended =
					rte_pktmbuf_append(mbuf, to_add);

				if (appended == 0) {
					/* This packet is in error,
					 * we cannot send it so just
					 * count it and delete it.
					 */
					queue->tx_errors += 1;
					rte_pktmbuf_free(mbuf);
					continue;
				}
				memset(appended, 0, to_add);
			}
		}

		if (unlikely(mbuf->nb_segs != 1)) {
			stat = eth_ark_tx_jumbo(queue, mbuf);
			if (unlikely(stat != 0))
				break;		/* Queue is full */
		} else {
			idx = queue->prod_index & queue->queue_mask;
			queue->bufs[idx] = mbuf;
			meta = &queue->meta_q[idx];
			eth_ark_tx_meta_from_mbuf(meta,
						  mbuf,
						  ARK_DDM_SOP |
						  ARK_DDM_EOP);
			queue->prod_index++;
		}
	}

	if (ARK_TX_DEBUG && (nb != nb_pkts)) {
		PMD_TX_LOG(DEBUG, "TX: Failure to send:"
			   " req: %" PRIU32
			   " sent: %" PRIU32
			   " prod: %" PRIU32
			   " cons: %" PRIU32
			   " free: %" PRIU32 "\n",
			   nb_pkts, nb,
			   queue->prod_index,
			   queue->cons_index,
			   queue->free_index);
		ark_mpu_dump(queue->mpu,
			     "TX Failure MPU: ",
			     queue->phys_qid);
	}

	/* let FPGA know producer index.  */
	if (likely(nb != 0))
		ark_mpu_set_producer(queue->mpu, queue->prod_index);

	return nb;
}

/* ************************************************************************* */
static uint32_t
eth_ark_tx_jumbo(struct ark_tx_queue *queue, struct rte_mbuf *mbuf)
{
	struct rte_mbuf *next;
	struct ark_tx_meta *meta;
	uint32_t free_queue_space;
	uint32_t idx;
	uint8_t flags = ARK_DDM_SOP;

	free_queue_space = queue->queue_mask -
		(queue->prod_index - queue->free_index);
	if (unlikely(free_queue_space < mbuf->nb_segs))
		return -1;

	while (mbuf != NULL) {
		next = mbuf->next;

		idx = queue->prod_index & queue->queue_mask;
		queue->bufs[idx] = mbuf;
		meta = &queue->meta_q[idx];

		flags |= (next == NULL) ? ARK_DDM_EOP : 0;
		eth_ark_tx_meta_from_mbuf(meta, mbuf, flags);
		queue->prod_index++;

		flags &= ~ARK_DDM_SOP;	/* drop SOP flags */
		mbuf = next;
	}

	return 0;
}

/* ************************************************************************* */
int
eth_ark_tx_queue_setup(struct rte_eth_dev *dev,
		       uint16_t queue_idx,
		       uint16_t nb_desc,
		       unsigned int socket_id,
		       const struct rte_eth_txconf *tx_conf __rte_unused)
{
	struct ark_adapter *ark = (struct ark_adapter *)dev->data->dev_private;
	struct ark_tx_queue *queue;
	int status;

	/* Future: divide the Q's evenly with multi-ports */
	int port = dev->data->port_id;
	int qidx = port + queue_idx;

	if (!rte_is_power_of_2(nb_desc)) {
		PMD_DRV_LOG(ERR,
			    "DPDK Arkville configuration queue size"
			    " must be power of two %u (%s)\n",
			    nb_desc, __func__);
		return -1;
	}

	/* Allocate queue struct */
	queue =	rte_zmalloc_socket("Ark_txqueue",
				   sizeof(struct ark_tx_queue),
				   64,
				   socket_id);
	if (queue == 0) {
		PMD_DRV_LOG(ERR, "Failed to allocate tx "
			    "queue memory in %s\n",
			    __func__);
		return -ENOMEM;
	}

	/* we use zmalloc no need to initialize fields */
	queue->queue_size = nb_desc;
	queue->queue_mask = nb_desc - 1;
	queue->phys_qid = qidx;
	queue->queue_index = queue_idx;
	dev->data->tx_queues[queue_idx] = queue;

	queue->meta_q =
		rte_zmalloc_socket("Ark_txqueue meta",
				   nb_desc * sizeof(struct ark_tx_meta),
				   64,
				   socket_id);
	queue->bufs =
		rte_zmalloc_socket("Ark_txqueue bufs",
				   nb_desc * sizeof(struct rte_mbuf *),
				   64,
				   socket_id);

	if (queue->meta_q == 0 || queue->bufs == 0) {
		PMD_DRV_LOG(ERR, "Failed to allocate "
			    "queue memory in %s\n", __func__);
		rte_free(queue->meta_q);
		rte_free(queue->bufs);
		rte_free(queue);
		return -ENOMEM;
	}

	queue->ddm = RTE_PTR_ADD(ark->ddm.v, qidx * ARK_DDM_QOFFSET);
	queue->mpu = RTE_PTR_ADD(ark->mputx.v, qidx * ARK_MPU_QOFFSET);

	status = eth_ark_tx_hw_queue_config(queue);

	if (unlikely(status != 0)) {
		rte_free(queue->meta_q);
		rte_free(queue->bufs);
		rte_free(queue);
		return -1;		/* ERROR CODE */
	}

	return 0;
}

/* ************************************************************************* */
static int
eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue)
{
	rte_iova_t queue_base, ring_base, cons_index_addr;
	uint32_t write_interval_ns;

	/* Verify HW -- MPU */
	if (ark_mpu_verify(queue->mpu, sizeof(struct ark_tx_meta)))
		return -1;

	queue_base = rte_malloc_virt2iova(queue);
	ring_base = rte_malloc_virt2iova(queue->meta_q);
	cons_index_addr =
		queue_base + offsetof(struct ark_tx_queue, cons_index);

	ark_mpu_stop(queue->mpu);
	ark_mpu_reset(queue->mpu);

	/* Stop and Reset and configure MPU */
	ark_mpu_configure(queue->mpu, ring_base, queue->queue_size, 1);

	/*
	 * Adjust the write interval based on queue size --
	 * increase pcie traffic  when low mbuf count
	 * Queue sizes less than 128 are not allowed
	 */
	switch (queue->queue_size) {
	case 128:
		write_interval_ns = 500;
		break;
	case 256:
		write_interval_ns = 500;
		break;
	case 512:
		write_interval_ns = 1000;
		break;
	default:
		write_interval_ns = 2000;
		break;
	}

	/* Completion address in UDM */
	ark_ddm_setup(queue->ddm, cons_index_addr, write_interval_ns);

	return 0;
}

/* ************************************************************************* */
void
eth_ark_tx_queue_release(void *vtx_queue)
{
	struct ark_tx_queue *queue;

	queue = (struct ark_tx_queue *)vtx_queue;

	ark_tx_hw_queue_stop(queue);

	queue->cons_index = queue->prod_index;
	free_completed_tx(queue);

	rte_free(queue->meta_q);
	rte_free(queue->bufs);
	rte_free(queue);
}

/* ************************************************************************* */
int
eth_ark_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id)
{
	struct ark_tx_queue *queue;
	int cnt = 0;

	queue = dev->data->tx_queues[queue_id];

	/* Wait for DDM to send out all packets. */
	while (queue->cons_index != queue->prod_index) {
		usleep(100);
		if (cnt++ > 10000)
			return -1;
	}

	ark_mpu_stop(queue->mpu);
	free_completed_tx(queue);

	dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;

	return 0;
}

int
eth_ark_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id)
{
	struct ark_tx_queue *queue;

	queue = dev->data->tx_queues[queue_id];
	if (dev->data->tx_queue_state[queue_id] == RTE_ETH_QUEUE_STATE_STARTED)
		return 0;

	ark_mpu_start(queue->mpu);
	dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED;

	return 0;
}

/* ************************************************************************* */
static void
free_completed_tx(struct ark_tx_queue *queue)
{
	struct rte_mbuf *mbuf;
	struct ark_tx_meta *meta;
	uint32_t top_index;

	top_index = queue->cons_index;	/* read once */
	while (queue->free_index != top_index) {
		meta = &queue->meta_q[queue->free_index & queue->queue_mask];
		mbuf = queue->bufs[queue->free_index & queue->queue_mask];

		if (likely((meta->flags & ARK_DDM_SOP) != 0)) {
			/* ref count of the mbuf is checked in this call. */
			rte_pktmbuf_free(mbuf);
		}
		queue->free_index++;
	}
}

/* ************************************************************************* */
void
eth_tx_queue_stats_get(void *vqueue, struct rte_eth_stats *stats)
{
	struct ark_tx_queue *queue;
	struct ark_ddm_t *ddm;
	uint64_t bytes, pkts;

	queue = vqueue;
	ddm = queue->ddm;

	bytes = ark_ddm_queue_byte_count(ddm);
	pkts = ark_ddm_queue_pkt_count(ddm);

	stats->q_opackets[queue->queue_index] = pkts;
	stats->q_obytes[queue->queue_index] = bytes;
	stats->opackets += pkts;
	stats->obytes += bytes;
	stats->oerrors += queue->tx_errors;
}

void
eth_tx_queue_stats_reset(void *vqueue)
{
	struct ark_tx_queue *queue;
	struct ark_ddm_t *ddm;

	queue = vqueue;
	ddm = queue->ddm;

	ark_ddm_queue_reset_stats(ddm);
	queue->tx_errors = 0;
}