tests/vpp/perf/crypto/10ge2p1x710-ethip4ipsec1tnlhw-ip4base-int-aes128cbc-hmac512sha-ndrpdr.robot


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143

# Copyright (c) 2019 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

*** Settings ***
| Resource | resources/libraries/robot/shared/default.robot
| Resource | resources/libraries/robot/crypto/ipsec.robot
| ...
| Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV | NDRPDR | TNL_1
| ... | IP4FWD | IPSEC | IPSECHW | IPSECINT | NIC_Intel-X710 | BASE
| ... | AES_128_CBC | HMAC_SHA_512 | HMAC | AES | DRV_VFIO_PCI
| ...
| Suite Setup | Setup suite single link | performance | ipsechw
| Suite Teardown | Tear down suite | performance
| Test Setup | Setup test
| Test Teardown | Tear down test | performance
| ...
| Test Template | Local Template
| ...
| Documentation | *RFC2544: Pkt throughput IPv4 IPsec tunnel mode.*
| ...
| ... | *[Top] Network Topologies:* TG-DUT1-DUT2-TG 3-node circular topology
| ... | with single links between nodes.
| ... | *[Enc] Packet Encapsulations:* Eth-IPv4 on TG-DUTn,
| ... | Eth-IPv4-IPSec on DUT1-DUT2
| ... | *[Cfg] DUT configuration:* DUT1 and DUT2 are configured with multiple
| ... | IPsec tunnels between them. DUTs get IPv4 traffic from TG, encrypt it
| ... | and send to another DUT, where packets are decrypted and sent back to TG
| ... | *[Ver] TG verification:* TG finds and reports throughput NDR (Non Drop\
| ... | Rate) with zero packet loss tolerance and throughput PDR (Partial Drop\
| ... | Rate) with non-zero packet loss tolerance (LT) expressed in percentage\
| ... | of packets transmitted. NDR and PDR are discovered for different\
| ... | Ethernet L2 frame sizes using MLRsearch library.\
| ... | Test packets are generated by TG on
| ... | links to DUTs. TG traffic profile contains two L3 flow-groups
| ... | (flow-group per direction, number of flows per flow-group equals to
| ... | number of IPSec tunnels) with all packets
| ... | containing Ethernet header, IPv4 header with IP protocol=61 and
| ... | static payload. MAC addresses are matching MAC addresses of the TG
| ... | node interfaces. Incrementing of IP.dst (IPv4 destination address) field
| ... | is applied to both streams.
| ... | *[Ref] Applicable standard specifications:* RFC4303 and RFC2544.

*** Variables ***
| @{plugins_to_enable}= | dpdk_plugin.so | crypto_ia32_plugin.so
| ... | crypto_ipsecmb_plugin.so | crypto_openssl_plugin.so
| ${crypto_type}= | HW_DH895xcc
| ${nic_name}= | Intel-X710
| ${nic_driver}= | vfio-pci
| ${osi_layer}= | L3
| ${overhead}= | ${58}
| ${tg_if1_ip4}= | 192.168.10.2
| ${dut1_if1_ip4}= | 192.168.10.1
| ${dut1_if2_ip4}= | 100.0.0.1
| ${dut2_if1_ip4}= | 200.0.0.2
| ${dut2_if2_ip4}= | 192.168.20.1
| ${tg_if2_ip4}= | 192.168.20.2
| ${raddr_ip4}= | 20.0.0.0
| ${laddr_ip4}= | 10.0.0.0
| ${addr_range}= | ${24}
| ${n_tunnels}= | ${1}
# Traffic profile:
| ${traffic_profile}= | trex-sl-3n-ethip4-ip4dst${n_tunnels}

*** Keywords ***
| Local Template
| | [Documentation]
| | ... | [Cfg] DUT runs IPSec tunneling AES_128_CBC / HMAC_SHA_512 config.
| | ... | Each DUT uses ${phy_cores} physical core(s) for worker threads.
| | ... | [Ver] Measure NDR and PDR values using MLRsearch algorithm.\
| | ...
| | ... | *Arguments:*
| | ... | - frame_size - Framesize in Bytes in integer or string (IMIX_v4_1).
| | ... | Type: integer, string
| | ... | - phy_cores - Number of physical cores. Type: integer
| | ... | - rxq - Number of RX queues, default value: ${None}. Type: integer
| | ...
| | [Arguments] | ${frame_size} | ${phy_cores} | ${rxq}=${None}
| | ...
| | Set Test Variable | \${frame_size}
| | ...
| | # These are enums (not strings) so they cannot be in Variables table.
| | ${encr_alg}= | Crypto Alg AES CBC 128
| | ${auth_alg}= | Integ Alg SHA 512 256
| | ...
| | Given Set Max Rate And Jumbo
| | And Add worker threads to all DUTs | ${phy_cores} | ${rxq}
| | And Pre-initialize layer driver | ${nic_driver}
| | And Apply startup configuration on all VPP DUTs
| | When Initialize layer driver | ${nic_driver}
| | And Initialize layer interface
| | And Initialize IPSec in 3-node circular topology
| | And VPP IPsec Create Tunnel Interfaces
| | ... | ${nodes} | ${dut1_if2_ip4} | ${dut2_if1_ip4} | ${dut1_if2}
| | ... | ${dut2_if1} | ${n_tunnels} | ${encr_alg} | ${auth_alg}
| | ... | ${laddr_ip4} | ${raddr_ip4} | ${addr_range}
| | Then Find NDR and PDR intervals using optimized search

*** Test Cases ***
| tc01-64B-1c-ethip4ipsec1tnlhw-ip4base-int-aes128cbc-hmac512sha-ndrpdr
| | [Tags] | 64B | 1C
| | frame_size=${64} | phy_cores=${1}

| tc02-64B-2c-ethip4ipsec1tnlhw-ip4base-int-aes128cbc-hmac512sha-ndrpdr
| | [Tags] | 64B | 2C
| | frame_size=${64} | phy_cores=${2}

| tc03-64B-4c-ethip4ipsec1tnlhw-ip4base-int-aes128cbc-hmac512sha-ndrpdr
| | [Tags] | 64B | 4C
| | frame_size=${64} | phy_cores=${4}

| tc04-1518B-1c-ethip4ipsec1tnlhw-ip4base-int-aes128cbc-hmac512sha-ndrpdr
| | [Tags] | 1518B | 1C
| | frame_size=${1518} | phy_cores=${1}

| tc05-1518B-2c-ethip4ipsec1tnlhw-ip4base-int-aes128cbc-hmac512sha-ndrpdr
| | [Tags] | 1518B | 2C
| | frame_size=${1518} | phy_cores=${2}

| tc06-1518B-4c-ethip4ipsec1tnlhw-ip4base-int-aes128cbc-hmac512sha-ndrpdr
| | [Tags] | 1518B | 4C
| | frame_size=${1518} | phy_cores=${4}

| tc10-IMIX-1c-ethip4ipsec1tnlhw-ip4base-int-aes128cbc-hmac512sha-ndrpdr
| | [Tags] | IMIX | 1C
| | frame_size=IMIX_v4_1 | phy_cores=${1}

| tc11-IMIX-2c-ethip4ipsec1tnlhw-ip4base-int-aes128cbc-hmac512sha-ndrpdr
| | [Tags] | IMIX | 2C
| | frame_size=IMIX_v4_1 | phy_cores=${2}

| tc12-IMIX-4c-ethip4ipsec1tnlhw-ip4base-int-aes128cbc-hmac512sha-ndrpdr
| | [Tags] | IMIX | 4C
| | frame_size=IMIX_v4_1 | phy_cores=${4}
/*-
 *   BSD LICENSE
 *
 *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
 *   All rights reserved.
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <pthread.h>
#include <sys/queue.h>
#include <stdarg.h>
#include <unistd.h>
#include <string.h>
#include <errno.h>
#include <inttypes.h>
#include <sys/epoll.h>
#include <sys/signalfd.h>
#include <sys/ioctl.h>
#include <sys/eventfd.h>
#include <assert.h>

#include <rte_common.h>
#include <rte_interrupts.h>
#include <rte_memory.h>
#include <rte_memzone.h>
#include <rte_launch.h>
#include <rte_eal.h>
#include <rte_per_lcore.h>
#include <rte_lcore.h>
#include <rte_atomic.h>
#include <rte_branch_prediction.h>
#include <rte_debug.h>
#include <rte_log.h>
#include <rte_pci.h>
#include <rte_malloc.h>
#include <rte_errno.h>
#include <rte_spinlock.h>

#include "eal_private.h"
#include "eal_vfio.h"
#include "eal_thread.h"

#define EAL_INTR_EPOLL_WAIT_FOREVER (-1)
#define NB_OTHER_INTR               1

static RTE_DEFINE_PER_LCORE(int, _epfd) = -1; /**< epoll fd per thread */

/**
 * union for pipe fds.
 */
union intr_pipefds{
	struct {
		int pipefd[2];
	};
	struct {
		int readfd;
		int writefd;
	};
};

/**
 * union buffer for reading on different devices
 */
union rte_intr_read_buffer {
	int uio_intr_count;              /* for uio device */
#ifdef VFIO_PRESENT
	uint64_t vfio_intr_count;        /* for vfio device */
#endif
	uint64_t timerfd_num;            /* for timerfd */
	char charbuf[16];                /* for others */
};

TAILQ_HEAD(rte_intr_cb_list, rte_intr_callback);
TAILQ_HEAD(rte_intr_source_list, rte_intr_source);

struct rte_intr_callback {
	TAILQ_ENTRY(rte_intr_callback) next;
	rte_intr_callback_fn cb_fn;  /**< callback address */
	void *cb_arg;                /**< parameter for callback */
};

struct rte_intr_source {
	TAILQ_ENTRY(rte_intr_source) next;
	struct rte_intr_handle intr_handle; /**< interrupt handle */
	struct rte_intr_cb_list callbacks;  /**< user callbacks */
	uint32_t active;
};

/* global spinlock for interrupt data operation */
static rte_spinlock_t intr_lock = RTE_SPINLOCK_INITIALIZER;

/* union buffer for pipe read/write */
static union intr_pipefds intr_pipe;

/* interrupt sources list */
static struct rte_intr_source_list intr_sources;

/* interrupt handling thread */
static pthread_t intr_thread;

/* VFIO interrupts */
#ifdef VFIO_PRESENT

#define IRQ_SET_BUF_LEN  (sizeof(struct vfio_irq_set) + sizeof(int))
/* irq set buffer length for queue interrupts and LSC interrupt */
#define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
			      sizeof(int) * (RTE_MAX_RXTX_INTR_VEC_ID + 1))

/* enable legacy (INTx) interrupts */
static int
vfio_enable_intx(struct rte_intr_handle *intr_handle) {
	struct vfio_irq_set *irq_set;
	char irq_set_buf[IRQ_SET_BUF_LEN];
	int len, ret;
	int *fd_ptr;

	len = sizeof(irq_set_buf);

	/* enable INTx */
	irq_set = (struct vfio_irq_set *) irq_set_buf;
	irq_set->argsz = len;
	irq_set->count = 1;
	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
	irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
	irq_set->start = 0;
	fd_ptr = (int *) &irq_set->data;
	*fd_ptr = intr_handle->fd;

	ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);

	if (ret) {
		RTE_LOG(ERR, EAL, "Error enabling INTx interrupts for fd %d\n",
						intr_handle->fd);
		return -1;
	}

	/* unmask INTx after enabling */
	memset(irq_set, 0, len);
	len = sizeof(struct vfio_irq_set);
	irq_set->argsz = len;
	irq_set->count = 1;
	irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK;
	irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
	irq_set->start = 0;

	ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);

	if (ret) {
		RTE_LOG(ERR, EAL, "Error unmasking INTx interrupts for fd %d\n",
						intr_handle->fd);
		return -1;
	}
	return 0;
}

/* disable legacy (INTx) interrupts */
static int
vfio_disable_intx(struct rte_intr_handle *intr_handle) {
	struct vfio_irq_set *irq_set;
	char irq_set_buf[IRQ_SET_BUF_LEN];
	int len, ret;

	len = sizeof(struct vfio_irq_set);

	/* mask interrupts before disabling */
	irq_set = (struct vfio_irq_set *) irq_set_buf;
	irq_set->argsz = len;
	irq_set->count = 1;
	irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_MASK;
	irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
	irq_set->start = 0;

	ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);

	if (ret) {
		RTE_LOG(ERR, EAL, "Error masking INTx interrupts for fd %d\n",
						intr_handle->fd);
		return -1;
	}

	/* disable INTx*/
	memset(irq_set, 0, len);
	irq_set->argsz = len;
	irq_set->count = 0;
	irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER;
	irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
	irq_set->start = 0;

	ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);

	if (ret) {
		RTE_LOG(ERR, EAL,
			"Error disabling INTx interrupts for fd %d\n", intr_handle->fd);
		return -1;
	}
	return 0;
}

/* enable MSI interrupts */
static int
vfio_enable_msi(struct rte_intr_handle *intr_handle) {
	int len, ret;
	char irq_set_buf[IRQ_SET_BUF_LEN];
	struct vfio_irq_set *irq_set;
	int *fd_ptr;

	len = sizeof(irq_set_buf);

	irq_set = (struct vfio_irq_set *) irq_set_buf;
	irq_set->argsz = len;
	irq_set->count = 1;
	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
	irq_set->index = VFIO_PCI_MSI_IRQ_INDEX;
	irq_set->start = 0;
	fd_ptr = (int *) &irq_set->data;
	*fd_ptr = intr_handle->fd;

	ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);

	if (ret) {
		RTE_LOG(ERR, EAL, "Error enabling MSI interrupts for fd %d\n",
						intr_handle->fd);
		return -1;
	}
	return 0;
}

/* disable MSI interrupts */
static int
vfio_disable_msi(struct rte_intr_handle *intr_handle) {
	struct vfio_irq_set *irq_set;
	char irq_set_buf[IRQ_SET_BUF_LEN];
	int len, ret;

	len = sizeof(struct vfio_irq_set);

	irq_set = (struct vfio_irq_set *) irq_set_buf;
	irq_set->argsz = len;
	irq_set->count = 0;
	irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER;
	irq_set->index = VFIO_PCI_MSI_IRQ_INDEX;
	irq_set->start = 0;

	ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);

	if (ret)
		RTE_LOG(ERR, EAL,
			"Error disabling MSI interrupts for fd %d\n", intr_handle->fd);

	return ret;
}

/* enable MSI-X interrupts */
static int
vfio_enable_msix(struct rte_intr_handle *intr_handle) {
	int len, ret;
	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
	struct vfio_irq_set *irq_set;
	int *fd_ptr;

	len = sizeof(irq_set_buf);

	irq_set = (struct vfio_irq_set *) irq_set_buf;
	irq_set->argsz = len;
	if (!intr_handle->max_intr)
		intr_handle->max_intr = 1;
	else if (intr_handle->max_intr > RTE_MAX_RXTX_INTR_VEC_ID)
		intr_handle->max_intr = RTE_MAX_RXTX_INTR_VEC_ID + 1;

	irq_set->count = intr_handle->max_intr;
	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
	irq_set->start = 0;
	fd_ptr = (int *) &irq_set->data;
	/* INTR vector offset 0 reserve for non-efds mapping */
	fd_ptr[RTE_INTR_VEC_ZERO_OFFSET] = intr_handle->fd;
	memcpy(&fd_ptr[RTE_INTR_VEC_RXTX_OFFSET], intr_handle->efds,
		sizeof(*intr_handle->efds) * intr_handle->nb_efd);

	ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);

	if (ret) {
		RTE_LOG(ERR, EAL, "Error enabling MSI-X interrupts for fd %d\n",
						intr_handle->fd);
		return -1;
	}

	return 0;
}

/* disable MSI-X interrupts */
static int
vfio_disable_msix(struct rte_intr_handle *intr_handle) {
	struct vfio_irq_set *irq_set;
	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
	int len, ret;

	len = sizeof(struct vfio_irq_set);

	irq_set = (struct vfio_irq_set *) irq_set_buf;
	irq_set->argsz = len;
	irq_set->count = 0;
	irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER;
	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
	irq_set->start = 0;

	ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);

	if (ret)
		RTE_LOG(ERR, EAL,
			"Error disabling MSI-X interrupts for fd %d\n", intr_handle->fd);

	return ret;
}
#endif

static int
uio_intx_intr_disable(struct rte_intr_handle *intr_handle)
{
	unsigned char command_high;

	/* use UIO config file descriptor for uio_pci_generic */
	if (pread(intr_handle->uio_cfg_fd, &command_high, 1, 5) != 1) {
		RTE_LOG(ERR, EAL,
			"Error reading interrupts status for fd %d\n",
			intr_handle->uio_cfg_fd);
		return -1;
	}
	/* disable interrupts */
	command_high |= 0x4;
	if (pwrite(intr_handle->uio_cfg_fd, &command_high, 1, 5) != 1) {
		RTE_LOG(ERR, EAL,
			"Error disabling interrupts for fd %d\n",
			intr_handle->uio_cfg_fd);
		return -1;
	}

	return 0;
}

static int
uio_intx_intr_enable(struct rte_intr_handle *intr_handle)
{
	unsigned char command_high;

	/* use UIO config file descriptor for uio_pci_generic */
	if (pread(intr_handle->uio_cfg_fd, &command_high, 1, 5) != 1) {
		RTE_LOG(ERR, EAL,
			"Error reading interrupts status for fd %d\n",
			intr_handle->uio_cfg_fd);
		return -1;
	}
	/* enable interrupts */
	command_high &= ~0x4;
	if (pwrite(intr_handle->uio_cfg_fd, &command_high, 1, 5) != 1) {
		RTE_LOG(ERR, EAL,
			"Error enabling interrupts for fd %d\n",
			intr_handle->uio_cfg_fd);
		return -1;
	}

	return 0;
}

static int
uio_intr_disable(struct rte_intr_handle *intr_handle)
{
	const int value = 0;

	if (write(intr_handle->fd, &value, sizeof(value)) < 0) {
		RTE_LOG(ERR, EAL,
			"Error disabling interrupts for fd %d (%s)\n",
			intr_handle->fd, strerror(errno));
		return -1;
	}
	return 0;
}

static int
uio_intr_enable(struct rte_intr_handle *intr_handle)
{
	const int value = 1;

	if (write(intr_handle->fd, &value, sizeof(value)) < 0) {
		RTE_LOG(ERR, EAL,
			"Error enabling interrupts for fd %d (%s)\n",
			intr_handle->fd, strerror(errno));
		return -1;
	}
	return 0;
}

int
rte_intr_callback_register(struct rte_intr_handle *intr_handle,
			rte_intr_callback_fn cb, void *cb_arg)
{
	int ret, wake_thread;
	struct rte_intr_source *src;
	struct rte_intr_callback *callback;

	wake_thread = 0;

	/* first do parameter checking */
	if (intr_handle == NULL || intr_handle->fd < 0 || cb == NULL) {
		RTE_LOG(ERR, EAL,
			"Registering with invalid input parameter\n");
		return -EINVAL;
	}

	/* allocate a new interrupt callback entity */
	callback = rte_zmalloc("interrupt callback list",
				sizeof(*callback), 0);
	if (callback == NULL) {
		RTE_LOG(ERR, EAL, "Can not allocate memory\n");
		return -ENOMEM;
	}
	callback->cb_fn = cb;
	callback->cb_arg = cb_arg;

	rte_spinlock_lock(&intr_lock);

	/* check if there is at least one callback registered for the fd */
	TAILQ_FOREACH(src, &intr_sources, next) {
		if (src->intr_handle.fd == intr_handle->fd) {
			/* we had no interrupts for this */
			if (TAILQ_EMPTY(&src->callbacks))
				wake_thread = 1;

			TAILQ_INSERT_TAIL(&(src->callbacks), callback, next);
			ret = 0;
			break;
		}
	}

	/* no existing callbacks for this - add new source */
	if (src == NULL) {
		if ((src = rte_zmalloc("interrupt source list",
				sizeof(*src), 0)) == NULL) {
			RTE_LOG(ERR, EAL, "Can not allocate memory\n");
			rte_free(callback);
			ret = -ENOMEM;
		} else {
			src->intr_handle = *intr_handle;
			TAILQ_INIT(&src->callbacks);
			TAILQ_INSERT_TAIL(&(src->callbacks), callback, next);
			TAILQ_INSERT_TAIL(&intr_sources, src, next);
			wake_thread = 1;
			ret = 0;
		}
	}

	rte_spinlock_unlock(&intr_lock);

	/**
	 * check if need to notify the pipe fd waited by epoll_wait to
	 * rebuild the wait list.
	 */
	if (wake_thread)
		if (write(intr_pipe.writefd, "1", 1) < 0)
			return -EPIPE;

	return ret;
}

int
rte_intr_callback_unregister(struct rte_intr_handle *intr_handle,
			rte_intr_callback_fn cb_fn, void *cb_arg)
{
	int ret;
	struct rte_intr_source *src;
	struct rte_intr_callback *cb, *next;

	/* do parameter checking first */
	if (intr_handle == NULL || intr_handle->fd < 0) {
		RTE_LOG(ERR, EAL,
		"Unregistering with invalid input parameter\n");
		return -EINVAL;
	}

	rte_spinlock_lock(&intr_lock);

	/* check if the insterrupt source for the fd is existent */
	TAILQ_FOREACH(src, &intr_sources, next)
		if (src->intr_handle.fd == intr_handle->fd)
			break;

	/* No interrupt source registered for the fd */
	if (src == NULL) {
		ret = -ENOENT;

	/* interrupt source has some active callbacks right now. */
	} else if (src->active != 0) {
		ret = -EAGAIN;

	/* ok to remove. */
	} else {
		ret = 0;

		/*walk through the callbacks and remove all that match. */
		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {

			next = TAILQ_NEXT(cb, next);

			if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
					cb->cb_arg == cb_arg)) {
				TAILQ_REMOVE(&src->callbacks, cb, next);
				rte_free(cb);
				ret++;
			}
		}

		/* all callbacks for that source are removed. */
		if (TAILQ_EMPTY(&src->callbacks)) {
			TAILQ_REMOVE(&intr_sources, src, next);
			rte_free(src);
		}
	}

	rte_spinlock_unlock(&intr_lock);

	/* notify the pipe fd waited by epoll_wait to rebuild the wait list */
	if (ret >= 0 && write(intr_pipe.writefd, "1", 1) < 0) {
		ret = -EPIPE;
	}

	return ret;
}

int
rte_intr_enable(struct rte_intr_handle *intr_handle)
{
	if (!intr_handle || intr_handle->fd < 0 || intr_handle->uio_cfg_fd < 0)
		return -1;

	switch (intr_handle->type){
	/* write to the uio fd to enable the interrupt */
	case RTE_INTR_HANDLE_UIO:
		if (uio_intr_enable(intr_handle))
			return -1;
		break;
	case RTE_INTR_HANDLE_UIO_INTX:
		if (uio_intx_intr_enable(intr_handle))
			return -1;
		break;
	/* not used at this moment */
	case RTE_INTR_HANDLE_ALARM:
		return -1;
#ifdef VFIO_PRESENT
	case RTE_INTR_HANDLE_VFIO_MSIX:
		if (vfio_enable_msix(intr_handle))
			return -1;
		break;
	case RTE_INTR_HANDLE_VFIO_MSI:
		if (vfio_enable_msi(intr_handle))
			return -1;
		break;
	case RTE_INTR_HANDLE_VFIO_LEGACY:
		if (vfio_enable_intx(intr_handle))
			return -1;
		break;
#endif
	/* unknown handle type */
	default:
		RTE_LOG(ERR, EAL,
			"Unknown handle type of fd %d\n",
					intr_handle->fd);
		return -1;
	}

	return 0;
}

int
rte_intr_disable(struct rte_intr_handle *intr_handle)
{
	if (!intr_handle || intr_handle->fd < 0 || intr_handle->uio_cfg_fd < 0)
		return -1;

	switch (intr_handle->type){
	/* write to the uio fd to disable the interrupt */
	case RTE_INTR_HANDLE_UIO:
		if (uio_intr_disable(intr_handle))
			return -1;
		break;
	case RTE_INTR_HANDLE_UIO_INTX:
		if (uio_intx_intr_disable(intr_handle))
			return -1;
		break;
	/* not used at this moment */
	case RTE_INTR_HANDLE_ALARM:
		return -1;
#ifdef VFIO_PRESENT
	case RTE_INTR_HANDLE_VFIO_MSIX:
		if (vfio_disable_msix(intr_handle))
			return -1;
		break;
	case RTE_INTR_HANDLE_VFIO_MSI:
		if (vfio_disable_msi(intr_handle))
			return -1;
		break;
	case RTE_INTR_HANDLE_VFIO_LEGACY:
		if (vfio_disable_intx(intr_handle))
			return -1;
		break;
#endif
	/* unknown handle type */
	default:
		RTE_LOG(ERR, EAL,
			"Unknown handle type of fd %d\n",
					intr_handle->fd);
		return -1;
	}

	return 0;
}

static int
eal_intr_process_interrupts(struct epoll_event *events, int nfds)
{
	int n, bytes_read;
	struct rte_intr_source *src;
	struct rte_intr_callback *cb;
	union rte_intr_read_buffer buf;
	struct rte_intr_callback active_cb;

	for (n = 0; n < nfds; n++) {

		/**
		 * if the pipe fd is ready to read, return out to
		 * rebuild the wait list.
		 */
		if (events[n].data.fd == intr_pipe.readfd){
			int r = read(intr_pipe.readfd, buf.charbuf,
					sizeof(buf.charbuf));
			RTE_SET_USED(r);
			return -1;
		}
		rte_spinlock_lock(&intr_lock);
		TAILQ_FOREACH(src, &intr_sources, next)
			if (src->intr_handle.fd ==
					events[n].data.fd)
				break;
		if (src == NULL){
			rte_spinlock_unlock(&intr_lock);
			continue;
		}

		/* mark this interrupt source as active and release the lock. */
		src->active = 1;
		rte_spinlock_unlock(&intr_lock);

		/* set the length to be read dor different handle type */
		switch (src->intr_handle.type) {
		case RTE_INTR_HANDLE_UIO:
		case RTE_INTR_HANDLE_UIO_INTX:
			bytes_read = sizeof(buf.uio_intr_count);
			break;
		case RTE_INTR_HANDLE_ALARM:
			bytes_read = sizeof(buf.timerfd_num);
			break;
#ifdef VFIO_PRESENT
		case RTE_INTR_HANDLE_VFIO_MSIX:
		case RTE_INTR_HANDLE_VFIO_MSI:
		case RTE_INTR_HANDLE_VFIO_LEGACY:
			bytes_read = sizeof(buf.vfio_intr_count);
			break;
#endif
		case RTE_INTR_HANDLE_EXT:
		default:
			bytes_read = 1;
			break;
		}

		if (src->intr_handle.type != RTE_INTR_HANDLE_EXT) {
			/**
			 * read out to clear the ready-to-be-read flag
			 * for epoll_wait.
			 */
			bytes_read = read(events[n].data.fd, &buf, bytes_read);
			if (bytes_read < 0) {
				if (errno == EINTR || errno == EWOULDBLOCK)
					continue;

				RTE_LOG(ERR, EAL, "Error reading from file "
					"descriptor %d: %s\n",
					events[n].data.fd,
					strerror(errno));
			} else if (bytes_read == 0)
				RTE_LOG(ERR, EAL, "Read nothing from file "
					"descriptor %d\n", events[n].data.fd);
		}

		/* grab a lock, again to call callbacks and update status. */
		rte_spinlock_lock(&intr_lock);

		if (bytes_read > 0) {

			/* Finally, call all callbacks. */
			TAILQ_FOREACH(cb, &src->callbacks, next) {

				/* make a copy and unlock. */
				active_cb = *cb;
				rte_spinlock_unlock(&intr_lock);

				/* call the actual callback */
				active_cb.cb_fn(&src->intr_handle,
					active_cb.cb_arg);

				/*get the lock back. */
				rte_spinlock_lock(&intr_lock);
			}
		}

		/* we done with that interrupt source, release it. */
		src->active = 0;
		rte_spinlock_unlock(&intr_lock);
	}

	return 0;
}

/**
 * It handles all the interrupts.
 *
 * @param pfd
 *  epoll file descriptor.
 * @param totalfds
 *  The number of file descriptors added in epoll.
 *
 * @return
 *  void
 */
static void
eal_intr_handle_interrupts(int pfd, unsigned totalfds)
{
	struct epoll_event events[totalfds];
	int nfds = 0;

	for(;;) {
		nfds = epoll_wait(pfd, events, totalfds,
			EAL_INTR_EPOLL_WAIT_FOREVER);
		/* epoll_wait fail */
		if (nfds < 0) {
			if (errno == EINTR)
				continue;
			RTE_LOG(ERR, EAL,
				"epoll_wait returns with fail\n");
			return;
		}
		/* epoll_wait timeout, will never happens here */
		else if (nfds == 0)
			continue;
		/* epoll_wait has at least one fd ready to read */
		if (eal_intr_process_interrupts(events, nfds) < 0)
			return;
	}
}

/**
 * It builds/rebuilds up the epoll file descriptor with all the
 * file descriptors being waited on. Then handles the interrupts.
 *
 * @param arg
 *  pointer. (unused)
 *
 * @return
 *  never return;
 */
static __attribute__((noreturn)) void *
eal_intr_thread_main(__rte_unused void *arg)
{
	struct epoll_event ev;

	/* host thread, never break out */
	for (;;) {
		/* build up the epoll fd with all descriptors we are to
		 * wait on then pass it to the handle_interrupts function
		 */
		static struct epoll_event pipe_event = {
			.events = EPOLLIN | EPOLLPRI,
		};
		struct rte_intr_source *src;
		unsigned numfds = 0;

		/* create epoll fd */
		int pfd = epoll_create(1);
		if (pfd < 0)
			rte_panic("Cannot create epoll instance\n");

		pipe_event.data.fd = intr_pipe.readfd;
		/**
		 * add pipe fd into wait list, this pipe is used to
		 * rebuild the wait list.
		 */
		if (epoll_ctl(pfd, EPOLL_CTL_ADD, intr_pipe.readfd,
						&pipe_event) < 0) {
			rte_panic("Error adding fd to %d epoll_ctl, %s\n",
					intr_pipe.readfd, strerror(errno));
		}
		numfds++;

		rte_spinlock_lock(&intr_lock);

		TAILQ_FOREACH(src, &intr_sources, next) {
			if (src->callbacks.tqh_first == NULL)
				continue; /* skip those with no callbacks */
			ev.events = EPOLLIN | EPOLLPRI;
			ev.data.fd = src->intr_handle.fd;

			/**
			 * add all the uio device file descriptor
			 * into wait list.
			 */
			if (epoll_ctl(pfd, EPOLL_CTL_ADD,
					src->intr_handle.fd, &ev) < 0){
				rte_panic("Error adding fd %d epoll_ctl, %s\n",
					src->intr_handle.fd, strerror(errno));
			}
			else
				numfds++;
		}
		rte_spinlock_unlock(&intr_lock);
		/* serve the interrupt */
		eal_intr_handle_interrupts(pfd, numfds);

		/**
		 * when we return, we need to rebuild the
		 * list of fds to monitor.
		 */
		close(pfd);
	}
}

int
rte_eal_intr_init(void)
{
	int ret = 0, ret_1 = 0;
	char thread_name[RTE_MAX_THREAD_NAME_LEN];

	/* init the global interrupt source head */
	TAILQ_INIT(&intr_sources);

	/**
	 * create a pipe which will be waited by epoll and notified to
	 * rebuild the wait list of epoll.
	 */
	if (pipe(intr_pipe.pipefd) < 0)
		return -1;

	/* create the host thread to wait/handle the interrupt */
	ret = pthread_create(&intr_thread, NULL,
			eal_intr_thread_main, NULL);
	if (ret != 0) {
		RTE_LOG(ERR, EAL,
			"Failed to create thread for interrupt handling\n");
	} else {
		/* Set thread_name for aid in debugging. */
		snprintf(thread_name, RTE_MAX_THREAD_NAME_LEN,
			"eal-intr-thread");
		ret_1 = rte_thread_setname(intr_thread, thread_name);
		if (ret_1 != 0)
			RTE_LOG(DEBUG, EAL,
			"Failed to set thread name for interrupt handling\n");
	}

	return -ret;
}

static void
eal_intr_proc_rxtx_intr(int fd, const struct rte_intr_handle *intr_handle)
{
	union rte_intr_read_buffer buf;
	int bytes_read = 1;
	int nbytes;

	switch (intr_handle->type) {
	case RTE_INTR_HANDLE_UIO:
	case RTE_INTR_HANDLE_UIO_INTX:
		bytes_read = sizeof(buf.uio_intr_count);
		break;
#ifdef VFIO_PRESENT
	case RTE_INTR_HANDLE_VFIO_MSIX:
	case RTE_INTR_HANDLE_VFIO_MSI:
	case RTE_INTR_HANDLE_VFIO_LEGACY:
		bytes_read = sizeof(buf.vfio_intr_count);
		break;
#endif
	default:
		bytes_read = 1;
		RTE_LOG(INFO, EAL, "unexpected intr type\n");
		break;
	}

	/**
	 * read out to clear the ready-to-be-read flag
	 * for epoll_wait.
	 */
	do {
		nbytes = read(fd, &buf, bytes_read);
		if (nbytes < 0) {
			if (errno == EINTR || errno == EWOULDBLOCK ||
			    errno == EAGAIN)
				continue;
			RTE_LOG(ERR, EAL,
				"Error reading from fd %d: %s\n",
				fd, strerror(errno));
		} else if (nbytes == 0)
			RTE_LOG(ERR, EAL, "Read nothing from fd %d\n", fd);
		return;
	} while (1);
}

static int
eal_epoll_process_event(struct epoll_event *evs, unsigned int n,
			struct rte_epoll_event *events)
{
	unsigned int i, count = 0;
	struct rte_epoll_event *rev;

	for (i = 0; i < n; i++) {
		rev = evs[i].data.ptr;
		if (!rev || !rte_atomic32_cmpset(&rev->status, RTE_EPOLL_VALID,
						 RTE_EPOLL_EXEC))
			continue;

		events[count].status        = RTE_EPOLL_VALID;
		events[count].fd            = rev->fd;
		events[count].epfd          = rev->epfd;
		events[count].epdata.event  = rev->epdata.event;
		events[count].epdata.data   = rev->epdata.data;
		if (rev->epdata.cb_fun)
			rev->epdata.cb_fun(rev->fd,
					   rev->epdata.cb_arg);

		rte_compiler_barrier();
		rev->status = RTE_EPOLL_VALID;
		count++;
	}
	return count;
}

static inline int
eal_init_tls_epfd(void)
{
	int pfd = epoll_create(255);

	if (pfd < 0) {
		RTE_LOG(ERR, EAL,
			"Cannot create epoll instance\n");
		return -1;
	}
	return pfd;
}

int
rte_intr_tls_epfd(void)
{
	if (RTE_PER_LCORE(_epfd) == -1)
		RTE_PER_LCORE(_epfd) = eal_init_tls_epfd();

	return RTE_PER_LCORE(_epfd);
}

int
rte_epoll_wait(int epfd, struct rte_epoll_event *events,
	       int maxevents, int timeout)
{
	struct epoll_event evs[maxevents];
	int rc;

	if (!events) {
		RTE_LOG(ERR, EAL, "rte_epoll_event can't be NULL\n");
		return -1;
	}

	/* using per thread epoll fd */
	if (epfd == RTE_EPOLL_PER_THREAD)
		epfd = rte_intr_tls_epfd();

	while (1) {
		rc = epoll_wait(epfd, evs, maxevents, timeout);
		if (likely(rc > 0)) {
			/* epoll_wait has at least one fd ready to read */
			rc = eal_epoll_process_event(evs, rc, events);
			break;
		} else if (rc < 0) {
			if (errno == EINTR)
				continue;
			/* epoll_wait fail */
			RTE_LOG(ERR, EAL, "epoll_wait returns with fail %s\n",
				strerror(errno));
			rc = -1;
			break;
		} else {
			/* rc == 0, epoll_wait timed out */
			break;
		}
	}

	return rc;
}

static inline void
eal_epoll_data_safe_free(struct rte_epoll_event *ev)
{
	while (!rte_atomic32_cmpset(&ev->status, RTE_EPOLL_VALID,
				    RTE_EPOLL_INVALID))
		while (ev->status != RTE_EPOLL_VALID)
			rte_pause();
	memset(&ev->epdata, 0, sizeof(ev->epdata));
	ev->fd = -1;
	ev->epfd = -1;
}

int
rte_epoll_ctl(int epfd, int op, int fd,
	      struct rte_epoll_event *event)
{
	struct epoll_event ev;

	if (!event) {
		RTE_LOG(ERR, EAL, "rte_epoll_event can't be NULL\n");
		return -1;
	}

	/* using per thread epoll fd */
	if (epfd == RTE_EPOLL_PER_THREAD)
		epfd = rte_intr_tls_epfd();

	if (op == EPOLL_CTL_ADD) {
		event->status = RTE_EPOLL_VALID;
		event->fd = fd;  /* ignore fd in event */
		event->epfd = epfd;
		ev.data.ptr = (void *)event;
	}

	ev.events = event->epdata.event;
	if (epoll_ctl(epfd, op, fd, &ev) < 0) {
		RTE_LOG(ERR, EAL, "Error op %d fd %d epoll_ctl, %s\n",
			op, fd, strerror(errno));
		if (op == EPOLL_CTL_ADD)
			/* rollback status when CTL_ADD fail */
			event->status = RTE_EPOLL_INVALID;
		return -1;
	}

	if (op == EPOLL_CTL_DEL && event->status != RTE_EPOLL_INVALID)
		eal_epoll_data_safe_free(event);

	return 0;
}

int
rte_intr_rx_ctl(struct rte_intr_handle *intr_handle, int epfd,
		int op, unsigned int vec, void *data)
{
	struct rte_epoll_event *rev;
	struct rte_epoll_data *epdata;
	int epfd_op;
	unsigned int efd_idx;
	int rc = 0;

	efd_idx = (vec >= RTE_INTR_VEC_RXTX_OFFSET) ?
		(vec - RTE_INTR_VEC_RXTX_OFFSET) : vec;

	if (!intr_handle || intr_handle->nb_efd == 0 ||
	    efd_idx >= intr_handle->nb_efd) {
		RTE_LOG(ERR, EAL, "Wrong intr vector number.\n");
		return -EPERM;
	}

	switch (op) {
	case RTE_INTR_EVENT_ADD:
		epfd_op = EPOLL_CTL_ADD;
		rev = &intr_handle->elist[efd_idx];
		if (rev->status != RTE_EPOLL_INVALID) {
			RTE_LOG(INFO, EAL, "Event already been added.\n");
			return -EEXIST;
		}

		/* attach to intr vector fd */
		epdata = &rev->epdata;
		epdata->event  = EPOLLIN | EPOLLPRI | EPOLLET;
		epdata->data   = data;
		epdata->cb_fun = (rte_intr_event_cb_t)eal_intr_proc_rxtx_intr;
		epdata->cb_arg = (void *)intr_handle;
		rc = rte_epoll_ctl(epfd, epfd_op,
				   intr_handle->efds[efd_idx], rev);
		if (!rc)
			RTE_LOG(DEBUG, EAL,
				"efd %d associated with vec %d added on epfd %d"
				"\n", rev->fd, vec, epfd);
		else
			rc = -EPERM;
		break;
	case RTE_INTR_EVENT_DEL:
		epfd_op = EPOLL_CTL_DEL;
		rev = &intr_handle->elist[efd_idx];
		if (rev->status == RTE_EPOLL_INVALID) {
			RTE_LOG(INFO, EAL, "Event does not exist.\n");
			return -EPERM;
		}

		rc = rte_epoll_ctl(rev->epfd, epfd_op, rev->fd, rev);
		if (rc)
			rc = -EPERM;
		break;
	default:
		RTE_LOG(ERR, EAL, "event op type mismatch\n");
		rc = -EPERM;
	}

	return rc;
}

int
rte_intr_efd_enable(struct rte_intr_handle *intr_handle, uint32_t nb_efd)
{
	uint32_t i;
	int fd;
	uint32_t n = RTE_MIN(nb_efd, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);

	assert(nb_efd != 0);

	if (intr_handle->type == RTE_INTR_HANDLE_VFIO_MSIX) {
		for (i = 0; i < n; i++) {
			fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
			if (fd < 0) {
				RTE_LOG(ERR, EAL,
					"can't setup eventfd, error %i (%s)\n",
					errno, strerror(errno));
				return -1;
			}
			intr_handle->efds[i] = fd;
		}
		intr_handle->nb_efd   = n;
		intr_handle->max_intr = NB_OTHER_INTR + n;
	} else {
		intr_handle->efds[0]  = intr_handle->fd;
		intr_handle->nb_efd   = RTE_MIN(nb_efd, 1U);
		intr_handle->max_intr = NB_OTHER_INTR;
	}

	return 0;
}

void
rte_intr_efd_disable(struct rte_intr_handle *intr_handle)
{
	uint32_t i;
	struct rte_epoll_event *rev;

	for (i = 0; i < intr_handle->nb_efd; i++) {
		rev = &intr_handle->elist[i];
		if (rev->status == RTE_EPOLL_INVALID)
			continue;
		if (rte_epoll_ctl(rev->epfd, EPOLL_CTL_DEL, rev->fd, rev)) {
			/* force free if the entry valid */
			eal_epoll_data_safe_free(rev);
			rev->status = RTE_EPOLL_INVALID;
		}
	}

	if (intr_handle->max_intr > intr_handle->nb_efd) {
		for (i = 0; i < intr_handle->nb_efd; i++)
			close(intr_handle->efds[i]);
	}
	intr_handle->nb_efd = 0;
	intr_handle->max_intr = 0;
}

int
rte_intr_dp_is_en(struct rte_intr_handle *intr_handle)
{
	return !(!intr_handle->nb_efd);
}

int
rte_intr_allow_others(struct rte_intr_handle *intr_handle)
{
	if (!rte_intr_dp_is_en(intr_handle))
		return 1;
	else
		return !!(intr_handle->max_intr - intr_handle->nb_efd);
}

int
rte_intr_cap_multiple(struct rte_intr_handle *intr_handle)
{
	if (intr_handle->type == RTE_INTR_HANDLE_VFIO_MSIX)
		return 1;

	return 0;
}