New upstream version 18.02upstream/18.02

Change-Id: I89ed24cb2a49b78fe5be6970b99dd46c1499fcc3
Signed-off-by: Luca Boccassi <luca.boccassi@gmail.com>

1 files changed, 1126 insertions, 0 deletions

diff --git a/lib/librte_bbdev/rte_bbdev.c b/lib/librte_bbdev/rte_bbdev.c
new file mode 100644
index 00000000..74ecc490
--- /dev/null
+++ b/lib/librte_bbdev/rte_bbdev.c
@@ -0,0 +1,1126 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <string.h>
+#include <stdbool.h>
+
+#include <rte_compat.h>
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_log.h>
+#include <rte_debug.h>
+#include <rte_eal.h>
+#include <rte_malloc.h>
+#include <rte_mempool.h>
+#include <rte_memzone.h>
+#include <rte_lcore.h>
+#include <rte_dev.h>
+#include <rte_spinlock.h>
+#include <rte_tailq.h>
+#include <rte_interrupts.h>
+
+#include "rte_bbdev_op.h"
+#include "rte_bbdev.h"
+#include "rte_bbdev_pmd.h"
+
+#define DEV_NAME "BBDEV"
+
+
+/* BBDev library logging ID */
+static int bbdev_logtype;
+
+/* Helper macro for logging */
+#define rte_bbdev_log(level, fmt, ...) \
+	rte_log(RTE_LOG_ ## level, bbdev_logtype, fmt "\n", ##__VA_ARGS__)
+
+#define rte_bbdev_log_debug(fmt, ...) \
+	rte_bbdev_log(DEBUG, RTE_STR(__LINE__) ":%s() " fmt, __func__, \
+		##__VA_ARGS__)
+
+/* Helper macro to check dev_id is valid */
+#define VALID_DEV_OR_RET_ERR(dev, dev_id) do { \
+	if (dev == NULL) { \
+		rte_bbdev_log(ERR, "device %u is invalid", dev_id); \
+		return -ENODEV; \
+	} \
+} while (0)
+
+/* Helper macro to check dev_ops is valid */
+#define VALID_DEV_OPS_OR_RET_ERR(dev, dev_id) do { \
+	if (dev->dev_ops == NULL) { \
+		rte_bbdev_log(ERR, "NULL dev_ops structure in device %u", \
+				dev_id); \
+		return -ENODEV; \
+	} \
+} while (0)
+
+/* Helper macro to check that driver implements required function pointer */
+#define VALID_FUNC_OR_RET_ERR(func, dev_id) do { \
+	if (func == NULL) { \
+		rte_bbdev_log(ERR, "device %u does not support %s", \
+				dev_id, #func); \
+		return -ENOTSUP; \
+	} \
+} while (0)
+
+/* Helper macro to check that queue is valid */
+#define VALID_QUEUE_OR_RET_ERR(queue_id, dev) do { \
+	if (queue_id >= dev->data->num_queues) { \
+		rte_bbdev_log(ERR, "Invalid queue_id %u for device %u", \
+				queue_id, dev->data->dev_id); \
+		return -ERANGE; \
+	} \
+} while (0)
+
+/* List of callback functions registered by an application */
+struct rte_bbdev_callback {
+	TAILQ_ENTRY(rte_bbdev_callback) next;  /* Callbacks list */
+	rte_bbdev_cb_fn cb_fn;  /* Callback address */
+	void *cb_arg;  /* Parameter for callback */
+	void *ret_param;  /* Return parameter */
+	enum rte_bbdev_event_type event; /* Interrupt event type */
+	uint32_t active; /* Callback is executing */
+};
+
+/* spinlock for bbdev device callbacks */
+static rte_spinlock_t rte_bbdev_cb_lock = RTE_SPINLOCK_INITIALIZER;
+
+/*
+ * Global array of all devices. This is not static because it's used by the
+ * inline enqueue and dequeue functions
+ */
+struct rte_bbdev rte_bbdev_devices[RTE_BBDEV_MAX_DEVS];
+
+/* Global array with rte_bbdev_data structures */
+static struct rte_bbdev_data *rte_bbdev_data;
+
+/* Memzone name for global bbdev data pool */
+static const char *MZ_RTE_BBDEV_DATA = "rte_bbdev_data";
+
+/* Number of currently valid devices */
+static uint16_t num_devs;
+
+/* Return pointer to device structure, with validity check */
+static struct rte_bbdev *
+get_dev(uint16_t dev_id)
+{
+	if (rte_bbdev_is_valid(dev_id))
+		return &rte_bbdev_devices[dev_id];
+	return NULL;
+}
+
+/* Allocate global data array */
+static int
+rte_bbdev_data_alloc(void)
+{
+	const unsigned int flags = 0;
+	const struct rte_memzone *mz;
+
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		mz = rte_memzone_reserve(MZ_RTE_BBDEV_DATA,
+				RTE_BBDEV_MAX_DEVS * sizeof(*rte_bbdev_data),
+				rte_socket_id(), flags);
+	} else
+		mz = rte_memzone_lookup(MZ_RTE_BBDEV_DATA);
+	if (mz == NULL) {
+		rte_bbdev_log(CRIT,
+				"Cannot allocate memzone for bbdev port data");
+		return -ENOMEM;
+	}
+
+	rte_bbdev_data = mz->addr;
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+		memset(rte_bbdev_data, 0,
+				RTE_BBDEV_MAX_DEVS * sizeof(*rte_bbdev_data));
+	return 0;
+}
+
+/*
+ * Find data alocated for the device or if not found return first unused bbdev
+ * data. If all structures are in use and none is used by the device return
+ * NULL.
+ */
+static struct rte_bbdev_data *
+find_bbdev_data(const char *name)
+{
+	uint16_t data_id;
+
+	for (data_id = 0; data_id < RTE_BBDEV_MAX_DEVS; ++data_id) {
+		if (strlen(rte_bbdev_data[data_id].name) == 0) {
+			memset(&rte_bbdev_data[data_id], 0,
+					sizeof(struct rte_bbdev_data));
+			return &rte_bbdev_data[data_id];
+		} else if (strncmp(rte_bbdev_data[data_id].name, name,
+				RTE_BBDEV_NAME_MAX_LEN) == 0)
+			return &rte_bbdev_data[data_id];
+	}
+
+	return NULL;
+}
+
+/* Find lowest device id with no attached device */
+static uint16_t
+find_free_dev_id(void)
+{
+	uint16_t i;
+	for (i = 0; i < RTE_BBDEV_MAX_DEVS; i++) {
+		if (rte_bbdev_devices[i].state == RTE_BBDEV_UNUSED)
+			return i;
+	}
+	return RTE_BBDEV_MAX_DEVS;
+}
+
+struct rte_bbdev * __rte_experimental
+rte_bbdev_allocate(const char *name)
+{
+	int ret;
+	struct rte_bbdev *bbdev;
+	uint16_t dev_id;
+
+	if (name == NULL) {
+		rte_bbdev_log(ERR, "Invalid null device name");
+		return NULL;
+	}
+
+	if (rte_bbdev_get_named_dev(name) != NULL) {
+		rte_bbdev_log(ERR, "Device \"%s\" is already allocated", name);
+		return NULL;
+	}
+
+	dev_id = find_free_dev_id();
+	if (dev_id == RTE_BBDEV_MAX_DEVS) {
+		rte_bbdev_log(ERR, "Reached maximum number of devices");
+		return NULL;
+	}
+
+	bbdev = &rte_bbdev_devices[dev_id];
+
+	if (rte_bbdev_data == NULL) {
+		ret = rte_bbdev_data_alloc();
+		if (ret != 0)
+			return NULL;
+	}
+
+	bbdev->data = find_bbdev_data(name);
+	if (bbdev->data == NULL) {
+		rte_bbdev_log(ERR,
+				"Max BBDevs already allocated in multi-process environment!");
+		return NULL;
+	}
+
+	rte_atomic16_inc(&bbdev->data->process_cnt);
+	bbdev->data->dev_id = dev_id;
+	bbdev->state = RTE_BBDEV_INITIALIZED;
+
+	ret = snprintf(bbdev->data->name, RTE_BBDEV_NAME_MAX_LEN, "%s", name);
+	if ((ret < 0) || (ret >= RTE_BBDEV_NAME_MAX_LEN)) {
+		rte_bbdev_log(ERR, "Copying device name \"%s\" failed", name);
+		return NULL;
+	}
+
+	/* init user callbacks */
+	TAILQ_INIT(&(bbdev->list_cbs));
+
+	num_devs++;
+
+	rte_bbdev_log_debug("Initialised device %s (id = %u). Num devices = %u",
+			name, dev_id, num_devs);
+
+	return bbdev;
+}
+
+int __rte_experimental
+rte_bbdev_release(struct rte_bbdev *bbdev)
+{
+	uint16_t dev_id;
+	struct rte_bbdev_callback *cb, *next;
+
+	if (bbdev == NULL) {
+		rte_bbdev_log(ERR, "NULL bbdev");
+		return -ENODEV;
+	}
+	dev_id = bbdev->data->dev_id;
+
+	/* free all callbacks from the device's list */
+	for (cb = TAILQ_FIRST(&bbdev->list_cbs); cb != NULL; cb = next) {
+
+		next = TAILQ_NEXT(cb, next);
+		TAILQ_REMOVE(&(bbdev->list_cbs), cb, next);
+		rte_free(cb);
+	}
+
+	/* clear shared BBDev Data if no process is using the device anymore */
+	if (rte_atomic16_dec_and_test(&bbdev->data->process_cnt))
+		memset(bbdev->data, 0, sizeof(*bbdev->data));
+
+	memset(bbdev, 0, sizeof(*bbdev));
+	num_devs--;
+	bbdev->state = RTE_BBDEV_UNUSED;
+
+	rte_bbdev_log_debug(
+			"Un-initialised device id = %u. Num devices = %u",
+			dev_id, num_devs);
+	return 0;
+}
+
+struct rte_bbdev * __rte_experimental
+rte_bbdev_get_named_dev(const char *name)
+{
+	unsigned int i;
+
+	if (name == NULL) {
+		rte_bbdev_log(ERR, "NULL driver name");
+		return NULL;
+	}
+
+	for (i = 0; i < RTE_BBDEV_MAX_DEVS; i++) {
+		struct rte_bbdev *dev = get_dev(i);
+		if (dev && (strncmp(dev->data->name,
+				name, RTE_BBDEV_NAME_MAX_LEN) == 0))
+			return dev;
+	}
+
+	return NULL;
+}
+
+uint16_t __rte_experimental
+rte_bbdev_count(void)
+{
+	return num_devs;
+}
+
+bool __rte_experimental
+rte_bbdev_is_valid(uint16_t dev_id)
+{
+	if ((dev_id < RTE_BBDEV_MAX_DEVS) &&
+		rte_bbdev_devices[dev_id].state == RTE_BBDEV_INITIALIZED)
+		return true;
+	return false;
+}
+
+uint16_t __rte_experimental
+rte_bbdev_find_next(uint16_t dev_id)
+{
+	dev_id++;
+	for (; dev_id < RTE_BBDEV_MAX_DEVS; dev_id++)
+		if (rte_bbdev_is_valid(dev_id))
+			break;
+	return dev_id;
+}
+
+int __rte_experimental
+rte_bbdev_setup_queues(uint16_t dev_id, uint16_t num_queues, int socket_id)
+{
+	unsigned int i;
+	int ret;
+	struct rte_bbdev_driver_info dev_info;
+	struct rte_bbdev *dev = get_dev(dev_id);
+	VALID_DEV_OR_RET_ERR(dev, dev_id);
+
+	VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
+
+	if (dev->data->started) {
+		rte_bbdev_log(ERR,
+				"Device %u cannot be configured when started",
+				dev_id);
+		return -EBUSY;
+	}
+
+	/* Get device driver information to get max number of queues */
+	VALID_FUNC_OR_RET_ERR(dev->dev_ops->info_get, dev_id);
+	memset(&dev_info, 0, sizeof(dev_info));
+	dev->dev_ops->info_get(dev, &dev_info);
+
+	if ((num_queues == 0) || (num_queues > dev_info.max_num_queues)) {
+		rte_bbdev_log(ERR,
+				"Device %u supports 0 < N <= %u queues, not %u",
+				dev_id, dev_info.max_num_queues, num_queues);
+		return -EINVAL;
+	}
+
+	/* If re-configuration, get driver to free existing internal memory */
+	if (dev->data->queues != NULL) {
+		VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_release, dev_id);
+		for (i = 0; i < dev->data->num_queues; i++) {
+			int ret = dev->dev_ops->queue_release(dev, i);
+			if (ret < 0) {
+				rte_bbdev_log(ERR,
+						"Device %u queue %u release failed",
+						dev_id, i);
+				return ret;
+			}
+		}
+		/* Call optional device close */
+		if (dev->dev_ops->close) {
+			ret = dev->dev_ops->close(dev);
+			if (ret < 0) {
+				rte_bbdev_log(ERR,
+						"Device %u couldn't be closed",
+						dev_id);
+				return ret;
+			}
+		}
+		rte_free(dev->data->queues);
+	}
+
+	/* Allocate queue pointers */
+	dev->data->queues = rte_calloc_socket(DEV_NAME, num_queues,
+			sizeof(dev->data->queues[0]), RTE_CACHE_LINE_SIZE,
+				dev->data->socket_id);
+	if (dev->data->queues == NULL) {
+		rte_bbdev_log(ERR,
+				"calloc of %u queues for device %u on socket %i failed",
+				num_queues, dev_id, dev->data->socket_id);
+		return -ENOMEM;
+	}
+
+	dev->data->num_queues = num_queues;
+
+	/* Call optional device configuration */
+	if (dev->dev_ops->setup_queues) {
+		ret = dev->dev_ops->setup_queues(dev, num_queues, socket_id);
+		if (ret < 0) {
+			rte_bbdev_log(ERR,
+					"Device %u memory configuration failed",
+					dev_id);
+			goto error;
+		}
+	}
+
+	rte_bbdev_log_debug("Device %u set up with %u queues", dev_id,
+			num_queues);
+	return 0;
+
+error:
+	dev->data->num_queues = 0;
+	rte_free(dev->data->queues);
+	dev->data->queues = NULL;
+	return ret;
+}
+
+int __rte_experimental
+rte_bbdev_intr_enable(uint16_t dev_id)
+{
+	int ret;
+	struct rte_bbdev *dev = get_dev(dev_id);
+	VALID_DEV_OR_RET_ERR(dev, dev_id);
+
+	VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
+
+	if (dev->data->started) {
+		rte_bbdev_log(ERR,
+				"Device %u cannot be configured when started",
+				dev_id);
+		return -EBUSY;
+	}
+
+	if (dev->dev_ops->intr_enable) {
+		ret = dev->dev_ops->intr_enable(dev);
+		if (ret < 0) {
+			rte_bbdev_log(ERR,
+					"Device %u interrupts configuration failed",
+					dev_id);
+			return ret;
+		}
+		rte_bbdev_log_debug("Enabled interrupts for dev %u", dev_id);
+		return 0;
+	}
+
+	rte_bbdev_log(ERR, "Device %u doesn't support interrupts", dev_id);
+	return -ENOTSUP;
+}
+
+int __rte_experimental
+rte_bbdev_queue_configure(uint16_t dev_id, uint16_t queue_id,
+		const struct rte_bbdev_queue_conf *conf)
+{
+	int ret = 0;
+	struct rte_bbdev_driver_info dev_info;
+	struct rte_bbdev *dev = get_dev(dev_id);
+	const struct rte_bbdev_op_cap *p;
+	struct rte_bbdev_queue_conf *stored_conf;
+	const char *op_type_str;
+	VALID_DEV_OR_RET_ERR(dev, dev_id);
+
+	VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
+
+	VALID_QUEUE_OR_RET_ERR(queue_id, dev);
+
+	if (dev->data->queues[queue_id].started || dev->data->started) {
+		rte_bbdev_log(ERR,
+				"Queue %u of device %u cannot be configured when started",
+				queue_id, dev_id);
+		return -EBUSY;
+	}
+
+	VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_release, dev_id);
+	VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_setup, dev_id);
+
+	/* Get device driver information to verify config is valid */
+	VALID_FUNC_OR_RET_ERR(dev->dev_ops->info_get, dev_id);
+	memset(&dev_info, 0, sizeof(dev_info));
+	dev->dev_ops->info_get(dev, &dev_info);
+
+	/* Check configuration is valid */
+	if (conf != NULL) {
+		if ((conf->op_type == RTE_BBDEV_OP_NONE) &&
+				(dev_info.capabilities[0].type ==
+				RTE_BBDEV_OP_NONE)) {
+			ret = 1;
+		} else {
+			for (p = dev_info.capabilities;
+					p->type != RTE_BBDEV_OP_NONE; p++) {
+				if (conf->op_type == p->type) {
+					ret = 1;
+					break;
+				}
+			}
+		}
+		if (ret == 0) {
+			rte_bbdev_log(ERR, "Invalid operation type");
+			return -EINVAL;
+		}
+		if (conf->queue_size > dev_info.queue_size_lim) {
+			rte_bbdev_log(ERR,
+					"Size (%u) of queue %u of device %u must be: <= %u",
+					conf->queue_size, queue_id, dev_id,
+					dev_info.queue_size_lim);
+			return -EINVAL;
+		}
+		if (!rte_is_power_of_2(conf->queue_size)) {
+			rte_bbdev_log(ERR,
+					"Size (%u) of queue %u of device %u must be a power of 2",
+					conf->queue_size, queue_id, dev_id);
+			return -EINVAL;
+		}
+		if (conf->priority > dev_info.max_queue_priority) {
+			rte_bbdev_log(ERR,
+					"Priority (%u) of queue %u of bdev %u must be <= %u",
+					conf->priority, queue_id, dev_id,
+					dev_info.max_queue_priority);
+			return -EINVAL;
+		}
+	}
+
+	/* Release existing queue (in case of queue reconfiguration) */
+	if (dev->data->queues[queue_id].queue_private != NULL) {
+		ret = dev->dev_ops->queue_release(dev, queue_id);
+		if (ret < 0) {
+			rte_bbdev_log(ERR, "Device %u queue %u release failed",
+					dev_id, queue_id);
+			return ret;
+		}
+	}
+
+	/* Get driver to setup the queue */
+	ret = dev->dev_ops->queue_setup(dev, queue_id, (conf != NULL) ?
+			conf : &dev_info.default_queue_conf);
+	if (ret < 0) {
+		rte_bbdev_log(ERR,
+				"Device %u queue %u setup failed", dev_id,
+				queue_id);
+		return ret;
+	}
+
+	/* Store configuration */
+	stored_conf = &dev->data->queues[queue_id].conf;
+	memcpy(stored_conf,
+			(conf != NULL) ? conf : &dev_info.default_queue_conf,
+			sizeof(*stored_conf));
+
+	op_type_str = rte_bbdev_op_type_str(stored_conf->op_type);
+	if (op_type_str == NULL)
+		return -EINVAL;
+
+	rte_bbdev_log_debug("Configured dev%uq%u (size=%u, type=%s, prio=%u)",
+			dev_id, queue_id, stored_conf->queue_size, op_type_str,
+			stored_conf->priority);
+
+	return 0;
+}
+
+int __rte_experimental
+rte_bbdev_start(uint16_t dev_id)
+{
+	int i;
+	struct rte_bbdev *dev = get_dev(dev_id);
+	VALID_DEV_OR_RET_ERR(dev, dev_id);
+
+	VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
+
+	if (dev->data->started) {
+		rte_bbdev_log_debug("Device %u is already started", dev_id);
+		return 0;
+	}
+
+	if (dev->dev_ops->start) {
+		int ret = dev->dev_ops->start(dev);
+		if (ret < 0) {
+			rte_bbdev_log(ERR, "Device %u start failed", dev_id);
+			return ret;
+		}
+	}
+
+	/* Store new state */
+	for (i = 0; i < dev->data->num_queues; i++)
+		if (!dev->data->queues[i].conf.deferred_start)
+			dev->data->queues[i].started = true;
+	dev->data->started = true;
+
+	rte_bbdev_log_debug("Started device %u", dev_id);
+	return 0;
+}
+
+int __rte_experimental
+rte_bbdev_stop(uint16_t dev_id)
+{
+	struct rte_bbdev *dev = get_dev(dev_id);
+	VALID_DEV_OR_RET_ERR(dev, dev_id);
+
+	VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
+
+	if (!dev->data->started) {
+		rte_bbdev_log_debug("Device %u is already stopped", dev_id);
+		return 0;
+	}
+
+	if (dev->dev_ops->stop)
+		dev->dev_ops->stop(dev);
+	dev->data->started = false;
+
+	rte_bbdev_log_debug("Stopped device %u", dev_id);
+	return 0;
+}
+
+int __rte_experimental
+rte_bbdev_close(uint16_t dev_id)
+{
+	int ret;
+	uint16_t i;
+	struct rte_bbdev *dev = get_dev(dev_id);
+	VALID_DEV_OR_RET_ERR(dev, dev_id);
+
+	VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
+
+	if (dev->data->started) {
+		ret = rte_bbdev_stop(dev_id);
+		if (ret < 0) {
+			rte_bbdev_log(ERR, "Device %u stop failed", dev_id);
+			return ret;
+		}
+	}
+
+	/* Free memory used by queues */
+	for (i = 0; i < dev->data->num_queues; i++) {
+		ret = dev->dev_ops->queue_release(dev, i);
+		if (ret < 0) {
+			rte_bbdev_log(ERR, "Device %u queue %u release failed",
+					dev_id, i);
+			return ret;
+		}
+	}
+	rte_free(dev->data->queues);
+
+	if (dev->dev_ops->close) {
+		ret = dev->dev_ops->close(dev);
+		if (ret < 0) {
+			rte_bbdev_log(ERR, "Device %u close failed", dev_id);
+			return ret;
+		}
+	}
+
+	/* Clear configuration */
+	dev->data->queues = NULL;
+	dev->data->num_queues = 0;
+
+	rte_bbdev_log_debug("Closed device %u", dev_id);
+	return 0;
+}
+
+int __rte_experimental
+rte_bbdev_queue_start(uint16_t dev_id, uint16_t queue_id)
+{
+	struct rte_bbdev *dev = get_dev(dev_id);
+	VALID_DEV_OR_RET_ERR(dev, dev_id);
+
+	VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
+
+	VALID_QUEUE_OR_RET_ERR(queue_id, dev);
+
+	if (dev->data->queues[queue_id].started) {
+		rte_bbdev_log_debug("Queue %u of device %u already started",
+				queue_id, dev_id);
+		return 0;
+	}
+
+	if (dev->dev_ops->queue_start) {
+		int ret = dev->dev_ops->queue_start(dev, queue_id);
+		if (ret < 0) {
+			rte_bbdev_log(ERR, "Device %u queue %u start failed",
+					dev_id, queue_id);
+			return ret;
+		}
+	}
+	dev->data->queues[queue_id].started = true;
+
+	rte_bbdev_log_debug("Started queue %u of device %u", queue_id, dev_id);
+	return 0;
+}
+
+int __rte_experimental
+rte_bbdev_queue_stop(uint16_t dev_id, uint16_t queue_id)
+{
+	struct rte_bbdev *dev = get_dev(dev_id);
+	VALID_DEV_OR_RET_ERR(dev, dev_id);
+
+	VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
+
+	VALID_QUEUE_OR_RET_ERR(queue_id, dev);
+
+	if (!dev->data->queues[queue_id].started) {
+		rte_bbdev_log_debug("Queue %u of device %u already stopped",
+				queue_id, dev_id);
+		return 0;
+	}
+
+	if (dev->dev_ops->queue_stop) {
+		int ret = dev->dev_ops->queue_stop(dev, queue_id);
+		if (ret < 0) {
+			rte_bbdev_log(ERR, "Device %u queue %u stop failed",
+					dev_id, queue_id);
+			return ret;
+		}
+	}
+	dev->data->queues[queue_id].started = false;
+
+	rte_bbdev_log_debug("Stopped queue %u of device %u", queue_id, dev_id);
+	return 0;
+}
+
+/* Get device statistics */
+static void
+get_stats_from_queues(struct rte_bbdev *dev, struct rte_bbdev_stats *stats)
+{
+	unsigned int q_id;
+	for (q_id = 0; q_id < dev->data->num_queues; q_id++) {
+		struct rte_bbdev_stats *q_stats =
+				&dev->data->queues[q_id].queue_stats;
+
+		stats->enqueued_count += q_stats->enqueued_count;
+		stats->dequeued_count += q_stats->dequeued_count;
+		stats->enqueue_err_count += q_stats->enqueue_err_count;
+		stats->dequeue_err_count += q_stats->dequeue_err_count;
+	}
+	rte_bbdev_log_debug("Got stats on %u", dev->data->dev_id);
+}
+
+static void
+reset_stats_in_queues(struct rte_bbdev *dev)
+{
+	unsigned int q_id;
+	for (q_id = 0; q_id < dev->data->num_queues; q_id++) {
+		struct rte_bbdev_stats *q_stats =
+				&dev->data->queues[q_id].queue_stats;
+
+		memset(q_stats, 0, sizeof(*q_stats));
+	}
+	rte_bbdev_log_debug("Reset stats on %u", dev->data->dev_id);
+}
+
+int __rte_experimental
+rte_bbdev_stats_get(uint16_t dev_id, struct rte_bbdev_stats *stats)
+{
+	struct rte_bbdev *dev = get_dev(dev_id);
+	VALID_DEV_OR_RET_ERR(dev, dev_id);
+
+	VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
+
+	if (stats == NULL) {
+		rte_bbdev_log(ERR, "NULL stats structure");
+		return -EINVAL;
+	}
+
+	memset(stats, 0, sizeof(*stats));
+	if (dev->dev_ops->stats_get != NULL)
+		dev->dev_ops->stats_get(dev, stats);
+	else
+		get_stats_from_queues(dev, stats);
+
+	rte_bbdev_log_debug("Retrieved stats of device %u", dev_id);
+	return 0;
+}
+
+int __rte_experimental
+rte_bbdev_stats_reset(uint16_t dev_id)
+{
+	struct rte_bbdev *dev = get_dev(dev_id);
+	VALID_DEV_OR_RET_ERR(dev, dev_id);
+
+	VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
+
+	if (dev->dev_ops->stats_reset != NULL)
+		dev->dev_ops->stats_reset(dev);
+	else
+		reset_stats_in_queues(dev);
+
+	rte_bbdev_log_debug("Reset stats of device %u", dev_id);
+	return 0;
+}
+
+int __rte_experimental
+rte_bbdev_info_get(uint16_t dev_id, struct rte_bbdev_info *dev_info)
+{
+	struct rte_bbdev *dev = get_dev(dev_id);
+	VALID_DEV_OR_RET_ERR(dev, dev_id);
+
+	VALID_FUNC_OR_RET_ERR(dev->dev_ops->info_get, dev_id);
+
+	if (dev_info == NULL) {
+		rte_bbdev_log(ERR, "NULL dev info structure");
+		return -EINVAL;
+	}
+
+	/* Copy data maintained by device interface layer */
+	memset(dev_info, 0, sizeof(*dev_info));
+	dev_info->dev_name = dev->data->name;
+	dev_info->num_queues = dev->data->num_queues;
+	dev_info->bus = rte_bus_find_by_device(dev->device);
+	dev_info->socket_id = dev->data->socket_id;
+	dev_info->started = dev->data->started;
+
+	/* Copy data maintained by device driver layer */
+	dev->dev_ops->info_get(dev, &dev_info->drv);
+
+	rte_bbdev_log_debug("Retrieved info of device %u", dev_id);
+	return 0;
+}
+
+int __rte_experimental
+rte_bbdev_queue_info_get(uint16_t dev_id, uint16_t queue_id,
+		struct rte_bbdev_queue_info *queue_info)
+{
+	struct rte_bbdev *dev = get_dev(dev_id);
+	VALID_DEV_OR_RET_ERR(dev, dev_id);
+
+	VALID_QUEUE_OR_RET_ERR(queue_id, dev);
+
+	if (queue_info == NULL) {
+		rte_bbdev_log(ERR, "NULL queue info structure");
+		return -EINVAL;
+	}
+
+	/* Copy data to output */
+	memset(queue_info, 0, sizeof(*queue_info));
+	queue_info->conf = dev->data->queues[queue_id].conf;
+	queue_info->started = dev->data->queues[queue_id].started;
+
+	rte_bbdev_log_debug("Retrieved info of queue %u of device %u",
+			queue_id, dev_id);
+	return 0;
+}
+
+/* Calculate size needed to store bbdev_op, depending on type */
+static unsigned int
+get_bbdev_op_size(enum rte_bbdev_op_type type)
+{
+	unsigned int result = 0;
+	switch (type) {
+	case RTE_BBDEV_OP_NONE:
+		result = RTE_MAX(sizeof(struct rte_bbdev_dec_op),
+				sizeof(struct rte_bbdev_enc_op));
+		break;
+	case RTE_BBDEV_OP_TURBO_DEC:
+		result = sizeof(struct rte_bbdev_dec_op);
+		break;
+	case RTE_BBDEV_OP_TURBO_ENC:
+		result = sizeof(struct rte_bbdev_enc_op);
+		break;
+	default:
+		break;
+	}
+
+	return result;
+}
+
+/* Initialise a bbdev_op structure */
+static void
+bbdev_op_init(struct rte_mempool *mempool, void *arg, void *element,
+		__rte_unused unsigned int n)
+{
+	enum rte_bbdev_op_type type = *(enum rte_bbdev_op_type *)arg;
+
+	if (type == RTE_BBDEV_OP_TURBO_DEC) {
+		struct rte_bbdev_dec_op *op = element;
+		memset(op, 0, mempool->elt_size);
+		op->mempool = mempool;
+	} else if (type == RTE_BBDEV_OP_TURBO_ENC) {
+		struct rte_bbdev_enc_op *op = element;
+		memset(op, 0, mempool->elt_size);
+		op->mempool = mempool;
+	}
+}
+
+struct rte_mempool * __rte_experimental
+rte_bbdev_op_pool_create(const char *name, enum rte_bbdev_op_type type,
+		unsigned int num_elements, unsigned int cache_size,
+		int socket_id)
+{
+	struct rte_bbdev_op_pool_private *priv;
+	struct rte_mempool *mp;
+	const char *op_type_str;
+
+	if (name == NULL) {
+		rte_bbdev_log(ERR, "NULL name for op pool");
+		return NULL;
+	}
+
+	if (type >= RTE_BBDEV_OP_TYPE_COUNT) {
+		rte_bbdev_log(ERR,
+				"Invalid op type (%u), should be less than %u",
+				type, RTE_BBDEV_OP_TYPE_COUNT);
+		return NULL;
+	}
+
+	mp = rte_mempool_create(name, num_elements, get_bbdev_op_size(type),
+			cache_size, sizeof(struct rte_bbdev_op_pool_private),
+			NULL, NULL, bbdev_op_init, &type, socket_id, 0);
+	if (mp == NULL) {
+		rte_bbdev_log(ERR,
+				"Failed to create op pool %s (num ops=%u, op size=%u) with error: %s",
+				name, num_elements, get_bbdev_op_size(type),
+				rte_strerror(rte_errno));
+		return NULL;
+	}
+
+	op_type_str = rte_bbdev_op_type_str(type);
+	if (op_type_str == NULL)
+		return NULL;
+
+	rte_bbdev_log_debug(
+			"Op pool %s created for %u ops (type=%s, cache=%u, socket=%u, size=%u)",
+			name, num_elements, op_type_str, cache_size, socket_id,
+			get_bbdev_op_size(type));
+
+	priv = (struct rte_bbdev_op_pool_private *)rte_mempool_get_priv(mp);
+	priv->type = type;
+
+	return mp;
+}
+
+int __rte_experimental
+rte_bbdev_callback_register(uint16_t dev_id, enum rte_bbdev_event_type event,
+		rte_bbdev_cb_fn cb_fn, void *cb_arg)
+{
+	struct rte_bbdev_callback *user_cb;
+	struct rte_bbdev *dev = get_dev(dev_id);
+	VALID_DEV_OR_RET_ERR(dev, dev_id);
+
+	if (event >= RTE_BBDEV_EVENT_MAX) {
+		rte_bbdev_log(ERR,
+				"Invalid event type (%u), should be less than %u",
+				event, RTE_BBDEV_EVENT_MAX);
+		return -EINVAL;
+	}
+
+	if (cb_fn == NULL) {
+		rte_bbdev_log(ERR, "NULL callback function");
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&rte_bbdev_cb_lock);
+
+	TAILQ_FOREACH(user_cb, &(dev->list_cbs), next) {
+		if (user_cb->cb_fn == cb_fn &&
+				user_cb->cb_arg == cb_arg &&
+				user_cb->event == event)
+			break;
+	}
+
+	/* create a new callback. */
+	if (user_cb == NULL) {
+		user_cb = rte_zmalloc("INTR_USER_CALLBACK",
+				sizeof(struct rte_bbdev_callback), 0);
+		if (user_cb != NULL) {
+			user_cb->cb_fn = cb_fn;
+			user_cb->cb_arg = cb_arg;
+			user_cb->event = event;
+			TAILQ_INSERT_TAIL(&(dev->list_cbs), user_cb, next);
+		}
+	}
+
+	rte_spinlock_unlock(&rte_bbdev_cb_lock);
+	return (user_cb == NULL) ? -ENOMEM : 0;
+}
+
+int __rte_experimental
+rte_bbdev_callback_unregister(uint16_t dev_id, enum rte_bbdev_event_type event,
+		rte_bbdev_cb_fn cb_fn, void *cb_arg)
+{
+	int ret = 0;
+	struct rte_bbdev_callback *cb, *next;
+	struct rte_bbdev *dev = get_dev(dev_id);
+	VALID_DEV_OR_RET_ERR(dev, dev_id);
+
+	if (event >= RTE_BBDEV_EVENT_MAX) {
+		rte_bbdev_log(ERR,
+				"Invalid event type (%u), should be less than %u",
+				event, RTE_BBDEV_EVENT_MAX);
+		return -EINVAL;
+	}
+
+	if (cb_fn == NULL) {
+		rte_bbdev_log(ERR,
+				"NULL callback function cannot be unregistered");
+		return -EINVAL;
+	}
+
+	dev = &rte_bbdev_devices[dev_id];
+	rte_spinlock_lock(&rte_bbdev_cb_lock);
+
+	for (cb = TAILQ_FIRST(&dev->list_cbs); cb != NULL; cb = next) {
+
+		next = TAILQ_NEXT(cb, next);
+
+		if (cb->cb_fn != cb_fn || cb->event != event ||
+				(cb_arg != (void *)-1 && cb->cb_arg != cb_arg))
+			continue;
+
+		/* If this callback is not executing right now, remove it. */
+		if (cb->active == 0) {
+			TAILQ_REMOVE(&(dev->list_cbs), cb, next);
+			rte_free(cb);
+		} else
+			ret = -EAGAIN;
+	}
+
+	rte_spinlock_unlock(&rte_bbdev_cb_lock);
+	return ret;
+}
+
+void __rte_experimental
+rte_bbdev_pmd_callback_process(struct rte_bbdev *dev,
+	enum rte_bbdev_event_type event, void *ret_param)
+{
+	struct rte_bbdev_callback *cb_lst;
+	struct rte_bbdev_callback dev_cb;
+
+	if (dev == NULL) {
+		rte_bbdev_log(ERR, "NULL device");
+		return;
+	}
+
+	if (dev->data == NULL) {
+		rte_bbdev_log(ERR, "NULL data structure");
+		return;
+	}
+
+	if (event >= RTE_BBDEV_EVENT_MAX) {
+		rte_bbdev_log(ERR,
+				"Invalid event type (%u), should be less than %u",
+				event, RTE_BBDEV_EVENT_MAX);
+		return;
+	}
+
+	rte_spinlock_lock(&rte_bbdev_cb_lock);
+	TAILQ_FOREACH(cb_lst, &(dev->list_cbs), next) {
+		if (cb_lst->cb_fn == NULL || cb_lst->event != event)
+			continue;
+		dev_cb = *cb_lst;
+		cb_lst->active = 1;
+		if (ret_param != NULL)
+			dev_cb.ret_param = ret_param;
+
+		rte_spinlock_unlock(&rte_bbdev_cb_lock);
+		dev_cb.cb_fn(dev->data->dev_id, dev_cb.event,
+				dev_cb.cb_arg, dev_cb.ret_param);
+		rte_spinlock_lock(&rte_bbdev_cb_lock);
+		cb_lst->active = 0;
+	}
+	rte_spinlock_unlock(&rte_bbdev_cb_lock);
+}
+
+int __rte_experimental
+rte_bbdev_queue_intr_enable(uint16_t dev_id, uint16_t queue_id)
+{
+	struct rte_bbdev *dev = get_dev(dev_id);
+	VALID_DEV_OR_RET_ERR(dev, dev_id);
+	VALID_QUEUE_OR_RET_ERR(queue_id, dev);
+	VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
+	VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_intr_enable, dev_id);
+	return dev->dev_ops->queue_intr_enable(dev, queue_id);
+}
+
+int __rte_experimental
+rte_bbdev_queue_intr_disable(uint16_t dev_id, uint16_t queue_id)
+{
+	struct rte_bbdev *dev = get_dev(dev_id);
+	VALID_DEV_OR_RET_ERR(dev, dev_id);
+	VALID_QUEUE_OR_RET_ERR(queue_id, dev);
+	VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
+	VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_intr_disable, dev_id);
+	return dev->dev_ops->queue_intr_disable(dev, queue_id);
+}
+
+int __rte_experimental
+rte_bbdev_queue_intr_ctl(uint16_t dev_id, uint16_t queue_id, int epfd, int op,
+		void *data)
+{
+	uint32_t vec;
+	struct rte_bbdev *dev = get_dev(dev_id);
+	struct rte_intr_handle *intr_handle;
+	int ret;
+
+	VALID_DEV_OR_RET_ERR(dev, dev_id);
+	VALID_QUEUE_OR_RET_ERR(queue_id, dev);
+
+	intr_handle = dev->intr_handle;
+	if (!intr_handle || !intr_handle->intr_vec) {
+		rte_bbdev_log(ERR, "Device %u intr handle unset\n", dev_id);
+		return -ENOTSUP;
+	}
+
+	if (queue_id >= RTE_MAX_RXTX_INTR_VEC_ID) {
+		rte_bbdev_log(ERR, "Device %u queue_id %u is too big\n",
+				dev_id, queue_id);
+		return -ENOTSUP;
+	}
+
+	vec = intr_handle->intr_vec[queue_id];
+	ret = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
+	if (ret && (ret != -EEXIST)) {
+		rte_bbdev_log(ERR,
+				"dev %u q %u int ctl error op %d epfd %d vec %u\n",
+				dev_id, queue_id, op, epfd, vec);
+		return ret;
+	}
+
+	return 0;
+}
+
+
+const char * __rte_experimental
+rte_bbdev_op_type_str(enum rte_bbdev_op_type op_type)
+{
+	static const char * const op_types[] = {
+		"RTE_BBDEV_OP_NONE",
+		"RTE_BBDEV_OP_TURBO_DEC",
+		"RTE_BBDEV_OP_TURBO_ENC",
+	};
+
+	if (op_type < RTE_BBDEV_OP_TYPE_COUNT)
+		return op_types[op_type];
+
+	rte_bbdev_log(ERR, "Invalid operation type");
+	return NULL;
+}
+
+RTE_INIT(rte_bbdev_init_log);
+static void
+rte_bbdev_init_log(void)
+{
+	bbdev_logtype = rte_log_register("lib.bbdev");
+	if (bbdev_logtype >= 0)
+		rte_log_set_level(bbdev_logtype, RTE_LOG_NOTICE);
+}