diff options
Diffstat (limited to 'drivers/compress/isal')
| -rw-r--r-- | drivers/compress/isal/Makefile | 31 | ||||
| -rw-r--r-- | drivers/compress/isal/isal_compress_pmd.c | 694 | ||||
| -rw-r--r-- | drivers/compress/isal/isal_compress_pmd_ops.c | 351 | ||||
| -rw-r--r-- | drivers/compress/isal/isal_compress_pmd_private.h | 57 | ||||
| -rw-r--r-- | drivers/compress/isal/meson.build | 14 | ||||
| -rw-r--r-- | drivers/compress/isal/rte_pmd_isal_version.map | 3 |
6 files changed, 1150 insertions, 0 deletions
diff --git a/drivers/compress/isal/Makefile b/drivers/compress/isal/Makefile new file mode 100644 index 00000000..95904f64 --- /dev/null +++ b/drivers/compress/isal/Makefile @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Intel Corporation + +include $(RTE_SDK)/mk/rte.vars.mk + +# library name +LIB = librte_pmd_isal_comp.a + +# build flags +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) +CFLAGS += -DALLOW_EXPERIMENTAL_API + +# external library dependencies +LDLIBS += -lisal +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_compressdev +LDLIBS += -lrte_bus_vdev + +# library version +LIBABIVER := 1 + +# versioning export map +EXPORT_MAP := rte_pmd_isal_version.map + +# library source files +SRCS-$(CONFIG_RTE_LIBRTE_PMD_ISAL) += isal_compress_pmd.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_ISAL) += isal_compress_pmd_ops.c + +# export include files +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/drivers/compress/isal/isal_compress_pmd.c b/drivers/compress/isal/isal_compress_pmd.c new file mode 100644 index 00000000..e943336b --- /dev/null +++ b/drivers/compress/isal/isal_compress_pmd.c @@ -0,0 +1,694 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation + */ +#include <isa-l.h> + +#include <rte_bus_vdev.h> +#include <rte_common.h> +#include <rte_malloc.h> +#include <rte_mbuf.h> +#include <rte_compressdev_pmd.h> + +#include "isal_compress_pmd_private.h" + +#define RTE_COMP_ISAL_WINDOW_SIZE 15 +#define RTE_COMP_ISAL_LEVEL_ZERO 0 /* ISA-L Level 0 used for fixed Huffman */ +#define RTE_COMP_ISAL_LEVEL_ONE 1 +#define RTE_COMP_ISAL_LEVEL_TWO 2 +#define RTE_COMP_ISAL_LEVEL_THREE 3 /* Optimised for AVX512 & AVX2 only */ + +int isal_logtype_driver; + +/* Verify and set private xform parameters */ +int +isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform, + const struct rte_comp_xform *xform) +{ + if (xform == NULL) + return -EINVAL; + + /* Set compression private xform variables */ + if (xform->type == RTE_COMP_COMPRESS) { + /* Set private xform type - COMPRESS/DECOMPRESS */ + priv_xform->type = RTE_COMP_COMPRESS; + + /* Set private xform algorithm */ + if (xform->compress.algo != RTE_COMP_ALGO_DEFLATE) { + if (xform->compress.algo == RTE_COMP_ALGO_NULL) { + ISAL_PMD_LOG(ERR, "By-pass not supported\n"); + return -ENOTSUP; + } + ISAL_PMD_LOG(ERR, "Algorithm not supported\n"); + return -ENOTSUP; + } + priv_xform->compress.algo = RTE_COMP_ALGO_DEFLATE; + + /* Set private xform checksum - raw deflate by default */ + if (xform->compress.chksum != RTE_COMP_CHECKSUM_NONE) { + ISAL_PMD_LOG(ERR, "Checksum not supported\n"); + return -ENOTSUP; + } + + /* Set private xform window size, 32K supported */ + if (xform->compress.window_size == RTE_COMP_ISAL_WINDOW_SIZE) + priv_xform->compress.window_size = + RTE_COMP_ISAL_WINDOW_SIZE; + else { + ISAL_PMD_LOG(ERR, "Window size not supported\n"); + return -ENOTSUP; + } + + /* Set private xform huffman type */ + switch (xform->compress.deflate.huffman) { + case(RTE_COMP_HUFFMAN_DEFAULT): + priv_xform->compress.deflate.huffman = + RTE_COMP_HUFFMAN_DEFAULT; + break; + case(RTE_COMP_HUFFMAN_FIXED): + priv_xform->compress.deflate.huffman = + RTE_COMP_HUFFMAN_FIXED; + break; + case(RTE_COMP_HUFFMAN_DYNAMIC): + priv_xform->compress.deflate.huffman = + RTE_COMP_HUFFMAN_DYNAMIC; + break; + default: + ISAL_PMD_LOG(ERR, "Huffman code not supported\n"); + return -ENOTSUP; + } + + /* Set private xform level. + * Checking compliance with compressdev API, -1 <= level => 9 + */ + if (xform->compress.level < RTE_COMP_LEVEL_PMD_DEFAULT || + xform->compress.level > RTE_COMP_LEVEL_MAX) { + ISAL_PMD_LOG(ERR, "Compression level out of range\n"); + return -EINVAL; + } + /* Check for Compressdev API level 0, No compression + * not supported in ISA-L + */ + else if (xform->compress.level == RTE_COMP_LEVEL_NONE) { + ISAL_PMD_LOG(ERR, "No Compression not supported\n"); + return -ENOTSUP; + } + /* If using fixed huffman code, level must be 0 */ + else if (priv_xform->compress.deflate.huffman == + RTE_COMP_HUFFMAN_FIXED) { + ISAL_PMD_LOG(DEBUG, "ISA-L level 0 used due to a" + " fixed huffman code\n"); + priv_xform->compress.level = RTE_COMP_ISAL_LEVEL_ZERO; + priv_xform->level_buffer_size = + ISAL_DEF_LVL0_DEFAULT; + } else { + /* Mapping API levels to ISA-L levels 1,2 & 3 */ + switch (xform->compress.level) { + case RTE_COMP_LEVEL_PMD_DEFAULT: + /* Default is 1 if not using fixed huffman */ + priv_xform->compress.level = + RTE_COMP_ISAL_LEVEL_ONE; + priv_xform->level_buffer_size = + ISAL_DEF_LVL1_DEFAULT; + break; + case RTE_COMP_LEVEL_MIN: + priv_xform->compress.level = + RTE_COMP_ISAL_LEVEL_ONE; + priv_xform->level_buffer_size = + ISAL_DEF_LVL1_DEFAULT; + break; + case RTE_COMP_ISAL_LEVEL_TWO: + priv_xform->compress.level = + RTE_COMP_ISAL_LEVEL_TWO; + priv_xform->level_buffer_size = + ISAL_DEF_LVL2_DEFAULT; + break; + /* Level 3 or higher requested */ + default: + /* Check for AVX512, to use ISA-L level 3 */ + if (rte_cpu_get_flag_enabled( + RTE_CPUFLAG_AVX512F)) { + priv_xform->compress.level = + RTE_COMP_ISAL_LEVEL_THREE; + priv_xform->level_buffer_size = + ISAL_DEF_LVL3_DEFAULT; + } + /* Check for AVX2, to use ISA-L level 3 */ + else if (rte_cpu_get_flag_enabled( + RTE_CPUFLAG_AVX2)) { + priv_xform->compress.level = + RTE_COMP_ISAL_LEVEL_THREE; + priv_xform->level_buffer_size = + ISAL_DEF_LVL3_DEFAULT; + } else { + ISAL_PMD_LOG(DEBUG, "Requested ISA-L level" + " 3 or above; Level 3 optimized" + " for AVX512 & AVX2 only." + " level changed to 2.\n"); + priv_xform->compress.level = + RTE_COMP_ISAL_LEVEL_TWO; + priv_xform->level_buffer_size = + ISAL_DEF_LVL2_DEFAULT; + } + } + } + } + + /* Set decompression private xform variables */ + else if (xform->type == RTE_COMP_DECOMPRESS) { + + /* Set private xform type - COMPRESS/DECOMPRESS */ + priv_xform->type = RTE_COMP_DECOMPRESS; + + /* Set private xform algorithm */ + if (xform->decompress.algo != RTE_COMP_ALGO_DEFLATE) { + if (xform->decompress.algo == RTE_COMP_ALGO_NULL) { + ISAL_PMD_LOG(ERR, "By pass not supported\n"); + return -ENOTSUP; + } + ISAL_PMD_LOG(ERR, "Algorithm not supported\n"); + return -ENOTSUP; + } + priv_xform->decompress.algo = RTE_COMP_ALGO_DEFLATE; + + /* Set private xform checksum - raw deflate by default */ + if (xform->compress.chksum != RTE_COMP_CHECKSUM_NONE) { + ISAL_PMD_LOG(ERR, "Checksum not supported\n"); + return -ENOTSUP; + } + + /* Set private xform window size, 32K supported */ + if (xform->decompress.window_size == RTE_COMP_ISAL_WINDOW_SIZE) + priv_xform->decompress.window_size = + RTE_COMP_ISAL_WINDOW_SIZE; + else { + ISAL_PMD_LOG(ERR, "Window size not supported\n"); + return -ENOTSUP; + } + } + return 0; +} + +/* Compression using chained mbufs for input/output data */ +static int +chained_mbuf_compression(struct rte_comp_op *op, struct isal_comp_qp *qp) +{ + int ret; + uint32_t remaining_offset; + uint32_t remaining_data = op->src.length; + struct rte_mbuf *src = op->m_src; + struct rte_mbuf *dst = op->m_dst; + + /* check for source/destination offset passing multiple segments + * and point compression stream to input/output buffer. + */ + remaining_offset = op->src.offset; + while (remaining_offset >= src->data_len) { + remaining_offset -= src->data_len; + src = src->next; + } + qp->stream->avail_in = RTE_MIN(src->data_len - remaining_offset, + op->src.length); + qp->stream->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *, + remaining_offset); + + remaining_offset = op->dst.offset; + while (remaining_offset >= dst->data_len) { + remaining_offset -= dst->data_len; + dst = dst->next; + } + qp->stream->avail_out = dst->data_len - remaining_offset; + qp->stream->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *, + remaining_offset); + + if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) { + ISAL_PMD_LOG(ERR, "Invalid source or destination buffer\n"); + op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + return -1; + } + + while (qp->stream->internal_state.state != ZSTATE_END) { + /* Last segment of data */ + if (remaining_data <= src->data_len) + qp->stream->end_of_stream = 1; + + /* Execute compression operation */ + ret = isal_deflate(qp->stream); + + remaining_data = op->src.length - qp->stream->total_in; + + if (ret != COMP_OK) { + ISAL_PMD_LOG(ERR, "Compression operation failed\n"); + op->status = RTE_COMP_OP_STATUS_ERROR; + return ret; + } + + if (qp->stream->avail_in == 0 && + qp->stream->total_in != op->src.length) { + if (src->next != NULL) { + src = src->next; + qp->stream->next_in = + rte_pktmbuf_mtod(src, uint8_t *); + qp->stream->avail_in = + RTE_MIN(remaining_data, src->data_len); + } else { + ISAL_PMD_LOG(ERR, + "Not enough input buffer segments\n"); + op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + return -1; + } + } + + if (qp->stream->avail_out == 0 && + qp->stream->internal_state.state != ZSTATE_END) { + if (dst->next != NULL) { + dst = dst->next; + qp->stream->next_out = + rte_pktmbuf_mtod(dst, uint8_t *); + qp->stream->avail_out = dst->data_len; + } else { + ISAL_PMD_LOG(ERR, + "Not enough output buffer segments\n"); + op->status = + RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; + return -1; + } + } + } + + return 0; +} + +/* Decompression using chained mbufs for input/output data */ +static int +chained_mbuf_decompression(struct rte_comp_op *op, struct isal_comp_qp *qp) +{ + int ret; + uint32_t consumed_data, src_remaining_offset, dst_remaining_offset; + uint32_t remaining_data = op->src.length; + struct rte_mbuf *src = op->m_src; + struct rte_mbuf *dst = op->m_dst; + + /* check for offset passing multiple segments + * and point decompression state to input/output buffer + */ + src_remaining_offset = op->src.offset; + while (src_remaining_offset >= src->data_len) { + src_remaining_offset -= src->data_len; + src = src->next; + } + qp->state->avail_in = RTE_MIN(src->data_len - src_remaining_offset, + op->src.length); + qp->state->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *, + src_remaining_offset); + + dst_remaining_offset = op->dst.offset; + while (dst_remaining_offset >= dst->data_len) { + dst_remaining_offset -= dst->data_len; + dst = dst->next; + } + qp->state->avail_out = dst->data_len - dst_remaining_offset; + qp->state->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *, + dst_remaining_offset); + + while (qp->state->block_state != ISAL_BLOCK_FINISH) { + + ret = isal_inflate(qp->state); + + /* Check for first segment, offset needs to be accounted for */ + if (remaining_data == op->src.length) { + consumed_data = src->data_len - qp->state->avail_in - + src_remaining_offset; + } else + consumed_data = src->data_len - qp->state->avail_in; + + op->consumed += consumed_data; + remaining_data -= consumed_data; + + if (ret != ISAL_DECOMP_OK) { + ISAL_PMD_LOG(ERR, "Decompression operation failed\n"); + op->status = RTE_COMP_OP_STATUS_ERROR; + return ret; + } + + if (qp->state->avail_in == 0 + && op->consumed != op->src.length) { + if (src->next != NULL) { + src = src->next; + qp->state->next_in = + rte_pktmbuf_mtod(src, uint8_t *); + qp->state->avail_in = + RTE_MIN(remaining_data, src->data_len); + } + } + + if (qp->state->avail_out == 0 && + qp->state->block_state != ISAL_BLOCK_FINISH) { + if (dst->next != NULL) { + dst = dst->next; + qp->state->next_out = + rte_pktmbuf_mtod(dst, uint8_t *); + qp->state->avail_out = dst->data_len; + } else { + ISAL_PMD_LOG(ERR, + "Not enough output buffer segments\n"); + op->status = + RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; + return -1; + } + } + } + + return 0; +} + +/* Stateless Compression Function */ +static int +process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp, + struct isal_priv_xform *priv_xform) +{ + int ret = 0; + op->status = RTE_COMP_OP_STATUS_SUCCESS; + + /* Required due to init clearing level_buf */ + uint8_t *temp_level_buf = qp->stream->level_buf; + + /* Initialize compression stream */ + isal_deflate_stateless_init(qp->stream); + + qp->stream->level_buf = temp_level_buf; + + /* Stateless operation, input will be consumed in one go */ + qp->stream->flush = NO_FLUSH; + + /* set compression level & intermediate level buffer size */ + qp->stream->level = priv_xform->compress.level; + qp->stream->level_buf_size = priv_xform->level_buffer_size; + + /* Set op huffman code */ + if (priv_xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED) + isal_deflate_set_hufftables(qp->stream, NULL, + IGZIP_HUFFTABLE_STATIC); + else if (priv_xform->compress.deflate.huffman == + RTE_COMP_HUFFMAN_DEFAULT) + isal_deflate_set_hufftables(qp->stream, NULL, + IGZIP_HUFFTABLE_DEFAULT); + /* Dynamically change the huffman code to suit the input data */ + else if (priv_xform->compress.deflate.huffman == + RTE_COMP_HUFFMAN_DYNAMIC) + isal_deflate_set_hufftables(qp->stream, NULL, + IGZIP_HUFFTABLE_DEFAULT); + + if (op->m_src->pkt_len < (op->src.length + op->src.offset)) { + ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n"); + op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + return -1; + } + + if (op->dst.offset >= op->m_dst->pkt_len) { + ISAL_PMD_LOG(ERR, "Output mbuf(s) not big enough" + " for offset provided.\n"); + op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + return -1; + } + + /* Chained mbufs */ + if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) { + ret = chained_mbuf_compression(op, qp); + if (ret < 0) + return ret; + } else { + /* Linear buffer */ + qp->stream->end_of_stream = 1; /* All input consumed in one */ + /* Point compression stream to input buffer */ + qp->stream->avail_in = op->src.length; + qp->stream->next_in = rte_pktmbuf_mtod_offset(op->m_src, + uint8_t *, op->src.offset); + + /* Point compression stream to output buffer */ + qp->stream->avail_out = op->m_dst->data_len - op->dst.offset; + qp->stream->next_out = rte_pktmbuf_mtod_offset(op->m_dst, + uint8_t *, op->dst.offset); + + if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) { + ISAL_PMD_LOG(ERR, "Invalid source or destination" + " buffers\n"); + op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + return -1; + } + + /* Execute compression operation */ + ret = isal_deflate_stateless(qp->stream); + + /* Check that output buffer did not run out of space */ + if (ret == STATELESS_OVERFLOW) { + ISAL_PMD_LOG(ERR, "Output buffer not big enough\n"); + op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; + return ret; + } + + /* Check that input buffer has been fully consumed */ + if (qp->stream->avail_in != (uint32_t)0) { + ISAL_PMD_LOG(ERR, "Input buffer could not be read" + " entirely\n"); + op->status = RTE_COMP_OP_STATUS_ERROR; + return -1; + } + + if (ret != COMP_OK) { + ISAL_PMD_LOG(ERR, "Compression operation failed\n"); + op->status = RTE_COMP_OP_STATUS_ERROR; + return ret; + } + } + op->consumed = qp->stream->total_in; + op->produced = qp->stream->total_out; + + return ret; +} + +/* Stateless Decompression Function */ +static int +process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp) +{ + int ret = 0; + + op->status = RTE_COMP_OP_STATUS_SUCCESS; + + /* Initialize decompression state */ + isal_inflate_init(qp->state); + + if (op->m_src->pkt_len < (op->src.length + op->src.offset)) { + ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n"); + op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + return -1; + } + + if (op->dst.offset >= op->m_dst->pkt_len) { + ISAL_PMD_LOG(ERR, "Output mbuf not big enough for " + "offset provided.\n"); + op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + return -1; + } + + /* Chained mbufs */ + if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) { + ret = chained_mbuf_decompression(op, qp); + if (ret != 0) + return ret; + } else { + /* Linear buffer */ + /* Point decompression state to input buffer */ + qp->state->avail_in = op->src.length; + qp->state->next_in = rte_pktmbuf_mtod_offset(op->m_src, + uint8_t *, op->src.offset); + + /* Point decompression state to output buffer */ + qp->state->avail_out = op->m_dst->data_len - op->dst.offset; + qp->state->next_out = rte_pktmbuf_mtod_offset(op->m_dst, + uint8_t *, op->dst.offset); + + if (unlikely(!qp->state->next_in || !qp->state->next_out)) { + ISAL_PMD_LOG(ERR, "Invalid source or destination" + " buffers\n"); + op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + return -1; + } + + /* Execute decompression operation */ + ret = isal_inflate_stateless(qp->state); + + if (ret == ISAL_OUT_OVERFLOW) { + ISAL_PMD_LOG(ERR, "Output buffer not big enough\n"); + op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; + return ret; + } + + /* Check that input buffer has been fully consumed */ + if (qp->state->avail_in != (uint32_t)0) { + ISAL_PMD_LOG(ERR, "Input buffer could not be read" + " entirely\n"); + op->status = RTE_COMP_OP_STATUS_ERROR; + return -1; + } + + if (ret != ISAL_DECOMP_OK) { + op->status = RTE_COMP_OP_STATUS_ERROR; + return ret; + } + op->consumed = op->src.length - qp->state->avail_in; + } + op->produced = qp->state->total_out; + + return ret; +} + +/* Process compression/decompression operation */ +static int +process_op(struct isal_comp_qp *qp, struct rte_comp_op *op, + struct isal_priv_xform *priv_xform) +{ + switch (priv_xform->type) { + case RTE_COMP_COMPRESS: + process_isal_deflate(op, qp, priv_xform); + break; + case RTE_COMP_DECOMPRESS: + process_isal_inflate(op, qp); + break; + default: + ISAL_PMD_LOG(ERR, "Operation Not Supported\n"); + return -ENOTSUP; + } + return 0; +} + +/* Enqueue burst */ +static uint16_t +isal_comp_pmd_enqueue_burst(void *queue_pair, struct rte_comp_op **ops, + uint16_t nb_ops) +{ + struct isal_comp_qp *qp = queue_pair; + uint16_t i; + int retval; + int16_t num_enq = RTE_MIN(qp->num_free_elements, nb_ops); + + for (i = 0; i < num_enq; i++) { + if (unlikely(ops[i]->op_type != RTE_COMP_OP_STATELESS)) { + ops[i]->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + ISAL_PMD_LOG(ERR, "Stateful operation not Supported\n"); + qp->qp_stats.enqueue_err_count++; + continue; + } + retval = process_op(qp, ops[i], ops[i]->private_xform); + if (unlikely(retval < 0) || + ops[i]->status != RTE_COMP_OP_STATUS_SUCCESS) { + qp->qp_stats.enqueue_err_count++; + } + } + + retval = rte_ring_enqueue_burst(qp->processed_pkts, (void *)ops, + num_enq, NULL); + qp->num_free_elements -= retval; + qp->qp_stats.enqueued_count += retval; + + return retval; +} + +/* Dequeue burst */ +static uint16_t +isal_comp_pmd_dequeue_burst(void *queue_pair, struct rte_comp_op **ops, + uint16_t nb_ops) +{ + struct isal_comp_qp *qp = queue_pair; + uint16_t nb_dequeued; + + nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts, (void **)ops, + nb_ops, NULL); + qp->num_free_elements += nb_dequeued; + qp->qp_stats.dequeued_count += nb_dequeued; + + return nb_dequeued; +} + +/* Create ISA-L compression device */ +static int +compdev_isal_create(const char *name, struct rte_vdev_device *vdev, + struct rte_compressdev_pmd_init_params *init_params) +{ + struct rte_compressdev *dev; + + dev = rte_compressdev_pmd_create(name, &vdev->device, + sizeof(struct isal_comp_private), init_params); + if (dev == NULL) { + ISAL_PMD_LOG(ERR, "failed to create compressdev vdev"); + return -EFAULT; + } + + dev->dev_ops = isal_compress_pmd_ops; + + /* register rx/tx burst functions for data path */ + dev->dequeue_burst = isal_comp_pmd_dequeue_burst; + dev->enqueue_burst = isal_comp_pmd_enqueue_burst; + + return 0; +} + +/** Remove compression device */ +static int +compdev_isal_remove_dev(struct rte_vdev_device *vdev) +{ + struct rte_compressdev *compdev; + const char *name; + + name = rte_vdev_device_name(vdev); + if (name == NULL) + return -EINVAL; + + compdev = rte_compressdev_pmd_get_named_dev(name); + if (compdev == NULL) + return -ENODEV; + + return rte_compressdev_pmd_destroy(compdev); +} + +/** Initialise ISA-L compression device */ +static int +compdev_isal_probe(struct rte_vdev_device *dev) +{ + struct rte_compressdev_pmd_init_params init_params = { + "", + rte_socket_id(), + }; + const char *name, *args; + int retval; + + name = rte_vdev_device_name(dev); + if (name == NULL) + return -EINVAL; + + args = rte_vdev_device_args(dev); + + retval = rte_compressdev_pmd_parse_input_args(&init_params, args); + if (retval) { + ISAL_PMD_LOG(ERR, + "Failed to parse initialisation arguments[%s]\n", args); + return -EINVAL; + } + + return compdev_isal_create(name, dev, &init_params); +} + +static struct rte_vdev_driver compdev_isal_pmd_drv = { + .probe = compdev_isal_probe, + .remove = compdev_isal_remove_dev, +}; + +RTE_PMD_REGISTER_VDEV(COMPDEV_NAME_ISAL_PMD, compdev_isal_pmd_drv); +RTE_PMD_REGISTER_PARAM_STRING(COMPDEV_NAME_ISAL_PMD, + "socket_id=<int>"); + +RTE_INIT(isal_init_log) +{ + isal_logtype_driver = rte_log_register("pmd.compress.isal"); + if (isal_logtype_driver >= 0) + rte_log_set_level(isal_logtype_driver, RTE_LOG_INFO); +} diff --git a/drivers/compress/isal/isal_compress_pmd_ops.c b/drivers/compress/isal/isal_compress_pmd_ops.c new file mode 100644 index 00000000..41cade87 --- /dev/null +++ b/drivers/compress/isal/isal_compress_pmd_ops.c @@ -0,0 +1,351 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation + */ +#include <isa-l.h> + +#include <rte_common.h> +#include <rte_compressdev_pmd.h> +#include <rte_malloc.h> + +#include "isal_compress_pmd_private.h" + +static const struct rte_compressdev_capabilities isal_pmd_capabilities[] = { + { + .algo = RTE_COMP_ALGO_DEFLATE, + .comp_feature_flags = RTE_COMP_FF_OOP_SGL_IN_SGL_OUT | + RTE_COMP_FF_OOP_SGL_IN_LB_OUT | + RTE_COMP_FF_OOP_LB_IN_SGL_OUT | + RTE_COMP_FF_SHAREABLE_PRIV_XFORM | + RTE_COMP_FF_HUFFMAN_FIXED | + RTE_COMP_FF_HUFFMAN_DYNAMIC, + .window_size = { + .min = 15, + .max = 15, + .increment = 0 + }, + }, + RTE_COMP_END_OF_CAPABILITIES_LIST() +}; + +/** Configure device */ +static int +isal_comp_pmd_config(struct rte_compressdev *dev, + struct rte_compressdev_config *config) +{ + int ret = 0; + unsigned int n; + char mp_name[RTE_COMPRESSDEV_NAME_MAX_LEN]; + unsigned int elt_size = sizeof(struct isal_priv_xform); + struct isal_comp_private *internals = dev->data->dev_private; + + n = snprintf(mp_name, sizeof(mp_name), "compdev_%d_xform_mp", + dev->data->dev_id); + if (n > sizeof(mp_name)) { + ISAL_PMD_LOG(ERR, + "Unable to create unique name for xform mempool"); + return -ENOMEM; + } + + internals->priv_xform_mp = rte_mempool_lookup(mp_name); + + if (internals->priv_xform_mp != NULL) { + if (((internals->priv_xform_mp)->elt_size != elt_size) || + ((internals->priv_xform_mp)->size < + config->max_nb_priv_xforms)) { + + ISAL_PMD_LOG(ERR, "%s mempool already exists with different" + " initialization parameters", mp_name); + internals->priv_xform_mp = NULL; + return -ENOMEM; + } + } else { /* First time configuration */ + internals->priv_xform_mp = rte_mempool_create( + mp_name, /* mempool name */ + /* number of elements*/ + config->max_nb_priv_xforms, + elt_size, /* element size*/ + 0, /* Cache size*/ + 0, /* private data size */ + NULL, /* obj initialization constructor */ + NULL, /* obj initialization constructor arg */ + NULL, /**< obj constructor*/ + NULL, /* obj constructor arg */ + config->socket_id, /* socket id */ + 0); /* flags */ + } + + if (internals->priv_xform_mp == NULL) { + ISAL_PMD_LOG(ERR, "%s mempool allocation failed", mp_name); + return -ENOMEM; + } + + dev->data->dev_private = internals; + + return ret; +} + +/** Start device */ +static int +isal_comp_pmd_start(__rte_unused struct rte_compressdev *dev) +{ + return 0; +} + +/** Stop device */ +static void +isal_comp_pmd_stop(__rte_unused struct rte_compressdev *dev) +{ +} + +/** Close device */ +static int +isal_comp_pmd_close(struct rte_compressdev *dev) +{ + /* Free private data */ + struct isal_comp_private *internals = dev->data->dev_private; + + rte_mempool_free(internals->priv_xform_mp); + return 0; +} + +/** Get device statistics */ +static void +isal_comp_pmd_stats_get(struct rte_compressdev *dev, + struct rte_compressdev_stats *stats) +{ + uint16_t qp_id; + + for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { + struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id]; + + stats->enqueued_count += qp->qp_stats.enqueued_count; + stats->dequeued_count += qp->qp_stats.dequeued_count; + + stats->enqueue_err_count += qp->qp_stats.enqueue_err_count; + stats->dequeue_err_count += qp->qp_stats.dequeue_err_count; + } +} + +/** Get device info */ +static void +isal_comp_pmd_info_get(struct rte_compressdev *dev __rte_unused, + struct rte_compressdev_info *dev_info) +{ + if (dev_info != NULL) { + dev_info->capabilities = isal_pmd_capabilities; + dev_info->feature_flags = RTE_COMPDEV_FF_CPU_AVX512 | + RTE_COMPDEV_FF_CPU_AVX2 | + RTE_COMPDEV_FF_CPU_AVX | + RTE_COMPDEV_FF_CPU_SSE; + } +} + +/** Reset device statistics */ +static void +isal_comp_pmd_stats_reset(struct rte_compressdev *dev) +{ + uint16_t qp_id; + + for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { + struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id]; + memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); + } +} + +/** Release queue pair */ +static int +isal_comp_pmd_qp_release(struct rte_compressdev *dev, uint16_t qp_id) +{ + struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id]; + + if (qp == NULL) + return -EINVAL; + + if (qp->stream != NULL) + rte_free(qp->stream); + + if (qp->stream->level_buf != NULL) + rte_free(qp->stream->level_buf); + + if (qp->state != NULL) + rte_free(qp->state); + + if (qp->processed_pkts != NULL) + rte_ring_free(qp->processed_pkts); + + rte_free(qp); + dev->data->queue_pairs[qp_id] = NULL; + + return 0; +} + +/** Create a ring to place process packets on */ +static struct rte_ring * +isal_comp_pmd_qp_create_processed_pkts_ring(struct isal_comp_qp *qp, + unsigned int ring_size, int socket_id) +{ + struct rte_ring *r; + + r = rte_ring_lookup(qp->name); + if (r) { + if (rte_ring_get_size(r) >= ring_size) { + ISAL_PMD_LOG(DEBUG, + "Reusing existing ring %s for processed packets", + qp->name); + return r; + } + + ISAL_PMD_LOG(ERR, + "Unable to reuse existing ring %s" + " for processed packets", + qp->name); + return NULL; + } + + return rte_ring_create(qp->name, ring_size, socket_id, + RING_F_SP_ENQ | RING_F_SC_DEQ); +} + +/** set a unique name for the queue pair based on its name, dev_id and qp_id */ +static int +isal_comp_pmd_qp_set_unique_name(struct rte_compressdev *dev, +struct isal_comp_qp *qp) +{ + unsigned int n = snprintf(qp->name, sizeof(qp->name), + "isal_compression_pmd_%u_qp_%u", + dev->data->dev_id, qp->id); + + if (n >= sizeof(qp->name)) + return -1; + + return 0; +} + +/* Setup a queue pair */ +static int +isal_comp_pmd_qp_setup(struct rte_compressdev *dev, uint16_t qp_id, + uint32_t max_inflight_ops, int socket_id) +{ + struct isal_comp_qp *qp = NULL; + int retval; + + /* Free memory prior to re-allocation if needed. */ + if (dev->data->queue_pairs[qp_id] != NULL) + isal_comp_pmd_qp_release(dev, qp_id); + + /* Allocate the queue pair data structure. */ + qp = rte_zmalloc_socket("Isa-l compression PMD Queue Pair", sizeof(*qp), + RTE_CACHE_LINE_SIZE, socket_id); + if (qp == NULL) { + ISAL_PMD_LOG(ERR, "Failed to allocate queue pair memory"); + return (-ENOMEM); + } + + /* Initialize memory for compression stream structure */ + qp->stream = rte_zmalloc_socket("Isa-l compression stream ", + sizeof(struct isal_zstream), RTE_CACHE_LINE_SIZE, + socket_id); + + /* Initialize memory for compression level buffer */ + qp->stream->level_buf = rte_zmalloc_socket("Isa-l compression lev_buf", + ISAL_DEF_LVL3_DEFAULT, RTE_CACHE_LINE_SIZE, + socket_id); + + /* Initialize memory for decompression state structure */ + qp->state = rte_zmalloc_socket("Isa-l decompression state", + sizeof(struct inflate_state), RTE_CACHE_LINE_SIZE, + socket_id); + + qp->id = qp_id; + dev->data->queue_pairs[qp_id] = qp; + + retval = isal_comp_pmd_qp_set_unique_name(dev, qp); + if (retval) { + ISAL_PMD_LOG(ERR, "Failed to create unique name for isal " + "compression device"); + goto qp_setup_cleanup; + } + + qp->processed_pkts = isal_comp_pmd_qp_create_processed_pkts_ring(qp, + max_inflight_ops, socket_id); + if (qp->processed_pkts == NULL) { + ISAL_PMD_LOG(ERR, "Failed to create unique name for isal " + "compression device"); + goto qp_setup_cleanup; + } + + qp->num_free_elements = rte_ring_free_count(qp->processed_pkts); + + memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); + return 0; + +qp_setup_cleanup: + if (qp) + rte_free(qp); + + return -1; +} + +/** Set private xform data*/ +static int +isal_comp_pmd_priv_xform_create(struct rte_compressdev *dev, + const struct rte_comp_xform *xform, void **priv_xform) +{ + int ret; + struct isal_comp_private *internals = dev->data->dev_private; + + if (xform == NULL) { + ISAL_PMD_LOG(ERR, "Invalid Xform struct"); + return -EINVAL; + } + + if (rte_mempool_get(internals->priv_xform_mp, priv_xform)) { + ISAL_PMD_LOG(ERR, + "Couldn't get object from private xform mempool"); + return -ENOMEM; + } + + ret = isal_comp_set_priv_xform_parameters(*priv_xform, xform); + if (ret != 0) { + ISAL_PMD_LOG(ERR, "Failed to configure private xform parameters"); + + /* Return private xform to mempool */ + rte_mempool_put(internals->priv_xform_mp, priv_xform); + return ret; + } + return 0; +} + +/** Clear memory of the private xform so it doesn't leave key material behind */ +static int +isal_comp_pmd_priv_xform_free(struct rte_compressdev *dev, void *priv_xform) +{ + struct isal_comp_private *internals = dev->data->dev_private; + + /* Zero out the whole structure */ + if (priv_xform) { + memset(priv_xform, 0, sizeof(struct isal_priv_xform)); + rte_mempool_put(internals->priv_xform_mp, priv_xform); + } + return 0; +} + +struct rte_compressdev_ops isal_pmd_ops = { + .dev_configure = isal_comp_pmd_config, + .dev_start = isal_comp_pmd_start, + .dev_stop = isal_comp_pmd_stop, + .dev_close = isal_comp_pmd_close, + + .stats_get = isal_comp_pmd_stats_get, + .stats_reset = isal_comp_pmd_stats_reset, + + .dev_infos_get = isal_comp_pmd_info_get, + + .queue_pair_setup = isal_comp_pmd_qp_setup, + .queue_pair_release = isal_comp_pmd_qp_release, + + .private_xform_create = isal_comp_pmd_priv_xform_create, + .private_xform_free = isal_comp_pmd_priv_xform_free, +}; + +struct rte_compressdev_ops *isal_compress_pmd_ops = &isal_pmd_ops; diff --git a/drivers/compress/isal/isal_compress_pmd_private.h b/drivers/compress/isal/isal_compress_pmd_private.h new file mode 100644 index 00000000..46e9fcfa --- /dev/null +++ b/drivers/compress/isal/isal_compress_pmd_private.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation + */ + +#ifndef _ISAL_COMP_PMD_PRIVATE_H_ +#define _ISAL_COMP_PMD_PRIVATE_H_ + +#define COMPDEV_NAME_ISAL_PMD compress_isal +/**< ISA-L comp PMD device name */ + +extern int isal_logtype_driver; +#define ISAL_PMD_LOG(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, isal_logtype_driver, "%s(): "fmt "\n", \ + __func__, ##args) + +/* private data structure for each ISA-L compression device */ +struct isal_comp_private { + struct rte_mempool *priv_xform_mp; +}; + +/** ISA-L queue pair */ +struct isal_comp_qp { + /* Queue Pair Identifier */ + uint16_t id; + /* Unique Queue Pair Name */ + char name[RTE_COMPRESSDEV_NAME_MAX_LEN]; + /* Ring for placing process packets */ + struct rte_ring *processed_pkts; + /* Queue pair statistics */ + struct rte_compressdev_stats qp_stats; + /* Compression stream information*/ + struct isal_zstream *stream; + /* Decompression state information*/ + struct inflate_state *state; + /* Number of free elements on ring */ + uint16_t num_free_elements; +} __rte_cache_aligned; + +/** ISA-L private xform structure */ +struct isal_priv_xform { + enum rte_comp_xform_type type; + union { + struct rte_comp_compress_xform compress; + struct rte_comp_decompress_xform decompress; + }; + uint32_t level_buffer_size; +} __rte_cache_aligned; + +/** Set and validate NULL comp private xform parameters */ +extern int +isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform, + const struct rte_comp_xform *xform); + +/** device specific operations function pointer structure */ +extern struct rte_compressdev_ops *isal_compress_pmd_ops; + +#endif /* _ISAL_COMP_PMD_PRIVATE_H_ */ diff --git a/drivers/compress/isal/meson.build b/drivers/compress/isal/meson.build new file mode 100644 index 00000000..94c10fd6 --- /dev/null +++ b/drivers/compress/isal/meson.build @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2018 Intel Corporation + +dep = dependency('libisal', required: false) +if not dep.found() + build =false +endif + +deps += 'bus_vdev' +sources = files('isal_compress_pmd.c', 'isal_compress_pmd_ops.c') +ext_deps += dep +pkgconfig_extra_libs += '-lisal' + +allow_experimental_apis = true diff --git a/drivers/compress/isal/rte_pmd_isal_version.map b/drivers/compress/isal/rte_pmd_isal_version.map new file mode 100644 index 00000000..de8e412f --- /dev/null +++ b/drivers/compress/isal/rte_pmd_isal_version.map @@ -0,0 +1,3 @@ +DPDK_18.05 { + local: *; +}; |