2 files changed, 298 insertions, 67 deletions

diff --git a/drivers/compress/isal/isal_compress_pmd.c b/drivers/compress/isal/isal_compress_pmd.c
index 0f025a3b..e943336b 100644
--- a/drivers/compress/isal/isal_compress_pmd.c
+++ b/drivers/compress/isal/isal_compress_pmd.c
@@ -188,6 +188,179 @@ isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform,
 	return 0;
 }
 
+/* Compression using chained mbufs for input/output data */
+static int
+chained_mbuf_compression(struct rte_comp_op *op, struct isal_comp_qp *qp)
+{
+	int ret;
+	uint32_t remaining_offset;
+	uint32_t remaining_data = op->src.length;
+	struct rte_mbuf *src = op->m_src;
+	struct rte_mbuf *dst = op->m_dst;
+
+	/* check for source/destination offset passing multiple segments
+	 * and point compression stream to input/output buffer.
+	 */
+	remaining_offset = op->src.offset;
+	while (remaining_offset >= src->data_len) {
+		remaining_offset -= src->data_len;
+		src = src->next;
+	}
+	qp->stream->avail_in = RTE_MIN(src->data_len - remaining_offset,
+			op->src.length);
+	qp->stream->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *,
+			remaining_offset);
+
+	remaining_offset = op->dst.offset;
+	while (remaining_offset >= dst->data_len) {
+		remaining_offset -= dst->data_len;
+		dst = dst->next;
+	}
+	qp->stream->avail_out = dst->data_len - remaining_offset;
+	qp->stream->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *,
+			remaining_offset);
+
+	if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
+		ISAL_PMD_LOG(ERR, "Invalid source or destination buffer\n");
+		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+		return -1;
+	}
+
+	while (qp->stream->internal_state.state != ZSTATE_END) {
+		/* Last segment of data */
+		if (remaining_data <= src->data_len)
+			qp->stream->end_of_stream = 1;
+
+		/* Execute compression operation */
+		ret = isal_deflate(qp->stream);
+
+		remaining_data = op->src.length - qp->stream->total_in;
+
+		if (ret != COMP_OK) {
+			ISAL_PMD_LOG(ERR, "Compression operation failed\n");
+			op->status = RTE_COMP_OP_STATUS_ERROR;
+			return ret;
+		}
+
+		if (qp->stream->avail_in == 0 &&
+				qp->stream->total_in != op->src.length) {
+			if (src->next != NULL) {
+				src = src->next;
+				qp->stream->next_in =
+						rte_pktmbuf_mtod(src, uint8_t *);
+				qp->stream->avail_in =
+					RTE_MIN(remaining_data, src->data_len);
+			} else {
+				ISAL_PMD_LOG(ERR,
+				"Not enough input buffer segments\n");
+				op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+				return -1;
+			}
+		}
+
+		if (qp->stream->avail_out == 0 &&
+				qp->stream->internal_state.state != ZSTATE_END) {
+			if (dst->next != NULL) {
+				dst = dst->next;
+				qp->stream->next_out =
+						rte_pktmbuf_mtod(dst, uint8_t *);
+				qp->stream->avail_out = dst->data_len;
+			} else {
+				ISAL_PMD_LOG(ERR,
+				"Not enough output buffer segments\n");
+				op->status =
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+				return -1;
+			}
+		}
+	}
+
+	return 0;
+}
+
+/* Decompression using chained mbufs for input/output data */
+static int
+chained_mbuf_decompression(struct rte_comp_op *op, struct isal_comp_qp *qp)
+{
+	int ret;
+	uint32_t consumed_data, src_remaining_offset, dst_remaining_offset;
+	uint32_t remaining_data = op->src.length;
+	struct rte_mbuf *src = op->m_src;
+	struct rte_mbuf *dst = op->m_dst;
+
+	/* check for offset passing multiple segments
+	 * and point decompression state to input/output buffer
+	 */
+	src_remaining_offset = op->src.offset;
+	while (src_remaining_offset >= src->data_len) {
+		src_remaining_offset -= src->data_len;
+		src = src->next;
+	}
+	qp->state->avail_in = RTE_MIN(src->data_len - src_remaining_offset,
+			op->src.length);
+	qp->state->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *,
+			src_remaining_offset);
+
+	dst_remaining_offset = op->dst.offset;
+	while (dst_remaining_offset >= dst->data_len) {
+		dst_remaining_offset -= dst->data_len;
+		dst = dst->next;
+	}
+	qp->state->avail_out = dst->data_len - dst_remaining_offset;
+	qp->state->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *,
+			dst_remaining_offset);
+
+	while (qp->state->block_state != ISAL_BLOCK_FINISH) {
+
+		ret = isal_inflate(qp->state);
+
+		/* Check for first segment, offset needs to be accounted for */
+		if (remaining_data == op->src.length) {
+			consumed_data = src->data_len - qp->state->avail_in -
+					src_remaining_offset;
+		} else
+			consumed_data = src->data_len - qp->state->avail_in;
+
+		op->consumed += consumed_data;
+		remaining_data -= consumed_data;
+
+		if (ret != ISAL_DECOMP_OK) {
+			ISAL_PMD_LOG(ERR, "Decompression operation failed\n");
+			op->status = RTE_COMP_OP_STATUS_ERROR;
+			return ret;
+		}
+
+		if (qp->state->avail_in == 0
+				&& op->consumed != op->src.length) {
+			if (src->next != NULL) {
+				src = src->next;
+				qp->state->next_in =
+						rte_pktmbuf_mtod(src, uint8_t *);
+				qp->state->avail_in =
+					RTE_MIN(remaining_data, src->data_len);
+			}
+		}
+
+		if (qp->state->avail_out == 0 &&
+				qp->state->block_state != ISAL_BLOCK_FINISH) {
+			if (dst->next != NULL) {
+				dst = dst->next;
+				qp->state->next_out =
+						rte_pktmbuf_mtod(dst, uint8_t *);
+				qp->state->avail_out = dst->data_len;
+			} else {
+				ISAL_PMD_LOG(ERR,
+				"Not enough output buffer segments\n");
+				op->status =
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+				return -1;
+			}
+		}
+	}
+
+	return 0;
+}
+
 /* Stateless Compression Function */
 static int
 process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
@@ -207,23 +380,10 @@ process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
 	/* Stateless operation, input will be consumed in one go */
 	qp->stream->flush = NO_FLUSH;
 
-	/* set op level & intermediate level buffer */
+	/* set compression level & intermediate level buffer size */
 	qp->stream->level = priv_xform->compress.level;
 	qp->stream->level_buf_size = priv_xform->level_buffer_size;
 
-	/* Point compression stream structure to input/output buffers */
-	qp->stream->avail_in = op->src.length;
-	qp->stream->next_in = rte_pktmbuf_mtod(op->m_src, uint8_t *);
-	qp->stream->avail_out = op->m_dst->data_len;
-	qp->stream->next_out  = rte_pktmbuf_mtod(op->m_dst, uint8_t *);
-	qp->stream->end_of_stream = 1; /* All input consumed in one go */
-
-	if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
-		ISAL_PMD_LOG(ERR, "Invalid source or destination buffers\n");
-		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
-		return -1;
-	}
-
 	/* Set op huffman code */
 	if (priv_xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED)
 		isal_deflate_set_hufftables(qp->stream, NULL,
@@ -238,30 +398,70 @@ process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
 		isal_deflate_set_hufftables(qp->stream, NULL,
 				IGZIP_HUFFTABLE_DEFAULT);
 
-	/* Execute compression operation */
-	ret =  isal_deflate_stateless(qp->stream);
-
-	/* Check that output buffer did not run out of space */
-	if (ret == STATELESS_OVERFLOW) {
-		ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
-		op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
-		return ret;
+	if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
+		ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
+		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+		return -1;
 	}
 
-	/* Check that input buffer has been fully consumed */
-	if (qp->stream->avail_in != (uint32_t)0) {
-		ISAL_PMD_LOG(ERR, "Input buffer could not be read entirely\n");
-		op->status = RTE_COMP_OP_STATUS_ERROR;
+	if (op->dst.offset >= op->m_dst->pkt_len) {
+		ISAL_PMD_LOG(ERR, "Output mbuf(s) not big enough"
+				" for offset provided.\n");
+		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 		return -1;
 	}
 
-	if (ret != COMP_OK) {
-		op->status = RTE_COMP_OP_STATUS_ERROR;
-		return ret;
-	}
+	/* Chained mbufs */
+	if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) {
+		ret = chained_mbuf_compression(op, qp);
+		if (ret < 0)
+			return ret;
+	} else {
+		/* Linear buffer */
+		qp->stream->end_of_stream = 1; /* All input consumed in one */
+		/* Point compression stream to input buffer */
+		qp->stream->avail_in = op->src.length;
+		qp->stream->next_in = rte_pktmbuf_mtod_offset(op->m_src,
+				uint8_t *, op->src.offset);
+
+		/*  Point compression stream to output buffer */
+		qp->stream->avail_out = op->m_dst->data_len - op->dst.offset;
+		qp->stream->next_out  = rte_pktmbuf_mtod_offset(op->m_dst,
+				uint8_t *, op->dst.offset);
+
+		if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
+			ISAL_PMD_LOG(ERR, "Invalid source or destination"
+					" buffers\n");
+			op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+			return -1;
+		}
 
-	op->consumed = qp->stream->total_in;
-	op->produced = qp->stream->total_out;
+		/* Execute compression operation */
+		ret =  isal_deflate_stateless(qp->stream);
+
+		/* Check that output buffer did not run out of space */
+		if (ret == STATELESS_OVERFLOW) {
+			ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
+			op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+			return ret;
+		}
+
+		/* Check that input buffer has been fully consumed */
+		if (qp->stream->avail_in != (uint32_t)0) {
+			ISAL_PMD_LOG(ERR, "Input buffer could not be read"
+					" entirely\n");
+			op->status = RTE_COMP_OP_STATUS_ERROR;
+			return -1;
+		}
+
+		if (ret != COMP_OK) {
+			ISAL_PMD_LOG(ERR, "Compression operation failed\n");
+			op->status = RTE_COMP_OP_STATUS_ERROR;
+			return ret;
+		}
+	}
+		op->consumed = qp->stream->total_in;
+		op->produced = qp->stream->total_out;
 
 	return ret;
 }
@@ -277,43 +477,69 @@ process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp)
 	/* Initialize decompression state */
 	isal_inflate_init(qp->state);
 
-	/* Point decompression state structure to input/output buffers */
-	qp->state->avail_in = op->src.length;
-	qp->state->next_in = rte_pktmbuf_mtod(op->m_src, uint8_t *);
-	qp->state->avail_out = op->m_dst->data_len;
-	qp->state->next_out  = rte_pktmbuf_mtod(op->m_dst, uint8_t *);
+	if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
+		ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
+		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+		return -1;
+	}
 
-	if (unlikely(!qp->state->next_in || !qp->state->next_out)) {
-		ISAL_PMD_LOG(ERR, "Invalid source or destination buffers\n");
+	if (op->dst.offset >= op->m_dst->pkt_len) {
+		ISAL_PMD_LOG(ERR, "Output mbuf not big enough for "
+				"offset provided.\n");
 		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 		return -1;
 	}
 
-	/* Execute decompression operation */
-	ret = isal_inflate_stateless(qp->state);
+	/* Chained mbufs */
+	if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) {
+		ret = chained_mbuf_decompression(op, qp);
+		if (ret !=  0)
+			return ret;
+	} else {
+		/* Linear buffer */
+		/* Point decompression state to input buffer */
+		qp->state->avail_in = op->src.length;
+		qp->state->next_in = rte_pktmbuf_mtod_offset(op->m_src,
+				uint8_t *, op->src.offset);
+
+		/* Point decompression state to output buffer */
+		qp->state->avail_out = op->m_dst->data_len - op->dst.offset;
+		qp->state->next_out  = rte_pktmbuf_mtod_offset(op->m_dst,
+				uint8_t *, op->dst.offset);
+
+		if (unlikely(!qp->state->next_in || !qp->state->next_out)) {
+			ISAL_PMD_LOG(ERR, "Invalid source or destination"
+					" buffers\n");
+			op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+			return -1;
+		}
 
-	if (ret == ISAL_OUT_OVERFLOW) {
-		ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
-		op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
-		return ret;
-	}
+		/* Execute decompression operation */
+		ret = isal_inflate_stateless(qp->state);
 
-	/* Check that input buffer has been fully consumed */
-	if (qp->state->avail_in != (uint32_t)0) {
-		ISAL_PMD_LOG(ERR, "Input buffer could not be read entirely\n");
-		op->status = RTE_COMP_OP_STATUS_ERROR;
-		return -1;
-	}
+		if (ret == ISAL_OUT_OVERFLOW) {
+			ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
+			op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+			return ret;
+		}
 
-	if (ret != ISAL_DECOMP_OK) {
-		op->status = RTE_COMP_OP_STATUS_ERROR;
-		return ret;
-	}
+		/* Check that input buffer has been fully consumed */
+		if (qp->state->avail_in != (uint32_t)0) {
+			ISAL_PMD_LOG(ERR, "Input buffer could not be read"
+					" entirely\n");
+			op->status = RTE_COMP_OP_STATUS_ERROR;
+			return -1;
+		}
 
-	op->consumed = op->src.length - qp->state->avail_in;
-	op->produced = qp->state->total_out;
+		if (ret != ISAL_DECOMP_OK) {
+			op->status = RTE_COMP_OP_STATUS_ERROR;
+			return ret;
+		}
+		op->consumed = op->src.length - qp->state->avail_in;
+	}
+		op->produced = qp->state->total_out;
 
-return ret;
+	return ret;
 }
 
 /* Process compression/decompression operation */
@@ -460,12 +686,9 @@ RTE_PMD_REGISTER_VDEV(COMPDEV_NAME_ISAL_PMD, compdev_isal_pmd_drv);
 RTE_PMD_REGISTER_PARAM_STRING(COMPDEV_NAME_ISAL_PMD,
 	"socket_id=<int>");
 
-RTE_INIT(isal_init_log);
-
-static void
-isal_init_log(void)
+RTE_INIT(isal_init_log)
 {
-	isal_logtype_driver = rte_log_register("comp_isal");
+	isal_logtype_driver = rte_log_register("pmd.compress.isal");
 	if (isal_logtype_driver >= 0)
 		rte_log_set_level(isal_logtype_driver, RTE_LOG_INFO);
 }
diff --git a/drivers/compress/isal/isal_compress_pmd_ops.c b/drivers/compress/isal/isal_compress_pmd_ops.c
index 970a0413..41cade87 100644
--- a/drivers/compress/isal/isal_compress_pmd_ops.c
+++ b/drivers/compress/isal/isal_compress_pmd_ops.c
@@ -12,7 +12,12 @@
 static const struct rte_compressdev_capabilities isal_pmd_capabilities[] = {
 	{
 		.algo = RTE_COMP_ALGO_DEFLATE,
-		.comp_feature_flags =	RTE_COMP_FF_SHAREABLE_PRIV_XFORM,
+		.comp_feature_flags =	RTE_COMP_FF_OOP_SGL_IN_SGL_OUT |
+					RTE_COMP_FF_OOP_SGL_IN_LB_OUT |
+					RTE_COMP_FF_OOP_LB_IN_SGL_OUT |
+					RTE_COMP_FF_SHAREABLE_PRIV_XFORM |
+					RTE_COMP_FF_HUFFMAN_FIXED |
+					RTE_COMP_FF_HUFFMAN_DYNAMIC,
 		.window_size = {
 			.min = 15,
 			.max = 15,
@@ -165,8 +170,11 @@ isal_comp_pmd_qp_release(struct rte_compressdev *dev, uint16_t qp_id)
 	if (qp->state != NULL)
 		rte_free(qp->state);
 
-	if (dev->data->queue_pairs[qp_id] != NULL)
-		rte_free(dev->data->queue_pairs[qp_id]);
+	if (qp->processed_pkts != NULL)
+		rte_ring_free(qp->processed_pkts);
+
+	rte_free(qp);
+	dev->data->queue_pairs[qp_id] = NULL;
 
 	return 0;
 }