From 8d01b9cd70a67cdafd5b965a70420c3bd7fb3f82 Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Thu, 1 Nov 2018 11:59:50 +0000 Subject: New upstream version 18.11-rc1 Change-Id: Iaa71986dd6332e878d8f4bf493101b2bbc6313bb Signed-off-by: Luca Boccassi --- drivers/compress/octeontx/include/zip_regs.h | 4 +- drivers/compress/octeontx/otx_zip.h | 6 +- drivers/compress/octeontx/otx_zip_pmd.c | 2 +- drivers/compress/qat/qat_comp.c | 45 ++++++--- drivers/compress/qat/qat_comp.h | 13 +++ drivers/compress/qat/qat_comp_pmd.c | 144 ++++++++++++++++++++++++++- drivers/compress/qat/qat_comp_pmd.h | 3 + 7 files changed, 195 insertions(+), 22 deletions(-) (limited to 'drivers/compress') diff --git a/drivers/compress/octeontx/include/zip_regs.h b/drivers/compress/octeontx/include/zip_regs.h index 1e74db43..04c3d75e 100644 --- a/drivers/compress/octeontx/include/zip_regs.h +++ b/drivers/compress/octeontx/include/zip_regs.h @@ -12,12 +12,12 @@ * ZIP compression coding Enumeration * Enumerates ZIP_INST_S[CC]. */ -enum { +enum zip_cc { ZIP_CC_DEFAULT = 0, ZIP_CC_DYN_HUFF, ZIP_CC_FIXED_HUFF, ZIP_CC_LZS -} zip_cc; +}; /** * Register (NCB) zip_vq#_ena diff --git a/drivers/compress/octeontx/otx_zip.h b/drivers/compress/octeontx/otx_zip.h index 99a38d00..3abefd1d 100644 --- a/drivers/compress/octeontx/otx_zip.h +++ b/drivers/compress/octeontx/otx_zip.h @@ -79,7 +79,7 @@ int octtx_zip_logtype_driver; ZIP_PMD_LOG(ERR, fmt, ## args) /* resources required to process stream */ -enum { +enum NUM_BUFS_PER_STREAM { RES_BUF = 0, CMD_BUF, HASH_CTX_BUF, @@ -88,7 +88,7 @@ enum { OUT_DATA_BUF, HISTORY_DATA_BUF, MAX_BUFS_PER_STREAM -} NUM_BUFS_PER_STREAM; +}; struct zip_stream; struct zipvf_qp; @@ -106,7 +106,7 @@ struct zip_stream { comp_func_t func; /* function to process comp operation */ void *bufs[MAX_BUFS_PER_STREAM]; -} _rte_cache_aligned; +} __rte_cache_aligned; /** diff --git a/drivers/compress/octeontx/otx_zip_pmd.c b/drivers/compress/octeontx/otx_zip_pmd.c index 9d13f933..67ff5066 100644 --- a/drivers/compress/octeontx/otx_zip_pmd.c +++ b/drivers/compress/octeontx/otx_zip_pmd.c @@ -533,7 +533,7 @@ zip_pmd_dequeue_burst_sync(void *queue_pair, return nb_dequeued; } -struct rte_compressdev_ops octtx_zip_pmd_ops = { +static struct rte_compressdev_ops octtx_zip_pmd_ops = { .dev_configure = zip_pmd_config, .dev_start = zip_pmd_start, .dev_stop = zip_pmd_stop, diff --git a/drivers/compress/qat/qat_comp.c b/drivers/compress/qat/qat_comp.c index 38c8a5b8..d70c5949 100644 --- a/drivers/compress/qat/qat_comp.c +++ b/drivers/compress/qat/qat_comp.c @@ -145,7 +145,6 @@ qat_comp_process_response(void **op, uint8_t *resp) rx_op->debug_status = *((uint16_t *)(&resp_msg->comn_resp.comn_error)); } else { - struct qat_comp_xform *qat_xform = rx_op->private_xform; struct icp_qat_fw_resp_comp_pars *comp_resp = (struct icp_qat_fw_resp_comp_pars *)&resp_msg->comp_resp_pars; @@ -193,7 +192,7 @@ static void qat_comp_create_req_hdr(struct icp_qat_fw_comn_req_hdr *header, } static int qat_comp_create_templates(struct qat_comp_xform *qat_xform, - const struct rte_memzone *interm_buff_mz __rte_unused, + const struct rte_memzone *interm_buff_mz, const struct rte_comp_xform *xform) { struct icp_qat_fw_comp_req *comp_req; @@ -281,10 +280,20 @@ static int qat_comp_create_templates(struct qat_comp_xform *qat_xform, ICP_QAT_FW_COMN_CURR_ID_SET(&comp_req->comp_cd_ctrl, ICP_QAT_FW_SLICE_COMP); } else if (qat_xform->qat_comp_request_type == - QAT_COMP_REQUEST_DYNAMIC_COMP_STATELESS) { + QAT_COMP_REQUEST_DYNAMIC_COMP_STATELESS) { - QAT_LOG(ERR, "Dynamic huffman encoding not supported"); - return -EINVAL; + ICP_QAT_FW_COMN_NEXT_ID_SET(&comp_req->comp_cd_ctrl, + ICP_QAT_FW_SLICE_XLAT); + ICP_QAT_FW_COMN_CURR_ID_SET(&comp_req->comp_cd_ctrl, + ICP_QAT_FW_SLICE_COMP); + + ICP_QAT_FW_COMN_NEXT_ID_SET(&comp_req->u2.xlt_cd_ctrl, + ICP_QAT_FW_SLICE_DRAM_WR); + ICP_QAT_FW_COMN_CURR_ID_SET(&comp_req->u2.xlt_cd_ctrl, + ICP_QAT_FW_SLICE_XLAT); + + comp_req->u1.xlt_pars.inter_buff_ptr = + interm_buff_mz->phys_addr; } #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG @@ -335,27 +344,35 @@ qat_comp_private_xform_create(struct rte_compressdev *dev, (struct qat_comp_xform *)*private_xform; if (xform->type == RTE_COMP_COMPRESS) { - if (xform->compress.deflate.huffman == - RTE_COMP_HUFFMAN_DYNAMIC) { - QAT_LOG(ERR, - "QAT device doesn't support dynamic compression"); - return -ENOTSUP; - } if (xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED || ((xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_DEFAULT) && qat->interm_buff_mz == NULL)) - qat_xform->qat_comp_request_type = QAT_COMP_REQUEST_FIXED_COMP_STATELESS; + else if ((xform->compress.deflate.huffman == + RTE_COMP_HUFFMAN_DYNAMIC || + xform->compress.deflate.huffman == + RTE_COMP_HUFFMAN_DEFAULT) && + qat->interm_buff_mz != NULL) + + qat_xform->qat_comp_request_type = + QAT_COMP_REQUEST_DYNAMIC_COMP_STATELESS; + + else { + QAT_LOG(ERR, + "IM buffers needed for dynamic deflate. Set size in config file"); + return -EINVAL; + } + + qat_xform->checksum_type = xform->compress.chksum; } else { qat_xform->qat_comp_request_type = QAT_COMP_REQUEST_DECOMPRESS; + qat_xform->checksum_type = xform->decompress.chksum; } - qat_xform->checksum_type = xform->compress.chksum; - if (qat_comp_create_templates(qat_xform, qat->interm_buff_mz, xform)) { QAT_LOG(ERR, "QAT: Problem with setting compression"); return -EINVAL; diff --git a/drivers/compress/qat/qat_comp.h b/drivers/compress/qat/qat_comp.h index 8d315efb..99a4462e 100644 --- a/drivers/compress/qat/qat_comp.h +++ b/drivers/compress/qat/qat_comp.h @@ -15,6 +15,10 @@ #include "icp_qat_fw_comp.h" #include "icp_qat_fw_la.h" +#define QAT_64_BYTE_ALIGN_MASK (~0x3f) +#define QAT_64_BYTE_ALIGN (64) +#define QAT_NUM_BUFS_IN_IM_SGL 1 + #define ERR_CODE_QAT_COMP_WRONG_FW -99 enum qat_comp_request_type { @@ -24,6 +28,15 @@ enum qat_comp_request_type { REQ_COMP_END }; +struct array_of_ptrs { + phys_addr_t pointer[0]; +}; + +struct qat_inter_sgl { + qat_sgl_hdr; + struct qat_flat_buf buffers[QAT_NUM_BUFS_IN_IM_SGL]; +} __rte_packed __rte_cache_aligned; + struct qat_comp_sgl { qat_sgl_hdr; struct qat_flat_buf buffers[RTE_PMD_QAT_COMP_SGL_MAX_SEGMENTS]; diff --git a/drivers/compress/qat/qat_comp_pmd.c b/drivers/compress/qat/qat_comp_pmd.c index b89975fc..01dd7361 100644 --- a/drivers/compress/qat/qat_comp_pmd.c +++ b/drivers/compress/qat/qat_comp_pmd.c @@ -14,6 +14,7 @@ static const struct rte_compressdev_capabilities qat_comp_gen_capabilities[] = { RTE_COMP_FF_CRC32_ADLER32_CHECKSUM | RTE_COMP_FF_SHAREABLE_PRIV_XFORM | RTE_COMP_FF_HUFFMAN_FIXED | + RTE_COMP_FF_HUFFMAN_DYNAMIC | RTE_COMP_FF_OOP_SGL_IN_SGL_OUT | RTE_COMP_FF_OOP_SGL_IN_LB_OUT | RTE_COMP_FF_OOP_LB_IN_SGL_OUT, @@ -112,7 +113,7 @@ qat_comp_qp_setup(struct rte_compressdev *dev, uint16_t qp_id, /* store a link to the qp in the qat_pci_device */ qat_private->qat_dev->qps_in_use[QAT_SERVICE_COMPRESSION][qp_id] - = *qp_addr; + = *qp_addr; qp = (struct qat_qp *)*qp_addr; @@ -135,6 +136,103 @@ qat_comp_qp_setup(struct rte_compressdev *dev, uint16_t qp_id, return ret; } + +#define QAT_IM_BUFFER_DEBUG 0 +static const struct rte_memzone * +qat_comp_setup_inter_buffers(struct qat_comp_dev_private *comp_dev, + uint32_t buff_size) +{ + char inter_buff_mz_name[RTE_MEMZONE_NAMESIZE]; + const struct rte_memzone *memzone; + uint8_t *mz_start = NULL; + rte_iova_t mz_start_phys = 0; + struct array_of_ptrs *array_of_pointers; + int size_of_ptr_array; + uint32_t full_size; + uint32_t offset_of_sgls, offset_of_flat_buffs = 0; + int i; + int num_im_sgls = qat_gen_config[ + comp_dev->qat_dev->qat_dev_gen].comp_num_im_bufs_required; + + QAT_LOG(DEBUG, "QAT COMP device %s needs %d sgls", + comp_dev->qat_dev->name, num_im_sgls); + snprintf(inter_buff_mz_name, RTE_MEMZONE_NAMESIZE, + "%s_inter_buff", comp_dev->qat_dev->name); + memzone = rte_memzone_lookup(inter_buff_mz_name); + if (memzone != NULL) { + QAT_LOG(DEBUG, "QAT COMP im buffer memzone created already"); + return memzone; + } + + /* Create a memzone to hold intermediate buffers and associated + * meta-data needed by the firmware. The memzone contains: + * - a list of num_im_sgls physical pointers to sgls + * - the num_im_sgl sgl structures, each pointing to 2 flat buffers + * - the flat buffers: num_im_sgl * 2 + * where num_im_sgls depends on the hardware generation of the device + */ + + size_of_ptr_array = num_im_sgls * sizeof(phys_addr_t); + offset_of_sgls = (size_of_ptr_array + (~QAT_64_BYTE_ALIGN_MASK)) + & QAT_64_BYTE_ALIGN_MASK; + offset_of_flat_buffs = + offset_of_sgls + num_im_sgls * sizeof(struct qat_inter_sgl); + full_size = offset_of_flat_buffs + + num_im_sgls * buff_size * QAT_NUM_BUFS_IN_IM_SGL; + + memzone = rte_memzone_reserve_aligned(inter_buff_mz_name, full_size, + comp_dev->compressdev->data->socket_id, + RTE_MEMZONE_2MB, QAT_64_BYTE_ALIGN); + if (memzone == NULL) { + QAT_LOG(ERR, "Can't allocate intermediate buffers" + " for device %s", comp_dev->qat_dev->name); + return NULL; + } + + mz_start = (uint8_t *)memzone->addr; + mz_start_phys = memzone->phys_addr; + QAT_LOG(DEBUG, "Memzone %s: addr = %p, phys = 0x%"PRIx64 + ", size required %d, size created %zu", + inter_buff_mz_name, mz_start, mz_start_phys, + full_size, memzone->len); + + array_of_pointers = (struct array_of_ptrs *)mz_start; + for (i = 0; i < num_im_sgls; i++) { + uint32_t curr_sgl_offset = + offset_of_sgls + i * sizeof(struct qat_inter_sgl); + struct qat_inter_sgl *sgl = + (struct qat_inter_sgl *)(mz_start + curr_sgl_offset); + array_of_pointers->pointer[i] = mz_start_phys + curr_sgl_offset; + + sgl->num_bufs = QAT_NUM_BUFS_IN_IM_SGL; + sgl->num_mapped_bufs = 0; + sgl->resrvd = 0; + sgl->buffers[0].addr = mz_start_phys + offset_of_flat_buffs + + ((i * QAT_NUM_BUFS_IN_IM_SGL) * buff_size); + sgl->buffers[0].len = buff_size; + sgl->buffers[0].resrvd = 0; + sgl->buffers[1].addr = mz_start_phys + offset_of_flat_buffs + + (((i * QAT_NUM_BUFS_IN_IM_SGL) + 1) * buff_size); + sgl->buffers[1].len = buff_size; + sgl->buffers[1].resrvd = 0; + +#if QAT_IM_BUFFER_DEBUG + QAT_LOG(DEBUG, " : phys addr of sgl[%i] in array_of_pointers" + "= 0x%"PRIx64, i, array_of_pointers->pointer[i]); + QAT_LOG(DEBUG, " : virt address of sgl[%i] = %p", i, sgl); + QAT_LOG(DEBUG, " : sgl->buffers[0].addr = 0x%"PRIx64", len=%d", + sgl->buffers[0].addr, sgl->buffers[0].len); + QAT_LOG(DEBUG, " : sgl->buffers[1].addr = 0x%"PRIx64", len=%d", + sgl->buffers[1].addr, sgl->buffers[1].len); +#endif + } +#if QAT_IM_BUFFER_DEBUG + QAT_DP_HEXDUMP_LOG(DEBUG, "IM buffer memzone start:", + mz_start, offset_of_flat_buffs + 32); +#endif + return memzone; +} + static struct rte_mempool * qat_comp_create_xform_pool(struct qat_comp_dev_private *comp_dev, uint32_t num_elements) @@ -176,6 +274,12 @@ qat_comp_create_xform_pool(struct qat_comp_dev_private *comp_dev, static void _qat_comp_dev_config_clear(struct qat_comp_dev_private *comp_dev) { + /* Free intermediate buffers */ + if (comp_dev->interm_buff_mz) { + rte_memzone_free(comp_dev->interm_buff_mz); + comp_dev->interm_buff_mz = NULL; + } + /* Free private_xform pool */ if (comp_dev->xformpool) { /* Free internal mempool for private xforms */ @@ -197,6 +301,21 @@ qat_comp_dev_config(struct rte_compressdev *dev, return -EINVAL; } + if (RTE_PMD_QAT_COMP_IM_BUFFER_SIZE == 0) { + QAT_LOG(WARNING, + "RTE_PMD_QAT_COMP_IM_BUFFER_SIZE = 0 in config file, so" + " QAT device can't be used for Dynamic Deflate. " + "Did you really intend to do this?"); + } else { + comp_dev->interm_buff_mz = + qat_comp_setup_inter_buffers(comp_dev, + RTE_PMD_QAT_COMP_IM_BUFFER_SIZE); + if (comp_dev->interm_buff_mz == NULL) { + ret = -ENOMEM; + goto error_out; + } + } + comp_dev->xformpool = qat_comp_create_xform_pool(comp_dev, config->max_nb_priv_xforms); if (comp_dev->xformpool == NULL) { @@ -348,6 +467,16 @@ static struct rte_compressdev_ops compress_qat_ops = { .private_xform_free = qat_comp_private_xform_free }; +/* An rte_driver is needed in the registration of the device with compressdev. + * The actual qat pci's rte_driver can't be used as its name represents + * the whole pci device with all services. Think of this as a holder for a name + * for the compression part of the pci device. + */ +static const char qat_comp_drv_name[] = RTE_STR(COMPRESSDEV_NAME_QAT_PMD); +static const struct rte_driver compdev_qat_driver = { + .name = qat_comp_drv_name, + .alias = qat_comp_drv_name +}; int qat_comp_dev_create(struct qat_pci_device *qat_pci_dev) { @@ -355,6 +484,10 @@ qat_comp_dev_create(struct qat_pci_device *qat_pci_dev) QAT_LOG(ERR, "Compression PMD not supported on QAT dh895xcc"); return 0; } + if (qat_pci_dev->qat_dev_gen == QAT_GEN3) { + QAT_LOG(ERR, "Compression PMD not supported on QAT c4xxx"); + return 0; + } struct rte_compressdev_pmd_init_params init_params = { .name = "", @@ -368,8 +501,14 @@ qat_comp_dev_create(struct qat_pci_device *qat_pci_dev) qat_pci_dev->name, "comp"); QAT_LOG(DEBUG, "Creating QAT COMP device %s", name); + /* Populate subset device to use in compressdev device creation */ + qat_pci_dev->comp_rte_dev.driver = &compdev_qat_driver; + qat_pci_dev->comp_rte_dev.numa_node = + qat_pci_dev->pci_dev->device.numa_node; + qat_pci_dev->comp_rte_dev.devargs = NULL; + compressdev = rte_compressdev_pmd_create(name, - &qat_pci_dev->pci_dev->device, + &(qat_pci_dev->comp_rte_dev), sizeof(struct qat_comp_dev_private), &init_params); @@ -391,6 +530,7 @@ qat_comp_dev_create(struct qat_pci_device *qat_pci_dev) switch (qat_pci_dev->qat_dev_gen) { case QAT_GEN1: case QAT_GEN2: + case QAT_GEN3: comp_dev->qat_dev_capabilities = qat_comp_gen_capabilities; break; default: diff --git a/drivers/compress/qat/qat_comp_pmd.h b/drivers/compress/qat/qat_comp_pmd.h index 9ad2a283..b8299d43 100644 --- a/drivers/compress/qat/qat_comp_pmd.h +++ b/drivers/compress/qat/qat_comp_pmd.h @@ -12,6 +12,9 @@ #include "qat_device.h" +/**< Intel(R) QAT Compression PMD driver name */ +#define COMPRESSDEV_NAME_QAT_PMD compress_qat + /** private data structure for a QAT compression device. * This QAT device is a device offering only a compression service, * there can be one of these on each qat_pci_device (VF). -- cgit 1.2.3-korg