Skip to contentSkip to conversation

compress/isal: enable checksum support in driver

PATCH Superseded Lee Daly 3 participants 15 messages Dec 14, 2018 – Jan 8, 2019 1 message not in the archive

lore
Share & tools
lore

A message that is not in the archive (sent off-list or not yet received): [email protected]

Lee Daly

author
#1 lore raw

PATCH v4 1/3 compress/isal: enable checksum support in driver

This patch adds checksum support in the ISA-L PMD for both compression and decompression.
CRC32 is supported as well as Adler32.

V2:
Documentation Changes
V3:
Added Release note
V4:
Removed dependency on offset unit test

---
 drivers/compress/isal/isal_compress_pmd.c     | 80 ++++++++++++++++++++++-----
 drivers/compress/isal/isal_compress_pmd_ops.c |  4 +-
 2 files changed, 69 insertions(+), 15 deletions(-)
drivers/compress/isal/isal_compress_pmd.c +66 -14
index 9f1e968..b087ed8 100644
@@ -16,6 +16,8 @@
 #define RTE_COMP_ISAL_LEVEL_ONE 1
 #define RTE_COMP_ISAL_LEVEL_TWO 2
 #define RTE_COMP_ISAL_LEVEL_THREE 3 /* Optimised for AVX512 & AVX2 only */
+#define CHKSUM_SZ_CRC 8
+#define CHKSUM_SZ_ADLER 4
 
 int isal_logtype_driver;
 
@@ -43,12 +45,6 @@ isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform,
 		}
 		priv_xform->compress.algo = RTE_COMP_ALGO_DEFLATE;
 
-		/* Set private xform checksum - raw deflate by default */
-		if (xform->compress.chksum != RTE_COMP_CHECKSUM_NONE) {
-			ISAL_PMD_LOG(ERR, "Checksum not supported\n");
-			return -ENOTSUP;
-		}
-
 		/* Set private xform window size, 32K supported */
 		if (xform->compress.window_size == RTE_COMP_ISAL_WINDOW_SIZE)
 			priv_xform->compress.window_size =
@@ -77,6 +73,27 @@ isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform,
 			return -ENOTSUP;
 		}
 
+		/* Set private xform checksum */
+		switch (xform->compress.chksum) {
+		/* Raw deflate by default */
+		case(RTE_COMP_CHECKSUM_NONE):
+			priv_xform->compress.chksum = IGZIP_DEFLATE;
+			break;
+		case(RTE_COMP_CHECKSUM_CRC32):
+			priv_xform->compress.chksum = IGZIP_GZIP_NO_HDR;
+			break;
+		case(RTE_COMP_CHECKSUM_ADLER32):
+			priv_xform->compress.chksum = IGZIP_ZLIB_NO_HDR;
+			break;
+		case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
+			ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not"
+					" supported\n");
+			return -ENOTSUP;
+		default:
+			ISAL_PMD_LOG(ERR, "Checksum type not supported\n");
+			return -ENOTSUP;
+		}
+
 		/* Set private xform level.
 		 * Checking compliance with compressdev API, -1 <= level => 9
 		 */
@@ -170,9 +187,24 @@ isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform,
 		}
 		priv_xform->decompress.algo = RTE_COMP_ALGO_DEFLATE;
 
-		/* Set private xform checksum - raw deflate by default */
-		if (xform->compress.chksum != RTE_COMP_CHECKSUM_NONE) {
-			ISAL_PMD_LOG(ERR, "Checksum not supported\n");
+		/* Set private xform checksum */
+		switch (xform->decompress.chksum) {
+		/* Raw deflate by default */
+		case(RTE_COMP_CHECKSUM_NONE):
+			priv_xform->decompress.chksum = ISAL_DEFLATE;
+			break;
+		case(RTE_COMP_CHECKSUM_CRC32):
+			priv_xform->decompress.chksum = ISAL_GZIP_NO_HDR_VER;
+			break;
+		case(RTE_COMP_CHECKSUM_ADLER32):
+			priv_xform->decompress.chksum = ISAL_ZLIB_NO_HDR_VER;
+			break;
+		case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
+			ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not"
+					" supported\n");
+			return -ENOTSUP;
+		default:
+			ISAL_PMD_LOG(ERR, "Checksum type not supported\n");
 			return -ENOTSUP;
 		}
 
@@ -376,6 +408,9 @@ process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
 
 	qp->stream->level_buf = temp_level_buf;
 
+	/* Set Checksum flag */
+	qp->stream->gzip_flag = priv_xform->compress.chksum;
+
 	/* Stateless operation, input will be consumed in one go */
 	qp->stream->flush = NO_FLUSH;
 
@@ -459,8 +494,18 @@ process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
 			return ret;
 		}
 	}
+
 	op->consumed = qp->stream->total_in;
-	op->produced = qp->stream->total_out;
+	if (qp->stream->gzip_flag == IGZIP_DEFLATE) {
+		op->produced = qp->stream->total_out;
+	} else if (qp->stream->gzip_flag == IGZIP_ZLIB_NO_HDR) {
+		op->produced = qp->stream->total_out - CHKSUM_SZ_ADLER;
+		op->output_chksum = qp->stream->internal_state.crc + 1;
+	} else {
+		op->produced = qp->stream->total_out - CHKSUM_SZ_CRC;
+		op->output_chksum = qp->stream->internal_state.crc;
+	}
+
 	isal_deflate_reset(qp->stream);
 
 	return ret;
@@ -468,7 +513,8 @@ process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
 
 /* Stateless Decompression Function */
 static int
-process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp)
+process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
+		struct isal_priv_xform *priv_xform)
 {
 	int ret = 0;
 
@@ -477,6 +523,9 @@ process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp)
 	/* Initialize decompression state */
 	isal_inflate_init(qp->state);
 
+	/* Set Checksum flag */
+	qp->state->crc_flag = priv_xform->decompress.chksum;
+
 	if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
 		ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
 		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
@@ -531,13 +580,16 @@ process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp)
 			return -1;
 		}
 
-		if (ret != ISAL_DECOMP_OK) {
+		if (ret != ISAL_DECOMP_OK && ret != ISAL_END_INPUT) {
+			ISAL_PMD_LOG(ERR, "Decompression operation failed\n");
 			op->status = RTE_COMP_OP_STATUS_ERROR;
 			return ret;
 		}
 		op->consumed = op->src.length - qp->state->avail_in;
 	}
-		op->produced = qp->state->total_out;
+	op->produced = qp->state->total_out;
+	op->output_chksum = qp->state->crc;
+
 	isal_inflate_reset(qp->state);
 
 	return ret;
@@ -553,7 +605,7 @@ process_op(struct isal_comp_qp *qp, struct rte_comp_op *op,
 		process_isal_deflate(op, qp, priv_xform);
 		break;
 	case RTE_COMP_DECOMPRESS:
-		process_isal_inflate(op, qp);
+		process_isal_inflate(op, qp, priv_xform);
 		break;
 	default:
 		ISAL_PMD_LOG(ERR, "Operation Not Supported\n");
drivers/compress/isal/isal_compress_pmd_ops.c +3 -1
index 41cade8..7b91849 100644
@@ -17,7 +17,9 @@ static const struct rte_compressdev_capabilities isal_pmd_capabilities[] = {
 					RTE_COMP_FF_OOP_LB_IN_SGL_OUT |
 					RTE_COMP_FF_SHAREABLE_PRIV_XFORM |
 					RTE_COMP_FF_HUFFMAN_FIXED |
-					RTE_COMP_FF_HUFFMAN_DYNAMIC,
+					RTE_COMP_FF_HUFFMAN_DYNAMIC |
+					RTE_COMP_FF_CRC32_CHECKSUM |
+					RTE_COMP_FF_ADLER32_CHECKSUM,
 		.window_size = {
 			.min = 15,
 			.max = 15,
Signature · 1 line
2.7.4
author
#2 lore raw

PATCH v4 2/3 test/compress: add checksum tests

This patch adds a test which examines what type of checksum the PMD supports, adler, crc32 or alder32_crc32
and tests that feature if the PMD supports it.

V2:
Documentation Changes
V3:
Added Release note
V4:
Removed dependency on offset unit test

---
 test/test/test_compressdev.c | 206 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 203 insertions(+), 3 deletions(-)
test/test/test_compressdev.c +203 -3
index dab3c12..2e70ca3 100644
@@ -36,6 +36,12 @@
 #define NUM_MAX_INFLIGHT_OPS 128
 #define CACHE_SIZE 0
 
+#define ZLIB_CRC_CHECKSUM_WINDOW_BITS 31
+#define ZLIB_HEADER_SIZE 2
+#define ZLIB_TRAILER_SIZE 4
+#define GZIP_HEADER_SIZE 10
+#define GZIP_TRAILER_SIZE 8
+
 const char *
 huffman_type_strings[] = {
 	[RTE_COMP_HUFFMAN_DEFAULT]	= "PMD default",
@@ -318,6 +324,10 @@ compress_zlib(struct rte_comp_op *op,
 	 * When doing raw DEFLATE, this number will be negative.
 	 */
 	window_bits = -(xform->compress.window_size);
+	if (xform->compress.chksum == RTE_COMP_CHECKSUM_ADLER32)
+		window_bits *= -1;
+	else if (xform->compress.chksum == RTE_COMP_CHECKSUM_CRC32)
+		window_bits = ZLIB_CRC_CHECKSUM_WINDOW_BITS;
 
 	comp_level = xform->compress.level;
 
@@ -401,8 +411,19 @@ compress_zlib(struct rte_comp_op *op,
 	}
 
 	op->consumed = stream.total_in;
-	op->produced = stream.total_out;
+	if (xform->compress.chksum == RTE_COMP_CHECKSUM_ADLER32) {
+		rte_pktmbuf_adj(op->m_dst, ZLIB_HEADER_SIZE);
+		op->produced = stream.total_out -
+				(ZLIB_HEADER_SIZE + ZLIB_TRAILER_SIZE);
+	} else if (xform->compress.chksum == RTE_COMP_CHECKSUM_CRC32) {
+		rte_pktmbuf_adj(op->m_dst, GZIP_HEADER_SIZE);
+		op->produced = stream.total_out -
+				(GZIP_HEADER_SIZE + GZIP_TRAILER_SIZE);
+	} else
+		op->produced = stream.total_out;
+
 	op->status = RTE_COMP_OP_STATUS_SUCCESS;
+	op->output_chksum = stream.adler;
 
 	deflateReset(&stream);
 
@@ -436,7 +457,6 @@ decompress_zlib(struct rte_comp_op *op,
 	 * When doing raw DEFLATE, this number will be negative.
 	 */
 	window_bits = -(xform->decompress.window_size);
-
 	ret = inflateInit2(&stream, window_bits);
 
 	if (ret != Z_OK) {
@@ -652,6 +672,7 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 	const struct rte_compressdev_capabilities *capa =
 		rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
 	char *contig_buf = NULL;
+	uint64_t compress_checksum[num_bufs];
 
 	/* Initialize all arrays to NULL */
 	memset(uncomp_bufs, 0, sizeof(struct rte_mbuf *) * num_bufs);
@@ -838,6 +859,7 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 					&ops_processed[num_total_deqd], num_bufs);
 			num_total_deqd += num_deqd;
 			deqd_retries++;
+
 		} while (num_total_deqd < num_enqd);
 
 		deqd_retries = 0;
@@ -871,6 +893,8 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 			ops_processed[i]->consumed == 0 ? 0 :
 			(float)ops_processed[i]->produced /
 			ops_processed[i]->consumed * 100);
+		if (compress_xform->chksum != RTE_COMP_CHECKSUM_NONE)
+			compress_checksum[i] = ops_processed[i]->output_chksum;
 		ops[i] = NULL;
 	}
 
@@ -1106,11 +1130,23 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 
 		buf2 = rte_pktmbuf_read(ops_processed[i]->m_dst, 0,
 				ops_processed[i]->produced, contig_buf);
-
 		if (compare_buffers(buf1, strlen(buf1) + 1,
 				buf2, ops_processed[i]->produced) < 0)
 			goto exit;
 
+		/* Test checksums */
+		if (compress_xforms[0]->compress.chksum !=
+				RTE_COMP_CHECKSUM_NONE) {
+			if (ops_processed[i]->output_chksum !=
+					compress_checksum[i]) {
+				RTE_LOG(ERR, USER1, "The checksums differ\n"
+			"Compression Checksum: %" PRIu64 "\tDecompression "
+			"Checksum: %" PRIu64 "\n", compress_checksum[i],
+			ops_processed[i]->output_chksum);
+				goto exit;
+			}
+		}
+
 		rte_free(contig_buf);
 		contig_buf = NULL;
 	}
@@ -1462,6 +1498,168 @@ test_compressdev_deflate_stateless_sgl(void)
 	}
 
 	return TEST_SUCCESS;
+
+}
+
+static int
+test_compressdev_deflate_stateless_checksum(void)
+{
+	struct comp_testsuite_params *ts_params = &testsuite_params;
+	const char *test_buffer;
+	uint16_t i;
+	int ret;
+	const struct rte_compressdev_capabilities *capab;
+
+	capab = rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
+	TEST_ASSERT(capab != NULL, "Failed to retrieve device capabilities");
+
+	/* Check if driver supports any checksum */
+	if ((capab->comp_feature_flags & RTE_COMP_FF_CRC32_CHECKSUM) == 0 &&
+			(capab->comp_feature_flags &
+			RTE_COMP_FF_ADLER32_CHECKSUM) == 0 &&
+			(capab->comp_feature_flags &
+			RTE_COMP_FF_CRC32_ADLER32_CHECKSUM) == 0)
+		return -ENOTSUP;
+
+	struct rte_comp_xform *compress_xform =
+			rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
+	if (compress_xform == NULL) {
+		RTE_LOG(ERR, USER1, "Compress xform could not be created\n");
+		ret = TEST_FAILED;
+		goto exit;
+	}
+
+	memcpy(compress_xform, ts_params->def_comp_xform,
+			sizeof(struct rte_comp_xform));
+
+	struct rte_comp_xform *decompress_xform =
+			rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
+	if (decompress_xform == NULL) {
+		RTE_LOG(ERR, USER1, "Decompress xform could not be created\n");
+		ret = TEST_FAILED;
+		goto exit;
+	}
+
+	memcpy(decompress_xform, ts_params->def_decomp_xform,
+			sizeof(struct rte_comp_xform));
+
+	/* Check if driver supports crc32 checksum and test */
+	if ((capab->comp_feature_flags & RTE_COMP_FF_CRC32_CHECKSUM)) {
+		compress_xform->compress.chksum = RTE_COMP_CHECKSUM_CRC32;
+		decompress_xform->decompress.chksum = RTE_COMP_CHECKSUM_CRC32;
+
+		for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
+			test_buffer = compress_test_bufs[i];
+
+			/* Generate zlib checksum and test against selected
+			 * drivers decompression checksum
+			 */
+			if (test_deflate_comp_decomp(&test_buffer, 1,
+					&i,
+					&compress_xform,
+					&decompress_xform,
+					1,
+					RTE_COMP_OP_STATELESS,
+					0,
+					ZLIB_COMPRESS,
+					0) < 0) {
+				ret = TEST_FAILED;
+				goto exit;
+			}
+
+			/* Generate compression and decompression
+			 * checksum of selected driver
+			 */
+			if (test_deflate_comp_decomp(&test_buffer, 1,
+					&i,
+					&compress_xform,
+					&decompress_xform,
+					1,
+					RTE_COMP_OP_STATELESS,
+					0,
+					ZLIB_NONE,
+					0) < 0) {
+				ret = TEST_FAILED;
+				goto exit;
+			}
+		}
+	}
+
+	/* Check if driver supports adler32 checksum and test */
+	if ((capab->comp_feature_flags & RTE_COMP_FF_ADLER32_CHECKSUM)) {
+		compress_xform->compress.chksum = RTE_COMP_CHECKSUM_ADLER32;
+		decompress_xform->decompress.chksum = RTE_COMP_CHECKSUM_ADLER32;
+
+		for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
+			test_buffer = compress_test_bufs[i];
+
+			/* Generate zlib checksum and test against selected
+			 * drivers decompression checksum
+			 */
+			if (test_deflate_comp_decomp(&test_buffer, 1,
+					&i,
+					&compress_xform,
+					&decompress_xform,
+					1,
+					RTE_COMP_OP_STATELESS,
+					0,
+					ZLIB_COMPRESS,
+					0) < 0) {
+				ret = TEST_FAILED;
+				goto exit;
+			}
+			/* Generate compression and decompression
+			 * checksum of selected driver
+			 */
+			if (test_deflate_comp_decomp(&test_buffer, 1,
+					&i,
+					&compress_xform,
+					&decompress_xform,
+					1,
+					RTE_COMP_OP_STATELESS,
+					0,
+					ZLIB_NONE,
+					0) < 0) {
+				ret = TEST_FAILED;
+				goto exit;
+			}
+		}
+	}
+
+	/* Check if driver supports combined crc and adler checksum and test */
+	if ((capab->comp_feature_flags & RTE_COMP_FF_CRC32_ADLER32_CHECKSUM)) {
+		compress_xform->compress.chksum =
+				RTE_COMP_CHECKSUM_CRC32_ADLER32;
+		decompress_xform->decompress.chksum =
+				RTE_COMP_CHECKSUM_CRC32_ADLER32;
+
+		for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
+			test_buffer = compress_test_bufs[i];
+
+			/* Generate compression and decompression
+			 * checksum of selected driver
+			 */
+			if (test_deflate_comp_decomp(&test_buffer, 1,
+					&i,
+					&compress_xform,
+					&decompress_xform,
+					1,
+					RTE_COMP_OP_STATELESS,
+					0,
+					ZLIB_NONE,
+					0) < 0) {
+				ret = TEST_FAILED;
+				goto exit;
+			}
+		}
+	}
+
+	ret = TEST_SUCCESS;
+
+exit:
+	rte_free(compress_xform);
+	rte_free(decompress_xform);
+	return ret;
 }
 
 static struct unit_test_suite compressdev_testsuite  = {
@@ -1483,6 +1681,8 @@ static struct unit_test_suite compressdev_testsuite  = {
 			test_compressdev_deflate_stateless_multi_xform),
 		TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
 			test_compressdev_deflate_stateless_sgl),
+		TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
+			test_compressdev_deflate_stateless_checksum),
 		TEST_CASES_END() /**< NULL terminate unit test array */
 	}
 };
Signature · 1 line
2.7.4
author
#3 lore raw

PATCH v4 3/3 doc: update ISA-L guide to reflect checksum support

In reply to Lee Daly

This updates the ISA-L compression driver guide on how to enable and use checksums.

This also updates the compression drivers features matrix.

V2:
Documentation Changes
V3:
Added Release note
V4:
Removed depedency on offset unit test

---
 doc/guides/compressdevs/features/isal.ini |  2 ++
 doc/guides/compressdevs/isal.rst          | 30 ++++++++++++++++++++++++++++--
 doc/guides/rel_notes/release_19_02.rst    |  5 +++++
 3 files changed, 35 insertions(+), 2 deletions(-)
doc/guides/compressdevs/features/isal.ini +2 -0
index 919cf70..e705031 100644
@@ -12,5 +12,7 @@ OOP SGL In SGL Out = Y
 OOP SGL In LB  Out = Y
 OOP LB  In SGL Out = Y
 Deflate            = Y
+Adler32            = Y
+Crc32              = Y
 Fixed              = Y
 Dynamic            = Y
doc/guides/compressdevs/isal.rst +28 -2
index 3bc3022..af1f41f 100644
@@ -27,6 +27,33 @@ Window size support:
 
     * 32K
 
+Checksum:
+
+    * CRC32
+    * ADLER32
+
+To enable a checksum in the driver, the compression and/or decompression xform
+structure, rte_comp_xform, must be filled with either of the CompressDev
+checksum flags supported. ::
+
+ compress_xform->compress.chksum = RTE_COMP_CHECKSUM_CRC32
+
+ decompress_xform->decompress.chksum = RTE_COMP_CHECKSUM_CRC32
+
+::
+
+ compress_xform->compress.chksum = RTE_COMP_CHECKSUM_ADLER32
+
+ decompress_xform->decompress.chksum = RTE_COMP_CHECKSUM_ADLER32
+
+If you request a checksum for compression or decompression,
+the checksum field in the operation structure,  ``op->output_chksum``,
+will be filled with the checksum.
+
+.. Note::
+
+ For the compression case above, your output buffer will need to be large enough to hold the compressed data plus a scratchpad for the checksum at the end, the scratchpad is 8 bytes for CRC32 and 4 bytes for Adler32.
+
 Level guide:
 
 The ISA-L levels have been mapped to somewhat correspond to the same ZLIB level,
@@ -75,13 +102,12 @@ As a result the level mappings from the API to the PMD are shown below.
  The above table only shows mapping when API calls for dynamic compression.
  For fixed compression, regardless of API level, internally ISA-L level 0 is always used.
 
+
 Limitations
 -----------
 
 * Compressdev level 0, no compression, is not supported.
 
-* Checksums will not be supported until future release.
-
 Installation
 ------------
 
doc/guides/rel_notes/release_19_02.rst +5 -0
index a94fa86..3a4d264 100644
@@ -54,6 +54,11 @@ New Features
      Also, make sure to start the actual text at the margin.
      =========================================================
 
+   * **Enabled checksum support in the ISA-L compressdev driver.**
+
+     Added support for both adler and crc32 checksums in the ISA-L PMD.
+     This aids data integrity across both compression and decompression.
+
 
 Removed Items
 -------------
Signature · 1 line
2.7.4
#4 lore raw
In reply to Lee Daly

Hi Lee,

Apologies if the logic is already done for my query

snipped

drivers/compress/isal/isal_compress_pmd.c isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform, · line 78

Quoted 3 lines
drivers/compress/isal/isal_compress_pmd.c +3 -0
+		/* Set private xform checksum */
+		switch (xform->compress.chksum) {
+		/* Raw deflate by default */

Does the user have to choose RTE_COMP_CHECKSUM_NONE while creating compress/isal instance?

Quoted 17 lines
drivers/compress/isal/isal_compress_pmd.c +11 -0
+		case(RTE_COMP_CHECKSUM_NONE):
+			priv_xform->compress.chksum = IGZIP_DEFLATE;
+			break;
+		case(RTE_COMP_CHECKSUM_CRC32):
+			priv_xform->compress.chksum = IGZIP_GZIP_NO_HDR;
+			break;
+		case(RTE_COMP_CHECKSUM_ADLER32):
+			priv_xform->compress.chksum = IGZIP_ZLIB_NO_HDR;
+			break;
+		case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
+			ISAL_PMD_LOG(ERR, "Combined CRC and ADLER

checksum not"

drivers/compress/isal/isal_compress_pmd.c +4 -0
+					" supported\n");
+			return -ENOTSUP;
+		default:
+			ISAL_PMD_LOG(ERR, "Checksum type not

supported\n");

Review

Quoted 1 line
(quoted diff) +1 -0
+			return -ENOTSUP;

Do we not fall back to RTE_COMP_CHECKSUM_NONE if the configuration is wrong and report warning?

snipped

author
#5 lore raw

Hi Vipin, thanks for the question.

Quoted 10 lines

-----Original Message-----
From: Varghese, Vipin
Sent: Monday, December 17, 2018 8:20 AM
To: Daly, Lee <[email protected]>; [email protected]
Cc: [email protected]; Daly, Lee <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v4 1/3] compress/isal: enable checksum support in driver

Hi Lee,

Quoted 8 lines

Apologies if the logic is already done for my query

snipped

drivers/compress/isal/isal_compress_pmd.c isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform, · line 78

Quoted 3 lines
drivers/compress/isal/isal_compress_pmd.c +3 -0
+		/* Set private xform checksum */
+		switch (xform->compress.chksum) {
+		/* Raw deflate by default */

Does the user have to choose RTE_COMP_CHECKSUM_NONE while creating compress/isal instance?

At the moment, yes the application must fill out the xform/instance.

Quoted 11 lines

drivers/compress/isal/isal_compress_pmd.c isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform,

Quoted 6 lines
drivers/compress/isal/isal_compress_pmd.c +5 -0
+		case(RTE_COMP_CHECKSUM_NONE):
+			priv_xform->compress.chksum = IGZIP_DEFLATE;
+			break;
+		case(RTE_COMP_CHECKSUM_CRC32):
+			priv_xform->compress.chksum =

IGZIP_GZIP_NO_HDR;

drivers/compress/isal/isal_compress_pmd.c isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform,

Quoted 3 lines
drivers/compress/isal/isal_compress_pmd.c +3 -0
+			break;
+		case(RTE_COMP_CHECKSUM_ADLER32):
+			priv_xform->compress.chksum =

IGZIP_ZLIB_NO_HDR;

Quoted 13 lines

Review

Quoted 10 lines
drivers/compress/isal/isal_compress_pmd.c +3 -0
+			break;
+		case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
+			ISAL_PMD_LOG(ERR, "Combined CRC and ADLER

checksum not"

drivers/compress/isal/isal_compress_pmd.c +4 -0
+					" supported\n");
+			return -ENOTSUP;
+		default:
+			ISAL_PMD_LOG(ERR, "Checksum type not

supported\n");

(quoted diff) +1 -0
+			return -ENOTSUP;

Do we not fall back to RTE_COMP_CHECKSUM_NONE if the configuration is wrong and report warning?

Right now, the xform must be filled out correctly before compression can be executed. Perhaps we could do as you say and fall back to a default config if the non-essential params are incorrect.

Quoted 1 line

snipped

#6 lore raw

Thanks for the update

Quoted 60 lines

-----Original Message-----
From: Daly, Lee
Sent: Monday, December 17, 2018 2:50 PM
To: Varghese, Vipin <[email protected]>; [email protected]
Cc: [email protected]; Trahe, Fiona <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v4 1/3] compress/isal: enable checksum support in driver

Hi Vipin, thanks for the question.

Quoted 10 lines

-----Original Message-----
From: Varghese, Vipin
Sent: Monday, December 17, 2018 8:20 AM
To: Daly, Lee <[email protected]>; [email protected]
Cc: [email protected]; Daly, Lee <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v4 1/3] compress/isal: enable checksum support in driver

Hi Lee,

Quoted 8 lines

Apologies if the logic is already done for my query

snipped

drivers/compress/isal/isal_compress_pmd.c isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform, · line 78

Quoted 3 lines
drivers/compress/isal/isal_compress_pmd.c +3 -0
+		/* Set private xform checksum */
+		switch (xform->compress.chksum) {
+		/* Raw deflate by default */

Does the user have to choose RTE_COMP_CHECKSUM_NONE while creating compress/isal instance?

At the moment, yes the application must fill out the xform/instance.

Quoted 11 lines

drivers/compress/isal/isal_compress_pmd.c isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform,

Quoted 6 lines
drivers/compress/isal/isal_compress_pmd.c +5 -0
+		case(RTE_COMP_CHECKSUM_NONE):
+			priv_xform->compress.chksum = IGZIP_DEFLATE;
+			break;
+		case(RTE_COMP_CHECKSUM_CRC32):
+			priv_xform->compress.chksum =

IGZIP_GZIP_NO_HDR;

drivers/compress/isal/isal_compress_pmd.c isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform,

Quoted 3 lines
drivers/compress/isal/isal_compress_pmd.c +3 -0
+			break;
+		case(RTE_COMP_CHECKSUM_ADLER32):
+			priv_xform->compress.chksum =

IGZIP_ZLIB_NO_HDR;

Quoted 13 lines

Review

Quoted 10 lines
drivers/compress/isal/isal_compress_pmd.c +3 -0
+			break;
+		case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
+			ISAL_PMD_LOG(ERR, "Combined CRC and ADLER

checksum not"

drivers/compress/isal/isal_compress_pmd.c +4 -0
+					" supported\n");
+			return -ENOTSUP;
+		default:
+			ISAL_PMD_LOG(ERR, "Checksum type not

supported\n");

(quoted diff) +1 -0
+			return -ENOTSUP;

Do we not fall back to RTE_COMP_CHECKSUM_NONE if the configuration is wrong and report warning?

Right now, the xform must be filled out correctly before compression can be executed.
Perhaps we could do as you say and fall back to a default config if the non- essential params are incorrect.

Quoted 1 line

snipped

author
#7 lore raw

PATCH v5 1/3 compress/isal: enable checksum support in driver

In reply to Lee Daly

This patch adds checksum support in the ISA-L PMD for both compression and decompression.
CRC32 is supported as well as Adler32.

V2:
Documentation Changes
V3:
Added Release note
V4:
Removed dependency on offset unit test
V5:
If checksum type is incorrect, compression will continue without checksum, rather than failing now.

---
 drivers/compress/isal/isal_compress_pmd.c     | 82 ++++++++++++++++++++++-----
 drivers/compress/isal/isal_compress_pmd_ops.c |  4 +-
 2 files changed, 71 insertions(+), 15 deletions(-)
drivers/compress/isal/isal_compress_pmd.c +68 -14
index 9f1e968..b610e90 100644
@@ -16,6 +16,8 @@
 #define RTE_COMP_ISAL_LEVEL_ONE 1
 #define RTE_COMP_ISAL_LEVEL_TWO 2
 #define RTE_COMP_ISAL_LEVEL_THREE 3 /* Optimised for AVX512 & AVX2 only */
+#define CHKSUM_SZ_CRC 8
+#define CHKSUM_SZ_ADLER 4
 
 int isal_logtype_driver;
 
@@ -43,12 +45,6 @@ isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform,
 		}
 		priv_xform->compress.algo = RTE_COMP_ALGO_DEFLATE;
 
-		/* Set private xform checksum - raw deflate by default */
-		if (xform->compress.chksum != RTE_COMP_CHECKSUM_NONE) {
-			ISAL_PMD_LOG(ERR, "Checksum not supported\n");
-			return -ENOTSUP;
-		}
-
 		/* Set private xform window size, 32K supported */
 		if (xform->compress.window_size == RTE_COMP_ISAL_WINDOW_SIZE)
 			priv_xform->compress.window_size =
@@ -77,6 +73,28 @@ isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform,
 			return -ENOTSUP;
 		}
 
+		/* Set private xform checksum */
+		switch (xform->compress.chksum) {
+		/* Raw deflate by default */
+		case(RTE_COMP_CHECKSUM_NONE):
+			priv_xform->compress.chksum = IGZIP_DEFLATE;
+			break;
+		case(RTE_COMP_CHECKSUM_CRC32):
+			priv_xform->compress.chksum = IGZIP_GZIP_NO_HDR;
+			break;
+		case(RTE_COMP_CHECKSUM_ADLER32):
+			priv_xform->compress.chksum = IGZIP_ZLIB_NO_HDR;
+			break;
+		case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
+			ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not"
+					" supported\n");
+			return -ENOTSUP;
+		default:
+			ISAL_PMD_LOG(ERR, "Checksum type not supported\n");
+			priv_xform->compress.chksum = IGZIP_DEFLATE;
+			break;
+		}
+
 		/* Set private xform level.
 		 * Checking compliance with compressdev API, -1 <= level => 9
 		 */
@@ -170,10 +188,26 @@ isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform,
 		}
 		priv_xform->decompress.algo = RTE_COMP_ALGO_DEFLATE;
 
-		/* Set private xform checksum - raw deflate by default */
-		if (xform->compress.chksum != RTE_COMP_CHECKSUM_NONE) {
-			ISAL_PMD_LOG(ERR, "Checksum not supported\n");
+		/* Set private xform checksum */
+		switch (xform->decompress.chksum) {
+		/* Raw deflate by default */
+		case(RTE_COMP_CHECKSUM_NONE):
+			priv_xform->decompress.chksum = ISAL_DEFLATE;
+			break;
+		case(RTE_COMP_CHECKSUM_CRC32):
+			priv_xform->decompress.chksum = ISAL_GZIP_NO_HDR_VER;
+			break;
+		case(RTE_COMP_CHECKSUM_ADLER32):
+			priv_xform->decompress.chksum = ISAL_ZLIB_NO_HDR_VER;
+			break;
+		case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
+			ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not"
+					" supported\n");
 			return -ENOTSUP;
+		default:
+			ISAL_PMD_LOG(ERR, "Checksum type not supported\n");
+			priv_xform->decompress.chksum = ISAL_DEFLATE;
+			break;
 		}
 
 		/* Set private xform window size, 32K supported */
@@ -376,6 +410,9 @@ process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
 
 	qp->stream->level_buf = temp_level_buf;
 
+	/* Set Checksum flag */
+	qp->stream->gzip_flag = priv_xform->compress.chksum;
+
 	/* Stateless operation, input will be consumed in one go */
 	qp->stream->flush = NO_FLUSH;
 
@@ -459,8 +496,18 @@ process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
 			return ret;
 		}
 	}
+
 	op->consumed = qp->stream->total_in;
-	op->produced = qp->stream->total_out;
+	if (qp->stream->gzip_flag == IGZIP_DEFLATE) {
+		op->produced = qp->stream->total_out;
+	} else if (qp->stream->gzip_flag == IGZIP_ZLIB_NO_HDR) {
+		op->produced = qp->stream->total_out - CHKSUM_SZ_ADLER;
+		op->output_chksum = qp->stream->internal_state.crc + 1;
+	} else {
+		op->produced = qp->stream->total_out - CHKSUM_SZ_CRC;
+		op->output_chksum = qp->stream->internal_state.crc;
+	}
+
 	isal_deflate_reset(qp->stream);
 
 	return ret;
@@ -468,7 +515,8 @@ process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
 
 /* Stateless Decompression Function */
 static int
-process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp)
+process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
+		struct isal_priv_xform *priv_xform)
 {
 	int ret = 0;
 
@@ -477,6 +525,9 @@ process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp)
 	/* Initialize decompression state */
 	isal_inflate_init(qp->state);
 
+	/* Set Checksum flag */
+	qp->state->crc_flag = priv_xform->decompress.chksum;
+
 	if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
 		ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
 		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
@@ -531,13 +582,16 @@ process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp)
 			return -1;
 		}
 
-		if (ret != ISAL_DECOMP_OK) {
+		if (ret != ISAL_DECOMP_OK && ret != ISAL_END_INPUT) {
+			ISAL_PMD_LOG(ERR, "Decompression operation failed\n");
 			op->status = RTE_COMP_OP_STATUS_ERROR;
 			return ret;
 		}
 		op->consumed = op->src.length - qp->state->avail_in;
 	}
-		op->produced = qp->state->total_out;
+	op->produced = qp->state->total_out;
+	op->output_chksum = qp->state->crc;
+
 	isal_inflate_reset(qp->state);
 
 	return ret;
@@ -553,7 +607,7 @@ process_op(struct isal_comp_qp *qp, struct rte_comp_op *op,
 		process_isal_deflate(op, qp, priv_xform);
 		break;
 	case RTE_COMP_DECOMPRESS:
-		process_isal_inflate(op, qp);
+		process_isal_inflate(op, qp, priv_xform);
 		break;
 	default:
 		ISAL_PMD_LOG(ERR, "Operation Not Supported\n");
drivers/compress/isal/isal_compress_pmd_ops.c +3 -1
index 41cade8..7b91849 100644
@@ -17,7 +17,9 @@ static const struct rte_compressdev_capabilities isal_pmd_capabilities[] = {
 					RTE_COMP_FF_OOP_LB_IN_SGL_OUT |
 					RTE_COMP_FF_SHAREABLE_PRIV_XFORM |
 					RTE_COMP_FF_HUFFMAN_FIXED |
-					RTE_COMP_FF_HUFFMAN_DYNAMIC,
+					RTE_COMP_FF_HUFFMAN_DYNAMIC |
+					RTE_COMP_FF_CRC32_CHECKSUM |
+					RTE_COMP_FF_ADLER32_CHECKSUM,
 		.window_size = {
 			.min = 15,
 			.max = 15,
Signature · 1 line
2.7.4
author
#8 lore raw

PATCH v5 2/3 test/compress: add checksum tests

This patch adds a test which examines what type of checksum the PMD supports, adler, crc32 or alder32_crc32
and tests that feature if the PMD supports it.

V2:
Documentation Changes
V3:
Added Release note
V4:
Removed dependency on offset unit test
V5:
If checksum type is incorrect, compression will continue without checksum, rather than failing now.

---
 test/test/test_compressdev.c | 201 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 198 insertions(+), 3 deletions(-)
test/test/test_compressdev.c +198 -3
index 5d5e519..6358e0f 100644
@@ -36,6 +36,12 @@
 #define NUM_MAX_INFLIGHT_OPS 128
 #define CACHE_SIZE 0
 
+#define ZLIB_CRC_CHECKSUM_WINDOW_BITS 31
+#define ZLIB_HEADER_SIZE 2
+#define ZLIB_TRAILER_SIZE 4
+#define GZIP_HEADER_SIZE 10
+#define GZIP_TRAILER_SIZE 8
+
 const char *
 huffman_type_strings[] = {
 	[RTE_COMP_HUFFMAN_DEFAULT]	= "PMD default",
@@ -318,6 +324,10 @@ compress_zlib(struct rte_comp_op *op,
 	 * When doing raw DEFLATE, this number will be negative.
 	 */
 	window_bits = -(xform->compress.window_size);
+	if (xform->compress.chksum == RTE_COMP_CHECKSUM_ADLER32)
+		window_bits *= -1;
+	else if (xform->compress.chksum == RTE_COMP_CHECKSUM_CRC32)
+		window_bits = ZLIB_CRC_CHECKSUM_WINDOW_BITS;
 
 	comp_level = xform->compress.level;
 
@@ -401,8 +411,19 @@ compress_zlib(struct rte_comp_op *op,
 	}
 
 	op->consumed = stream.total_in;
-	op->produced = stream.total_out;
+	if (xform->compress.chksum == RTE_COMP_CHECKSUM_ADLER32) {
+		rte_pktmbuf_adj(op->m_dst, ZLIB_HEADER_SIZE);
+		op->produced = stream.total_out -
+				(ZLIB_HEADER_SIZE + ZLIB_TRAILER_SIZE);
+	} else if (xform->compress.chksum == RTE_COMP_CHECKSUM_CRC32) {
+		rte_pktmbuf_adj(op->m_dst, GZIP_HEADER_SIZE);
+		op->produced = stream.total_out -
+				(GZIP_HEADER_SIZE + GZIP_TRAILER_SIZE);
+	} else
+		op->produced = stream.total_out;
+
 	op->status = RTE_COMP_OP_STATUS_SUCCESS;
+	op->output_chksum = stream.adler;
 
 	deflateReset(&stream);
 
@@ -436,7 +457,6 @@ decompress_zlib(struct rte_comp_op *op,
 	 * When doing raw DEFLATE, this number will be negative.
 	 */
 	window_bits = -(xform->decompress.window_size);
-
 	ret = inflateInit2(&stream, window_bits);
 
 	if (ret != Z_OK) {
@@ -652,6 +672,7 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 	const struct rte_compressdev_capabilities *capa =
 		rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
 	char *contig_buf = NULL;
+	uint64_t compress_checksum[num_bufs];
 
 	/* Initialize all arrays to NULL */
 	memset(uncomp_bufs, 0, sizeof(struct rte_mbuf *) * num_bufs);
@@ -838,6 +859,7 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 					&ops_processed[num_total_deqd], num_bufs);
 			num_total_deqd += num_deqd;
 			deqd_retries++;
+
 		} while (num_total_deqd < num_enqd);
 
 		deqd_retries = 0;
@@ -873,6 +895,8 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 			ops_processed[i]->consumed == 0 ? 0 :
 			(float)ops_processed[i]->produced /
 			ops_processed[i]->consumed * 100);
+		if (compress_xform->chksum != RTE_COMP_CHECKSUM_NONE)
+			compress_checksum[i] = ops_processed[i]->output_chksum;
 		ops[i] = NULL;
 	}
 
@@ -1110,11 +1134,23 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 
 		buf2 = rte_pktmbuf_read(ops_processed[i]->m_dst, 0,
 				ops_processed[i]->produced, contig_buf);
-
 		if (compare_buffers(buf1, strlen(buf1) + 1,
 				buf2, ops_processed[i]->produced) < 0)
 			goto exit;
 
+		/* Test checksums */
+		if (compress_xforms[0]->compress.chksum !=
+				RTE_COMP_CHECKSUM_NONE) {
+			if (ops_processed[i]->output_chksum !=
+					compress_checksum[i]) {
+				RTE_LOG(ERR, USER1, "The checksums differ\n"
+			"Compression Checksum: %" PRIu64 "\tDecompression "
+			"Checksum: %" PRIu64 "\n", compress_checksum[i],
+			ops_processed[i]->output_chksum);
+				goto exit;
+			}
+		}
+
 		rte_free(contig_buf);
 		contig_buf = NULL;
 	}
@@ -1466,6 +1502,163 @@ test_compressdev_deflate_stateless_sgl(void)
 	}
 
 	return TEST_SUCCESS;
+
+}
+
+static int
+test_compressdev_deflate_stateless_checksum(void)
+{
+	struct comp_testsuite_params *ts_params = &testsuite_params;
+	const char *test_buffer;
+	uint16_t i;
+	int ret;
+	const struct rte_compressdev_capabilities *capab;
+
+	capab = rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
+	TEST_ASSERT(capab != NULL, "Failed to retrieve device capabilities");
+
+	/* Check if driver supports any checksum */
+	if ((capab->comp_feature_flags & RTE_COMP_FF_CRC32_CHECKSUM) == 0 &&
+			(capab->comp_feature_flags &
+			RTE_COMP_FF_ADLER32_CHECKSUM) == 0 &&
+			(capab->comp_feature_flags &
+			RTE_COMP_FF_CRC32_ADLER32_CHECKSUM) == 0)
+		return -ENOTSUP;
+
+	struct rte_comp_xform *compress_xform =
+			rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
+	if (compress_xform == NULL) {
+		RTE_LOG(ERR, USER1, "Compress xform could not be created\n");
+		ret = TEST_FAILED;
+		goto exit;
+	}
+
+	memcpy(compress_xform, ts_params->def_comp_xform,
+			sizeof(struct rte_comp_xform));
+
+	struct rte_comp_xform *decompress_xform =
+			rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
+	if (decompress_xform == NULL) {
+		RTE_LOG(ERR, USER1, "Decompress xform could not be created\n");
+		ret = TEST_FAILED;
+		goto exit;
+	}
+
+	memcpy(decompress_xform, ts_params->def_decomp_xform,
+			sizeof(struct rte_comp_xform));
+
+	/* Check if driver supports crc32 checksum and test */
+	if ((capab->comp_feature_flags & RTE_COMP_FF_CRC32_CHECKSUM)) {
+		compress_xform->compress.chksum = RTE_COMP_CHECKSUM_CRC32;
+		decompress_xform->decompress.chksum = RTE_COMP_CHECKSUM_CRC32;
+
+		for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
+			test_buffer = compress_test_bufs[i];
+
+			/* Generate zlib checksum and test against selected
+			 * drivers decompression checksum
+			 */
+			if (test_deflate_comp_decomp(&test_buffer, 1,
+					&i,
+					&compress_xform,
+					&decompress_xform,
+					1,
+					RTE_COMP_OP_STATELESS,
+					0,
+					ZLIB_COMPRESS) < 0) {
+				ret = TEST_FAILED;
+				goto exit;
+			}
+
+			/* Generate compression and decompression
+			 * checksum of selected driver
+			 */
+			if (test_deflate_comp_decomp(&test_buffer, 1,
+					&i,
+					&compress_xform,
+					&decompress_xform,
+					1,
+					RTE_COMP_OP_STATELESS,
+					0,
+					ZLIB_NONE) < 0) {
+				ret = TEST_FAILED;
+				goto exit;
+			}
+		}
+	}
+
+	/* Check if driver supports adler32 checksum and test */
+	if ((capab->comp_feature_flags & RTE_COMP_FF_ADLER32_CHECKSUM)) {
+		compress_xform->compress.chksum = RTE_COMP_CHECKSUM_ADLER32;
+		decompress_xform->decompress.chksum = RTE_COMP_CHECKSUM_ADLER32;
+
+		for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
+			test_buffer = compress_test_bufs[i];
+
+			/* Generate zlib checksum and test against selected
+			 * drivers decompression checksum
+			 */
+			if (test_deflate_comp_decomp(&test_buffer, 1,
+					&i,
+					&compress_xform,
+					&decompress_xform,
+					1,
+					RTE_COMP_OP_STATELESS,
+					0,
+					ZLIB_COMPRESS) < 0) {
+				ret = TEST_FAILED;
+				goto exit;
+			}
+			/* Generate compression and decompression
+			 * checksum of selected driver
+			 */
+			if (test_deflate_comp_decomp(&test_buffer, 1,
+					&i,
+					&compress_xform,
+					&decompress_xform,
+					1,
+					RTE_COMP_OP_STATELESS,
+					0,
+					ZLIB_NONE) < 0) {
+				ret = TEST_FAILED;
+				goto exit;
+			}
+		}
+	}
+
+	/* Check if driver supports combined crc and adler checksum and test */
+	if ((capab->comp_feature_flags & RTE_COMP_FF_CRC32_ADLER32_CHECKSUM)) {
+		compress_xform->compress.chksum =
+				RTE_COMP_CHECKSUM_CRC32_ADLER32;
+		decompress_xform->decompress.chksum =
+				RTE_COMP_CHECKSUM_CRC32_ADLER32;
+
+		for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
+			test_buffer = compress_test_bufs[i];
+
+			/* Generate compression and decompression
+			 * checksum of selected driver
+			 */
+			if (test_deflate_comp_decomp(&test_buffer, 1,
+					&i,
+					&compress_xform,
+					&decompress_xform,
+					1,
+					RTE_COMP_OP_STATELESS,
+					0,
+					ZLIB_NONE) < 0) {
+				ret = TEST_FAILED;
+				goto exit;
+			}
+		}
+	}
+
+	ret = TEST_SUCCESS;
+
+exit:
+	rte_free(compress_xform);
+	rte_free(decompress_xform);
+	return ret;
 }
 
 static struct unit_test_suite compressdev_testsuite  = {
@@ -1487,6 +1680,8 @@ static struct unit_test_suite compressdev_testsuite  = {
 			test_compressdev_deflate_stateless_multi_xform),
 		TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
 			test_compressdev_deflate_stateless_sgl),
+		TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
+			test_compressdev_deflate_stateless_checksum),
 		TEST_CASES_END() /**< NULL terminate unit test array */
 	}
 };
Signature · 1 line
2.7.4
author
#9 lore raw

PATCH v5 3/3 doc: update ISA-L guide to reflect checksum support

In reply to Lee Daly

This updates the ISA-L compression driver guide on how to enable and use checksums and release notes.

---
 doc/guides/compressdevs/features/isal.ini |  2 ++
 doc/guides/compressdevs/isal.rst          | 30 ++++++++++++++++++++++++++++--
 doc/guides/rel_notes/release_19_02.rst    |  4 ++++
 3 files changed, 34 insertions(+), 2 deletions(-)
doc/guides/compressdevs/features/isal.ini +2 -0
index 919cf70..e705031 100644
@@ -12,5 +12,7 @@ OOP SGL In SGL Out = Y
 OOP SGL In LB  Out = Y
 OOP LB  In SGL Out = Y
 Deflate            = Y
+Adler32            = Y
+Crc32              = Y
 Fixed              = Y
 Dynamic            = Y
doc/guides/compressdevs/isal.rst +28 -2
index 3bc3022..af1f41f 100644
@@ -27,6 +27,33 @@ Window size support:
 
     * 32K
 
+Checksum:
+
+    * CRC32
+    * ADLER32
+
+To enable a checksum in the driver, the compression and/or decompression xform
+structure, rte_comp_xform, must be filled with either of the CompressDev
+checksum flags supported. ::
+
+ compress_xform->compress.chksum = RTE_COMP_CHECKSUM_CRC32
+
+ decompress_xform->decompress.chksum = RTE_COMP_CHECKSUM_CRC32
+
+::
+
+ compress_xform->compress.chksum = RTE_COMP_CHECKSUM_ADLER32
+
+ decompress_xform->decompress.chksum = RTE_COMP_CHECKSUM_ADLER32
+
+If you request a checksum for compression or decompression,
+the checksum field in the operation structure,  ``op->output_chksum``,
+will be filled with the checksum.
+
+.. Note::
+
+ For the compression case above, your output buffer will need to be large enough to hold the compressed data plus a scratchpad for the checksum at the end, the scratchpad is 8 bytes for CRC32 and 4 bytes for Adler32.
+
 Level guide:
 
 The ISA-L levels have been mapped to somewhat correspond to the same ZLIB level,
@@ -75,13 +102,12 @@ As a result the level mappings from the API to the PMD are shown below.
  The above table only shows mapping when API calls for dynamic compression.
  For fixed compression, regardless of API level, internally ISA-L level 0 is always used.
 
+
 Limitations
 -----------
 
 * Compressdev level 0, no compression, is not supported.
 
-* Checksums will not be supported until future release.
-
 Installation
 ------------
 
doc/guides/rel_notes/release_19_02.rst +4 -0
index 8deb68b..286c060 100644
@@ -60,6 +60,10 @@ New Features
   * Added the handler to get firmware version string.
   * Added support for multicast filtering.
 
+  * **Enabled checksum support in the ISA-L compressdev driver.**
+
+    Added support for both adler and crc32 checksums in the ISA-L PMD.
+    This aids data integrity across both compression and decompression.
 
 Removed Items
 -------------
Signature · 1 line
2.7.4
author
#10 lore raw

PATCH v6 1/2 test/compress: add checksum tests

In reply to Lee Daly

This patch adds a test which examines what type of checksum the PMD supports, adler, crc32 or alder32_crc32
and tests that feature if the PMD supports it.

V2:
Documentation Changes
V3:
Added Release note
V4:
Removed dependency on offset unit test
V5:
If checksum type is incorrect, compression will continue without checksum, rather than failing now.

---
 test/test/test_compressdev.c | 201 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 198 insertions(+), 3 deletions(-)
test/test/test_compressdev.c +198 -3
index 5d5e519..6358e0f 100644
@@ -36,6 +36,12 @@
 #define NUM_MAX_INFLIGHT_OPS 128
 #define CACHE_SIZE 0
 
+#define ZLIB_CRC_CHECKSUM_WINDOW_BITS 31
+#define ZLIB_HEADER_SIZE 2
+#define ZLIB_TRAILER_SIZE 4
+#define GZIP_HEADER_SIZE 10
+#define GZIP_TRAILER_SIZE 8
+
 const char *
 huffman_type_strings[] = {
 	[RTE_COMP_HUFFMAN_DEFAULT]	= "PMD default",
@@ -318,6 +324,10 @@ compress_zlib(struct rte_comp_op *op,
 	 * When doing raw DEFLATE, this number will be negative.
 	 */
 	window_bits = -(xform->compress.window_size);
+	if (xform->compress.chksum == RTE_COMP_CHECKSUM_ADLER32)
+		window_bits *= -1;
+	else if (xform->compress.chksum == RTE_COMP_CHECKSUM_CRC32)
+		window_bits = ZLIB_CRC_CHECKSUM_WINDOW_BITS;
 
 	comp_level = xform->compress.level;
 
@@ -401,8 +411,19 @@ compress_zlib(struct rte_comp_op *op,
 	}
 
 	op->consumed = stream.total_in;
-	op->produced = stream.total_out;
+	if (xform->compress.chksum == RTE_COMP_CHECKSUM_ADLER32) {
+		rte_pktmbuf_adj(op->m_dst, ZLIB_HEADER_SIZE);
+		op->produced = stream.total_out -
+				(ZLIB_HEADER_SIZE + ZLIB_TRAILER_SIZE);
+	} else if (xform->compress.chksum == RTE_COMP_CHECKSUM_CRC32) {
+		rte_pktmbuf_adj(op->m_dst, GZIP_HEADER_SIZE);
+		op->produced = stream.total_out -
+				(GZIP_HEADER_SIZE + GZIP_TRAILER_SIZE);
+	} else
+		op->produced = stream.total_out;
+
 	op->status = RTE_COMP_OP_STATUS_SUCCESS;
+	op->output_chksum = stream.adler;
 
 	deflateReset(&stream);
 
@@ -436,7 +457,6 @@ decompress_zlib(struct rte_comp_op *op,
 	 * When doing raw DEFLATE, this number will be negative.
 	 */
 	window_bits = -(xform->decompress.window_size);
-
 	ret = inflateInit2(&stream, window_bits);
 
 	if (ret != Z_OK) {
@@ -652,6 +672,7 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 	const struct rte_compressdev_capabilities *capa =
 		rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
 	char *contig_buf = NULL;
+	uint64_t compress_checksum[num_bufs];
 
 	/* Initialize all arrays to NULL */
 	memset(uncomp_bufs, 0, sizeof(struct rte_mbuf *) * num_bufs);
@@ -838,6 +859,7 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 					&ops_processed[num_total_deqd], num_bufs);
 			num_total_deqd += num_deqd;
 			deqd_retries++;
+
 		} while (num_total_deqd < num_enqd);
 
 		deqd_retries = 0;
@@ -873,6 +895,8 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 			ops_processed[i]->consumed == 0 ? 0 :
 			(float)ops_processed[i]->produced /
 			ops_processed[i]->consumed * 100);
+		if (compress_xform->chksum != RTE_COMP_CHECKSUM_NONE)
+			compress_checksum[i] = ops_processed[i]->output_chksum;
 		ops[i] = NULL;
 	}
 
@@ -1110,11 +1134,23 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 
 		buf2 = rte_pktmbuf_read(ops_processed[i]->m_dst, 0,
 				ops_processed[i]->produced, contig_buf);
-
 		if (compare_buffers(buf1, strlen(buf1) + 1,
 				buf2, ops_processed[i]->produced) < 0)
 			goto exit;
 
+		/* Test checksums */
+		if (compress_xforms[0]->compress.chksum !=
+				RTE_COMP_CHECKSUM_NONE) {
+			if (ops_processed[i]->output_chksum !=
+					compress_checksum[i]) {
+				RTE_LOG(ERR, USER1, "The checksums differ\n"
+			"Compression Checksum: %" PRIu64 "\tDecompression "
+			"Checksum: %" PRIu64 "\n", compress_checksum[i],
+			ops_processed[i]->output_chksum);
+				goto exit;
+			}
+		}
+
 		rte_free(contig_buf);
 		contig_buf = NULL;
 	}
@@ -1466,6 +1502,163 @@ test_compressdev_deflate_stateless_sgl(void)
 	}
 
 	return TEST_SUCCESS;
+
+}
+
+static int
+test_compressdev_deflate_stateless_checksum(void)
+{
+	struct comp_testsuite_params *ts_params = &testsuite_params;
+	const char *test_buffer;
+	uint16_t i;
+	int ret;
+	const struct rte_compressdev_capabilities *capab;
+
+	capab = rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
+	TEST_ASSERT(capab != NULL, "Failed to retrieve device capabilities");
+
+	/* Check if driver supports any checksum */
+	if ((capab->comp_feature_flags & RTE_COMP_FF_CRC32_CHECKSUM) == 0 &&
+			(capab->comp_feature_flags &
+			RTE_COMP_FF_ADLER32_CHECKSUM) == 0 &&
+			(capab->comp_feature_flags &
+			RTE_COMP_FF_CRC32_ADLER32_CHECKSUM) == 0)
+		return -ENOTSUP;
+
+	struct rte_comp_xform *compress_xform =
+			rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
+	if (compress_xform == NULL) {
+		RTE_LOG(ERR, USER1, "Compress xform could not be created\n");
+		ret = TEST_FAILED;
+		goto exit;
+	}
+
+	memcpy(compress_xform, ts_params->def_comp_xform,
+			sizeof(struct rte_comp_xform));
+
+	struct rte_comp_xform *decompress_xform =
+			rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
+	if (decompress_xform == NULL) {
+		RTE_LOG(ERR, USER1, "Decompress xform could not be created\n");
+		ret = TEST_FAILED;
+		goto exit;
+	}
+
+	memcpy(decompress_xform, ts_params->def_decomp_xform,
+			sizeof(struct rte_comp_xform));
+
+	/* Check if driver supports crc32 checksum and test */
+	if ((capab->comp_feature_flags & RTE_COMP_FF_CRC32_CHECKSUM)) {
+		compress_xform->compress.chksum = RTE_COMP_CHECKSUM_CRC32;
+		decompress_xform->decompress.chksum = RTE_COMP_CHECKSUM_CRC32;
+
+		for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
+			test_buffer = compress_test_bufs[i];
+
+			/* Generate zlib checksum and test against selected
+			 * drivers decompression checksum
+			 */
+			if (test_deflate_comp_decomp(&test_buffer, 1,
+					&i,
+					&compress_xform,
+					&decompress_xform,
+					1,
+					RTE_COMP_OP_STATELESS,
+					0,
+					ZLIB_COMPRESS) < 0) {
+				ret = TEST_FAILED;
+				goto exit;
+			}
+
+			/* Generate compression and decompression
+			 * checksum of selected driver
+			 */
+			if (test_deflate_comp_decomp(&test_buffer, 1,
+					&i,
+					&compress_xform,
+					&decompress_xform,
+					1,
+					RTE_COMP_OP_STATELESS,
+					0,
+					ZLIB_NONE) < 0) {
+				ret = TEST_FAILED;
+				goto exit;
+			}
+		}
+	}
+
+	/* Check if driver supports adler32 checksum and test */
+	if ((capab->comp_feature_flags & RTE_COMP_FF_ADLER32_CHECKSUM)) {
+		compress_xform->compress.chksum = RTE_COMP_CHECKSUM_ADLER32;
+		decompress_xform->decompress.chksum = RTE_COMP_CHECKSUM_ADLER32;
+
+		for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
+			test_buffer = compress_test_bufs[i];
+
+			/* Generate zlib checksum and test against selected
+			 * drivers decompression checksum
+			 */
+			if (test_deflate_comp_decomp(&test_buffer, 1,
+					&i,
+					&compress_xform,
+					&decompress_xform,
+					1,
+					RTE_COMP_OP_STATELESS,
+					0,
+					ZLIB_COMPRESS) < 0) {
+				ret = TEST_FAILED;
+				goto exit;
+			}
+			/* Generate compression and decompression
+			 * checksum of selected driver
+			 */
+			if (test_deflate_comp_decomp(&test_buffer, 1,
+					&i,
+					&compress_xform,
+					&decompress_xform,
+					1,
+					RTE_COMP_OP_STATELESS,
+					0,
+					ZLIB_NONE) < 0) {
+				ret = TEST_FAILED;
+				goto exit;
+			}
+		}
+	}
+
+	/* Check if driver supports combined crc and adler checksum and test */
+	if ((capab->comp_feature_flags & RTE_COMP_FF_CRC32_ADLER32_CHECKSUM)) {
+		compress_xform->compress.chksum =
+				RTE_COMP_CHECKSUM_CRC32_ADLER32;
+		decompress_xform->decompress.chksum =
+				RTE_COMP_CHECKSUM_CRC32_ADLER32;
+
+		for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
+			test_buffer = compress_test_bufs[i];
+
+			/* Generate compression and decompression
+			 * checksum of selected driver
+			 */
+			if (test_deflate_comp_decomp(&test_buffer, 1,
+					&i,
+					&compress_xform,
+					&decompress_xform,
+					1,
+					RTE_COMP_OP_STATELESS,
+					0,
+					ZLIB_NONE) < 0) {
+				ret = TEST_FAILED;
+				goto exit;
+			}
+		}
+	}
+
+	ret = TEST_SUCCESS;
+
+exit:
+	rte_free(compress_xform);
+	rte_free(decompress_xform);
+	return ret;
 }
 
 static struct unit_test_suite compressdev_testsuite  = {
@@ -1487,6 +1680,8 @@ static struct unit_test_suite compressdev_testsuite  = {
 			test_compressdev_deflate_stateless_multi_xform),
 		TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
 			test_compressdev_deflate_stateless_sgl),
+		TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
+			test_compressdev_deflate_stateless_checksum),
 		TEST_CASES_END() /**< NULL terminate unit test array */
 	}
 };
Signature · 1 line
2.7.4
author
#11 lore raw

PATCH v6 2/2 compress/isal: enable checksum support in driver

This patch adds checksum support in the ISA-L PMD for both compression and decompression.
CRC32 is supported as well as Adler32.

V2:
Documentation Changes
V3:
Added Release note
V4:
Removed dependency on offset unit test
V5:
If checksum type is incorrect, compression will continue without checksum, rather than failing now.
V6:
Include documentation in this patch rather than separate doc patch.

---
 doc/guides/compressdevs/features/isal.ini     |  2 +
 doc/guides/compressdevs/isal.rst              | 30 +++++++++-
 doc/guides/rel_notes/release_19_02.rst        |  4 ++
 drivers/compress/isal/isal_compress_pmd.c     | 82 ++++++++++++++++++++++-----
 drivers/compress/isal/isal_compress_pmd_ops.c |  4 +-
 5 files changed, 105 insertions(+), 17 deletions(-)
doc/guides/compressdevs/features/isal.ini +2 -0
index 919cf70..e705031 100644
@@ -12,5 +12,7 @@ OOP SGL In SGL Out = Y
 OOP SGL In LB  Out = Y
 OOP LB  In SGL Out = Y
 Deflate            = Y
+Adler32            = Y
+Crc32              = Y
 Fixed              = Y
 Dynamic            = Y
doc/guides/compressdevs/isal.rst +28 -2
index 3bc3022..af1f41f 100644
@@ -27,6 +27,33 @@ Window size support:
 
     * 32K
 
+Checksum:
+
+    * CRC32
+    * ADLER32
+
+To enable a checksum in the driver, the compression and/or decompression xform
+structure, rte_comp_xform, must be filled with either of the CompressDev
+checksum flags supported. ::
+
+ compress_xform->compress.chksum = RTE_COMP_CHECKSUM_CRC32
+
+ decompress_xform->decompress.chksum = RTE_COMP_CHECKSUM_CRC32
+
+::
+
+ compress_xform->compress.chksum = RTE_COMP_CHECKSUM_ADLER32
+
+ decompress_xform->decompress.chksum = RTE_COMP_CHECKSUM_ADLER32
+
+If you request a checksum for compression or decompression,
+the checksum field in the operation structure,  ``op->output_chksum``,
+will be filled with the checksum.
+
+.. Note::
+
+ For the compression case above, your output buffer will need to be large enough to hold the compressed data plus a scratchpad for the checksum at the end, the scratchpad is 8 bytes for CRC32 and 4 bytes for Adler32.
+
 Level guide:
 
 The ISA-L levels have been mapped to somewhat correspond to the same ZLIB level,
@@ -75,13 +102,12 @@ As a result the level mappings from the API to the PMD are shown below.
  The above table only shows mapping when API calls for dynamic compression.
  For fixed compression, regardless of API level, internally ISA-L level 0 is always used.
 
+
 Limitations
 -----------
 
 * Compressdev level 0, no compression, is not supported.
 
-* Checksums will not be supported until future release.
-
 Installation
 ------------
 
doc/guides/rel_notes/release_19_02.rst +4 -0
index 8deb68b..286c060 100644
@@ -60,6 +60,10 @@ New Features
   * Added the handler to get firmware version string.
   * Added support for multicast filtering.
 
+  * **Enabled checksum support in the ISA-L compressdev driver.**
+
+    Added support for both adler and crc32 checksums in the ISA-L PMD.
+    This aids data integrity across both compression and decompression.
 
 Removed Items
 -------------
drivers/compress/isal/isal_compress_pmd.c +68 -14

149 lines not on this page · raw message

drivers/compress/isal/isal_compress_pmd_ops.c +3 -1
index 41cade8..7b91849 100644
@@ -17,7 +17,9 @@ static const struct rte_compressdev_capabilities isal_pmd_capabilities[] = {
 					RTE_COMP_FF_OOP_LB_IN_SGL_OUT |
 					RTE_COMP_FF_SHAREABLE_PRIV_XFORM |
 					RTE_COMP_FF_HUFFMAN_FIXED |
-					RTE_COMP_FF_HUFFMAN_DYNAMIC,
+					RTE_COMP_FF_HUFFMAN_DYNAMIC |
+					RTE_COMP_FF_CRC32_CHECKSUM |
+					RTE_COMP_FF_ADLER32_CHECKSUM,
 		.window_size = {
 			.min = 15,
 			.max = 15,
Signature · 1 line
2.7.4
#12 lore raw

Re: [PATCH v6 1/2] test/compress: add checksum tests

In reply to Lee Daly

Hi Lee,

Quoted 14 lines

-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Lee Daly
Sent: Tuesday, January 8, 2019 10:29 AM
To: [email protected]
Cc: [email protected]; Varghese, Vipin <[email protected]>; Daly, Lee <[email protected]>
Subject: [dpdk-dev] [PATCH v6 1/2] test/compress: add checksum tests

test/test/test_compressdev.c +5 -0
+
+	struct rte_comp_xform *compress_xform =
+			rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
+	if (compress_xform == NULL) {
+		RTE_LOG(ERR, USER1, "Compress xform could not be
Quoted 1 line

created\n");

test/test/test_compressdev.c:1531:6: error: variable 'decompress_xform' is used uninitialized whenever 'if' condition is true

      [-Werror,-Wsometimes-uninitialized]
        if (compress_xform == NULL) {
            ^~~~~~~~~~~~~~~~~~~~~~
test/test/test_compressdev.c:1661:11: note: uninitialized use occurs here
        rte_free(decompress_xform);
                 ^~~~~~~~~~~~~~~~

There are clang compilation issues iwith this patch. Could you fix them?

Thanks,
Pablo

author
#13 lore raw

PATCH v7 1/2 test/compress: add checksum tests

In reply to Lee Daly

This patch adds a test which examines what type of checksum the PMD supports, adler, crc32 or alder32_crc32
and tests that feature if the PMD supports it.

V2:
Documentation Changes
V3:
Added Release note
V4:
Removed dependency on offset unit test
V5:
If checksum type is incorrect, compression will continue without checksum, rather than failing now.
V7:
Fix possible freeing of uninitialized memory

---
 test/test/test_compressdev.c | 202 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 199 insertions(+), 3 deletions(-)
test/test/test_compressdev.c +199 -3
index 5d5e519..37901a9 100644
@@ -36,6 +36,12 @@
 #define NUM_MAX_INFLIGHT_OPS 128
 #define CACHE_SIZE 0
 
+#define ZLIB_CRC_CHECKSUM_WINDOW_BITS 31
+#define ZLIB_HEADER_SIZE 2
+#define ZLIB_TRAILER_SIZE 4
+#define GZIP_HEADER_SIZE 10
+#define GZIP_TRAILER_SIZE 8
+
 const char *
 huffman_type_strings[] = {
 	[RTE_COMP_HUFFMAN_DEFAULT]	= "PMD default",
@@ -318,6 +324,10 @@ compress_zlib(struct rte_comp_op *op,
 	 * When doing raw DEFLATE, this number will be negative.
 	 */
 	window_bits = -(xform->compress.window_size);
+	if (xform->compress.chksum == RTE_COMP_CHECKSUM_ADLER32)
+		window_bits *= -1;
+	else if (xform->compress.chksum == RTE_COMP_CHECKSUM_CRC32)
+		window_bits = ZLIB_CRC_CHECKSUM_WINDOW_BITS;
 
 	comp_level = xform->compress.level;
 
@@ -401,8 +411,19 @@ compress_zlib(struct rte_comp_op *op,
 	}
 
 	op->consumed = stream.total_in;
-	op->produced = stream.total_out;
+	if (xform->compress.chksum == RTE_COMP_CHECKSUM_ADLER32) {
+		rte_pktmbuf_adj(op->m_dst, ZLIB_HEADER_SIZE);
+		op->produced = stream.total_out -
+				(ZLIB_HEADER_SIZE + ZLIB_TRAILER_SIZE);
+	} else if (xform->compress.chksum == RTE_COMP_CHECKSUM_CRC32) {
+		rte_pktmbuf_adj(op->m_dst, GZIP_HEADER_SIZE);
+		op->produced = stream.total_out -
+				(GZIP_HEADER_SIZE + GZIP_TRAILER_SIZE);
+	} else
+		op->produced = stream.total_out;
+
 	op->status = RTE_COMP_OP_STATUS_SUCCESS;
+	op->output_chksum = stream.adler;
 
 	deflateReset(&stream);
 
@@ -436,7 +457,6 @@ decompress_zlib(struct rte_comp_op *op,
 	 * When doing raw DEFLATE, this number will be negative.
 	 */
 	window_bits = -(xform->decompress.window_size);
-
 	ret = inflateInit2(&stream, window_bits);
 
 	if (ret != Z_OK) {
@@ -652,6 +672,7 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 	const struct rte_compressdev_capabilities *capa =
 		rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
 	char *contig_buf = NULL;
+	uint64_t compress_checksum[num_bufs];
 
 	/* Initialize all arrays to NULL */
 	memset(uncomp_bufs, 0, sizeof(struct rte_mbuf *) * num_bufs);
@@ -838,6 +859,7 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 					&ops_processed[num_total_deqd], num_bufs);
 			num_total_deqd += num_deqd;
 			deqd_retries++;
+
 		} while (num_total_deqd < num_enqd);
 
 		deqd_retries = 0;
@@ -873,6 +895,8 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 			ops_processed[i]->consumed == 0 ? 0 :
 			(float)ops_processed[i]->produced /
 			ops_processed[i]->consumed * 100);
+		if (compress_xform->chksum != RTE_COMP_CHECKSUM_NONE)
+			compress_checksum[i] = ops_processed[i]->output_chksum;
 		ops[i] = NULL;
 	}
 
@@ -1110,11 +1134,23 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 
 		buf2 = rte_pktmbuf_read(ops_processed[i]->m_dst, 0,
 				ops_processed[i]->produced, contig_buf);
-
 		if (compare_buffers(buf1, strlen(buf1) + 1,
 				buf2, ops_processed[i]->produced) < 0)
 			goto exit;
 
+		/* Test checksums */
+		if (compress_xforms[0]->compress.chksum !=
+				RTE_COMP_CHECKSUM_NONE) {
+			if (ops_processed[i]->output_chksum !=
+					compress_checksum[i]) {
+				RTE_LOG(ERR, USER1, "The checksums differ\n"
+			"Compression Checksum: %" PRIu64 "\tDecompression "
+			"Checksum: %" PRIu64 "\n", compress_checksum[i],
+			ops_processed[i]->output_chksum);
+				goto exit;
+			}
+		}
+
 		rte_free(contig_buf);
 		contig_buf = NULL;
 	}
@@ -1466,6 +1502,164 @@ test_compressdev_deflate_stateless_sgl(void)
 	}
 
 	return TEST_SUCCESS;
+
+}
+
+static int
+test_compressdev_deflate_stateless_checksum(void)
+{
+	struct comp_testsuite_params *ts_params = &testsuite_params;
+	const char *test_buffer;
+	uint16_t i;
+	int ret;
+	const struct rte_compressdev_capabilities *capab;
+
+	capab = rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
+	TEST_ASSERT(capab != NULL, "Failed to retrieve device capabilities");
+
+	/* Check if driver supports any checksum */
+	if ((capab->comp_feature_flags & RTE_COMP_FF_CRC32_CHECKSUM) == 0 &&
+			(capab->comp_feature_flags &
+			RTE_COMP_FF_ADLER32_CHECKSUM) == 0 &&
+			(capab->comp_feature_flags &
+			RTE_COMP_FF_CRC32_ADLER32_CHECKSUM) == 0)
+		return -ENOTSUP;
+
+	struct rte_comp_xform *compress_xform =
+			rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
+	if (compress_xform == NULL) {
+		RTE_LOG(ERR, USER1, "Compress xform could not be created\n");
+		ret = TEST_FAILED;
+		return ret;
+	}
+
+	memcpy(compress_xform, ts_params->def_comp_xform,
+			sizeof(struct rte_comp_xform));
+
+	struct rte_comp_xform *decompress_xform =
+			rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
+	if (decompress_xform == NULL) {
+		RTE_LOG(ERR, USER1, "Decompress xform could not be created\n");
+		rte_free(compress_xform);
+		ret = TEST_FAILED;
+		return ret;
+	}
+
+	memcpy(decompress_xform, ts_params->def_decomp_xform,
+			sizeof(struct rte_comp_xform));
+
+	/* Check if driver supports crc32 checksum and test */
+	if ((capab->comp_feature_flags & RTE_COMP_FF_CRC32_CHECKSUM)) {
+		compress_xform->compress.chksum = RTE_COMP_CHECKSUM_CRC32;
+		decompress_xform->decompress.chksum = RTE_COMP_CHECKSUM_CRC32;
+
+		for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
+			test_buffer = compress_test_bufs[i];
+
+			/* Generate zlib checksum and test against selected
+			 * drivers decompression checksum
+			 */
+			if (test_deflate_comp_decomp(&test_buffer, 1,
+					&i,
+					&compress_xform,
+					&decompress_xform,
+					1,
+					RTE_COMP_OP_STATELESS,
+					0,
+					ZLIB_COMPRESS) < 0) {
+				ret = TEST_FAILED;
+				goto exit;
+			}
+
+			/* Generate compression and decompression
+			 * checksum of selected driver
+			 */
+			if (test_deflate_comp_decomp(&test_buffer, 1,
+					&i,
+					&compress_xform,
+					&decompress_xform,
+					1,
+					RTE_COMP_OP_STATELESS,
+					0,
+					ZLIB_NONE) < 0) {
+				ret = TEST_FAILED;
+				goto exit;
+			}
+		}
+	}
+
+	/* Check if driver supports adler32 checksum and test */
+	if ((capab->comp_feature_flags & RTE_COMP_FF_ADLER32_CHECKSUM)) {
+		compress_xform->compress.chksum = RTE_COMP_CHECKSUM_ADLER32;
+		decompress_xform->decompress.chksum = RTE_COMP_CHECKSUM_ADLER32;
+
+		for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
+			test_buffer = compress_test_bufs[i];
+
+			/* Generate zlib checksum and test against selected
+			 * drivers decompression checksum
+			 */
+			if (test_deflate_comp_decomp(&test_buffer, 1,
+					&i,
+					&compress_xform,
+					&decompress_xform,
+					1,
+					RTE_COMP_OP_STATELESS,
+					0,
+					ZLIB_COMPRESS) < 0) {
+				ret = TEST_FAILED;
+				goto exit;
+			}
+			/* Generate compression and decompression
+			 * checksum of selected driver
+			 */
+			if (test_deflate_comp_decomp(&test_buffer, 1,
+					&i,
+					&compress_xform,
+					&decompress_xform,
+					1,
+					RTE_COMP_OP_STATELESS,
+					0,
+					ZLIB_NONE) < 0) {
+				ret = TEST_FAILED;
+				goto exit;
+			}
+		}
+	}
+
+	/* Check if driver supports combined crc and adler checksum and test */
+	if ((capab->comp_feature_flags & RTE_COMP_FF_CRC32_ADLER32_CHECKSUM)) {
+		compress_xform->compress.chksum =
+				RTE_COMP_CHECKSUM_CRC32_ADLER32;
+		decompress_xform->decompress.chksum =
+				RTE_COMP_CHECKSUM_CRC32_ADLER32;
+
+		for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
+			test_buffer = compress_test_bufs[i];
+
+			/* Generate compression and decompression
+			 * checksum of selected driver
+			 */
+			if (test_deflate_comp_decomp(&test_buffer, 1,
+					&i,
+					&compress_xform,
+					&decompress_xform,
+					1,
+					RTE_COMP_OP_STATELESS,
+					0,
+					ZLIB_NONE) < 0) {
+				ret = TEST_FAILED;
+				goto exit;
+			}
+		}
+	}
+
+	ret = TEST_SUCCESS;
+
+exit:
+	rte_free(compress_xform);
+	rte_free(decompress_xform);
+	return ret;
 }
 
 static struct unit_test_suite compressdev_testsuite  = {
@@ -1487,6 +1681,8 @@ static struct unit_test_suite compressdev_testsuite  = {
 			test_compressdev_deflate_stateless_multi_xform),
 		TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
 			test_compressdev_deflate_stateless_sgl),
+		TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
+			test_compressdev_deflate_stateless_checksum),
 		TEST_CASES_END() /**< NULL terminate unit test array */
 	}
 };
Signature · 1 line
2.7.4
author
#14 lore raw

PATCH v7 2/2 compress/isal: enable checksum support in driver

This patch adds checksum support in the ISA-L PMD for both compression and decompression.
CRC32 is supported as well as Adler32.

V2:
Documentation Changes
V3:
Added Release note
V4:
Removed dependency on offset unit test
V5:
If checksum type is incorrect, compression will continue without checksum, rather than failing now.
V6:
Include documentation in this patch rather than separate doc patch.

---
 doc/guides/compressdevs/features/isal.ini     |  2 +
 doc/guides/compressdevs/isal.rst              | 30 +++++++++-
 doc/guides/rel_notes/release_19_02.rst        |  4 ++
 drivers/compress/isal/isal_compress_pmd.c     | 82 ++++++++++++++++++++++-----
 drivers/compress/isal/isal_compress_pmd_ops.c |  4 +-
 5 files changed, 105 insertions(+), 17 deletions(-)
doc/guides/compressdevs/features/isal.ini +2 -0
index 919cf70..e705031 100644
@@ -12,5 +12,7 @@ OOP SGL In SGL Out = Y
 OOP SGL In LB  Out = Y
 OOP LB  In SGL Out = Y
 Deflate            = Y
+Adler32            = Y
+Crc32              = Y
 Fixed              = Y
 Dynamic            = Y
doc/guides/compressdevs/isal.rst +28 -2
index 3bc3022..af1f41f 100644
@@ -27,6 +27,33 @@ Window size support:
 
     * 32K
 
+Checksum:
+
+    * CRC32
+    * ADLER32
+
+To enable a checksum in the driver, the compression and/or decompression xform
+structure, rte_comp_xform, must be filled with either of the CompressDev
+checksum flags supported. ::
+
+ compress_xform->compress.chksum = RTE_COMP_CHECKSUM_CRC32
+
+ decompress_xform->decompress.chksum = RTE_COMP_CHECKSUM_CRC32
+
+::
+
+ compress_xform->compress.chksum = RTE_COMP_CHECKSUM_ADLER32
+
+ decompress_xform->decompress.chksum = RTE_COMP_CHECKSUM_ADLER32
+
+If you request a checksum for compression or decompression,
+the checksum field in the operation structure,  ``op->output_chksum``,
+will be filled with the checksum.
+
+.. Note::
+
+ For the compression case above, your output buffer will need to be large enough to hold the compressed data plus a scratchpad for the checksum at the end, the scratchpad is 8 bytes for CRC32 and 4 bytes for Adler32.
+
 Level guide:
 
 The ISA-L levels have been mapped to somewhat correspond to the same ZLIB level,
@@ -75,13 +102,12 @@ As a result the level mappings from the API to the PMD are shown below.
  The above table only shows mapping when API calls for dynamic compression.
  For fixed compression, regardless of API level, internally ISA-L level 0 is always used.
 
+
 Limitations
 -----------
 
 * Compressdev level 0, no compression, is not supported.
 
-* Checksums will not be supported until future release.
-
 Installation
 ------------
 
doc/guides/rel_notes/release_19_02.rst +4 -0
index 8deb68b..286c060 100644
@@ -60,6 +60,10 @@ New Features
   * Added the handler to get firmware version string.
   * Added support for multicast filtering.
 
+  * **Enabled checksum support in the ISA-L compressdev driver.**
+
+    Added support for both adler and crc32 checksums in the ISA-L PMD.
+    This aids data integrity across both compression and decompression.
 
 Removed Items
 -------------
drivers/compress/isal/isal_compress_pmd.c +68 -14

149 lines not on this page · raw message

drivers/compress/isal/isal_compress_pmd_ops.c +3 -1
index 41cade8..7b91849 100644
@@ -17,7 +17,9 @@ static const struct rte_compressdev_capabilities isal_pmd_capabilities[] = {
 					RTE_COMP_FF_OOP_LB_IN_SGL_OUT |
 					RTE_COMP_FF_SHAREABLE_PRIV_XFORM |
 					RTE_COMP_FF_HUFFMAN_FIXED |
-					RTE_COMP_FF_HUFFMAN_DYNAMIC,
+					RTE_COMP_FF_HUFFMAN_DYNAMIC |
+					RTE_COMP_FF_CRC32_CHECKSUM |
+					RTE_COMP_FF_ADLER32_CHECKSUM,
 		.window_size = {
 			.min = 15,
 			.max = 15,
Signature · 1 line
2.7.4
#15 lore raw

Re: [PATCH v7 1/2] test/compress: add checksum tests

In reply to Lee Daly
Quoted 10 lines

-----Original Message-----
From: Daly, Lee
Sent: Tuesday, January 8, 2019 4:21 PM
To: [email protected]; De Lara Guarch, Pablo <[email protected]>
Cc: [email protected]; Daly, Lee <[email protected]>
Subject: [PATCH v7 1/2] test/compress: add checksum tests

Quoted 4 lines

This patch adds a test which examines what type of checksum the PMD supports, adler, crc32 or alder32_crc32 and tests that feature if the PMD supports it.

Series applied to dpdk-next-crypto.

Thanks,
Pablo