cryptodev: add compile support for AMD CCP crypto PMD

1,776 lines not on this page · raw message

364 lines not on this page · raw message

791 lines not on this page · raw message

530 lines not on this page · raw message

328 lines not on this page · raw message

55 lines not on this page · raw message

645 lines not on this page · raw message

128 lines not on this page · raw message

281 lines not on this page · raw message

index c981f0b..03e154f 100644
@@ -273,6 +273,24 @@ enum rte_crypto_auth_algorithm {
 	RTE_CRYPTO_AUTH_ZUC_EIA3,
 	/**< ZUC algorithm in EIA3 mode */
 
+	/**< SHA3 algorithm support*/
+	RTE_CRYPTO_AUTH_SHA3_224,
+	/**< 224 bit SHA3 algorithm. */
+	RTE_CRYPTO_AUTH_SHA3_224_HMAC,
+	/**< HMAC using 224 bit SHA3 algorithm. */
+	RTE_CRYPTO_AUTH_SHA3_256,
+	/**< 256 bit SHA3 algorithm. */
+	RTE_CRYPTO_AUTH_SHA3_256_HMAC,
+	/**< HMAC using 256 bit SHA3 algorithm. */
+	RTE_CRYPTO_AUTH_SHA3_384,
+	/**< 384 bit SHA3 algorithm. */
+	RTE_CRYPTO_AUTH_SHA3_384_HMAC,
+	/**< HMAC using 384 bit SHA3 algorithm. */
+	RTE_CRYPTO_AUTH_SHA3_512,
+	/**< 512 bit SHA3 algorithm. */
+	RTE_CRYPTO_AUTH_SHA3_512_HMAC,
+	/**< HMAC using 512 bit SHA3 algorithm. */
+
 	RTE_CRYPTO_AUTH_LIST_END
 };
 

index 68e1169..4d71ec1 100644
@@ -51,6 +51,7 @@
 #include <ccp_pmd_private.h>
 
 #include <openssl/sha.h> /*partial hash apis*/
+#include <openssl/cmac.h> /*sub key apis*/
 #include <openssl/evp.h> /*sub key apis*/
 
 /* SHA initial context values */

@@ -276,6 +277,84 @@ static int generate_partial_hash(struct ccp_session *sess)
 	}
 }
 
+/* prepare temporary keys K1 and K2 */
+static void prepare_key(unsigned char *k, unsigned char *l, int bl)
+{
+	int i;
+	/* Shift block to left, including carry */
+	for (i = 0; i < bl; i++) {
+		k[i] = l[i] << 1;
+		if (i < bl - 1 && l[i + 1] & 0x80)
+			k[i] |= 1;
+	}
+	/* If MSB set fixup with R */
+	if (l[0] & 0x80)
+		k[bl - 1] ^= bl == 16 ? 0x87 : 0x1b;
+}
+
+/**subkeys K1 and K2 generation for CMAC*/
+static int
+generate_cmac_subkeys(struct ccp_session *sess)
+{
+	const EVP_CIPHER *algo;
+	EVP_CIPHER_CTX *ctx;
+	unsigned char *ccp_ctx;
+	size_t i;
+	int dstlen, totlen;
+	unsigned char zero_iv[AES_BLOCK_SIZE] = {0};
+	unsigned char dst[2 * AES_BLOCK_SIZE] = {0};
+	unsigned char k1[AES_BLOCK_SIZE] = {0};
+	unsigned char k2[AES_BLOCK_SIZE] = {0};
+
+	if (sess->auth.ut.aes_type == CCP_AES_TYPE_128)
+		algo =  EVP_aes_128_cbc();
+	else if (sess->auth.ut.aes_type == CCP_AES_TYPE_192)
+		algo =  EVP_aes_192_cbc();
+	else if (sess->auth.ut.aes_type == CCP_AES_TYPE_256)
+		algo =  EVP_aes_256_cbc();
+	else {
+		CCP_LOG_ERR("Invalid CMAC type length");
+		return -1;
+	}
+
+	ctx = EVP_CIPHER_CTX_new();
+	if (!ctx) {
+		CCP_LOG_ERR("ctx creation failed");
+		return -1;
+	}
+	if (EVP_EncryptInit(ctx, algo, (unsigned char *)sess->auth.key,
+			    (unsigned char *)zero_iv) <= 0)
+		goto key_generate_err;
+	if (EVP_CIPHER_CTX_set_padding(ctx, 0) <= 0)
+		goto key_generate_err;
+	if (EVP_EncryptUpdate(ctx, dst, &dstlen, zero_iv,
+			      AES_BLOCK_SIZE) <= 0)
+		goto key_generate_err;
+	if (EVP_EncryptFinal_ex(ctx, dst + dstlen, &totlen) <= 0)
+		goto key_generate_err;
+
+	memset(sess->auth.pre_compute, 0, CCP_SB_BYTES * 2);
+
+	ccp_ctx = (unsigned char *)(sess->auth.pre_compute + CCP_SB_BYTES - 1);
+	prepare_key(k1, dst, AES_BLOCK_SIZE);
+	for (i = 0; i < AES_BLOCK_SIZE;  i++, ccp_ctx--)
+		*ccp_ctx = k1[i];
+
+	ccp_ctx = (unsigned char *)(sess->auth.pre_compute +
+				   (2 * CCP_SB_BYTES) - 1);
+	prepare_key(k2, k1, AES_BLOCK_SIZE);
+	for (i = 0; i < AES_BLOCK_SIZE;  i++, ccp_ctx--)
+		*ccp_ctx = k2[i];
+
+	EVP_CIPHER_CTX_free(ctx);
+
+	return 0;
+
+key_generate_err:
+	CCP_LOG_ERR("CMAC Init failed");
+		return -1;
+}
+
 /**configure session*/
 static int
 ccp_configure_session_cipher(struct ccp_session *sess,

@@ -373,6 +452,7 @@ ccp_configure_session_auth(struct ccp_session *sess,
 			   const struct rte_crypto_sym_xform *xform)
 {
 	const struct rte_crypto_auth_xform *auth_xform = NULL;
+	size_t i;
 
 	auth_xform = &xform->auth;
 

@@ -507,6 +587,33 @@ ccp_configure_session_auth(struct ccp_session *sess,
 		if (generate_partial_hash(sess))
 			return -1;
 		break;
+	case RTE_CRYPTO_AUTH_AES_CMAC:
+		sess->auth.algo = CCP_AUTH_ALGO_AES_CMAC;
+		sess->auth.engine = CCP_ENGINE_AES;
+		sess->auth.um.aes_mode = CCP_AES_MODE_CMAC;
+		sess->auth.key_length = auth_xform->key.length;
+		/**<padding and hash result*/
+		sess->auth.ctx_len = CCP_SB_BYTES << 1;
+		sess->auth.offset = AES_BLOCK_SIZE;
+		sess->auth.block_size = AES_BLOCK_SIZE;
+		if (sess->auth.key_length == 16)
+			sess->auth.ut.aes_type = CCP_AES_TYPE_128;
+		else if (sess->auth.key_length == 24)
+			sess->auth.ut.aes_type = CCP_AES_TYPE_192;
+		else if (sess->auth.key_length == 32)
+			sess->auth.ut.aes_type = CCP_AES_TYPE_256;
+		else {
+			CCP_LOG_ERR("Invalid CMAC key length");
+			return -1;
+		}
+		rte_memcpy(sess->auth.key, auth_xform->key.data,
+			   sess->auth.key_length);
+		for (i = 0; i < sess->auth.key_length; i++)
+			sess->auth.key_ccp[sess->auth.key_length - i - 1] =
+				sess->auth.key[i];
+		if (generate_cmac_subkeys(sess))
+			return -1;
+		break;
 	default:
 		CCP_LOG_ERR("Unsupported hash algo");
 		return -1;

@@ -697,6 +804,15 @@ ccp_auth_slot(struct ccp_session *session)
 		 * 6. Retrieve HMAC output from LSB to host memory
 		 */
 		break;
+	case CCP_AUTH_ALGO_AES_CMAC:
+		count = 4;
+		/**
+		 * op
+		 * extra descriptor in padding case
+		 * (k1/k2(255:128) with iv(127:0))
+		 * Retrieve result
+		 */
+		break;
 	default:
 		CCP_LOG_ERR("Unsupported ALGO %d", session->cipher.algo);
 	}

@@ -1077,6 +1193,158 @@ ccp_perform_sha(struct rte_crypto_op *op,
 }
 
 static int
+ccp_perform_aes_cmac(struct rte_crypto_op *op,
+		     struct ccp_queue *cmd_q)
+{
+	struct ccp_session *session;
+	union ccp_function function;
+	struct ccp_passthru pst;
+	struct ccp_desc *desc;
+	uint32_t tail;
+	uint8_t *src_tb, *append_ptr, *ctx_addr;
+	phys_addr_t src_addr, dest_addr, key_addr;
+	int length, non_align_len;
+
+	session = (struct ccp_session *)get_session_private_data(
+					 op->sym->session,
+					 cryptodev_driver_id);
+	key_addr = rte_mem_virt2phy(session->auth.key_ccp);
+
+	src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,
+					      op->sym->auth.data.offset);
+	append_ptr = (uint8_t *)rte_pktmbuf_append(op->sym->m_src,
+						session->auth.ctx_len);
+	dest_addr = (phys_addr_t)rte_mem_virt2phy((void *)append_ptr);
+
+	function.raw = 0;
+	CCP_AES_ENCRYPT(&function) = CCP_CIPHER_DIR_ENCRYPT;
+	CCP_AES_MODE(&function) = session->auth.um.aes_mode;
+	CCP_AES_TYPE(&function) = session->auth.ut.aes_type;
+
+	if (op->sym->auth.data.length % session->auth.block_size == 0) {
+
+		ctx_addr = session->auth.pre_compute;
+		memset(ctx_addr, 0, AES_BLOCK_SIZE);
+		pst.src_addr = (phys_addr_t)rte_mem_virt2phy((void *)ctx_addr);
+		pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
+		pst.len = CCP_SB_BYTES;
+		pst.dir = 1;
+		pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
+		pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
+		ccp_perform_passthru(&pst, cmd_q);
+
+		desc = &cmd_q->qbase_desc[cmd_q->qidx];
+		memset(desc, 0, Q_DESC_SIZE);
+
+		/* prepare desc for aes-cmac command */
+		CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
+		CCP_CMD_EOM(desc) = 1;
+		CCP_CMD_FUNCTION(desc) = function.raw;
+
+		CCP_CMD_LEN(desc) = op->sym->auth.data.length;
+		CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
+		CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
+		CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
+
+		CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
+		CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
+		CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
+		CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
+
+		cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
+
+		rte_wmb();
+
+		tail =
+		(uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
+		CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
+		CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
+			      cmd_q->qcontrol | CMD_Q_RUN);
+	} else {
+		ctx_addr = session->auth.pre_compute + CCP_SB_BYTES;
+		memset(ctx_addr, 0, AES_BLOCK_SIZE);
+		pst.src_addr = (phys_addr_t)rte_mem_virt2phy((void *)ctx_addr);
+		pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
+		pst.len = CCP_SB_BYTES;
+		pst.dir = 1;
+		pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
+		pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
+		ccp_perform_passthru(&pst, cmd_q);
+
+		length = (op->sym->auth.data.length / AES_BLOCK_SIZE);
+		length *= AES_BLOCK_SIZE;
+		non_align_len = op->sym->auth.data.length - length;
+		/* prepare desc for aes-cmac command */
+		/*Command 1*/
+		desc = &cmd_q->qbase_desc[cmd_q->qidx];
+		memset(desc, 0, Q_DESC_SIZE);
+
+		CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
+		CCP_CMD_INIT(desc) = 1;
+		CCP_CMD_FUNCTION(desc) = function.raw;
+
+		CCP_CMD_LEN(desc) = length;
+		CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
+		CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
+		CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
+
+		CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
+		CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
+		CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
+		CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
+
+		cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
+
+		/*Command 2*/
+		append_ptr = append_ptr + CCP_SB_BYTES;
+		memset(append_ptr, 0, AES_BLOCK_SIZE);
+		src_tb = rte_pktmbuf_mtod_offset(op->sym->m_src,
+						 uint8_t *,
+						 op->sym->auth.data.offset +
+						 length);
+		rte_memcpy(append_ptr, src_tb, non_align_len);
+		append_ptr[non_align_len] = CMAC_PAD_VALUE;
+
+		desc = &cmd_q->qbase_desc[cmd_q->qidx];
+		memset(desc, 0, Q_DESC_SIZE);
+
+		CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
+		CCP_CMD_EOM(desc) = 1;
+		CCP_CMD_FUNCTION(desc) = function.raw;
+		CCP_CMD_LEN(desc) = AES_BLOCK_SIZE;
+
+		CCP_CMD_SRC_LO(desc) = ((uint32_t)(dest_addr + CCP_SB_BYTES));
+		CCP_CMD_SRC_HI(desc) = high32_value(dest_addr + CCP_SB_BYTES);
+		CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
+
+		CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
+		CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
+		CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
+		CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
+
+		cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
+
+		rte_wmb();
+		tail =
+		(uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
+		CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
+		CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
+			      cmd_q->qcontrol | CMD_Q_RUN);
+	}
+	/* Retrieve result */
+	pst.dest_addr = dest_addr;
+	pst.src_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
+	pst.len = CCP_SB_BYTES;
+	pst.dir = 0;
+	pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
+	pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
+	ccp_perform_passthru(&pst, cmd_q);
+
+	op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
+	return 0;
+}
+
+static int
 ccp_perform_aes(struct rte_crypto_op *op,
 		struct ccp_queue *cmd_q,
 		struct ccp_batch_info *b_info)

@@ -1513,6 +1781,10 @@ ccp_crypto_auth(struct rte_crypto_op *op,
 		result = ccp_perform_hmac(op, cmd_q);
 		b_info->desccnt += 7;
 		break;
+	case CCP_AUTH_ALGO_AES_CMAC:
+		result = ccp_perform_aes_cmac(op, cmd_q);
+		b_info->desccnt += 4;
+		break;
 	default:
 		CCP_LOG_ERR("Unsupported Cipher algo");
 		result = -1;

index 675b5ae..21cc99f 100644
@@ -47,6 +47,7 @@
 #include <ccp_dev.h>
 
 #define AES_BLOCK_SIZE 16
+#define CMAC_PAD_VALUE 0x80
 #define CTR_NONCE_SIZE 4
 #define CTR_IV_SIZE 8
 #define CCP_SHA_CTX_SIZE 200

@@ -233,6 +234,7 @@ enum ccp_hash_algo {
 	CCP_AUTH_ALGO_SHA384_HMAC,
 	CCP_AUTH_ALGO_SHA512,
 	CCP_AUTH_ALGO_SHA512_HMAC,
+	CCP_AUTH_ALGO_AES_CMAC,
 	CCP_AUTH_ALGO_AES_GCM,
 };
 

@@ -301,6 +303,7 @@ struct ccp_session {
 		int block_size;
 		/**<Buffer to store  Software generated precomute values*/
 		/**< For HMAC H(ipad ^ key) and H(opad ^ key) */
+		/**< For CMAC K1 IV and K2 IV*/
 		uint8_t pre_compute[2 * CCP_SHA_CTX_SIZE];
 		int aad_length;
 	} auth;

index bc26020..af4d2b0 100644
@@ -50,6 +50,7 @@
 #include <ccp_pmd_private.h>
 
 #include <openssl/sha.h> /*partial hash apis*/
+#include <openssl/cmac.h> /*sub key apis*/
 #include <openssl/evp.h> /*sub key apis*/
 
 struct ccp_list ccp_list = TAILQ_HEAD_INITIALIZER(ccp_list);

index 02080a5..3a5e03c 100644
@@ -249,6 +249,27 @@ static const struct rte_cryptodev_capabilities ccp_pmd_capabilities[] = {
 			 }, }
 		}, }
 	},
+	{	/*AES-CMAC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				 .algo = RTE_CRYPTO_AUTH_AES_CMAC,
+				 .block_size = 16,
+				 .key_size = {
+					 .min = 16,
+					 .max = 32,
+					 .increment = 8
+				 },
+				 .digest_size = {
+					 .min = 16,
+					 .max = 16,
+					 .increment = 0
+				 },
+				 .aad_size = { 0 }
+			}, }
+		}, }
+	},
 	{       /* AES ECB */
 		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 		{.sym = {

384 lines not on this page · raw message

index 21cc99f..a350dfd 100644
@@ -68,6 +68,11 @@
 #define HMAC_IPAD_VALUE 0x36
 #define HMAC_OPAD_VALUE 0x5c
 
+#ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
+#define MD5_DIGEST_SIZE         16
+#define MD5_BLOCK_SIZE          64
+#endif
+
 /**SHA */
 #define SHA1_DIGEST_SIZE        20
 #define SHA1_BLOCK_SIZE         64

@@ -236,6 +241,9 @@ enum ccp_hash_algo {
 	CCP_AUTH_ALGO_SHA512_HMAC,
 	CCP_AUTH_ALGO_AES_CMAC,
 	CCP_AUTH_ALGO_AES_GCM,
+#ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
+	CCP_AUTH_ALGO_MD5_HMAC,
+#endif
 };
 
 /**

index 3a5e03c..e0160b3 100644
@@ -39,6 +39,29 @@
 #include <ccp_crypto.h>
 
 static const struct rte_cryptodev_capabilities ccp_pmd_capabilities[] = {
+#ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
+	{	/* MD5 HMAC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				 .algo = RTE_CRYPTO_AUTH_MD5_HMAC,
+				 .block_size = 64,
+				 .key_size = {
+					 .min = 1,
+					 .max = 64,
+					 .increment = 1
+				 },
+				 .digest_size = {
+					 .min = 16,
+					 .max = 16,
+					 .increment = 0
+				 },
+				 .aad_size = { 0 }
+			}, }
+		}, }
+	},
+#endif
 	{	/* SHA1 */
 		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 		{.sym = {

index 95433d7..9e288c5 100644
@@ -88,6 +88,10 @@ struct ccp_batch_info {
 	phys_addr_t lsb_buf_phys;
 	/**< LSB intermediate buf for passthru */
 	int lsb_buf_idx;
+#ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
+	uint16_t auth_ctr;
+	/**< auth only ops batch */
+#endif
 } __rte_cache_aligned;
 
 /**< CCP crypto queue pair */

index 1833929..1fc6569 100644
@@ -98,6 +98,74 @@ uint64_t ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(uint64_t)] = {
 };
 #endif
 
+#if defined(_MSC_VER)
+#define SHA3_CONST(x) x
+#else
+#define SHA3_CONST(x) x##L
+#endif
+
+/** 'Words' here refers to uint64_t */
+#define SHA3_KECCAK_SPONGE_WORDS \
+	(((1600) / 8) / sizeof(uint64_t))
+typedef struct sha3_context_ {
+	uint64_t saved;
+	/**
+	 * The portion of the input message that we
+	 * didn't consume yet
+	 */
+	union {
+		uint64_t s[SHA3_KECCAK_SPONGE_WORDS];
+		/* Keccak's state */
+		uint8_t sb[SHA3_KECCAK_SPONGE_WORDS * 8];
+		/**total 200 ctx size**/
+	};
+	unsigned int byteIndex;
+	/**
+	 * 0..7--the next byte after the set one
+	 * (starts from 0; 0--none are buffered)
+	 */
+	unsigned int wordIndex;
+	/**
+	 * 0..24--the next word to integrate input
+	 * (starts from 0)
+	 */
+	unsigned int capacityWords;
+	/**
+	 * the double size of the hash output in
+	 * words (e.g. 16 for Keccak 512)
+	 */
+} sha3_context;
+
+#ifndef SHA3_ROTL64
+#define SHA3_ROTL64(x, y) \
+	(((x) << (y)) | ((x) >> ((sizeof(uint64_t)*8) - (y))))
+#endif
+
+static const uint64_t keccakf_rndc[24] = {
+	SHA3_CONST(0x0000000000000001UL), SHA3_CONST(0x0000000000008082UL),
+	SHA3_CONST(0x800000000000808aUL), SHA3_CONST(0x8000000080008000UL),
+	SHA3_CONST(0x000000000000808bUL), SHA3_CONST(0x0000000080000001UL),
+	SHA3_CONST(0x8000000080008081UL), SHA3_CONST(0x8000000000008009UL),
+	SHA3_CONST(0x000000000000008aUL), SHA3_CONST(0x0000000000000088UL),
+	SHA3_CONST(0x0000000080008009UL), SHA3_CONST(0x000000008000000aUL),
+	SHA3_CONST(0x000000008000808bUL), SHA3_CONST(0x800000000000008bUL),
+	SHA3_CONST(0x8000000000008089UL), SHA3_CONST(0x8000000000008003UL),
+	SHA3_CONST(0x8000000000008002UL), SHA3_CONST(0x8000000000000080UL),
+	SHA3_CONST(0x000000000000800aUL), SHA3_CONST(0x800000008000000aUL),
+	SHA3_CONST(0x8000000080008081UL), SHA3_CONST(0x8000000000008080UL),
+	SHA3_CONST(0x0000000080000001UL), SHA3_CONST(0x8000000080008008UL)
+};
+
+static const unsigned int keccakf_rotc[24] = {
+	1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 2, 14, 27, 41, 56, 8, 25, 43, 62,
+	18, 39, 61, 20, 44
+};
+
+static const unsigned int keccakf_piln[24] = {
+	10, 7, 11, 17, 18, 3, 5, 16, 8, 21, 24, 4, 15, 23, 19, 13, 12, 2, 20,
+	14, 22, 9, 6, 1
+};
+
 static enum ccp_cmd_order
 ccp_get_cmd_id(const struct rte_crypto_sym_xform *xform)
 {

@@ -182,7 +250,226 @@ static int partial_hash_sha512(uint8_t *data_in, uint8_t *data_out)
 		   SHA512_DIGEST_LENGTH);
 	return 0;
 }
+#endif
+
+static void
+keccakf(uint64_t s[25])
+{
+	int i, j, round;
+	uint64_t t, bc[5];
+#define KECCAK_ROUNDS 24
+
+	for (round = 0; round < KECCAK_ROUNDS; round++) {
+
+		/* Theta */
+		for (i = 0; i < 5; i++)
+			bc[i] = s[i] ^ s[i + 5] ^ s[i + 10] ^ s[i + 15] ^
+				s[i + 20];
+
+		for (i = 0; i < 5; i++) {
+			t = bc[(i + 4) % 5] ^ SHA3_ROTL64(bc[(i + 1) % 5], 1);
+			for (j = 0; j < 25; j += 5)
+				s[j + i] ^= t;
+		}
+
+		/* Rho Pi */
+		t = s[1];
+		for (i = 0; i < 24; i++) {
+			j = keccakf_piln[i];
+			bc[0] = s[j];
+			s[j] = SHA3_ROTL64(t, keccakf_rotc[i]);
+			t = bc[0];
+		}
+
+		/* Chi */
+		for (j = 0; j < 25; j += 5) {
+			for (i = 0; i < 5; i++)
+				bc[i] = s[j + i];
+			for (i = 0; i < 5; i++)
+				s[j + i] ^= (~bc[(i + 1) % 5]) &
+					    bc[(i + 2) % 5];
+		}
+
+		/* Iota */
+		s[0] ^= keccakf_rndc[round];
+	}
+}
+
+static void
+sha3_Init224(void *priv)
+{
+	sha3_context *ctx = (sha3_context *) priv;
+
+	memset(ctx, 0, sizeof(*ctx));
+	ctx->capacityWords = 2 * 224 / (8 * sizeof(uint64_t));
+}
+
+static void
+sha3_Init256(void *priv)
+{
+	sha3_context *ctx = (sha3_context *) priv;
+
+	memset(ctx, 0, sizeof(*ctx));
+	ctx->capacityWords = 2 * 256 / (8 * sizeof(uint64_t));
+}
+
+static void
+sha3_Init384(void *priv)
+{
+	sha3_context *ctx = (sha3_context *) priv;
+
+	memset(ctx, 0, sizeof(*ctx));
+	ctx->capacityWords = 2 * 384 / (8 * sizeof(uint64_t));
+}
+
+static void
+sha3_Init512(void *priv)
+{
+	sha3_context *ctx = (sha3_context *) priv;
+
+	memset(ctx, 0, sizeof(*ctx));
+	ctx->capacityWords = 2 * 512 / (8 * sizeof(uint64_t));
+}
+
+
+/* This is simply the 'update' with the padding block.
+ * The padding block is 0x01 || 0x00* || 0x80. First 0x01 and last 0x80
+ * bytes are always present, but they can be the same byte.
+ */
+static void
+sha3_Update(void *priv, void const *bufIn, size_t len)
+{
+	sha3_context *ctx = (sha3_context *) priv;
+	unsigned int old_tail = (8 - ctx->byteIndex) & 7;
+	size_t words;
+	unsigned int tail;
+	size_t i;
+	const uint8_t *buf = bufIn;
+
+	if (len < old_tail) {
+		while (len--)
+			ctx->saved |= (uint64_t) (*(buf++)) <<
+				      ((ctx->byteIndex++) * 8);
+		return;
+	}
+
+	if (old_tail) {
+		len -= old_tail;
+		while (old_tail--)
+			ctx->saved |= (uint64_t) (*(buf++)) <<
+				      ((ctx->byteIndex++) * 8);
+
+		ctx->s[ctx->wordIndex] ^= ctx->saved;
+		ctx->byteIndex = 0;
+		ctx->saved = 0;
+		if (++ctx->wordIndex ==
+		   (SHA3_KECCAK_SPONGE_WORDS - ctx->capacityWords)) {
+			keccakf(ctx->s);
+			ctx->wordIndex = 0;
+		}
+	}
+
+	words = len / sizeof(uint64_t);
+	tail = len - words * sizeof(uint64_t);
+
+	for (i = 0; i < words; i++, buf += sizeof(uint64_t)) {
+		const uint64_t t = (uint64_t) (buf[0]) |
+			((uint64_t) (buf[1]) << 8 * 1) |
+			((uint64_t) (buf[2]) << 8 * 2) |
+			((uint64_t) (buf[3]) << 8 * 3) |
+			((uint64_t) (buf[4]) << 8 * 4) |
+			((uint64_t) (buf[5]) << 8 * 5) |
+			((uint64_t) (buf[6]) << 8 * 6) |
+			((uint64_t) (buf[7]) << 8 * 7);
+		ctx->s[ctx->wordIndex] ^= t;
+		if (++ctx->wordIndex ==
+		   (SHA3_KECCAK_SPONGE_WORDS - ctx->capacityWords)) {
+			keccakf(ctx->s);
+			ctx->wordIndex = 0;
+		}
+	}
+
+	while (tail--)
+		ctx->saved |= (uint64_t) (*(buf++)) << ((ctx->byteIndex++) * 8);
+}
+
+int partial_hash_sha3_224(uint8_t *data_in, uint8_t *data_out)
+{
+	sha3_context *ctx;
+	int i;
+
+	ctx = rte_zmalloc("sha3-ctx", sizeof(sha3_context), 0);
+	if (!ctx) {
+		CCP_LOG_ERR("sha3-ctx creation failed");
+		return -ENOMEM;
+	}
+	sha3_Init224(ctx);
+	sha3_Update(ctx, data_in, SHA3_224_BLOCK_SIZE);
+	for (i = 0; i < CCP_SHA3_CTX_SIZE; i++, data_out++)
+		*data_out = ctx->sb[CCP_SHA3_CTX_SIZE - i - 1];
+	rte_free(ctx);
+
+	return 0;
+}
+
+int partial_hash_sha3_256(uint8_t *data_in, uint8_t *data_out)
+{
+	sha3_context *ctx;
+	int i;
+
+	ctx = rte_zmalloc("sha3-ctx", sizeof(sha3_context), 0);
+	if (!ctx) {
+		CCP_LOG_ERR("sha3-ctx creation failed");
+		return -ENOMEM;
+	}
+	sha3_Init256(ctx);
+	sha3_Update(ctx, data_in, SHA3_256_BLOCK_SIZE);
+	for (i = 0; i < CCP_SHA3_CTX_SIZE; i++, data_out++)
+		*data_out = ctx->sb[CCP_SHA3_CTX_SIZE - i - 1];
+	rte_free(ctx);
+
+	return 0;
+}
+
+int partial_hash_sha3_384(uint8_t *data_in, uint8_t *data_out)
+{
+	sha3_context *ctx;
+	int i;
+
+	ctx = rte_zmalloc("sha3-ctx", sizeof(sha3_context), 0);
+	if (!ctx) {
+		CCP_LOG_ERR("sha3-ctx creation failed");
+		return -ENOMEM;
+	}
+	sha3_Init384(ctx);
+	sha3_Update(ctx, data_in, SHA3_384_BLOCK_SIZE);
+	for (i = 0; i < CCP_SHA3_CTX_SIZE; i++, data_out++)
+		*data_out = ctx->sb[CCP_SHA3_CTX_SIZE - i - 1];
+	rte_free(ctx);
+
+	return 0;
+}
+
+int partial_hash_sha3_512(uint8_t *data_in, uint8_t *data_out)
+{
+	sha3_context *ctx;
+	int i;
+
+	ctx = rte_zmalloc("sha3-ctx", sizeof(sha3_context), 0);
+	if (!ctx) {
+		CCP_LOG_ERR("sha3-ctx creation failed");
+		return -ENOMEM;
+	}
+	sha3_Init512(ctx);
+	sha3_Update(ctx, data_in, SHA3_512_BLOCK_SIZE);
+	for (i = 0; i < CCP_SHA3_CTX_SIZE; i++, data_out++)
+		*data_out = ctx->sb[CCP_SHA3_CTX_SIZE - i - 1];
+	rte_free(ctx);
+
+	return 0;
+}
 
+#ifndef RTE_LIBRTE_PMD_CCP_CPU_AUTH
 static int generate_partial_hash(struct ccp_session *sess)
 {
 

@@ -192,6 +479,7 @@ static int generate_partial_hash(struct ccp_session *sess)
 	uint32_t *hash_value_be32, hash_temp32[8];
 	uint64_t *hash_value_be64, hash_temp64[8];
 	int i, count;
+	uint8_t *hash_value_sha3;
 
 	opad_t = ipad_t = (uint8_t *)sess->auth.key;
 

@@ -235,6 +523,16 @@ static int generate_partial_hash(struct ccp_session *sess)
 		for (i = 0; i < count; i++, hash_value_be32++)
 			*hash_value_be32 = hash_temp32[count - 1 - i];
 		return 0;
+	case CCP_AUTH_ALGO_SHA3_224_HMAC:
+		hash_value_sha3 = sess->auth.pre_compute;
+		if (partial_hash_sha3_224(ipad, hash_value_sha3))
+			return -1;
+
+		hash_value_sha3 = (uint8_t *)(sess->auth.pre_compute
+					       + sess->auth.ctx_len);
+		if (partial_hash_sha3_224(opad, hash_value_sha3))
+			return -1;
+		return 0;
 	case CCP_AUTH_ALGO_SHA256_HMAC:
 		count = SHA256_DIGEST_SIZE >> 2;
 

@@ -250,6 +548,16 @@ static int generate_partial_hash(struct ccp_session *sess)
 		for (i = 0; i < count; i++, hash_value_be32++)
 			*hash_value_be32 = hash_temp32[count - 1 - i];
 		return 0;
+	case CCP_AUTH_ALGO_SHA3_256_HMAC:
+		hash_value_sha3 = sess->auth.pre_compute;
+		if (partial_hash_sha3_256(ipad, hash_value_sha3))
+			return -1;
+
+		hash_value_sha3 = (uint8_t *)(sess->auth.pre_compute
+					      + sess->auth.ctx_len);
+		if (partial_hash_sha3_256(opad, hash_value_sha3))
+			return -1;
+		return 0;
 	case CCP_AUTH_ALGO_SHA384_HMAC:
 		count = SHA512_DIGEST_SIZE >> 3;
 

@@ -265,6 +573,16 @@ static int generate_partial_hash(struct ccp_session *sess)
 		for (i = 0; i < count; i++, hash_value_be64++)
 			*hash_value_be64 = hash_temp64[count - 1 - i];
 		return 0;
+	case CCP_AUTH_ALGO_SHA3_384_HMAC:
+		hash_value_sha3 = sess->auth.pre_compute;
+		if (partial_hash_sha3_384(ipad, hash_value_sha3))
+			return -1;
+
+		hash_value_sha3 = (uint8_t *)(sess->auth.pre_compute
+					      + sess->auth.ctx_len);
+		if (partial_hash_sha3_384(opad, hash_value_sha3))
+			return -1;
+		return 0;
 	case CCP_AUTH_ALGO_SHA512_HMAC:
 		count = SHA512_DIGEST_SIZE >> 3;
 

@@ -280,6 +598,16 @@ static int generate_partial_hash(struct ccp_session *sess)
 		for (i = 0; i < count; i++, hash_value_be64++)
 			*hash_value_be64 = hash_temp64[count - 1 - i];
 		return 0;
+	case CCP_AUTH_ALGO_SHA3_512_HMAC:
+		hash_value_sha3 = sess->auth.pre_compute;
+		if (partial_hash_sha3_512(ipad, hash_value_sha3))
+			return -1;
+
+		hash_value_sha3 = (uint8_t *)(sess->auth.pre_compute
+					      + sess->auth.ctx_len);
+		if (partial_hash_sha3_512(opad, hash_value_sha3))
+			return -1;
+		return 0;
 	default:
 		CCP_LOG_ERR("Invalid session algo");
 		return -1;

@@ -525,6 +853,30 @@ ccp_configure_session_auth(struct ccp_session *sess,
 		if (generate_partial_hash(sess))
 			return -1;
 		break;
+	case RTE_CRYPTO_AUTH_SHA3_224:
+		sess->auth.algo = CCP_AUTH_ALGO_SHA3_224;
+		sess->auth.engine = CCP_ENGINE_SHA;
+		sess->auth.ut.sha_type = CCP_SHA3_TYPE_224;
+		sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
+		sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA224_DIGEST_SIZE;
+		break;
+	case RTE_CRYPTO_AUTH_SHA3_224_HMAC:
+		if (auth_xform->key.length > SHA3_224_BLOCK_SIZE)
+			return -1;
+		sess->auth.algo = CCP_AUTH_ALGO_SHA3_224_HMAC;
+		sess->auth.engine = CCP_ENGINE_SHA;
+		sess->auth.ut.sha_type = CCP_SHA3_TYPE_224;
+		sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
+		sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA224_DIGEST_SIZE;
+		sess->auth.block_size = SHA3_224_BLOCK_SIZE;
+		sess->auth.key_length = auth_xform->key.length;
+		memset(sess->auth.key, 0, sess->auth.block_size);
+		memset(sess->auth.pre_compute, 0, 2 * sess->auth.ctx_len);
+		rte_memcpy(sess->auth.key, auth_xform->key.data,
+			   auth_xform->key.length);
+		if (generate_partial_hash(sess))
+			return -1;
+		break;
 	case RTE_CRYPTO_AUTH_SHA256:
 		sess->auth.algo = CCP_AUTH_ALGO_SHA256;
 		sess->auth.engine = CCP_ENGINE_SHA;

@@ -550,6 +902,30 @@ ccp_configure_session_auth(struct ccp_session *sess,
 		if (generate_partial_hash(sess))
 			return -1;
 		break;
+	case RTE_CRYPTO_AUTH_SHA3_256:
+		sess->auth.algo = CCP_AUTH_ALGO_SHA3_256;
+		sess->auth.engine = CCP_ENGINE_SHA;
+		sess->auth.ut.sha_type = CCP_SHA3_TYPE_256;
+		sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
+		sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA256_DIGEST_SIZE;
+		break;
+	case RTE_CRYPTO_AUTH_SHA3_256_HMAC:
+		if (auth_xform->key.length > SHA3_256_BLOCK_SIZE)
+			return -1;
+		sess->auth.algo = CCP_AUTH_ALGO_SHA3_256_HMAC;
+		sess->auth.engine = CCP_ENGINE_SHA;
+		sess->auth.ut.sha_type = CCP_SHA3_TYPE_256;
+		sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
+		sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA256_DIGEST_SIZE;
+		sess->auth.block_size = SHA3_256_BLOCK_SIZE;
+		sess->auth.key_length = auth_xform->key.length;
+		memset(sess->auth.key, 0, sess->auth.block_size);
+		memset(sess->auth.pre_compute, 0, 2 * sess->auth.ctx_len);
+		rte_memcpy(sess->auth.key, auth_xform->key.data,
+			   auth_xform->key.length);
+		if (generate_partial_hash(sess))
+			return -1;
+		break;
 	case RTE_CRYPTO_AUTH_SHA384:
 		sess->auth.algo = CCP_AUTH_ALGO_SHA384;
 		sess->auth.engine = CCP_ENGINE_SHA;

@@ -575,6 +951,30 @@ ccp_configure_session_auth(struct ccp_session *sess,
 		if (generate_partial_hash(sess))
 			return -1;
 		break;
+	case RTE_CRYPTO_AUTH_SHA3_384:
+		sess->auth.algo = CCP_AUTH_ALGO_SHA3_384;
+		sess->auth.engine = CCP_ENGINE_SHA;
+		sess->auth.ut.sha_type = CCP_SHA3_TYPE_384;
+		sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
+		sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA384_DIGEST_SIZE;
+		break;
+	case RTE_CRYPTO_AUTH_SHA3_384_HMAC:
+		if (auth_xform->key.length > SHA3_384_BLOCK_SIZE)
+			return -1;
+		sess->auth.algo = CCP_AUTH_ALGO_SHA3_384_HMAC;
+		sess->auth.engine = CCP_ENGINE_SHA;
+		sess->auth.ut.sha_type = CCP_SHA3_TYPE_384;
+		sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
+		sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA384_DIGEST_SIZE;
+		sess->auth.block_size = SHA3_384_BLOCK_SIZE;
+		sess->auth.key_length = auth_xform->key.length;
+		memset(sess->auth.key, 0, sess->auth.block_size);
+		memset(sess->auth.pre_compute, 0, 2 * sess->auth.ctx_len);
+		rte_memcpy(sess->auth.key, auth_xform->key.data,
+			   auth_xform->key.length);
+		if (generate_partial_hash(sess))
+			return -1;
+		break;
 	case RTE_CRYPTO_AUTH_SHA512:
 		sess->auth.algo = CCP_AUTH_ALGO_SHA512;
 		sess->auth.engine = CCP_ENGINE_SHA;

@@ -600,6 +1000,30 @@ ccp_configure_session_auth(struct ccp_session *sess,
 		if (generate_partial_hash(sess))
 			return -1;
 		break;
+	case RTE_CRYPTO_AUTH_SHA3_512:
+		sess->auth.algo = CCP_AUTH_ALGO_SHA3_512;
+		sess->auth.engine = CCP_ENGINE_SHA;
+		sess->auth.ut.sha_type = CCP_SHA3_TYPE_512;
+		sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
+		sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA512_DIGEST_SIZE;
+		break;
+	case RTE_CRYPTO_AUTH_SHA3_512_HMAC:
+		if (auth_xform->key.length > SHA3_512_BLOCK_SIZE)
+			return -1;
+		sess->auth.algo = CCP_AUTH_ALGO_SHA3_512_HMAC;
+		sess->auth.engine = CCP_ENGINE_SHA;
+		sess->auth.ut.sha_type = CCP_SHA3_TYPE_512;
+		sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
+		sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA512_DIGEST_SIZE;
+		sess->auth.block_size = SHA3_512_BLOCK_SIZE;
+		sess->auth.key_length = auth_xform->key.length;
+		memset(sess->auth.key, 0, sess->auth.block_size);
+		memset(sess->auth.pre_compute, 0, 2 * sess->auth.ctx_len);
+		rte_memcpy(sess->auth.key, auth_xform->key.data,
+			   auth_xform->key.length);
+		if (generate_partial_hash(sess))
+			return -1;
+		break;
 	case RTE_CRYPTO_AUTH_AES_CMAC:
 		sess->auth.algo = CCP_AUTH_ALGO_AES_CMAC;
 		sess->auth.engine = CCP_ENGINE_AES;

@@ -888,6 +1312,26 @@ ccp_auth_slot(struct ccp_session *session)
 		 * 6. Retrieve HMAC output from LSB to host memory
 		 */
 		break;
+	case CCP_AUTH_ALGO_SHA3_224:
+	case CCP_AUTH_ALGO_SHA3_256:
+	case CCP_AUTH_ALGO_SHA3_384:
+	case CCP_AUTH_ALGO_SHA3_512:
+		count = 1;
+		/**< only op ctx and dst in host memory*/
+		break;
+	case CCP_AUTH_ALGO_SHA3_224_HMAC:
+	case CCP_AUTH_ALGO_SHA3_256_HMAC:
+		count = 3;
+		break;
+	case CCP_AUTH_ALGO_SHA3_384_HMAC:
+	case CCP_AUTH_ALGO_SHA3_512_HMAC:
+		count = 4;
+		/**
+		 * 1. Op to Perform Ihash
+		 * 2. Retrieve result from LSB to host memory
+		 * 3. Perform final hash
+		 */
+		break;
 	case CCP_AUTH_ALGO_AES_CMAC:
 		count = 4;
 		/**

@@ -1398,6 +1842,213 @@ ccp_perform_sha(struct rte_crypto_op *op,
 }
 
 static int
+ccp_perform_sha3_hmac(struct rte_crypto_op *op,
+		      struct ccp_queue *cmd_q)
+{
+	struct ccp_session *session;
+	struct ccp_passthru pst;
+	union ccp_function function;
+	struct ccp_desc *desc;
+	uint8_t *append_ptr;
+	uint32_t tail;
+	phys_addr_t src_addr, dest_addr, ctx_paddr, dest_addr_t;
+
+	session = (struct ccp_session *)get_session_private_data(
+					 op->sym->session,
+					 cryptodev_driver_id);
+
+	src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,
+					      op->sym->auth.data.offset);
+	append_ptr = (uint8_t *)rte_pktmbuf_append(op->sym->m_src,
+						session->auth.ctx_len);
+	if (!append_ptr) {
+		CCP_LOG_ERR("CCP MBUF append failed\n");
+		return -1;
+	}
+	dest_addr = (phys_addr_t)rte_mem_virt2phy((void *)append_ptr);
+	dest_addr_t = dest_addr + (session->auth.ctx_len / 2);
+	ctx_paddr = (phys_addr_t)rte_mem_virt2phy((void
+						   *)session->auth.pre_compute);
+	desc = &cmd_q->qbase_desc[cmd_q->qidx];
+	memset(desc, 0, Q_DESC_SIZE);
+
+	/*desc1 for SHA3-Ihash operation */
+	CCP_CMD_ENGINE(desc) = CCP_ENGINE_SHA;
+	CCP_CMD_INIT(desc) = 1;
+	CCP_CMD_EOM(desc) = 1;
+
+	function.raw = 0;
+	CCP_SHA_TYPE(&function) = session->auth.ut.sha_type;
+	CCP_CMD_FUNCTION(desc) = function.raw;
+	CCP_CMD_LEN(desc) = op->sym->auth.data.length;
+
+	CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
+	CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
+	CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
+
+	CCP_CMD_DST_LO(desc) = (cmd_q->sb_sha * CCP_SB_BYTES);
+	CCP_CMD_DST_HI(desc) = 0;
+	CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SB;
+
+	CCP_CMD_KEY_LO(desc) = ((uint32_t)ctx_paddr);
+	CCP_CMD_KEY_HI(desc) = high32_value(ctx_paddr);
+	CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
+
+	cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
+
+	rte_wmb();
+	tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
+	CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
+	CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
+		      cmd_q->qcontrol | CMD_Q_RUN);
+
+	/* Intermediate Hash value retrieve */
+	if ((session->auth.ut.sha_type == CCP_SHA3_TYPE_384) ||
+	    (session->auth.ut.sha_type == CCP_SHA3_TYPE_512)) {
+
+		pst.src_addr =
+			(phys_addr_t)((cmd_q->sb_sha + 1) * CCP_SB_BYTES);
+		pst.dest_addr = dest_addr_t;
+		pst.len = CCP_SB_BYTES;
+		pst.dir = 0;
+		pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
+		pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
+		ccp_perform_passthru(&pst, cmd_q);
+
+		pst.src_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
+		pst.dest_addr = dest_addr_t + CCP_SB_BYTES;
+		pst.len = CCP_SB_BYTES;
+		pst.dir = 0;
+		pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
+		pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
+		ccp_perform_passthru(&pst, cmd_q);
+
+	} else {
+		pst.src_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
+		pst.dest_addr = dest_addr_t;
+		pst.len = CCP_SB_BYTES;
+		pst.dir = 0;
+		pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
+		pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
+		ccp_perform_passthru(&pst, cmd_q);
+	}
+
+	/**sha engine command descriptor for FinalHash*/
+	ctx_paddr += CCP_SHA3_CTX_SIZE;
+	desc = &cmd_q->qbase_desc[cmd_q->qidx];
+	memset(desc, 0, Q_DESC_SIZE);
+
+	CCP_CMD_ENGINE(desc) = CCP_ENGINE_SHA;
+	CCP_CMD_INIT(desc) = 1;
+	CCP_CMD_EOM(desc) = 1;
+
+	function.raw = 0;
+	CCP_SHA_TYPE(&function) = session->auth.ut.sha_type;
+	CCP_CMD_FUNCTION(desc) = function.raw;
+
+	if (session->auth.ut.sha_type == CCP_SHA3_TYPE_224) {
+		dest_addr_t += (CCP_SB_BYTES - SHA224_DIGEST_SIZE);
+		CCP_CMD_LEN(desc) = SHA224_DIGEST_SIZE;
+	} else if (session->auth.ut.sha_type == CCP_SHA3_TYPE_256) {
+		CCP_CMD_LEN(desc) = SHA256_DIGEST_SIZE;
+	} else if (session->auth.ut.sha_type == CCP_SHA3_TYPE_384) {
+		dest_addr_t += (2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE);
+		CCP_CMD_LEN(desc) = SHA384_DIGEST_SIZE;
+	} else {
+		CCP_CMD_LEN(desc) = SHA512_DIGEST_SIZE;
+	}
+
+	CCP_CMD_SRC_LO(desc) = ((uint32_t)dest_addr_t);
+	CCP_CMD_SRC_HI(desc) = high32_value(dest_addr_t);
+	CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
+
+	CCP_CMD_DST_LO(desc) = (uint32_t)dest_addr;
+	CCP_CMD_DST_HI(desc) = high32_value(dest_addr);
+	CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SYSTEM;
+
+	CCP_CMD_KEY_LO(desc) = ((uint32_t)ctx_paddr);
+	CCP_CMD_KEY_HI(desc) = high32_value(ctx_paddr);
+	CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
+
+	cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
+
+	rte_wmb();
+	tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
+	CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
+	CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
+		      cmd_q->qcontrol | CMD_Q_RUN);
+
+	op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
+	return 0;
+}
+
+static int
+ccp_perform_sha3(struct rte_crypto_op *op,
+		 struct ccp_queue *cmd_q)
+{
+	struct ccp_session *session;
+	union ccp_function function;
+	struct ccp_desc *desc;
+	uint8_t *ctx_addr, *append_ptr;
+	uint32_t tail;
+	phys_addr_t src_addr, dest_addr, ctx_paddr;
+
+	session = (struct ccp_session *)get_session_private_data(
+					 op->sym->session,
+					 cryptodev_driver_id);
+
+	src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,
+					      op->sym->auth.data.offset);
+	append_ptr = (uint8_t *)rte_pktmbuf_append(op->sym->m_src,
+						session->auth.ctx_len);
+	if (!append_ptr) {
+		CCP_LOG_ERR("CCP MBUF append failed\n");
+		return -1;
+	}
+	dest_addr = (phys_addr_t)rte_mem_virt2phy((void *)append_ptr);
+	ctx_addr = session->auth.sha3_ctx;
+	ctx_paddr = (phys_addr_t)rte_mem_virt2phy((void *)ctx_addr);
+
+	desc = &cmd_q->qbase_desc[cmd_q->qidx];
+	memset(desc, 0, Q_DESC_SIZE);
+
+	/* prepare desc for SHA3 operation */
+	CCP_CMD_ENGINE(desc) = CCP_ENGINE_SHA;
+	CCP_CMD_INIT(desc) = 1;
+	CCP_CMD_EOM(desc) = 1;
+
+	function.raw = 0;
+	CCP_SHA_TYPE(&function) = session->auth.ut.sha_type;
+	CCP_CMD_FUNCTION(desc) = function.raw;
+
+	CCP_CMD_LEN(desc) = op->sym->auth.data.length;
+
+	CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
+	CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
+	CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
+
+	CCP_CMD_DST_LO(desc) = ((uint32_t)dest_addr);
+	CCP_CMD_DST_HI(desc) = high32_value(dest_addr);
+	CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SYSTEM;
+
+	CCP_CMD_KEY_LO(desc) = ((uint32_t)ctx_paddr);
+	CCP_CMD_KEY_HI(desc) = high32_value(ctx_paddr);
+	CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
+
+	cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
+
+	rte_wmb();
+
+	tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
+	CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
+	CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
+		      cmd_q->qcontrol | CMD_Q_RUN);
+
+	op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
+	return 0;
+}
+
+static int
 ccp_perform_aes_cmac(struct rte_crypto_op *op,
 		     struct ccp_queue *cmd_q)
 {

@@ -1986,6 +2637,23 @@ ccp_crypto_auth(struct rte_crypto_op *op,
 		result = ccp_perform_hmac(op, cmd_q);
 		b_info->desccnt += 7;
 		break;
+	case CCP_AUTH_ALGO_SHA3_224:
+	case CCP_AUTH_ALGO_SHA3_256:
+	case CCP_AUTH_ALGO_SHA3_384:
+	case CCP_AUTH_ALGO_SHA3_512:
+		result = ccp_perform_sha3(op, cmd_q);
+		b_info->desccnt += 1;
+		break;
+	case CCP_AUTH_ALGO_SHA3_224_HMAC:
+	case CCP_AUTH_ALGO_SHA3_256_HMAC:
+		result = ccp_perform_sha3_hmac(op, cmd_q);
+		b_info->desccnt += 3;
+		break;
+	case CCP_AUTH_ALGO_SHA3_384_HMAC:
+	case CCP_AUTH_ALGO_SHA3_512_HMAC:
+		result = ccp_perform_sha3_hmac(op, cmd_q);
+		b_info->desccnt += 4;
+		break;
 	case CCP_AUTH_ALGO_AES_CMAC:
 		result = ccp_perform_aes_cmac(op, cmd_q);
 		b_info->desccnt += 4;

index a350dfd..9d8d42e 100644
@@ -50,7 +50,7 @@
 #define CMAC_PAD_VALUE 0x80
 #define CTR_NONCE_SIZE 4
 #define CTR_IV_SIZE 8
-#define CCP_SHA_CTX_SIZE 200
+#define CCP_SHA3_CTX_SIZE 200
 
 /**Macro helpers for CCP command creation*/
 #define	CCP_AES_SIZE(p)		((p)->aes.size)

@@ -79,15 +79,19 @@
 
 #define SHA224_DIGEST_SIZE      28
 #define SHA224_BLOCK_SIZE       64
+#define SHA3_224_BLOCK_SIZE     144
 
 #define SHA256_DIGEST_SIZE      32
 #define SHA256_BLOCK_SIZE       64
+#define SHA3_256_BLOCK_SIZE     136
 
 #define SHA384_DIGEST_SIZE      48
 #define SHA384_BLOCK_SIZE       128
+#define SHA3_384_BLOCK_SIZE	104
 
 #define SHA512_DIGEST_SIZE      64
 #define SHA512_BLOCK_SIZE       128
+#define SHA3_512_BLOCK_SIZE     72
 
 /**SHA LSB intialiazation values*/
 

@@ -203,6 +207,10 @@ enum ccp_sha_type {
 	CCP_SHA_TYPE_512,
 	CCP_SHA_TYPE_RSVD1,
 	CCP_SHA_TYPE_RSVD2,
+	CCP_SHA3_TYPE_224,
+	CCP_SHA3_TYPE_256,
+	CCP_SHA3_TYPE_384,
+	CCP_SHA3_TYPE_512,
 	CCP_SHA_TYPE__LAST,
 };
 

@@ -233,12 +241,20 @@ enum ccp_hash_algo {
 	CCP_AUTH_ALGO_SHA1_HMAC,
 	CCP_AUTH_ALGO_SHA224,
 	CCP_AUTH_ALGO_SHA224_HMAC,
+	CCP_AUTH_ALGO_SHA3_224,
+	CCP_AUTH_ALGO_SHA3_224_HMAC,
 	CCP_AUTH_ALGO_SHA256,
 	CCP_AUTH_ALGO_SHA256_HMAC,
+	CCP_AUTH_ALGO_SHA3_256,
+	CCP_AUTH_ALGO_SHA3_256_HMAC,
 	CCP_AUTH_ALGO_SHA384,
 	CCP_AUTH_ALGO_SHA384_HMAC,
+	CCP_AUTH_ALGO_SHA3_384,
+	CCP_AUTH_ALGO_SHA3_384_HMAC,
 	CCP_AUTH_ALGO_SHA512,
 	CCP_AUTH_ALGO_SHA512_HMAC,
+	CCP_AUTH_ALGO_SHA3_512,
+	CCP_AUTH_ALGO_SHA3_512_HMAC,
 	CCP_AUTH_ALGO_AES_CMAC,
 	CCP_AUTH_ALGO_AES_GCM,
 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH

@@ -312,7 +328,9 @@ struct ccp_session {
 		/**<Buffer to store  Software generated precomute values*/
 		/**< For HMAC H(ipad ^ key) and H(opad ^ key) */
 		/**< For CMAC K1 IV and K2 IV*/
-		uint8_t pre_compute[2 * CCP_SHA_CTX_SIZE];
+		uint8_t pre_compute[2 * CCP_SHA3_CTX_SIZE];
+		/**<SHA3 initial ctx all zeros*/
+		uint8_t sha3_ctx[200];
 		int aad_length;
 	} auth;
 	/**< Authentication Parameters */

@@ -372,4 +390,20 @@ int process_ops_to_dequeue(struct ccp_qp *qp,
 			   uint16_t nb_ops);
 
 
+/**
+ * Apis for SHA3 partial hash generation
+ * @param data_in buffer pointer on which phash is applied
+ * @param data_out phash result in ccp be format is written
+ */
+int partial_hash_sha3_224(uint8_t *data_in,
+			  uint8_t *data_out);
+
+int partial_hash_sha3_256(uint8_t *data_in,
+			  uint8_t *data_out);
+
+int partial_hash_sha3_384(uint8_t *data_in,
+			  uint8_t *data_out);
+
+int partial_hash_sha3_512(uint8_t *data_in,
+			  uint8_t *data_out);
 #endif /* _CCP_CRYPTO_H_ */

192 lines not on this page · raw message

index 1fc6569..254fed8 100644
@@ -1593,7 +1593,7 @@ ccp_perform_hmac(struct rte_crypto_op *op,
 
 	session = (struct ccp_session *)get_session_private_data(
 					 op->sym->session,
-					 cryptodev_driver_id);
+					 ccp_cryptodev_driver_id);
 	addr = session->auth.pre_compute;
 
 	src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,

@@ -1766,7 +1766,7 @@ ccp_perform_sha(struct rte_crypto_op *op,
 
 	session = (struct ccp_session *)get_session_private_data(
 					 op->sym->session,
-					 cryptodev_driver_id);
+					ccp_cryptodev_driver_id);
 
 	src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,
 					      op->sym->auth.data.offset);

@@ -1855,7 +1855,7 @@ ccp_perform_sha3_hmac(struct rte_crypto_op *op,
 
 	session = (struct ccp_session *)get_session_private_data(
 					 op->sym->session,
-					 cryptodev_driver_id);
+					ccp_cryptodev_driver_id);
 
 	src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,
 					      op->sym->auth.data.offset);

@@ -1995,7 +1995,7 @@ ccp_perform_sha3(struct rte_crypto_op *op,
 
 	session = (struct ccp_session *)get_session_private_data(
 					 op->sym->session,
-					 cryptodev_driver_id);
+					ccp_cryptodev_driver_id);
 
 	src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,
 					      op->sym->auth.data.offset);

@@ -2063,7 +2063,7 @@ ccp_perform_aes_cmac(struct rte_crypto_op *op,
 
 	session = (struct ccp_session *)get_session_private_data(
 					 op->sym->session,
-					 cryptodev_driver_id);
+					ccp_cryptodev_driver_id);
 	key_addr = rte_mem_virt2phy(session->auth.key_ccp);
 
 	src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,

@@ -2215,7 +2215,7 @@ ccp_perform_aes(struct rte_crypto_op *op,
 
 	session = (struct ccp_session *)get_session_private_data(
 					 op->sym->session,
-					 cryptodev_driver_id);
+					ccp_cryptodev_driver_id);
 	function.raw = 0;
 
 	iv = rte_crypto_op_ctod_offset(op, uint8_t *, session->iv.offset);

@@ -2303,7 +2303,7 @@ ccp_perform_3des(struct rte_crypto_op *op,
 
 	session = (struct ccp_session *)get_session_private_data(
 					 op->sym->session,
-					 cryptodev_driver_id);
+					ccp_cryptodev_driver_id);
 
 	iv = rte_crypto_op_ctod_offset(op, uint8_t *, session->iv.offset);
 	switch (session->cipher.um.des_mode) {

@@ -2406,7 +2406,7 @@ ccp_perform_aes_gcm(struct rte_crypto_op *op, struct ccp_queue *cmd_q)
 
 	session = (struct ccp_session *)get_session_private_data(
 					 op->sym->session,
-					 cryptodev_driver_id);
+					 ccp_cryptodev_driver_id);
 	iv = rte_crypto_op_ctod_offset(op, uint8_t *, session->iv.offset);
 	key_addr = rte_mem_virt2phy(session->cipher.key_ccp);
 

@@ -2578,7 +2578,7 @@ ccp_crypto_cipher(struct rte_crypto_op *op,
 
 	session = (struct ccp_session *)get_session_private_data(
 					 op->sym->session,
-					 cryptodev_driver_id);
+					ccp_cryptodev_driver_id);
 
 	switch (session->cipher.algo) {
 	case CCP_CIPHER_ALGO_AES_CBC:

@@ -2615,7 +2615,7 @@ ccp_crypto_auth(struct rte_crypto_op *op,
 
 	session = (struct ccp_session *)get_session_private_data(
 					 op->sym->session,
-					 cryptodev_driver_id);
+					ccp_cryptodev_driver_id);
 
 	switch (session->auth.algo) {
 	case CCP_AUTH_ALGO_SHA1:

@@ -2676,7 +2676,7 @@ ccp_crypto_combined(struct rte_crypto_op *op,
 
 	session = (struct ccp_session *)get_session_private_data(
 					 op->sym->session,
-					 cryptodev_driver_id);
+					ccp_cryptodev_driver_id);
 
 	switch (session->auth.algo) {
 	case CCP_AUTH_ALGO_AES_GCM:

@@ -2734,7 +2734,7 @@ process_ops_to_enqueue(const struct ccp_qp *qp,
 	for (i = 0; i < nb_ops; i++) {
 		session = (struct ccp_session *)get_session_private_data(
 						 op[i]->sym->session,
-						 cryptodev_driver_id);
+						 ccp_cryptodev_driver_id);
 		switch (session->cmd_id) {
 		case CCP_CMD_CIPHER:
 			result = ccp_crypto_cipher(op[i], cmd_q, b_info);

@@ -2812,7 +2812,7 @@ static inline void ccp_auth_dq_prepare(struct rte_crypto_op *op)
 
 	session = (struct ccp_session *)get_session_private_data(
 					 op->sym->session,
-					 cryptodev_driver_id);
+					ccp_cryptodev_driver_id);
 
 	if (session->cmd_id == CCP_CMD_COMBINED) {
 		digest_data = op->sym->aead.digest.data;

@@ -2889,7 +2889,7 @@ ccp_prepare_ops(struct rte_crypto_op **op_d,
 		op_d[i] = b_info->op[b_info->op_idx++];
 		session = (struct ccp_session *)get_session_private_data(
 						 op_d[i]->sym->session,
-						 cryptodev_driver_id);
+						ccp_cryptodev_driver_id);
 		switch (session->cmd_id) {
 		case CCP_CMD_CIPHER:
 			op_d[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;

index 9d8d42e..ad3251b 100644
@@ -340,7 +340,7 @@ struct ccp_session {
 	uint32_t reserved;
 } __rte_cache_aligned;
 
-extern uint8_t cryptodev_driver_id;
+extern uint8_t ccp_cryptodev_driver_id;
 
 struct ccp_qp;
 

index 9410470..d46c07c 100644
@@ -44,7 +44,7 @@
  * Global static parameter used to find if CCP device is already initialized.
  */
 static unsigned int ccp_pmd_init_done;
-uint8_t cryptodev_driver_id;
+uint8_t ccp_cryptodev_driver_id;
 
 static struct ccp_session *
 get_ccp_session(struct ccp_qp *qp, struct rte_crypto_op *op)

@@ -58,7 +58,7 @@ get_ccp_session(struct ccp_qp *qp, struct rte_crypto_op *op)
 		sess = (struct ccp_session *)
 			get_session_private_data(
 				op->sym->session,
-				cryptodev_driver_id);
+				ccp_cryptodev_driver_id);
 	} else {
 		void *_sess;
 		void *_sess_private_data = NULL;

@@ -77,7 +77,7 @@ get_ccp_session(struct ccp_qp *qp, struct rte_crypto_op *op)
 			sess = NULL;
 		}
 		op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
-		set_session_private_data(op->sym->session, cryptodev_driver_id,
+		set_session_private_data(op->sym->session, ccp_cryptodev_driver_id,
 					 _sess_private_data);
 	}
 

@@ -203,7 +203,7 @@ cryptodev_ccp_create(const char *name,
 	}
 
 	printf("CCP : Crypto device count = %d\n", cryptodev_cnt);
-	dev->driver_id = cryptodev_driver_id;
+	dev->driver_id = ccp_cryptodev_driver_id;
 
 	/* register rx/tx burst functions for data path */
 	dev->dev_ops = ccp_pmd_ops;

@@ -278,4 +278,4 @@ static struct rte_vdev_driver cryptodev_ccp_pmd_drv = {
 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_CCP_PMD, cryptodev_ccp_pmd_drv);
 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_CCP_PMD,
 	"max_nb_queue_pairs=<int> max_nb_sessions=<int> socket_id=<int>");
-RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_ccp_pmd_drv, cryptodev_driver_id);
+RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_ccp_pmd_drv, ccp_cryptodev_driver_id);

index 9e94dcf..daeb59c 100644
@@ -647,8 +647,13 @@ ccp_pmd_info_get(struct rte_cryptodev *dev,
 static int
 ccp_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
 {
+	struct ccp_qp *qp;
+
 	if (dev->data->queue_pairs[qp_id] != NULL) {
-		rte_free(dev->data->queue_pairs[qp_id]);
+		qp = (struct ccp_qp *)dev->data->queue_pairs[qp_id];
+		rte_ring_free(qp->processed_pkts);
+		rte_mempool_free(qp->batch_mp);
+		rte_free(qp);
 		dev->data->queue_pairs[qp_id] = NULL;
 	}
 	return 0;

index 1bed65d..d99be69 100644
@@ -364,6 +364,23 @@ testsuite_setup(void)
 		}
 	}
 
+	/* Create an CCP device if required */
+	if (gbl_driver_id == rte_cryptodev_driver_id_get(
+			RTE_STR(CRYPTODEV_NAME_CCP_PMD))) {
+		nb_devs = rte_cryptodev_device_count_by_driver(
+				rte_cryptodev_driver_id_get(
+				RTE_STR(CRYPTODEV_NAME_CCP_PMD)));
+		if (nb_devs < 1) {
+			ret = rte_vdev_init(
+				RTE_STR(CRYPTODEV_NAME_CCP_PMD),
+				NULL);
+
+			TEST_ASSERT(ret == 0, "Failed to create "
+				"instance of pmd : %s",
+				RTE_STR(CRYPTODEV_NAME_CCP_PMD));
+		}
+	}
+
 #ifdef RTE_LIBRTE_PMD_CRYPTO_SCHEDULER
 	if (gbl_driver_id == rte_cryptodev_driver_id_get(
 			RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD))) {

@@ -1753,6 +1770,44 @@ test_AES_cipheronly_openssl_all(void)
 }
 
 static int
+test_AES_chain_ccp_all(void)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+	int status;
+
+	status = test_blockcipher_all_tests(ts_params->mbuf_pool,
+		ts_params->op_mpool,
+		ts_params->session_mpool,
+		ts_params->valid_devs[0],
+		rte_cryptodev_driver_id_get(
+		RTE_STR(CRYPTODEV_NAME_CCP_PMD)),
+		BLKCIPHER_AES_CHAIN_TYPE);
+
+	TEST_ASSERT_EQUAL(status, 0, "Test failed");
+
+	return TEST_SUCCESS;
+}
+
+static int
+test_AES_cipheronly_ccp_all(void)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+	int status;
+
+	status = test_blockcipher_all_tests(ts_params->mbuf_pool,
+		ts_params->op_mpool,
+		ts_params->session_mpool,
+		ts_params->valid_devs[0],
+		rte_cryptodev_driver_id_get(
+		RTE_STR(CRYPTODEV_NAME_CCP_PMD)),
+		BLKCIPHER_AES_CIPHERONLY_TYPE);
+
+	TEST_ASSERT_EQUAL(status, 0, "Test failed");
+
+	return TEST_SUCCESS;
+}
+
+static int
 test_AES_chain_qat_all(void)
 {
 	struct crypto_testsuite_params *ts_params = &testsuite_params;

@@ -1924,6 +1979,25 @@ test_authonly_openssl_all(void)
 }
 
 static int
+test_authonly_ccp_all(void)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+	int status;
+
+	status = test_blockcipher_all_tests(ts_params->mbuf_pool,
+		ts_params->op_mpool,
+		ts_params->session_mpool,
+		ts_params->valid_devs[0],
+		rte_cryptodev_driver_id_get(
+		RTE_STR(CRYPTODEV_NAME_CCP_PMD)),
+		BLKCIPHER_AUTHONLY_TYPE);
+
+	TEST_ASSERT_EQUAL(status, 0, "Test failed");
+
+	return TEST_SUCCESS;
+}
+
+static int
 test_AES_chain_armv8_all(void)
 {
 	struct crypto_testsuite_params *ts_params = &testsuite_params;

@@ -4999,6 +5073,44 @@ test_3DES_cipheronly_dpaa2_sec_all(void)
 }
 
 static int
+test_3DES_chain_ccp_all(void)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+	int status;
+
+	status = test_blockcipher_all_tests(ts_params->mbuf_pool,
+		ts_params->op_mpool,
+		ts_params->session_mpool,
+		ts_params->valid_devs[0],
+		rte_cryptodev_driver_id_get(
+		RTE_STR(CRYPTODEV_NAME_CCP_PMD)),
+		BLKCIPHER_3DES_CHAIN_TYPE);
+
+	TEST_ASSERT_EQUAL(status, 0, "Test failed");
+
+	return TEST_SUCCESS;
+}
+
+static int
+test_3DES_cipheronly_ccp_all(void)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+	int status;
+
+	status = test_blockcipher_all_tests(ts_params->mbuf_pool,
+		ts_params->op_mpool,
+		ts_params->session_mpool,
+		ts_params->valid_devs[0],
+		rte_cryptodev_driver_id_get(
+		RTE_STR(CRYPTODEV_NAME_CCP_PMD)),
+		BLKCIPHER_3DES_CIPHERONLY_TYPE);
+
+	TEST_ASSERT_EQUAL(status, 0, "Test failed");
+
+	return TEST_SUCCESS;
+}
+
+static int
 test_3DES_cipheronly_qat_all(void)
 {
 	struct crypto_testsuite_params *ts_params = &testsuite_params;

@@ -9580,6 +9692,38 @@ static struct unit_test_suite cryptodev_mrvl_testsuite  = {
 	}
 };
 
+static struct unit_test_suite cryptodev_ccp_testsuite  = {
+	.suite_name = "Crypto Device CCP Unit Test Suite",
+	.setup = testsuite_setup,
+	.teardown = testsuite_teardown,
+	.unit_test_cases = {
+		TEST_CASE_ST(ut_setup, ut_teardown, test_multi_session),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+				test_multi_session_random_usage),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+				test_AES_chain_ccp_all),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+				test_AES_cipheronly_ccp_all),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+				test_3DES_chain_ccp_all),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+				test_3DES_cipheronly_ccp_all),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+				test_authonly_ccp_all),
+
+		/** Negative tests */
+		TEST_CASE_ST(ut_setup, ut_teardown,
+			authentication_verify_HMAC_SHA1_fail_data_corrupt),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+			authentication_verify_HMAC_SHA1_fail_tag_corrupt),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+			auth_decryption_AES128CBC_HMAC_SHA1_fail_data_corrupt),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+			auth_decryption_AES128CBC_HMAC_SHA1_fail_tag_corrupt),
+
+		TEST_CASES_END() /**< NULL terminate unit test array */
+	}
+};
 
 static int
 test_cryptodev_qat(void /*argv __rte_unused, int argc __rte_unused*/)

@@ -9801,6 +9945,22 @@ test_cryptodev_dpaa_sec(void /*argv __rte_unused, int argc __rte_unused*/)
 	return unit_test_suite_runner(&cryptodev_dpaa_sec_testsuite);
 }
 
+static int
+test_cryptodev_ccp(void)
+{
+	gbl_driver_id = rte_cryptodev_driver_id_get(
+			RTE_STR(CRYPTODEV_NAME_CCP_PMD));
+
+	if (gbl_driver_id == -1) {
+		RTE_LOG(ERR, USER1, "CCP PMD must be loaded. Check if "
+				"CONFIG_RTE_LIBRTE_PMD_CCP is enabled "
+				"in config file to run this testsuite.\n");
+		return TEST_FAILED;
+	}
+
+	return unit_test_suite_runner(&cryptodev_ccp_testsuite);
+}
+
 REGISTER_TEST_COMMAND(cryptodev_qat_autotest, test_cryptodev_qat);
 REGISTER_TEST_COMMAND(cryptodev_aesni_mb_autotest, test_cryptodev_aesni_mb);
 REGISTER_TEST_COMMAND(cryptodev_openssl_autotest, test_cryptodev_openssl);

@@ -9813,3 +9973,4 @@ REGISTER_TEST_COMMAND(cryptodev_sw_armv8_autotest, test_cryptodev_armv8);
 REGISTER_TEST_COMMAND(cryptodev_sw_mrvl_autotest, test_cryptodev_mrvl);
 REGISTER_TEST_COMMAND(cryptodev_dpaa2_sec_autotest, test_cryptodev_dpaa2_sec);
 REGISTER_TEST_COMMAND(cryptodev_dpaa_sec_autotest, test_cryptodev_dpaa_sec);
+REGISTER_TEST_COMMAND(cryptodev_ccp_autotest, test_cryptodev_ccp);

index 26bfbe6..9fa68f9 100644
@@ -89,6 +89,7 @@
 #define CRYPTODEV_NAME_DPAA2_SEC_PMD	crypto_dpaa2_sec
 #define CRYPTODEV_NAME_SCHEDULER_PMD	crypto_scheduler
 #define CRYPTODEV_NAME_MRVL_PMD		crypto_mrvl
+#define CRYPTODEV_NAME_CCP_PMD		crypto_ccp
 
 /**
  * Write (spread) data from buffer to mbuf data

279 lines not on this page · raw message

index 20f3296..0091a81 100644
@@ -82,6 +82,8 @@ test_blockcipher_one_case(const struct blockcipher_test_case *t,
 
 	int openssl_pmd = rte_cryptodev_driver_id_get(
 			RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD));
+	int ccp_pmd = rte_cryptodev_driver_id_get(
+			RTE_STR(CRYPTODEV_NAME_CCP_PMD));
 	int scheduler_pmd = rte_cryptodev_driver_id_get(
 			RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD));
 	int armv8_pmd = rte_cryptodev_driver_id_get(

@@ -122,7 +124,8 @@ test_blockcipher_one_case(const struct blockcipher_test_case *t,
 			driver_id == qat_pmd ||
 			driver_id == openssl_pmd ||
 			driver_id == armv8_pmd ||
-			driver_id == mrvl_pmd) { /* Fall through */
+			driver_id == mrvl_pmd) ||
+			driver_id == ccp_pmd) { /* Fall through */
 		digest_len = tdata->digest.len;
 	} else if (driver_id == aesni_mb_pmd ||
 			driver_id == scheduler_pmd) {

@@ -583,6 +586,8 @@ test_blockcipher_all_tests(struct rte_mempool *mbuf_pool,
 
 	int openssl_pmd = rte_cryptodev_driver_id_get(
 			RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD));
+	int ccp_pmd = rte_cryptodev_driver_id_get(
+			RTE_STR(CRYPTODEV_NAME_CCP_PMD));
 	int dpaa2_sec_pmd = rte_cryptodev_driver_id_get(
 			RTE_STR(CRYPTODEV_NAME_DPAA2_SEC_PMD));
 	int dpaa_sec_pmd = rte_cryptodev_driver_id_get(

@@ -655,6 +660,8 @@ test_blockcipher_all_tests(struct rte_mempool *mbuf_pool,
 		target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER;
 	else if (driver_id == dpaa2_sec_pmd)
 		target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC;
+	else if (driver_id == ccp_pmd)
+		target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_CCP;
 	else if (driver_id == dpaa_sec_pmd)
 		target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC;
 	else if (driver_id == mrvl_pmd)

index 67c78d4..b6824f6 100644
@@ -55,6 +55,7 @@
 #define BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC	0x0020 /* DPAA2_SEC flag */
 #define BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC	0x0040 /* DPAA_SEC flag */
 #define BLOCKCIPHER_TEST_TARGET_PMD_MRVL	0x0080 /* Marvell flag */
+#define BLOCKCIPHER_TEST_TARGET_PMD_CCP		0x0040 /* CCP flag */
 
 #define BLOCKCIPHER_TEST_OP_CIPHER	(BLOCKCIPHER_TEST_OP_ENCRYPT | \
 					BLOCKCIPHER_TEST_OP_DECRYPT)

124 lines not on this page · raw message

171 lines not on this page · raw message

index 28eb68e..832407a 100644
@@ -36,6 +36,7 @@
 #include <stdint.h>
 #include <string.h>
 
+#include <rte_bus_pci.h>
 #include <rte_atomic.h>
 #include <rte_byteorder.h>
 #include <rte_io.h>

index a0edf75..292234d 100644
@@ -33,7 +33,7 @@
 
 #include <stdint.h>
 
-#include <rte_pci.h>
+#include <rte_bus_pci.h>
 
 #define SYSFS_PCI_DEVICES "/sys/bus/pci/devices"
 #define PROC_MODULES "/proc/modules"

index d46c07c..f1369be 100644
@@ -28,12 +28,14 @@
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include <rte_bus_pci.h>
+#include <rte_bus_vdev.h>
 #include <rte_common.h>
 #include <rte_config.h>
 #include <rte_cryptodev.h>
 #include <rte_cryptodev_pmd.h>
-#include <rte_cryptodev_vdev.h>
-#include <rte_vdev.h>
+#include <rte_pci.h>
+#include <rte_dev.h>
 #include <rte_malloc.h>
 
 #include <ccp_crypto.h>

@@ -176,7 +178,7 @@ cryptodev_ccp_remove(struct rte_vdev_device *dev)
 static int
 cryptodev_ccp_create(const char *name,
 		     struct rte_vdev_device *vdev,
-		     struct rte_crypto_vdev_init_params *init_params)
+		     struct rte_cryptodev_pmd_init_params *init_params)
 {
 	struct rte_cryptodev *dev;
 	struct ccp_private *internals;

@@ -186,10 +188,9 @@ cryptodev_ccp_create(const char *name,
 		snprintf(init_params->name, sizeof(init_params->name),
 				"%s", name);
 
-	dev = rte_cryptodev_vdev_pmd_init(init_params->name,
-			sizeof(struct ccp_private),
-			init_params->socket_id,
-			vdev);
+	dev = rte_cryptodev_pmd_create(init_params->name,
+				       &vdev->device,
+				       init_params);
 	if (dev == NULL) {
 		CCP_LOG_ERR("failed to create cryptodev vdev");
 		goto init_error;

@@ -236,11 +237,12 @@ cryptodev_ccp_probe(struct rte_vdev_device *vdev)
 {
 	int rc = 0;
 	const char *name;
-	struct rte_crypto_vdev_init_params init_params = {
-		CCP_PMD_MAX_QUEUE_PAIRS,
-		RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS,
+	struct rte_cryptodev_pmd_init_params init_params = {
+		"",
+		sizeof(struct ccp_private),
 		rte_socket_id(),
-		{0}
+		CCP_PMD_MAX_QUEUE_PAIRS,
+		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS
 	};
 	const char *input_args;
 

@@ -253,7 +255,7 @@ cryptodev_ccp_probe(struct rte_vdev_device *vdev)
 		return -EINVAL;
 
 	input_args = rte_vdev_device_args(vdev);
-	rte_cryptodev_vdev_parse_init_params(&init_params, input_args);
+	rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
 	init_params.max_nb_queue_pairs = CCP_PMD_MAX_QUEUE_PAIRS;
 
 	RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name,

@@ -275,7 +277,10 @@ static struct rte_vdev_driver cryptodev_ccp_pmd_drv = {
 	.remove = cryptodev_ccp_remove
 };
 
+static struct cryptodev_driver ccp_crypto_drv;
+
 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_CCP_PMD, cryptodev_ccp_pmd_drv);
 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_CCP_PMD,
 	"max_nb_queue_pairs=<int> max_nb_sessions=<int> socket_id=<int>");
-RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_ccp_pmd_drv, ccp_cryptodev_driver_id);
+RTE_PMD_REGISTER_CRYPTO_DRIVER(ccp_crypto_drv, cryptodev_ccp_pmd_drv,
+			       ccp_cryptodev_driver_id);

62 lines not on this page · raw message

787 lines not on this page · raw message

310 lines not on this page · raw message

262 lines not on this page · raw message

53 lines not on this page · raw message

55 lines not on this page · raw message

82 lines not on this page · raw message

172 lines not on this page · raw message

index 5af2b49..57bccf4 100644
@@ -52,6 +52,15 @@
 struct ccp_list ccp_list = TAILQ_HEAD_INITIALIZER(ccp_list);
 static int ccp_dev_id;
 
+int
+ccp_dev_start(struct rte_cryptodev *dev)
+{
+	struct ccp_private *priv = dev->data->dev_private;
+
+	priv->last_dev = TAILQ_FIRST(&ccp_list);
+	return 0;
+}
+
 static const struct rte_memzone *
 ccp_queue_dma_zone_reserve(const char *queue_name,
 			   uint32_t queue_size,

index fe05bf0..b321530 100644
@@ -104,6 +104,10 @@
 #define LSB_ITEM_SIZE                   32
 #define SLSB_MAP_SIZE                   (MAX_LSB_CNT * LSB_SIZE)
 
+/* General CCP Defines */
+
+#define CCP_SB_BYTES                    32
+
 /* bitmap */
 enum {
 	BITS_PER_WORD = sizeof(unsigned long) * CHAR_BIT

@@ -299,6 +303,11 @@ high32_value(unsigned long addr)
 	return ((uint64_t)addr >> 32) & 0x00000ffff;
 }
 
+/*
+ * Start CCP device
+ */
+int ccp_dev_start(struct rte_cryptodev *dev);
+
 /**
  * Detect ccp platform and initialize all ccp devices
  *

74 lines not on this page · raw message

56 lines not on this page · raw message

230 lines not on this page · raw message

267 lines not on this page · raw message

141 lines not on this page · raw message

79 lines not on this page · raw message

index ad0a670..a02aa6f 100644
@@ -82,6 +82,145 @@ ccp_pmd_info_get(struct rte_cryptodev *dev,
 	}
 }
 
+static int
+ccp_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
+{
+	struct ccp_qp *qp;
+
+	if (dev->data->queue_pairs[qp_id] != NULL) {
+		qp = (struct ccp_qp *)dev->data->queue_pairs[qp_id];
+		rte_ring_free(qp->processed_pkts);
+		rte_mempool_free(qp->batch_mp);
+		rte_free(qp);
+		dev->data->queue_pairs[qp_id] = NULL;
+	}
+	return 0;
+}
+
+static int
+ccp_pmd_qp_set_unique_name(struct rte_cryptodev *dev,
+		struct ccp_qp *qp)
+{
+	unsigned int n = snprintf(qp->name, sizeof(qp->name),
+			"ccp_pmd_%u_qp_%u",
+			dev->data->dev_id, qp->id);
+
+	if (n > sizeof(qp->name))
+		return -1;
+
+	return 0;
+}
+
+static struct rte_ring *
+ccp_pmd_qp_create_batch_info_ring(struct ccp_qp *qp,
+				  unsigned int ring_size, int socket_id)
+{
+	struct rte_ring *r;
+
+	r = rte_ring_lookup(qp->name);
+	if (r) {
+		if (r->size >= ring_size) {
+			CCP_LOG_INFO(
+				"Reusing ring %s for processed packets",
+				qp->name);
+			return r;
+		}
+		CCP_LOG_INFO(
+			"Unable to reuse ring %s for processed packets",
+			 qp->name);
+		return NULL;
+	}
+
+	return rte_ring_create(qp->name, ring_size, socket_id,
+			RING_F_SP_ENQ | RING_F_SC_DEQ);
+}
+
+static int
+ccp_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
+		 const struct rte_cryptodev_qp_conf *qp_conf,
+		 int socket_id, struct rte_mempool *session_pool)
+{
+	struct ccp_private *internals = dev->data->dev_private;
+	struct ccp_qp *qp;
+	int retval = 0;
+
+	if (qp_id >= internals->max_nb_qpairs) {
+		CCP_LOG_ERR("Invalid qp_id %u, should be less than %u",
+			    qp_id, internals->max_nb_qpairs);
+		return (-EINVAL);
+	}
+
+	/* Free memory prior to re-allocation if needed. */
+	if (dev->data->queue_pairs[qp_id] != NULL)
+		ccp_pmd_qp_release(dev, qp_id);
+
+	/* Allocate the queue pair data structure. */
+	qp = rte_zmalloc_socket("CCP Crypto PMD Queue Pair", sizeof(*qp),
+					RTE_CACHE_LINE_SIZE, socket_id);
+	if (qp == NULL) {
+		CCP_LOG_ERR("Failed to allocate queue pair memory");
+		return (-ENOMEM);
+	}
+
+	qp->dev = dev;
+	qp->id = qp_id;
+	dev->data->queue_pairs[qp_id] = qp;
+
+	retval = ccp_pmd_qp_set_unique_name(dev, qp);
+	if (retval) {
+		CCP_LOG_ERR("Failed to create unique name for ccp qp");
+		goto qp_setup_cleanup;
+	}
+
+	qp->processed_pkts = ccp_pmd_qp_create_batch_info_ring(qp,
+			qp_conf->nb_descriptors, socket_id);
+	if (qp->processed_pkts == NULL) {
+		CCP_LOG_ERR("Failed to create batch info ring");
+		goto qp_setup_cleanup;
+	}
+
+	qp->sess_mp = session_pool;
+
+	/* mempool for batch info */
+	qp->batch_mp = rte_mempool_create(
+				qp->name,
+				qp_conf->nb_descriptors,
+				sizeof(struct ccp_batch_info),
+				RTE_CACHE_LINE_SIZE,
+				0, NULL, NULL, NULL, NULL,
+				SOCKET_ID_ANY, 0);
+	if (qp->batch_mp == NULL)
+		goto qp_setup_cleanup;
+	memset(&qp->qp_stats, 0, sizeof(qp->qp_stats));
+	return 0;
+
+qp_setup_cleanup:
+	dev->data->queue_pairs[qp_id] = NULL;
+	if (qp)
+		rte_free(qp);
+	return -1;
+}
+
+static int
+ccp_pmd_qp_start(struct rte_cryptodev *dev __rte_unused,
+		 uint16_t queue_pair_id __rte_unused)
+{
+	return -ENOTSUP;
+}
+
+static int
+ccp_pmd_qp_stop(struct rte_cryptodev *dev __rte_unused,
+		uint16_t queue_pair_id __rte_unused)
+{
+	return -ENOTSUP;
+}
+
+static uint32_t
+ccp_pmd_qp_count(struct rte_cryptodev *dev)
+{
+	return dev->data->nb_queue_pairs;
+}
+
 static unsigned
 ccp_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused)
 {

@@ -147,11 +286,11 @@ struct rte_cryptodev_ops ccp_ops = {
 
 		.dev_infos_get		= ccp_pmd_info_get,
 
-		.queue_pair_setup	= NULL,
-		.queue_pair_release	= NULL,
-		.queue_pair_start	= NULL,
-		.queue_pair_stop	= NULL,
-		.queue_pair_count	= NULL,
+		.queue_pair_setup	= ccp_pmd_qp_setup,
+		.queue_pair_release	= ccp_pmd_qp_release,
+		.queue_pair_start	= ccp_pmd_qp_start,
+		.queue_pair_stop	= ccp_pmd_qp_stop,
+		.queue_pair_count	= ccp_pmd_qp_count,
 
 		.session_get_size	= ccp_pmd_session_get_size,
 		.session_configure	= ccp_pmd_session_configure,

index 80abcdc..c17e84f 100644
@@ -228,3 +228,362 @@ ccp_set_session_parameters(struct ccp_session *sess,
 	return ret;
 }
 
+/* calculate CCP descriptors requirement */
+static inline int
+ccp_cipher_slot(struct ccp_session *session)
+{
+	int count = 0;
+
+	switch (session->cipher.algo) {
+	default:
+		CCP_LOG_ERR("Unsupported cipher algo %d",
+			    session->cipher.algo);
+	}
+	return count;
+}
+
+static inline int
+ccp_auth_slot(struct ccp_session *session)
+{
+	int count = 0;
+
+	switch (session->auth.algo) {
+	default:
+		CCP_LOG_ERR("Unsupported auth algo %d",
+			    session->auth.algo);
+	}
+
+	return count;
+}
+
+static int
+ccp_aead_slot(struct ccp_session *session)
+{
+	int count = 0;
+
+	switch (session->aead_algo) {
+	default:
+		CCP_LOG_ERR("Unsupported aead algo %d",
+			    session->aead_algo);
+	}
+	return count;
+}
+
+int
+ccp_compute_slot_count(struct ccp_session *session)
+{
+	int count = 0;
+
+	switch (session->cmd_id) {
+	case CCP_CMD_CIPHER:
+		count = ccp_cipher_slot(session);
+		break;
+	case CCP_CMD_AUTH:
+		count = ccp_auth_slot(session);
+		break;
+	case CCP_CMD_CIPHER_HASH:
+	case CCP_CMD_HASH_CIPHER:
+		count = ccp_cipher_slot(session);
+		count += ccp_auth_slot(session);
+		break;
+	case CCP_CMD_COMBINED:
+		count = ccp_aead_slot(session);
+		break;
+	default:
+		CCP_LOG_ERR("Unsupported cmd_id");
+
+	}
+
+	return count;
+}
+
+static inline int
+ccp_crypto_cipher(struct rte_crypto_op *op,
+		  struct ccp_queue *cmd_q __rte_unused,
+		  struct ccp_batch_info *b_info __rte_unused)
+{
+	int result = 0;
+	struct ccp_session *session;
+
+	session = (struct ccp_session *)get_session_private_data(
+					 op->sym->session,
+					 ccp_cryptodev_driver_id);
+
+	switch (session->cipher.algo) {
+	default:
+		CCP_LOG_ERR("Unsupported cipher algo %d",
+			    session->cipher.algo);
+		return -ENOTSUP;
+	}
+	return result;
+}
+
+static inline int
+ccp_crypto_auth(struct rte_crypto_op *op,
+		struct ccp_queue *cmd_q __rte_unused,
+		struct ccp_batch_info *b_info __rte_unused)
+{
+
+	int result = 0;
+	struct ccp_session *session;
+
+	session = (struct ccp_session *)get_session_private_data(
+					 op->sym->session,
+					ccp_cryptodev_driver_id);
+
+	switch (session->auth.algo) {
+	default:
+		CCP_LOG_ERR("Unsupported auth algo %d",
+			    session->auth.algo);
+		return -ENOTSUP;
+	}
+
+	return result;
+}
+
+static inline int
+ccp_crypto_aead(struct rte_crypto_op *op,
+		struct ccp_queue *cmd_q __rte_unused,
+		struct ccp_batch_info *b_info __rte_unused)
+{
+	int result = 0;
+	struct ccp_session *session;
+
+	session = (struct ccp_session *)get_session_private_data(
+					 op->sym->session,
+					ccp_cryptodev_driver_id);
+
+	switch (session->aead_algo) {
+	default:
+		CCP_LOG_ERR("Unsupported aead algo %d",
+			    session->aead_algo);
+		return -ENOTSUP;
+	}
+	return result;
+}
+
+int
+process_ops_to_enqueue(const struct ccp_qp *qp,
+		       struct rte_crypto_op **op,
+		       struct ccp_queue *cmd_q,
+		       uint16_t nb_ops,
+		       int slots_req)
+{
+	int i, result = 0;
+	struct ccp_batch_info *b_info;
+	struct ccp_session *session;
+
+	if (rte_mempool_get(qp->batch_mp, (void **)&b_info)) {
+		CCP_LOG_ERR("batch info allocation failed");
+		return 0;
+	}
+	/* populate batch info necessary for dequeue */
+	b_info->op_idx = 0;
+	b_info->lsb_buf_idx = 0;
+	b_info->desccnt = 0;
+	b_info->cmd_q = cmd_q;
+	b_info->lsb_buf_phys =
+		(phys_addr_t)rte_mem_virt2phy((void *)b_info->lsb_buf);
+	rte_atomic64_sub(&b_info->cmd_q->free_slots, slots_req);
+
+	b_info->head_offset = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx *
+					 Q_DESC_SIZE);
+	for (i = 0; i < nb_ops; i++) {
+		session = (struct ccp_session *)get_session_private_data(
+						 op[i]->sym->session,
+						 ccp_cryptodev_driver_id);
+		switch (session->cmd_id) {
+		case CCP_CMD_CIPHER:
+			result = ccp_crypto_cipher(op[i], cmd_q, b_info);
+			break;
+		case CCP_CMD_AUTH:
+			result = ccp_crypto_auth(op[i], cmd_q, b_info);
+			break;
+		case CCP_CMD_CIPHER_HASH:
+			result = ccp_crypto_cipher(op[i], cmd_q, b_info);
+			if (result)
+				break;
+			result = ccp_crypto_auth(op[i], cmd_q, b_info);
+			break;
+		case CCP_CMD_HASH_CIPHER:
+			result = ccp_crypto_auth(op[i], cmd_q, b_info);
+			if (result)
+				break;
+			result = ccp_crypto_cipher(op[i], cmd_q, b_info);
+			break;
+		case CCP_CMD_COMBINED:
+			result = ccp_crypto_aead(op[i], cmd_q, b_info);
+			break;
+		default:
+			CCP_LOG_ERR("Unsupported cmd_id");
+			result = -1;
+		}
+		if (unlikely(result < 0)) {
+			rte_atomic64_add(&b_info->cmd_q->free_slots,
+					 (slots_req - b_info->desccnt));
+			break;
+		}
+		b_info->op[i] = op[i];
+	}
+
+	b_info->opcnt = i;
+	b_info->tail_offset = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx *
+					 Q_DESC_SIZE);
+
+	rte_wmb();
+	/* Write the new tail address back to the queue register */
+	CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE,
+			      b_info->tail_offset);
+	/* Turn the queue back on using our cached control register */
+	CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
+			      cmd_q->qcontrol | CMD_Q_RUN);
+
+	rte_ring_enqueue(qp->processed_pkts, (void *)b_info);
+
+	return i;
+}
+
+static inline void ccp_auth_dq_prepare(struct rte_crypto_op *op)
+{
+	struct ccp_session *session;
+	uint8_t *digest_data, *addr;
+	struct rte_mbuf *m_last;
+	int offset, digest_offset;
+	uint8_t digest_le[64];
+
+	session = (struct ccp_session *)get_session_private_data(
+					 op->sym->session,
+					ccp_cryptodev_driver_id);
+
+	if (session->cmd_id == CCP_CMD_COMBINED) {
+		digest_data = op->sym->aead.digest.data;
+		digest_offset = op->sym->aead.data.offset +
+					op->sym->aead.data.length;
+	} else {
+		digest_data = op->sym->auth.digest.data;
+		digest_offset = op->sym->auth.data.offset +
+					op->sym->auth.data.length;
+	}
+	m_last = rte_pktmbuf_lastseg(op->sym->m_src);
+	addr = (uint8_t *)((char *)m_last->buf_addr + m_last->data_off +
+			   m_last->data_len - session->auth.ctx_len);
+
+	rte_mb();
+	offset = session->auth.offset;
+
+	if (session->auth.engine == CCP_ENGINE_SHA)
+		if ((session->auth.ut.sha_type != CCP_SHA_TYPE_1) &&
+		    (session->auth.ut.sha_type != CCP_SHA_TYPE_224) &&
+		    (session->auth.ut.sha_type != CCP_SHA_TYPE_256)) {
+			/* All other algorithms require byte
+			 * swap done by host
+			 */
+			unsigned int i;
+
+			offset = session->auth.ctx_len -
+				session->auth.offset - 1;
+			for (i = 0; i < session->auth.digest_length; i++)
+				digest_le[i] = addr[offset - i];
+			offset = 0;
+			addr = digest_le;
+		}
+
+	op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+	if (session->auth.op == CCP_AUTH_OP_VERIFY) {
+		if (memcmp(addr + offset, digest_data,
+			   session->auth.digest_length) != 0)
+			op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
+
+	} else {
+		if (unlikely(digest_data == 0))
+			digest_data = rte_pktmbuf_mtod_offset(
+					op->sym->m_dst, uint8_t *,
+					digest_offset);
+		rte_memcpy(digest_data, addr + offset,
+			   session->auth.digest_length);
+	}
+	/* Trim area used for digest from mbuf. */
+	rte_pktmbuf_trim(op->sym->m_src,
+			 session->auth.ctx_len);
+}
+
+static int
+ccp_prepare_ops(struct rte_crypto_op **op_d,
+		struct ccp_batch_info *b_info,
+		uint16_t nb_ops)
+{
+	int i, min_ops;
+	struct ccp_session *session;
+
+	min_ops = RTE_MIN(nb_ops, b_info->opcnt);
+
+	for (i = 0; i < min_ops; i++) {
+		op_d[i] = b_info->op[b_info->op_idx++];
+		session = (struct ccp_session *)get_session_private_data(
+						 op_d[i]->sym->session,
+						ccp_cryptodev_driver_id);
+		switch (session->cmd_id) {
+		case CCP_CMD_CIPHER:
+			op_d[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+			break;
+		case CCP_CMD_AUTH:
+		case CCP_CMD_CIPHER_HASH:
+		case CCP_CMD_HASH_CIPHER:
+		case CCP_CMD_COMBINED:
+			ccp_auth_dq_prepare(op_d[i]);
+			break;
+		default:
+			CCP_LOG_ERR("Unsupported cmd_id");
+		}
+	}
+
+	b_info->opcnt -= min_ops;
+	return min_ops;
+}
+
+int
+process_ops_to_dequeue(struct ccp_qp *qp,
+		       struct rte_crypto_op **op,
+		       uint16_t nb_ops)
+{
+	struct ccp_batch_info *b_info;
+	uint32_t cur_head_offset;
+
+	if (qp->b_info != NULL) {
+		b_info = qp->b_info;
+		if (unlikely(b_info->op_idx > 0))
+			goto success;
+	} else if (rte_ring_dequeue(qp->processed_pkts,
+				    (void **)&b_info))
+		return 0;
+	cur_head_offset = CCP_READ_REG(b_info->cmd_q->reg_base,
+				       CMD_Q_HEAD_LO_BASE);
+
+	if (b_info->head_offset < b_info->tail_offset) {
+		if ((cur_head_offset >= b_info->head_offset) &&
+		    (cur_head_offset < b_info->tail_offset)) {
+			qp->b_info = b_info;
+			return 0;
+		}
+	} else {
+		if ((cur_head_offset >= b_info->head_offset) ||
+		    (cur_head_offset < b_info->tail_offset)) {
+			qp->b_info = b_info;
+			return 0;
+		}
+	}
+
+
+success:
+	nb_ops = ccp_prepare_ops(op, b_info, nb_ops);
+	rte_atomic64_add(&b_info->cmd_q->free_slots, b_info->desccnt);
+	b_info->desccnt = 0;
+	if (b_info->opcnt > 0) {
+		qp->b_info = b_info;
+	} else {
+		rte_mempool_put(qp->batch_mp, (void *)b_info);
+		qp->b_info = NULL;
+	}
+
+	return nb_ops;
+}

index 346d5ee..4455497 100644
@@ -264,4 +264,39 @@ struct ccp_qp;
 int ccp_set_session_parameters(struct ccp_session *sess,
 			       const struct rte_crypto_sym_xform *xform);
 
+/**
+ * Find count of slots
+ *
+ * @param session CCP private session
+ * @return count of free slots available
+ */
+int ccp_compute_slot_count(struct ccp_session *session);
+
+/**
+ * process crypto ops to be enqueued
+ *
+ * @param qp CCP crypto queue-pair
+ * @param op crypto ops table
+ * @param cmd_q CCP cmd queue
+ * @param nb_ops No. of ops to be submitted
+ * @return 0 on success otherwise -1
+ */
+int process_ops_to_enqueue(const struct ccp_qp *qp,
+			   struct rte_crypto_op **op,
+			   struct ccp_queue *cmd_q,
+			   uint16_t nb_ops,
+			   int slots_req);
+
+/**
+ * process crypto ops to be dequeued
+ *
+ * @param qp CCP crypto queue-pair
+ * @param op crypto ops table
+ * @param nb_ops requested no. of ops
+ * @return 0 on success otherwise -1
+ */
+int process_ops_to_dequeue(struct ccp_qp *qp,
+			   struct rte_crypto_op **op,
+			   uint16_t nb_ops);
+
 #endif /* _CCP_CRYPTO_H_ */

index 57bccf4..fee90e3 100644
@@ -61,6 +61,33 @@ ccp_dev_start(struct rte_cryptodev *dev)
 	return 0;
 }
 
+struct ccp_queue *
+ccp_allot_queue(struct rte_cryptodev *cdev, int slot_req)
+{
+	int i, ret = 0;
+	struct ccp_device *dev;
+	struct ccp_private *priv = cdev->data->dev_private;
+
+	dev = TAILQ_NEXT(priv->last_dev, next);
+	if (unlikely(dev == NULL))
+		dev = TAILQ_FIRST(&ccp_list);
+	priv->last_dev = dev;
+	if (dev->qidx >= dev->cmd_q_count)
+		dev->qidx = 0;
+	ret = rte_atomic64_read(&dev->cmd_q[dev->qidx].free_slots);
+	if (ret >= slot_req)
+		return &dev->cmd_q[dev->qidx];
+	for (i = 0; i < dev->cmd_q_count; i++) {
+		dev->qidx++;
+		if (dev->qidx >= dev->cmd_q_count)
+			dev->qidx = 0;
+		ret = rte_atomic64_read(&dev->cmd_q[dev->qidx].free_slots);
+		if (ret >= slot_req)
+			return &dev->cmd_q[dev->qidx];
+	}
+	return NULL;
+}
+
 static const struct rte_memzone *
 ccp_queue_dma_zone_reserve(const char *queue_name,
 			   uint32_t queue_size,

index a16ba81..cfb3b03 100644
@@ -445,4 +445,13 @@ int ccp_dev_start(struct rte_cryptodev *dev);
  */
 int ccp_probe_devices(const struct rte_pci_id *ccp_id);
 
+/**
+ * allocate a ccp command queue
+ *
+ * @dev rte crypto device
+ * @param slot_req number of required
+ * @return allotted CCP queue on success otherwise NULL
+ */
+struct ccp_queue *ccp_allot_queue(struct rte_cryptodev *dev, int slot_req);
+
 #endif /* _CCP_DEV_H_ */

86 lines not on this page · raw message

index fb6d41e..011240b 100644
@@ -49,7 +49,7 @@ static unsigned int ccp_pmd_init_done;
 uint8_t ccp_cryptodev_driver_id;
 
 static struct ccp_session *
-get_ccp_session(struct ccp_qp *qp __rte_unused, struct rte_crypto_op *op)
+get_ccp_session(struct ccp_qp *qp, struct rte_crypto_op *op)
 {
 	struct ccp_session *sess = NULL;
 

@@ -61,6 +61,27 @@ get_ccp_session(struct ccp_qp *qp __rte_unused, struct rte_crypto_op *op)
 			get_session_private_data(
 				op->sym->session,
 				ccp_cryptodev_driver_id);
+	} else if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
+		void *_sess;
+		void *_sess_private_data = NULL;
+
+		if (rte_mempool_get(qp->sess_mp, &_sess))
+			return NULL;
+		if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data))
+			return NULL;
+
+		sess = (struct ccp_session *)_sess_private_data;
+
+		if (unlikely(ccp_set_session_parameters(sess,
+							op->sym->xform) != 0)) {
+			rte_mempool_put(qp->sess_mp, _sess);
+			rte_mempool_put(qp->sess_mp, _sess_private_data);
+			sess = NULL;
+		}
+		op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
+		set_session_private_data(op->sym->session,
+					 ccp_cryptodev_driver_id,
+					 _sess_private_data);
 	}
 
 	return sess;

@@ -108,10 +129,18 @@ ccp_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 		uint16_t nb_ops)
 {
 	struct ccp_qp *qp = queue_pair;
-	uint16_t nb_dequeued = 0;
+	uint16_t nb_dequeued = 0, i;
 
 	nb_dequeued = process_ops_to_dequeue(qp, ops, nb_ops);
 
+	/* Free session if a session-less crypto op */
+	for (i = 0; i < nb_dequeued; i++)
+		if (unlikely(ops[i]->sess_type ==
+			     RTE_CRYPTO_OP_SESSIONLESS)) {
+			rte_mempool_put(qp->sess_mp,
+					ops[i]->sym->session);
+			ops[i]->sym->session = NULL;
+		}
 	qp->qp_stats.dequeued_count += nb_dequeued;
 
 	return nb_dequeued;

index a02aa6f..d483a74 100644
@@ -68,6 +68,36 @@ ccp_pmd_close(struct rte_cryptodev *dev __rte_unused)
 }
 
 static void
+ccp_pmd_stats_get(struct rte_cryptodev *dev,
+		  struct rte_cryptodev_stats *stats)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct ccp_qp *qp = dev->data->queue_pairs[qp_id];
+
+		stats->enqueued_count += qp->qp_stats.enqueued_count;
+		stats->dequeued_count += qp->qp_stats.dequeued_count;
+
+		stats->enqueue_err_count += qp->qp_stats.enqueue_err_count;
+		stats->dequeue_err_count += qp->qp_stats.dequeue_err_count;
+	}
+
+}
+
+static void
+ccp_pmd_stats_reset(struct rte_cryptodev *dev)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct ccp_qp *qp = dev->data->queue_pairs[qp_id];
+
+		memset(&qp->qp_stats, 0, sizeof(qp->qp_stats));
+	}
+}
+
+static void
 ccp_pmd_info_get(struct rte_cryptodev *dev,
 		 struct rte_cryptodev_info *dev_info)
 {

@@ -281,8 +311,8 @@ struct rte_cryptodev_ops ccp_ops = {
 		.dev_stop		= ccp_pmd_stop,
 		.dev_close		= ccp_pmd_close,
 
-		.stats_get		= NULL,
-		.stats_reset		= NULL,
+		.stats_get		= ccp_pmd_stats_get,
+		.stats_reset		= ccp_pmd_stats_reset,
 
 		.dev_infos_get		= ccp_pmd_info_get,
 

index fee90e3..d8c0ab4 100644
@@ -88,6 +88,26 @@ ccp_allot_queue(struct rte_cryptodev *cdev, int slot_req)
 	return NULL;
 }
 
+int
+ccp_read_hwrng(uint32_t *value)
+{
+	struct ccp_device *dev;
+
+	TAILQ_FOREACH(dev, &ccp_list, next) {
+		void *vaddr = (void *)(dev->pci.mem_resource[2].addr);
+
+		while (dev->hwrng_retries++ < CCP_MAX_TRNG_RETRIES) {
+			*value = CCP_READ_REG(vaddr, TRNG_OUT_REG);
+			if (*value) {
+				dev->hwrng_retries = 0;
+				return 0;
+			}
+		}
+		dev->hwrng_retries = 0;
+	}
+	return -1;
+}
+
 static const struct rte_memzone *
 ccp_queue_dma_zone_reserve(const char *queue_name,
 			   uint32_t queue_size,

index cfb3b03..a5c9ef3 100644
@@ -47,6 +47,7 @@
 
 /**< CCP sspecific */
 #define MAX_HW_QUEUES                   5
+#define CCP_MAX_TRNG_RETRIES		10
 
 /**< CCP Register Mappings */
 #define Q_MASK_REG                      0x000

@@ -223,6 +224,8 @@ struct ccp_device {
 	/**< protection for shared lsb region allocation */
 	int qidx;
 	/**< current queue index */
+	int hwrng_retries;
+	/**< retry counter for CCP TRNG */
 } __rte_cache_aligned;
 
 /**< CCP H/W engine related */

@@ -454,4 +457,12 @@ int ccp_probe_devices(const struct rte_pci_id *ccp_id);
  */
 struct ccp_queue *ccp_allot_queue(struct rte_cryptodev *dev, int slot_req);
 
+/**
+ * read hwrng value
+ *
+ * @param trng_value data pointer to write RNG value
+ * @return 0 on success otherwise -1
+ */
+int ccp_read_hwrng(uint32_t *trng_value);
+
 #endif /* _CCP_DEV_H_ */